Index: hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java =================================================================== --- hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java (revision 1552923) +++ hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java (working copy) @@ -159,11 +159,11 @@ private volatile Configuration conf; private final long pause; - private final int numRetries; + protected final int numRetries; // Some operations can take a long time such as disable of big table. // numRetries is for 'normal' stuff... Multiply by this factor when // want to wait a long time. - private final int retryLongerMultiplier; + protected final int retryLongerMultiplier; private boolean aborted; private boolean cleanupConnectionOnClose = false; // close the connection in close() private boolean closed = false; @@ -394,7 +394,7 @@ return getTableDescriptor(TableName.valueOf(tableName)); } - private long getPauseTime(int tries) { + protected long getPauseTime(int tries) { int triesCount = tries; if (triesCount >= HConstants.RETRY_BACKOFF.length) { triesCount = HConstants.RETRY_BACKOFF.length - 1; Index: hbase-client/src/main/java/org/apache/hadoop/hbase/filter/ByteArrayComparable.java =================================================================== --- hbase-client/src/main/java/org/apache/hadoop/hbase/filter/ByteArrayComparable.java (revision 1552923) +++ hbase-client/src/main/java/org/apache/hadoop/hbase/filter/ByteArrayComparable.java (working copy) @@ -32,7 +32,7 @@ @InterfaceStability.Stable public abstract class ByteArrayComparable implements Comparable { - byte[] value; + protected byte[] value; /** * Constructor. Index: hbase-client/src/main/java/org/apache/hadoop/hbase/filter/SingleColumnValueFilter.java =================================================================== --- hbase-client/src/main/java/org/apache/hadoop/hbase/filter/SingleColumnValueFilter.java (revision 1552923) +++ hbase-client/src/main/java/org/apache/hadoop/hbase/filter/SingleColumnValueFilter.java (working copy) @@ -303,7 +303,7 @@ return filter; } - FilterProtos.SingleColumnValueFilter convert() { + public FilterProtos.SingleColumnValueFilter convert() { FilterProtos.SingleColumnValueFilter.Builder builder = FilterProtos.SingleColumnValueFilter.newBuilder(); if (this.columnFamily != null) { Index: hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTableReadOnly.java =================================================================== --- hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTableReadOnly.java (revision 1552923) +++ hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTableReadOnly.java (working copy) @@ -26,11 +26,15 @@ import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos; +import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.Table.State; import org.apache.zookeeper.KeeperException; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; +import java.util.Map.Entry; /** * Non-instantiable class that provides helper functions for @@ -132,7 +136,31 @@ } return disabledTables; } + + /** + * Gets all the tables by state. + * @return map of state vs tables. + * @throws KeeperException + */ + public static Map> + getTableNamesByState(ZooKeeperWatcher zkw) throws KeeperException { + Map> tablesByState = new HashMap>(); + List children = ZKUtil.listChildrenNoWatch(zkw, zkw.tableZNode); + for (String child : children) { + TableName tableName = TableName.valueOf(child); + ZooKeeperProtos.Table.State state = getTableState(zkw, tableName); + if (tablesByState.containsKey(state)) { + tablesByState.get(state).add(tableName); + } else { + HashSet set = new HashSet(); + set.add(tableName); + tablesByState.put(state, set); + } + } + return tablesByState; + } + static boolean isTableState(final ZooKeeperProtos.Table.State expectedState, final ZooKeeperProtos.Table.State currentState) { return currentState != null && currentState.equals(expectedState); Index: hbase-common/src/main/resources/hbase-default.xml =================================================================== --- hbase-common/src/main/resources/hbase-default.xml (revision 1552923) +++ hbase-common/src/main/resources/hbase-default.xml (working copy) @@ -1112,4 +1112,23 @@ as the SimpleLoadBalancer). + + hbase.use.secondary.index + false + Enable this property when you are using secondary index. + + + + hbase.index.half.storefile.reader.class + org.apache.hadoop.hbase.index.io.IndexHalfStoreFileReader + + Do not change this. It is used internally when secondary + index is enabled + + + + hbase.index.loadbalancer.delegator.class + org.apache.hadoop.hbase.master.balancer.StochasticLoadBalancer + + Index: hbase-secondaryindex/pom.xml =================================================================== --- hbase-secondaryindex/pom.xml (revision 0) +++ hbase-secondaryindex/pom.xml (working copy) @@ -0,0 +1,197 @@ + + + + 4.0.0 + + hbase + org.apache.hbase + 0.99.0-SNAPSHOT + .. + + + hbase-secondaryindex + HBase - Secondary Index + Secondary index related funcationality for HBase + + + + + org.apache.maven.plugins + maven-site-plugin + + true + + + + + org.apache.maven.plugins + maven-source-plugin + + + + maven-assembly-plugin + ${maven.assembly.version} + + true + + + + maven-surefire-plugin + + + + secondPartTestsExecution + test + + test + + + true + + + + + + + + + + + org.apache.hbase + hbase-common + + + org.apache.hbase + hbase-client + + + org.apache.hbase + hbase-server + + + org.apache.hbase + hbase-common + test-jar + + + org.apache.hbase + hbase-server + test-jar + + + + commons-lang + commons-lang + + + commons-logging + commons-logging + + + + + + + skipSecondaryIndexTests + + + skipSecondaryIndexTests + + + + true + true + + + + + hadoop-1.1 + + + + hadoop.profile + 1.1 + + + + + org.apache.hadoop + hadoop-core + + + org.apache.hadoop + hadoop-test + + + + + + + hadoop-2.0 + + + + !hadoop.profile + + + + + org.apache.hadoop + hadoop-common + + + org.apache.hadoop + hadoop-mapreduce-client-core + + + + + + + hadoop-3.0 + + + hadoop.profile + 3.0 + + + + 3.0-SNAPSHOT + + + + org.apache.hadoop + hadoop-common + + + org.apache.hadoop + hadoop-mapreduce-client-core + + + + + Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/Column.java =================================================================== --- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/Column.java (revision 0) +++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/Column.java (working copy) @@ -0,0 +1,96 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index; + +import java.io.Serializable; + +import org.apache.hadoop.hbase.util.Bytes; + +public class Column implements Serializable{ + private static final long serialVersionUID = -1958705310924323448L; + + private byte[] cf; + private byte[] qualifier; + private ValuePartition valuePartition = null; + + public Column() { + } + + public Column(byte[] cf, byte[] qualifier) { + this.cf = cf; + this.qualifier = qualifier; + } + + public Column(byte[] cf, byte[] qualifier, ValuePartition vp){ + this.cf = cf; + this.qualifier = qualifier; + this.valuePartition = vp; + } + + public void setFamily(byte[] cf) { + this.cf = cf; + } + + public void setQualifier(byte[] qualifier) { + this.qualifier = qualifier; + } + + public byte[] getFamily() { + return cf; + } + + public byte[] getQualifier() { + return qualifier; + } + + public ValuePartition getValuePartition() { + return this.valuePartition; + } + + public void setValuePartition(ValuePartition vp) { + this.valuePartition = vp; + } + + public boolean equals(Object obj) { + if (!(obj instanceof Column)) + return false; + Column that = (Column) obj; + if (!(Bytes.equals(this.cf, that.cf))) + return false; + if (!(Bytes.equals(this.qualifier, that.qualifier))) + return false; + if (valuePartition == null && that.valuePartition == null) { + return true; + } else if (valuePartition != null && that.valuePartition != null) { + return valuePartition.equals(that.valuePartition); + } else { + return false; + } + } + + public int hashCode() { + int result = Bytes.hashCode(this.cf); + result ^= Bytes.hashCode(this.qualifier); + if (valuePartition != null) result ^= valuePartition.hashCode(); + return result; + } + + public String toString() { + return Bytes.toString(this.cf) + " : " + Bytes.toString(this.qualifier); + } +} Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/ColumnQualifier.java =================================================================== --- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/ColumnQualifier.java (revision 0) +++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/ColumnQualifier.java (working copy) @@ -0,0 +1,234 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +import org.apache.hadoop.hbase.exceptions.DeserializationException; +import org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition.PartitionType; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.io.WritableComparable; + +/** + * + * A ColumnQualifier contains information about column family name,qualifier and maximum value + * length used in index specification. maValueLength is used to make sure the common pattern for all + * the rowkeys in the index table. + * + * ColumnQualifier is used as input for index specification to specify column family and column + * qualifier on which we are going to do indexing. + * + */ +public class ColumnQualifier implements WritableComparable { + + private byte[] cfBytes; + + private byte[] qualifierBytes; + + private int maxValueLength; + + private ValuePartition valuePartition = null; + + private ValueType type; + + public ColumnQualifier() { + // Dummy constructor which is needed for the readFields + } + + public ColumnQualifier(String cf, String qualifier) { + this(Bytes.toBytes(cf), Bytes.toBytes(qualifier)); + } + + public ColumnQualifier(byte[] cf, byte[] qualifier) { + this(cf, qualifier, ValueType.String, 0, null); + } + + public ColumnQualifier(String cf, String qualifier, ValueType type, int maxValueLength) { + this(Bytes.toBytes(cf), Bytes.toBytes(qualifier), type, maxValueLength, null); + } + + public ColumnQualifier(String cf, String qualifier, ValueType type, int maxValueLength, + ValuePartition vp) { + this(Bytes.toBytes(cf), Bytes.toBytes(qualifier), type, maxValueLength, vp); + } + + public ColumnQualifier(byte[] cf, byte[] qualifier, ValueType type, int maxValueLength, + ValuePartition vp) { + this.cfBytes = cf; + this.qualifierBytes = qualifier; + this.type = type; + this.maxValueLength = maxValueLength; + this.valuePartition = vp; + } + + /** + * + * @return Column Family as string + */ + public String getColumnFamilyString() { + return Bytes.toString(this.cfBytes); + } + + /** + * + * @return Column qualifier as string + */ + public String getQualifierString() { + return Bytes.toString(this.qualifierBytes); + } + + /** + * + * @return Column family as byte array + */ + public byte[] getColumnFamily() { + return this.cfBytes; + } + + /** + * + * @return Column qualifier as byte array + */ + public byte[] getQualifier() { + return this.qualifierBytes; + } + + public ValuePartition getValuePartition(){ + return valuePartition; + } + + /** + * + * @param DataInput + * Stream + * @throws IOException + */ + public void readFields(DataInput in) throws IOException { + this.cfBytes = Bytes.readByteArray(in); + this.qualifierBytes = Bytes.readByteArray(in); + this.type = ValueType.valueOf(Bytes.toString(Bytes.readByteArray(in))); + this.maxValueLength = in.readInt(); + PartitionType p = PartitionType.valueOf(in.readUTF()); + try { + if (p.equals(PartitionType.SEPARATOR)) { + this.valuePartition = SeparatorPartition.parseFrom(Bytes.readByteArray(in)); + } else if (p.equals(PartitionType.SPATIAL)) { + this.valuePartition = SpatialPartition.parseFrom(Bytes.readByteArray(in)); + } + } catch (DeserializationException e) { + new IOException(e); + } + } + + /** + * + * @param DataOutput + * stream + * @throws IOException + */ + public void write(DataOutput out) throws IOException { + Bytes.writeByteArray(out, this.cfBytes); + Bytes.writeByteArray(out, this.qualifierBytes); + Bytes.writeByteArray(out, Bytes.toBytes(this.type.name())); + out.writeInt(maxValueLength); + if (this.valuePartition == null) { + out.writeUTF(PartitionType.NONE.name()); + } else { + out.writeUTF(valuePartition.getPartitionType().name()); + Bytes.writeByteArray(out, this.valuePartition.toByteArray()); + } + } + + /** + * + * @param ColumnQualifier + * with whom to compare + * @return return true if both objects are equal otherwise false + */ + @Override + public boolean equals(Object cq) { + if (this == cq) { + return true; + } + if (false == (cq instanceof ColumnQualifier)) { + return false; + } + return this.compareTo((ColumnQualifier) cq) == 0; + } + + /** + * return hashcode of object + */ + public int hashCode() { + int result = Bytes.hashCode(this.cfBytes); + result ^= Bytes.hashCode(this.qualifierBytes); + result ^= this.maxValueLength; + if (valuePartition != null) result ^= valuePartition.hashCode(); + return result; + } + + /** + * + * @param IndexSpecification + * @return int + */ + @Override + public int compareTo(ColumnQualifier cq) { + int diff = 0; + diff = Bytes.compareTo(this.cfBytes, cq.cfBytes); + if (0 == diff) { + diff = Bytes.compareTo(this.qualifierBytes, cq.qualifierBytes); + if (0 == diff) { + if (valuePartition != null && cq.valuePartition != null) { + return valuePartition.compareTo(cq.valuePartition); + } else if (valuePartition == null && cq.valuePartition == null) { + return 0; + } else { + return 1; + } + } + } + return diff; + } + + public int getMaxValueLength() { + return this.maxValueLength; + } + + public ValueType getType(){ + return this.type; + } + + public enum ValueType { + String, + Int, + Float, + Long, + Double, + Short, + Byte, + Char + }; + + //TODO - Include valuePartition also into this + public String toString() { + return "CF : " + getColumnFamilyString() + ",Qualifier : " + getQualifierString(); + } +} Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/Constants.java =================================================================== --- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/Constants.java (revision 0) +++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/Constants.java (working copy) @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index; + +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.util.Bytes; + +/** + * Constants contains all constants used in indexing + * + */ +public class Constants { + + public static final int DEFAULT_NUM_RETRIES = 10; + + public static final long DEFAULT_PAUSE = 1000; + + public static final int DEFAULT_RETRY_LONGER_MULTIPLIER = 10; + + public static final byte[] IDX_COL_FAMILY = Bytes.toBytes("d"); + + public static final byte[] IDX_COL_QUAL = new byte[0]; + + public static final String INDEX_TABLE_SUFFIX = "_idx"; + + public static final int DEF_MAX_INDEX_NAME_LENGTH = 18; + + public static final String INDEX_SPEC = "INDEX_SPEC"; + + public static final String INDEX_BALANCER_DELEGATOR_CLASS = + "hbase.index.loadbalancer.delegator.class"; + + /** + * Use this as a key to specify index details in {@link HTableDescriptor} + * @see HTableDescriptor#setValue(byte[], byte[]) + */ + public static final byte[] INDEX_SPEC_KEY = Bytes.toBytes(INDEX_SPEC); + + /** + * While scan the index(s) to be used can be explicitly passed from client application. + * Use this as the name to pass it in attributes + * @see Scan#setAttribute(String, byte[]) + */ + public static final String INDEX_EXPRESSION = "indexExpression"; + +} Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/GroupingCondition.java =================================================================== --- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/GroupingCondition.java (revision 0) +++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/GroupingCondition.java (working copy) @@ -0,0 +1,22 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index; + +public enum GroupingCondition { + AND, OR +} Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/IndexSpecification.java =================================================================== --- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/IndexSpecification.java (revision 0) +++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/IndexSpecification.java (working copy) @@ -0,0 +1,366 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.EOFException; +import java.io.IOException; +import java.util.LinkedHashSet; +import java.util.Set; + +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.index.ColumnQualifier.ValueType; +import org.apache.hadoop.hbase.index.util.IndexUtils; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.io.WritableComparable; +import org.mortbay.log.Log; + + +/** + * An IndexSpecification should have a unique name and can be created on 1 or more columns (Here + * column refers to columnfamily + qualifier) For each of such column a ColumnQualifier is provided + * which takes the column details. This includes the column family name and qualifier name. The + * additional columns are those columns in the main table whose value also will be captured in the + * secondary index table. + * Index Specfication name should not start with '.' and '-'. + * Can contain alphanumerics and '-','.','_'. + */ + +public class IndexSpecification implements WritableComparable { + + private byte[] name; + + private Set indexColumns = new LinkedHashSet(1); + + private ColumnQualifier lastColumn = null; + + private int totalValueLength = 0; + + private long ttl = -1; + + private int maxVersions = -1; + + // Empty constructor for serialization and deserialization. + public IndexSpecification() { + } + + /** + * @param name should not start with '.' and '-'. + * Can contain alphanumerics and '-','.','_'. + * @throws IllegalArgumentException if invalid table name is provided + */ + public IndexSpecification(String name) { + validateIndexSpecification(Bytes.toBytes(name)); + this.name = Bytes.toBytes(name); + } + + private void validateIndexSpecification(byte[] indexSpecName) { + //throws IllegalArgException if invalid index name is provided + IndexUtils.isLegalIndexName(indexSpecName); + } + + /** + * @param name should not start with '.' and '-'. + * Can contain alphanumerics and '-','.','_'. + * @throws IllegalArgumentException if invalid table name is provided + */ + public IndexSpecification(byte[] name) { + validateIndexSpecification(name); + this.name = name; + } + + /** + * + * @return index name + */ + public String getName() { + return Bytes.toString(this.name); + } + + /** + * @param cf + * column family + * @param qualifier + * @param type + * - If type is specified as null then by default ValueType will be taken as String. + * @param maxValueLength + * @throws IllegalArgumentException + * If column family name and/or qualifier is null or blanks.
+ * If column family name starts with '.',contains control characters or colons. + * @see ValueType + */ + public void addIndexColumn(HColumnDescriptor cf, String qualifier, ValueType type, int maxValueLength) + throws IllegalArgumentException { + type = checkForType(type); + isValidFamilyAndQualifier(cf, qualifier); + maxValueLength = getMaxLength(type, maxValueLength); + ColumnQualifier cq = new ColumnQualifier(cf.getNameAsString(), qualifier, type, maxValueLength); + isNotDuplicateEntry(cq); + formMinTTL(cf); + formMaxVersions(cf); + internalAdd(cq); + } + + private ValueType checkForType(ValueType type) { + if(type == null){ + type = ValueType.String; + } + return type; + } + + private int getMaxLength(ValueType type, int maxValueLength) { + if ((type == ValueType.Int || type == ValueType.Float) && maxValueLength != 4) { + Log.warn("With integer or float datatypes, the maxValueLength has to be 4 bytes"); + return 4; + } + if ((type == ValueType.Double || type == ValueType.Long) && maxValueLength != 8) { + Log.warn("With Double and Long datatypes, the maxValueLength has to be 8 bytes"); + return 8; + } + if ((type == ValueType.Short || type == ValueType.Char) && maxValueLength != 2) { + Log.warn("With Short and Char datatypes, the maxValueLength has to be 2 bytes"); + return 2; + } + if (type == ValueType.Byte && maxValueLength != 1) { + Log.warn("With Byte datatype, the maxValueLength has to be 1 bytes"); + return 1; + } + if (type == ValueType.String && maxValueLength == 0) { + Log.warn("With String datatype, the minimun value length is 2"); + maxValueLength = 2; + } + return maxValueLength; + } + + /** + * @param cf + * Column Family + * @param qualifier + * Column Qualifier + * @param ValuePortion + * vp + * @param type + * Data Type + * @param maxValueLength + * @throws IllegalArgumentException + */ + public void addIndexColumn(HColumnDescriptor cf, String qualifier, ValuePartition vp, + ValueType type, int maxValueLength) throws IllegalArgumentException { + checkForType(type); + isValidFamilyAndQualifier(cf, qualifier); + maxValueLength = getMaxLength(type, maxValueLength); + ColumnQualifier cq = new ColumnQualifier(cf.getNameAsString(), qualifier, type, maxValueLength, vp); + isNotDuplicateEntry(cq); + formMinTTL(cf); + formMaxVersions(cf); + internalAdd(cq); + } + + private void formMinTTL(HColumnDescriptor cf) { + int timeToLive = cf.getTimeToLive(); + if (ttl == -1) { + ttl = timeToLive; + } else if (timeToLive != HConstants.FOREVER && timeToLive != -1) { + if (timeToLive < ttl) { + ttl = timeToLive; + } + } + } + + private void formMaxVersions(HColumnDescriptor cf) { + int maxVersion = cf.getMaxVersions(); + if (maxVersions == -1) { + maxVersions = maxVersion; + } else if (maxVersion != HConstants.FOREVER && maxVersion != -1) { + if (maxVersion < maxVersions) { + maxVersions = maxVersion; + } + } + } + + private void internalAdd(ColumnQualifier cq) { + indexColumns.add(cq); + lastColumn = cq; + totalValueLength += cq.getMaxValueLength(); + } + + /** + * @return List of column specifiers + */ + public Set getIndexColumns() { + return this.indexColumns; + } + + /** + * + * @param cf + * column family + * @param qualifier + * @throws IllegalArgumentException + * If column family name and/or qualifier is null or blanks + * @throws IllegalArgumentException + * If column family name starts with '.',contains control characters or colons + * + */ + private void isValidFamilyAndQualifier(HColumnDescriptor cf, String qualifier) { + if (null == cf || null == qualifier) { + throw new IllegalArgumentException("Column family/qualifier should not be null."); + } + if (StringUtils.isBlank(cf.getNameAsString()) || StringUtils.isBlank(qualifier)) { + throw new IllegalArgumentException("Column family/qualifier should not be blank."); + } + } + + /** + * + * @param ColumnQualifier + * to check duplicate entry + */ + private void isNotDuplicateEntry(ColumnQualifier c) { + if (this.getIndexColumns().contains(c)) { + throw new IllegalArgumentException("Duplicate column family and qualifier " + + "combination should not be present."); + } + } + + /** + * + * @param Data + * Input Stream + * @throws IOException + */ + public void readFields(DataInput in) throws IOException { + this.name = Bytes.readByteArray(in); + try { + IndexUtils.isLegalIndexName(this.name); + } catch (IllegalArgumentException e) { + String msg = "Received unexpected data while parsing the column qualifiers :" + + Bytes.toString(this.name)+"."; + Log.warn(msg+" Could be an non-indexed table."); + throw new EOFException(msg); + } + int indexColsSize = in.readInt(); + indexColumns.clear(); + for (int i = 0; i < indexColsSize; i++) { + ColumnQualifier cq = new ColumnQualifier(); + // Need to revisit this place. May be some other valid value though invalid + // comes up. + try { + cq.readFields(in); + } catch (IllegalArgumentException e) { + throw new EOFException("Received unexpected data while parsing the column qualifiers."); + } + internalAdd(cq); + } + this.maxVersions = in.readInt(); + this.ttl = in.readLong(); + } + + /** + * + * @param Data + * Output Stream + * @throws IOException + */ + public void write(DataOutput out) throws IOException { + Bytes.writeByteArray(out, this.name); + out.writeInt(this.indexColumns.size()); + for (ColumnQualifier cq : this.indexColumns) { + cq.write(out); + } + out.writeInt(maxVersions); + out.writeLong(ttl); + } + + /** + * + * @param IndexSpecification + * @return int + */ + public int compareTo(IndexSpecification o) { + return 0; + } + + public String toString() { + return "Index : " + getName() + ",Index Columns : " + indexColumns; + } + + public boolean equals(Object obj) { + if (obj instanceof IndexSpecification) { + IndexSpecification other = (IndexSpecification) obj; + return Bytes.equals(this.name, other.name); + } + return false; + } + + public int hashCode() { + return Bytes.hashCode(this.name); + } + + public ColumnQualifier getLastColumn() { + return this.lastColumn; + } + + public boolean contains(byte[] family) { + for(ColumnQualifier qual : indexColumns) { + if(Bytes.equals(family, qual.getColumnFamily())){ + return true; + } + } + return false; + } + + public boolean contains(byte[] family, byte[] qualifier) { + if(qualifier == null || qualifier.length == 0) { + return contains(family); + } + for (ColumnQualifier qual : indexColumns) { + if (Bytes.equals(family, qual.getColumnFamily()) + && Bytes.equals(qualifier, qual.getQualifier())) { + return true; + } + } + return false; + } + + public int getTotalValueLength () { + return totalValueLength; + } + + /** + * Return the minimum of the timeToLive specified for the column families in the + * specifed index + * @return + */ + public long getTTL() { + return this.ttl; + } + + /** + * Return the minimum of the maxVersion specified for the column families in the specified + * index + * @return + */ + public int getMaxVersions() { + return this.maxVersions; + } + +} Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/SecIndexLoadBalancer.java =================================================================== --- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/SecIndexLoadBalancer.java (revision 0) +++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/SecIndexLoadBalancer.java (working copy) @@ -0,0 +1,693 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Set; +import java.util.Map.Entry; +import java.util.concurrent.ConcurrentHashMap; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.ClusterStatus; +import org.apache.hadoop.hbase.HBaseIOException; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.index.Constants; +import org.apache.hadoop.hbase.index.util.IndexUtils; +import org.apache.hadoop.hbase.master.LoadBalancer; +import org.apache.hadoop.hbase.master.MasterServices; +import org.apache.hadoop.hbase.master.RegionPlan; +import org.apache.hadoop.hbase.master.balancer.StochasticLoadBalancer; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.util.ReflectionUtils; + +/** + * + * This class is an extension of the load balancer class. It allows to colocate the regions of the + * actual table and the regions of the indexed table. + * + * roundRobinAssignment, retainAssignment -> index regions will follow the actual table regions. + * randomAssignment -> either index table or actual table region will follow each other based on + * which ever comes first. + * + * In case of master failover there is a chance that the znodes of the index table and actual table + * are left behind. Then in that scenario we may get randomAssignment for either the actual table + * region first or the index table region first. + * + */ + +public class SecIndexLoadBalancer implements LoadBalancer { + + private static final Log LOG = LogFactory.getLog(SecIndexLoadBalancer.class); + + private LoadBalancer delegator; + + private MasterServices master; + + private Configuration conf; + + private ClusterStatus clusterStatus; + + private static final Random RANDOM = new Random(System.currentTimeMillis()); + + /** + * Maintains colocation information of user regions and corresponding index regions. + * TODO: Change map key type to TableName. for performance reasons. + */ + private Map> colocationInfo = + new ConcurrentHashMap>(); + + /** + * To find indexed tables or index tables quickly. + */ + private Set indexedAndIndexTables = new HashSet(); + + private Set balancedTables = new HashSet(); + + private boolean stopped = false; + + @Override + public void initialize() throws HBaseIOException { + Class delegatorKlass = + conf.getClass(Constants.INDEX_BALANCER_DELEGATOR_CLASS, StochasticLoadBalancer.class, + LoadBalancer.class); + this.delegator = ReflectionUtils.newInstance(delegatorKlass, conf); + this.delegator.setClusterStatus(clusterStatus); + this.delegator.setMasterServices(this.master); + } + + @Override + public Configuration getConf() { + return conf; + } + + @Override + public void setConf(Configuration configuration) { + this.conf = configuration; + } + + @Override + public void setClusterStatus(ClusterStatus st) { + this.clusterStatus = st; + } + + public Map> getRegionLocation() { + return colocationInfo; + } + + @Override + public void setMasterServices(MasterServices masterServices) { + this.master = masterServices; + } + + @Override + public List balanceCluster(Map> clusterState) + throws HBaseIOException { + synchronized (this.colocationInfo) { + Map> userClusterState = + new HashMap>(1); + Map> indexClusterState = + new HashMap>(1); + boolean balanceByTable = conf.getBoolean("hbase.master.loadbalance.bytable", false); + + TableName tableName = null; + if (balanceByTable) { + // Check and modify the colocation info map based on values of cluster state because we will + // call balancer only when the cluster is in stable state and reliable. + Map regionMap = null; + for (Entry> serverVsRegionList : clusterState.entrySet()) { + ServerName sn = serverVsRegionList.getKey(); + List regionInfos = serverVsRegionList.getValue(); + if (regionInfos.isEmpty()) { + continue; + } + if (!this.indexedAndIndexTables.contains(regionInfos.get(0).getTable())) break; + // Just get the table name from any one of the values in the regioninfo list + if (null == tableName) { + tableName = regionInfos.get(0).getTable(); + regionMap = this.colocationInfo.get(tableName.getNameAsString()); + } + if (regionMap != null) { + for (HRegionInfo hri : regionInfos) { + updateServer(regionMap, sn, hri); + } + } + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Seperating user and index regions of each region server in the cluster."); + } + for (Entry> serverVsRegionList : clusterState.entrySet()) { + ServerName sn = serverVsRegionList.getKey(); + List regionsInfos = serverVsRegionList.getValue(); + List idxRegionsToBeMoved = new ArrayList(); + List uRegionsToBeMoved = new ArrayList(); + for (HRegionInfo hri : regionsInfos) { + if (hri.isMetaRegion()) { + continue; + } + tableName = hri.getTable(); + if (this.indexedAndIndexTables.contains(tableName)) { + // table name may change every time thats why always need to get table entries. + Map regionMap = + this.colocationInfo.get(tableName.getNameAsString()); + if (regionMap != null) { + updateServer(regionMap, sn, hri); + } + } + if (IndexUtils.isIndexTable(tableName)) { + idxRegionsToBeMoved.add(hri); + continue; + } + uRegionsToBeMoved.add(hri); + + } + // there may be dummy entries here if assignments by table is set + userClusterState.put(sn, uRegionsToBeMoved); + indexClusterState.put(sn, idxRegionsToBeMoved); + } + } + List regionPlanList = null; + + if (balanceByTable) { + if (!this.indexedAndIndexTables.contains(tableName)) { + return this.delegator.balanceCluster(clusterState); + } + TableName correspondingTableName = + IndexUtils.isIndexTable(tableName) ? IndexUtils.getActualTableName(tableName) + : TableName.valueOf(IndexUtils.getIndexTableName(tableName)); + // regionPlanList is null means skipping balancing. + if (balancedTables.contains(correspondingTableName)) { + balancedTables.remove(correspondingTableName); + regionPlanList = new ArrayList(1); + Map regionMap = + colocationInfo.get(correspondingTableName.getNameAsString()); + // no previous region plan for user table. + if (null == regionMap) { + if (LOG.isDebugEnabled()) { + LOG.debug("No region plans present for table " + correspondingTableName + '.'); + } + return null; + } + for (Entry e : regionMap.entrySet()) { + regionPlanList.add(new RegionPlan(e.getKey(), null, e.getValue())); + } + List correspondingTablePlans = new ArrayList(1); + // copy of region plan to iterate. + List regionPlanListCopy = new ArrayList(regionPlanList); + if (LOG.isDebugEnabled()) { + LOG.debug("Preparing region plans for table " + tableName + + " from regions' plans of table" + correspondingTableName + "."); + } + return prepareIndexPlan(clusterState, correspondingTablePlans, regionPlanListCopy); + } else { + balancedTables.add(tableName); + regionPlanList = this.delegator.balanceCluster(clusterState); + if (null == regionPlanList) { + if (LOG.isDebugEnabled()) { + LOG.debug(tableName + " regions already balanced."); + } + return null; + } else { + saveRegionPlanList(regionPlanList); + return regionPlanList; + } + } + } else { + regionPlanList = this.delegator.balanceCluster(userClusterState); + if (null == regionPlanList) { + if (LOG.isDebugEnabled()) { + LOG.debug("User region plan is null."); + } + regionPlanList = new ArrayList(1); + } else { + saveRegionPlanList(regionPlanList); + } + List userRegionPlans = new ArrayList(1); + + for (Entry> tableVsRegions : this.colocationInfo + .entrySet()) { + Map regionMap = colocationInfo.get(tableVsRegions.getKey()); + // no previous region plan for user table. + if (null == regionMap) { + if (LOG.isDebugEnabled()) { + LOG.debug("No user table region plans present for index table " + tableName + '.'); + } + } else { + for (Entry e : regionMap.entrySet()) { + userRegionPlans.add(new RegionPlan(e.getKey(), null, e.getValue())); + } + } + } + List regionPlanListCopy = new ArrayList(userRegionPlans); + if (LOG.isDebugEnabled()) { + LOG.debug("Preparing index region plans from user region plans for whole cluster."); + } + return prepareIndexPlan(indexClusterState, regionPlanList, regionPlanListCopy); + } + } + } + + private void updateServer(Map regionMap, ServerName sn, HRegionInfo hri) { + ServerName existingServer = regionMap.get(hri); + if (!sn.equals(existingServer)) { + if (LOG.isDebugEnabled()) { + LOG.debug("There is a mismatch in the existing server name for the region " + hri + + ". Replacing the server " + existingServer + " with " + sn + "."); + } + regionMap.put(hri, sn); + } + } + + // Creates the index region plan based on the corresponding user region plan + private List prepareIndexPlan(Map> indexClusterState, + List regionPlanList, List regionPlanListCopy) { + if (LOG.isDebugEnabled()) { + LOG.debug("Entered prepareIndexPlan"); + } + OUTER_LOOP: for (RegionPlan regionPlan : regionPlanListCopy) { + HRegionInfo hri = regionPlan.getRegionInfo(); + + MIDDLE_LOOP: for (Entry> serverVsRegionList : indexClusterState + .entrySet()) { + List indexRegions = serverVsRegionList.getValue(); + ServerName server = serverVsRegionList.getKey(); + if (regionPlan.getDestination().equals(server)) { + // desination server in the region plan is new and should not be same with this + // server in index cluster state.thats why skipping regions check in this server + continue MIDDLE_LOOP; + } + String actualTableName = null; + + for (HRegionInfo indexRegionInfo : indexRegions) { + String indexTableName = indexRegionInfo.getTable().getNameAsString(); + actualTableName = extractActualTableName(indexTableName); + if (false == hri.getTable().getNameAsString().equals(actualTableName)) { + continue; + } + if (0 != Bytes.compareTo(hri.getStartKey(), indexRegionInfo.getStartKey())) { + continue; + } + RegionPlan rp = new RegionPlan(indexRegionInfo, server, regionPlan.getDestination()); + if (LOG.isDebugEnabled()) { + LOG.debug("Selected server " + regionPlan.getDestination() + + " as destination for region " + indexRegionInfo.getRegionNameAsString() + + "from user region plan."); + } + + updateRegionLocation(indexRegionInfo, regionPlan.getDestination()); + regionPlanList.add(rp); + continue OUTER_LOOP; + } + } + } + regionPlanListCopy.clear(); + // if no user regions to balance then return newly formed index region plan. + if (LOG.isDebugEnabled()) { + LOG.debug("Exited prepareIndexPlan"); + } + return regionPlanList; + } + + private void saveRegionPlanList(List regionPlanList) { + for (RegionPlan regionPlan : regionPlanList) { + HRegionInfo hri = regionPlan.getRegionInfo(); + if (!this.indexedAndIndexTables.contains(hri.getTable())) continue; + if (LOG.isDebugEnabled()) { + LOG.debug("Saving region plan of region " + hri.getRegionNameAsString() + '.'); + } + updateRegionLocation(hri, regionPlan.getDestination()); + } + } + + @Override + public Map> roundRobinAssignment(List regions, + List servers) throws HBaseIOException { + List userRegions = new ArrayList(1); + List indexRegions = new ArrayList(1); + for (HRegionInfo hri : regions) { + seperateUserAndIndexRegion(hri, userRegions, indexRegions); + } + Map> bulkPlan = null; + if (false == userRegions.isEmpty()) { + bulkPlan = this.delegator.roundRobinAssignment(userRegions, servers); + if (null == bulkPlan) { + if (LOG.isDebugEnabled()) { + LOG.debug("No region plan for user regions."); + } + return null; + } + synchronized (this.colocationInfo) { + savePlan(bulkPlan); + } + } + bulkPlan = prepareIndexRegionsPlan(indexRegions, bulkPlan, servers); + return bulkPlan; + } + + private void seperateUserAndIndexRegion(HRegionInfo hri, List userRegions, + List indexRegions) { + if (IndexUtils.isIndexTable(hri.getTable().getNameAsString())) { + indexRegions.add(hri); + return; + } + userRegions.add(hri); + } + + private String extractActualTableName(String indexTableName) { + int endIndex = indexTableName.length() - Constants.INDEX_TABLE_SUFFIX.length(); + return indexTableName.substring(0, endIndex); + } + + private Map> prepareIndexRegionsPlan(List indexRegions, + Map> bulkPlan, List servers) + throws HBaseIOException { + if (null != indexRegions && !indexRegions.isEmpty()) { + if (null == bulkPlan) { + bulkPlan = new ConcurrentHashMap>(1); + } + for (HRegionInfo hri : indexRegions) { + if (LOG.isDebugEnabled()) { + LOG.debug("Preparing region plan for index region " + hri.getRegionNameAsString() + '.'); + } + ServerName destServer = getDestServerForIdxRegion(hri); + List destServerRegions = null; + if (null == destServer) { + destServer = this.randomAssignment(hri, servers); + } + if (null != destServer) { + destServerRegions = bulkPlan.get(destServer); + if (null == destServerRegions) { + destServerRegions = new ArrayList(1); + bulkPlan.put(destServer, destServerRegions); + } + if (LOG.isDebugEnabled()) { + LOG.debug("Server " + destServer + " selected for region " + + hri.getRegionNameAsString() + '.'); + } + destServerRegions.add(hri); + updateRegionLocation(hri, destServer); + } + } + } + return bulkPlan; + } + + private ServerName getDestServerForIdxRegion(HRegionInfo hri) { + // Every time we calculate the table name because in case of master restart the index regions + // may be coming for different index tables. + String indexTableName = hri.getTable().getNameAsString(); + String actualTableName = extractActualTableName(indexTableName); + synchronized (this.colocationInfo) { + Map regionMap = colocationInfo.get(actualTableName); + if (null == regionMap) { + // Can this case come + return null; + } + for (Map.Entry e : regionMap.entrySet()) { + HRegionInfo uHri = e.getKey(); + if (0 == Bytes.compareTo(uHri.getStartKey(), hri.getStartKey())) { + // put index region location if corresponding user region found in regionLocation map. + updateRegionLocation(hri, e.getValue()); + return e.getValue(); + } + } + } + return null; + } + + private void savePlan(Map> bulkPlan) { + for (Entry> e : bulkPlan.entrySet()) { + if (LOG.isDebugEnabled()) { + LOG.debug("Saving user regions' plans for server " + e.getKey() + '.'); + } + for (HRegionInfo hri : e.getValue()) { + if (!this.indexedAndIndexTables.contains(hri.getTable())) continue; + updateRegionLocation(hri, e.getKey()); + } + if (LOG.isDebugEnabled()) { + LOG.debug("Saved user regions' plans for server " + e.getKey() + '.'); + } + } + } + + @Override + public Map> retainAssignment(Map regions, + List servers) throws HBaseIOException { + Map userRegionsMap = new ConcurrentHashMap(1); + List indexRegions = new ArrayList(1); + for (Entry e : regions.entrySet()) { + seperateUserAndIndexRegion(e, userRegionsMap, indexRegions, servers); + } + Map> bulkPlan = null; + if (false == userRegionsMap.isEmpty()) { + bulkPlan = this.delegator.retainAssignment(userRegionsMap, servers); + if (null == bulkPlan) { + if (LOG.isDebugEnabled()) { + LOG.debug("Empty region plan for user regions."); + } + return null; + } + synchronized (this.colocationInfo) { + savePlan(bulkPlan); + } + } + bulkPlan = prepareIndexRegionsPlan(indexRegions, bulkPlan, servers); + return bulkPlan; + } + + private void seperateUserAndIndexRegion(Entry e, + Map userRegionsMap, List indexRegions, + List servers) { + HRegionInfo hri = e.getKey(); + if (IndexUtils.isIndexTable(hri.getTable().getNameAsString())) { + indexRegions.add(hri); + return; + } + if (e.getValue() == null) { + userRegionsMap.put(hri, servers.get(RANDOM.nextInt(servers.size()))); + } else { + userRegionsMap.put(hri, e.getValue()); + } + } + + @Override + public Map immediateAssignment(List regions, + List servers) throws HBaseIOException { + return this.delegator.immediateAssignment(regions, servers); + } + + @Override + public ServerName randomAssignment(HRegionInfo regionInfo, List servers) + throws HBaseIOException { + if (!this.indexedAndIndexTables.contains(regionInfo.getTable())) { + return this.delegator.randomAssignment(regionInfo, servers); + } + ServerName sn = null; + try { + sn = getServerNameFromMap(regionInfo, servers); + } catch (IOException e) { + if (LOG.isDebugEnabled()) { + LOG.debug("Not able to get server name.", e); + } + } catch (InterruptedException e) { + if (LOG.isDebugEnabled()) { + LOG.debug("Interrupted while getting region and location details.", e); + } + } + if (sn == null) { + if (LOG.isDebugEnabled()) { + LOG.debug("No server found for region " + regionInfo.getRegionNameAsString() + '.'); + } + sn = getRandomServer(regionInfo, servers); + } + if (LOG.isDebugEnabled()) { + LOG.debug("Destination server for region " + regionInfo.getRegionNameAsString() + " is " + + ((sn == null) ? "null" : sn.toString()) + '.'); + } + return sn; + } + + private ServerName getRandomServer(HRegionInfo regionInfo, List servers) + throws HBaseIOException { + ServerName sn = null; + sn = this.delegator.randomAssignment(regionInfo, servers); + if (sn == null) { + return null; + } + synchronized (this.colocationInfo) { + updateRegionLocation(regionInfo, sn); + } + return sn; + } + + private ServerName getServerNameFromMap(HRegionInfo regionInfo, List onlineServers) + throws IOException, InterruptedException { + String tableNameOfCurrentRegion = regionInfo.getTable().getNameAsString(); + String correspondingTableName = null; + if (false == tableNameOfCurrentRegion.endsWith(Constants.INDEX_TABLE_SUFFIX)) { + // if the region is user region need to check whether index region plan available or not. + correspondingTableName = tableNameOfCurrentRegion + Constants.INDEX_TABLE_SUFFIX; + } else { + // if the region is index region need to check whether user region plan available or not. + correspondingTableName = extractActualTableName(tableNameOfCurrentRegion); + } + synchronized (this.colocationInfo) { + + // skip if its in both index and user and same server + // I will always have the regionMapWithServerLocation for the correspondingTableName already + // populated. + // Only on the first time both the regionMapWithServerLocation and actualRegionMap may be + // null. + Map regionMapWithServerLocation = this.colocationInfo + .get(correspondingTableName); + Map actualRegionMap = this.colocationInfo + .get(tableNameOfCurrentRegion); + + if (null != regionMapWithServerLocation) { + for (Entry iHri : regionMapWithServerLocation.entrySet()) { + if (0 == Bytes.compareTo(iHri.getKey().getStartKey(), regionInfo.getStartKey())) { + ServerName previousServer = null; + if (null != actualRegionMap) { + previousServer = actualRegionMap.get(regionInfo); + } + ServerName sn = iHri.getValue(); + if (null != previousServer) { + // if servername of index region and user region are same in regionLocation clean + // previous plans and return null + if (previousServer.equals(sn)) { + regionMapWithServerLocation.remove(iHri.getKey()); + actualRegionMap.remove(regionInfo); + if (LOG.isDebugEnabled()) { + LOG.debug("Both user region plan and index region plan " + + "in regionLocation are same for the region." + + regionInfo.getRegionNameAsString() + " The location is " + sn + + ". Hence clearing from regionLocation."); + } + return null; + } + } + if (sn != null && onlineServers.contains(sn)) { + if (LOG.isDebugEnabled()) { + LOG.debug("Updating the region " + regionInfo.getRegionNameAsString() + + " with server " + sn); + } + updateRegionLocation(regionInfo, sn); + return sn; + } else if (sn != null) { + if (LOG.isDebugEnabled()) { + LOG.debug("The location " + sn + " of region " + + iHri.getKey().getRegionNameAsString() + + " is not in online. Selecting other region server."); + } + return null; + } + } + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("No region plans in regionLocation for table " + correspondingTableName); + } + } + return null; + } + } + + @Override + public void updateRegionLocation(HRegionInfo regionInfo, ServerName sn) { + String tableName = regionInfo.getTable().getNameAsString(); + synchronized (this.colocationInfo) { + Map regionMap = this.colocationInfo.get(tableName); + if (null == regionMap) { + if (LOG.isDebugEnabled()) { + LOG.debug("No regions of table " + tableName + " in the region plan."); + } + regionMap = new ConcurrentHashMap(1); + this.colocationInfo.put(tableName, regionMap); + } + regionMap.put(regionInfo, sn); + } + } + + public void clearTableRegionPlans(String tableName) { + if (LOG.isDebugEnabled()) { + LOG.debug("Clearing regions plans from regionLocation for table " + tableName); + } + synchronized (this.colocationInfo) { + this.colocationInfo.remove(tableName); + } + } + + /** + * Add the specified table to indexed tables set. + * @param tableName + */ + public void addIndexedTable(TableName tableName) { + String indexTable = IndexUtils.getIndexTableName(tableName); + this.indexedAndIndexTables.add(tableName); + this.indexedAndIndexTables.add(TableName.valueOf(indexTable)); + } + + /** + * Remove the specified table from indexed tables set. + * @param tableName + */ + public void removeIndexedTable(TableName tableName) { + this.indexedAndIndexTables.remove(tableName); + String indexTable = IndexUtils.getIndexTableName(tableName); + this.indexedAndIndexTables.remove(TableName.valueOf(indexTable)); + } + + @Override + public void removeRegionLocation(HRegionInfo regionInfo) { + String tableName = regionInfo.getTable().getNameAsString(); + synchronized (this.colocationInfo) { + Map regionMap = this.colocationInfo.get(tableName); + if (null == regionMap) { + if (LOG.isDebugEnabled()) { + LOG.debug("No regions of table " + tableName + " in the region plan."); + } + } else { + regionMap.remove(regionInfo); + if (LOG.isDebugEnabled()) { + LOG.debug("The regioninfo "+regionInfo+" removed from the region plan"); + } + } + } + } + + @Override + public boolean isStopped() { + return stopped ; + } + + @Override + public void stop(String why) { + LOG.info("Load Balancer stop requested: "+why); + stopped = true; + } +} Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/SeparatorPartition.java =================================================================== --- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/SeparatorPartition.java (revision 0) +++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/SeparatorPartition.java (working copy) @@ -0,0 +1,190 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index; + + +import org.apache.hadoop.hbase.exceptions.DeserializationException; +import org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos; +import org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition.Builder; +import org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition.PartitionType; +import org.apache.hadoop.hbase.util.Bytes; + +import com.google.protobuf.ByteString; +import com.google.protobuf.ExtensionRegistry; +import com.google.protobuf.InvalidProtocolBufferException; + +/** + * A column value is composed of many values separated using some known separator. + * Part of the column value to be indexed. This class specified how to get that value part. + * Takes the separator so as to split the value and the value position in the split. Note that + * the position index starts from '1' + */ +public class SeparatorPartition extends ValuePartition { + + private byte[] separator; + + private int position; + + public SeparatorPartition(String separator, int position) { + if ((null == separator || separator.length() == 0)) { + throw new IllegalArgumentException("Separator cannot be null"); + } + if ((null != separator) && position == 0) { + throw new IllegalArgumentException("With separator ,the position cannot be zero."); + } + this.separator = Bytes.toBytes(separator); + this.position = position; + } + + public SeparatorPartition(byte[] separator, int position) { + this.separator = separator; + this.position = position; + } + + @Override + public PartitionType getPartitionType() { + return PartitionType.SEPARATOR; + } + + public byte[] getSeparator() { + return this.separator; + } + + public int getPosition() { + return this.position; + } + + @Override + public byte[] getPartOfValue(byte[] value) { + // TODO check this method.. Seems so much of code! + int sepLastKnownPosition = -1; + int sepCurrPositon = -1; + int separatorOccurences = 0; + byte[] kvSubset = new byte[separator.length]; + for (int i = 0; i < value.length;) { + if ((value.length - i) >= separator.length) { + System.arraycopy(value, i, kvSubset, 0, separator.length); + if (Bytes.equals(kvSubset, separator)) { + separatorOccurences++; + sepLastKnownPosition = sepCurrPositon; + sepCurrPositon = i; + i += separator.length; + } else { + i++; + } + if (separatorOccurences < this.position) { + continue; + } + break; + } + break; + } + if (separatorOccurences < this.position - 1) { + return new byte[0]; + } + byte valuePart[] = null; + if (separatorOccurences == this.position - 1) { + if (sepCurrPositon == -1) { + valuePart = value; + } else { + valuePart = new byte[value.length - sepCurrPositon - separator.length]; + System.arraycopy(value, sepCurrPositon + separator.length, valuePart, 0, valuePart.length); + } + return valuePart; + } else if (separatorOccurences == this.position) { + if (sepLastKnownPosition == -1) { + valuePart = new byte[sepCurrPositon]; + System.arraycopy(value, 0, valuePart, 0, valuePart.length); + } else { + valuePart = new byte[sepCurrPositon - sepLastKnownPosition - separator.length]; + System.arraycopy(value, sepLastKnownPosition + separator.length, valuePart, 0, + valuePart.length); + } + return valuePart; + } + return valuePart; + } + + @Override + public ValuePartitionProtos.ValuePartition convert() { + Builder builder = ValuePartitionProtos.ValuePartition.newBuilder(); + builder.setPartitionType(PartitionType.SEPARATOR); + builder + .setExtension( + org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SeparatorPartition.separator, + ByteString.copyFrom(this.separator)); + builder + .setExtension( + org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SeparatorPartition.position, + this.position); + return builder.build(); + } + + @Override + public int compareTo(ValuePartition vp) { + if (!(vp instanceof SeparatorPartition)) return 1; + SeparatorPartition sp = (SeparatorPartition) vp; + int diff = Bytes.compareTo(this.separator, sp.separator); + if (diff == 0) return this.position - sp.position; + return diff; + } + + @Override + public boolean equals(Object that) { + if (this == that) { + return true; + } else if (that instanceof SeparatorPartition) { + SeparatorPartition sp = (SeparatorPartition) that; + return Bytes.compareTo(this.separator, sp.getSeparator()) == 0 + && this.position == sp.getPosition(); + } + return false; + } + + @Override + public int hashCode() { + int result = 13; + result ^= Bytes.hashCode(this.separator); + result ^= this.position; + return result; + } + + public static ValuePartition parseFrom(final byte[] bytes) throws DeserializationException { + ValuePartitionProtos.ValuePartition valuePartition; + try { + ExtensionRegistry registry = ExtensionRegistry.newInstance(); + registry.add(org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SeparatorPartition.separator); + registry.add(org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SeparatorPartition.position); + valuePartition = ValuePartitionProtos.ValuePartition.parseFrom(bytes, registry); + } catch (InvalidProtocolBufferException e) { + throw new DeserializationException(e); + } + return new SeparatorPartition( + valuePartition + .getExtension( + org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SeparatorPartition.separator) + .toByteArray(), + valuePartition + .getExtension(org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SeparatorPartition.position)); + } + + @Override + public byte[] toByteArray() { + return convert().toByteArray(); + } +} Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/SpatialPartition.java =================================================================== --- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/SpatialPartition.java (revision 0) +++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/SpatialPartition.java (working copy) @@ -0,0 +1,136 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index; + + +import org.apache.hadoop.hbase.exceptions.DeserializationException; +import org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos; +import org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition.Builder; +import org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition.PartitionType; + +import com.google.protobuf.ExtensionRegistry; +import com.google.protobuf.InvalidProtocolBufferException; + +/** + * Column value is composed of different values. The value to be indexed is in a known offset + */ +public class SpatialPartition extends ValuePartition { + + private int offset; + + private int length; + + public SpatialPartition(int offset, int length) { + if (offset < 0 ) { + throw new IllegalArgumentException("offset/length cannot be les than 0"); + } + this.offset = offset; + this.length = length; + } + + public int getOffset() { + return offset; + } + + public int getLength() { + return length; + } + + @Override + public PartitionType getPartitionType() { + return PartitionType.SPATIAL; + } + + @Override + public byte[] getPartOfValue(byte[] value) { + if (this.offset >= value.length) { + return new byte[0]; + } else { + int valueLength = (this.offset + this.length > value.length) ? value.length - this.offset + : this.length; + byte[] valuePart = new byte[valueLength]; + System.arraycopy(value, this.offset, valuePart, 0, valueLength); + return valuePart; + } + } + + @Override + public ValuePartitionProtos.ValuePartition convert() { + Builder builder = ValuePartitionProtos.ValuePartition.newBuilder(); + builder.setPartitionType(PartitionType.SPATIAL); + builder + .setExtension( + org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SpatialPartition.length, + this.length); + builder + .setExtension( + org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SpatialPartition.offset, + this.offset); + return builder.build(); + } + + @Override + public int compareTo(ValuePartition vp) { + if (!(vp instanceof SpatialPartition)) return 1; + SpatialPartition sp = (SpatialPartition) vp; + int diff = this.offset - sp.offset; + if (diff == 0) return this.length - sp.length; + return diff; + } + + @Override + public boolean equals(Object that) { + if (this == that) { + return true; + } else if (that instanceof SpatialPartition) { + SpatialPartition sp = (SpatialPartition) that; + return this.offset == sp.getOffset() && this.length == sp.getLength(); + } + return false; + } + + @Override + public int hashCode() { + int result = 13; + result ^= this.offset; + result ^= this.length; + return result; + } + + public static ValuePartition parseFrom(final byte[] bytes) throws DeserializationException { + ValuePartitionProtos.ValuePartition valuePartition = null; + try { + ExtensionRegistry registry = ExtensionRegistry.newInstance(); + registry.add(org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SpatialPartition.offset); + registry.add(org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SpatialPartition.length); + valuePartition = ValuePartitionProtos.ValuePartition.newBuilder().mergeFrom(bytes, registry).build(); + } catch (InvalidProtocolBufferException e) { + throw new DeserializationException(e); + } + return new SpatialPartition( + valuePartition + .getExtension(org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SpatialPartition.offset), + valuePartition + .getExtension(org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SpatialPartition.length)); + } + + @Override + public byte[] toByteArray() { + return convert().toByteArray(); + } +} Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/TableIndices.java =================================================================== --- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/TableIndices.java (revision 0) +++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/TableIndices.java (working copy) @@ -0,0 +1,125 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index; + +import java.io.DataOutput; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import com.google.common.io.ByteArrayDataInput; +import com.google.common.io.ByteArrayDataOutput; +import com.google.common.io.ByteStreams; + + +/** + * TableIndices contains indices to specify index name and column details. There can be one or more + * indices on one table. For each of the index on the table and IndexSpecification is to be created + * and added to the indices. + */ +public class TableIndices { + + private static final Log LOG = LogFactory.getLog(TableIndices.class); + + private List indices = new ArrayList(1); + + + public TableIndices() { + + } + + /** + * + * @param IndexSpecification + * to be added to indices + * @throws IllegalArgumentException + * if duplicate indexes for same table + */ + public void addIndex(IndexSpecification iSpec) throws IllegalArgumentException { + String indexName = iSpec.getName(); + if (null == indexName) { + String message = "Index name should not be null in Index Specification."; + LOG.error(message); + throw new IllegalArgumentException(message); + } + if (true == StringUtils.isBlank(indexName)) { + String message = "Index name should not be blank in Index Specification."; + LOG.error(message); + throw new IllegalArgumentException(message); + } + if (indexName.length() > Constants.DEF_MAX_INDEX_NAME_LENGTH) { + String message = "Index name length should not more than " + Constants.DEF_MAX_INDEX_NAME_LENGTH + + '.'; + LOG.error(message); + throw new IllegalArgumentException(message); + } + for (IndexSpecification is : indices) { + if (is.getName().equals(indexName)) { + String message = "Duplicate index names should not be present for same table."; + LOG.error(message); + throw new IllegalArgumentException(message); + } + } + indices.add(iSpec); + } + + /** + * + * @return IndexSpecification list + */ + public List getIndices() { + return (new ArrayList(this.indices)); + } + + /** + * @param DataOutput + * stream + */ + public void write(DataOutput out) throws IOException { + out.writeInt(this.indices.size()); + for (IndexSpecification index : indices) { + index.write(out); + } + } + + /** + * @param DataInput stream + * @throws IOException + */ + public void readFields(byte[] bytes) throws IOException { + ByteArrayDataInput in = ByteStreams.newDataInput(bytes); + int indicesSize = in.readInt(); + indices.clear(); + for (int i = 0; i < indicesSize; i++) { + IndexSpecification is = new IndexSpecification(); + is.readFields(in); + this.indices.add(is); + } + } + + + public byte[] toByteArray() throws IOException { + ByteArrayDataOutput out = ByteStreams.newDataOutput(); + write(out); + return out.toByteArray(); + } +} Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/ValuePartition.java =================================================================== --- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/ValuePartition.java (revision 0) +++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/ValuePartition.java (working copy) @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index; + +import java.io.Serializable; + +import org.apache.hadoop.hbase.exceptions.DeserializationException; +import org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos; +import org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition.PartitionType; + +/** + * Used to specify the part of a column which needs to be indexed. + */ +public abstract class ValuePartition implements Comparable, Serializable{ + + private static final long serialVersionUID = -3409814164480687975L; + + public abstract PartitionType getPartitionType(); + + public abstract byte[] getPartOfValue(byte[] value); + + public abstract byte[] toByteArray(); + + public abstract ValuePartitionProtos.ValuePartition convert(); + + public static ValuePartition parseFrom(final byte[] bytes) throws DeserializationException { + throw new DeserializationException( + "parseFrom called on base ValuePartition, but should be called on derived type"); + } + +} Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/EqualsExpression.java =================================================================== --- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/EqualsExpression.java (revision 0) +++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/EqualsExpression.java (working copy) @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.client; + +import java.io.Serializable; + +import org.apache.hadoop.hbase.index.Column; + +/** + * Can be used to specify an equals condition on a column associated with an index. + */ +public class EqualsExpression implements Serializable { + + private static final long serialVersionUID = -7130697408286943018L; + + private Column column; + private byte[] value; + + public EqualsExpression(Column column, byte[] value) { + this.column = column; + this.value = value; + } + + public Column getColumn() { + return this.column; + } + + public byte[] getValue() { + return value; + } + + @Override + public String toString() { + return "EqualsExpression : Column[" + this.column + ']'; + } +} Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/IndexAdmin.java =================================================================== --- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/IndexAdmin.java (revision 0) +++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/IndexAdmin.java (working copy) @@ -0,0 +1,221 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.client; + +import java.io.IOException; +import java.io.InterruptedIOException; +import java.net.SocketTimeoutException; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.MasterNotRunningException; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.TableNotEnabledException; +import org.apache.hadoop.hbase.TableNotFoundException; +import org.apache.hadoop.hbase.ZooKeeperConnectionException; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HConnection; +import org.apache.hadoop.hbase.client.MetaScanner; +import org.apache.hadoop.hbase.client.RegionOfflineException; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor; +import org.apache.hadoop.hbase.index.Constants; +import org.apache.hadoop.hbase.index.util.IndexUtils; +import org.apache.hadoop.hbase.util.Bytes; + +/** + * Extension of HBaseAdmin to perform index related operations. This also should be used when we + * create table with index details, and other admin operations on indexed table. + */ +public class IndexAdmin extends HBaseAdmin { + private final Log LOG = LogFactory.getLog(this.getClass().getName()); + + public IndexAdmin(Configuration c) throws IOException { + super(c); + } + + public IndexAdmin(HConnection connection) throws MasterNotRunningException, + ZooKeeperConnectionException { + super(connection); + } + + @Override + public void createTable(final HTableDescriptor desc, byte [][] splitKeys) + throws IOException { + try { + createTableAsync(desc, splitKeys); + } catch (SocketTimeoutException ste) { + LOG.warn("Creating " + desc.getNameAsString() + " took too long", ste); + } + int numRegs = splitKeys == null ? 1 : splitKeys.length + 1; + int prevRegCount = 0; + + MetaScannerVisitorBaseWithTableName userTableVisitor = null; + MetaScannerVisitorBaseWithTableName indexTableVisitor = null; + boolean indexedHTD = desc.getValue(Constants.INDEX_SPEC_KEY) != null; + + for (int tries = 0; tries < this.numRetries * this.retryLongerMultiplier; ++tries) { + + AtomicInteger actualRegCount = null; + // Wait for new table to come on-line + if (userTableVisitor == null) { + userTableVisitor = new MetaScannerVisitorBaseWithTableName(desc.getNameAsString()); + } + actualRegCount = userTableVisitor.getActualRgnCnt(); + actualRegCount.set(0); + MetaScanner.metaScan(getConfiguration(), getConnection(), userTableVisitor, + desc.getTableName()); + if (actualRegCount.get() != numRegs) { + if (tries == this.numRetries * this.retryLongerMultiplier - 1) { + throw new RegionOfflineException("Only " + actualRegCount.get() + " of " + numRegs + + " regions are online; retries exhausted."); + } + try { // Sleep + Thread.sleep(getPauseTime(tries)); + } catch (InterruptedException e) { + throw new InterruptedIOException("Interrupted when opening" + " regions; " + + actualRegCount.get() + " of " + numRegs + " regions processed so far"); + } + if (actualRegCount.get() > prevRegCount) { // Making progress + prevRegCount = actualRegCount.get(); + tries = -1; + } + } else { + if (indexedHTD) { + TableName indexTableName = TableName.valueOf(IndexUtils.getIndexTableName(desc.getName())); + if (indexTableVisitor == null) { + indexTableVisitor = + new MetaScannerVisitorBaseWithTableName(indexTableName.getNameAsString()); + } + actualRegCount = indexTableVisitor.getActualRgnCnt(); + actualRegCount.set(0); + MetaScanner.metaScan(getConfiguration(), getConnection(), indexTableVisitor, indexTableName); + if (actualRegCount.get() != numRegs) { + if (tries == this.numRetries * this.retryLongerMultiplier - 1) { + throw new RegionOfflineException("Only " + actualRegCount.get() + " of " + numRegs + + " regions are online; retries exhausted."); + } + try { // Sleep + Thread.sleep(getPauseTime(tries)); + } catch (InterruptedException e) { + throw new InterruptedIOException("Interrupted when opening" + " regions; " + + actualRegCount.get() + " of " + numRegs + " regions processed so far"); + } + if (actualRegCount.get() > prevRegCount) { // Making progress + prevRegCount = actualRegCount.get(); + tries = -1; + } + } else if (isTableEnabled(indexTableName)) { + return; + } + } else if (isTableEnabled(desc.getName())) { + return; + } + } + } + throw new TableNotEnabledException( + "Retries exhausted while still waiting for table: " + + desc.getNameAsString() + " to be enabled"); + } + + public class MetaScannerVisitorBaseWithTableName implements MetaScannerVisitor { + byte[] tableName = null; + AtomicInteger actualRegCount = new AtomicInteger(0); + + MetaScannerVisitorBaseWithTableName(String tableName) { + this.tableName = Bytes.toBytes(tableName); + } + + AtomicInteger getActualRgnCnt() { + return actualRegCount; + } + + @Override + public void close() throws IOException { + } + + @Override + public boolean processRow(Result rowResult) throws IOException { + HRegionInfo info = HRegionInfo.getHRegionInfo(rowResult); + // If regioninfo is null, skip this row + if (info == null) { + return true; + } + if (!(Bytes.equals(info.getTable().getName(), tableName))) { + return false; + } + ServerName serverName = HRegionInfo.getServerName(rowResult); + // Make sure that regions are assigned to server + if (!(info.isOffline() || info.isSplit()) && serverName != null + && serverName.getHostAndPort() != null) { + actualRegCount.incrementAndGet(); + } + return true; + } + } + + @Override + public boolean isTableEnabled(TableName tableName) throws IOException { + if (!tableExists(tableName)) { + throw new TableNotFoundException(tableName); + } + boolean indexEnabled = getConfiguration().getBoolean("hbase.use.secondary.index", false); + if (!indexEnabled) { + return getConnection().isTableEnabled(tableName); + } else { + boolean isTableEnabled = getConnection().isTableEnabled(tableName); + if (isTableEnabled && !IndexUtils.isIndexTable(tableName.getNameAsString())) { + TableName indexTableName = + TableName.valueOf(IndexUtils.getIndexTableName(tableName.getNameAsString())); + if (getConnection().isTableAvailable(indexTableName)) { + return getConnection().isTableEnabled(indexTableName); + } + return true; + } + return isTableEnabled; + } + } + + @Override + public boolean isTableDisabled(TableName tableName) throws IOException { + if (!tableExists(tableName)) { + throw new TableNotFoundException(tableName); + } + boolean indexEnabled = getConfiguration().getBoolean("hbase.use.secondary.index", false); + if (!indexEnabled) { + return getConnection().isTableDisabled(tableName); + } else { + boolean isTableDisabled = getConnection().isTableDisabled(tableName); + if (isTableDisabled && !IndexUtils.isIndexTable(tableName.getNameAsString())) { + TableName indexTableName = + TableName.valueOf(IndexUtils.getIndexTableName(tableName.getNameAsString())); + if (getConnection().isTableAvailable(indexTableName)) { + return getConnection().isTableDisabled(indexTableName); + } + return true; + } + return isTableDisabled; + } + } + +} Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/IndexExpression.java =================================================================== --- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/IndexExpression.java (revision 0) +++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/IndexExpression.java (working copy) @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.client; + +import java.io.Serializable; + +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.index.Constants; + +/** + * While scan the index(s) to be used can be explicitly passed from client application. + * Objects of IndexExpression can specify the index(s) details. + * + * @see SingleIndexExpression + * @see MultiIndexExpression + * @see NoIndexExpression + * @see Scan#setAttribute(String, byte[]) + * @see Constants#INDEX_EXPRESSION + * @see IndexUtils#toBytes(IndexExpression) + */ +public interface IndexExpression extends Serializable { + +} Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/IndexUtils.java =================================================================== --- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/IndexUtils.java (revision 0) +++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/IndexUtils.java (working copy) @@ -0,0 +1,64 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.client; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; + +/** + * Client side utility class for using Secondary Index. + */ +public class IndexUtils { + + private IndexUtils() { + // Avoid instantiation of this class. + } + + /** + * Utility to convert IndexExpression into byte[]. + * Can be used to pass the IndexExpression in the Scan attributes. + * @see Scan#setAttribute(String, byte[]) + * + * @param indexExpression + * @return byte[] + * @throws IOException + */ + public static byte[] toBytes(IndexExpression indexExpression) throws IOException { + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + ObjectOutputStream oos = new ObjectOutputStream(bos); + oos.writeObject(indexExpression); + return bos.toByteArray(); + } + + /** + * Creates back IndexExpression from byte[] + * @param bytes + * @return + * @throws IOException + * @throws ClassNotFoundException + */ + public static IndexExpression toIndexExpression(byte[] bytes) throws IOException, + ClassNotFoundException { + ByteArrayInputStream bis = new ByteArrayInputStream(bytes); + ObjectInputStream ois = new ObjectInputStream(bis); + return (IndexExpression) ois.readObject(); + } +} Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/MultiIndexExpression.java =================================================================== --- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/MultiIndexExpression.java (revision 0) +++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/MultiIndexExpression.java (working copy) @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.client; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.index.Constants; +import org.apache.hadoop.hbase.index.GroupingCondition; + +/** + * Can be used to group a set of {@link SingleIndexExpression} with AND/OR {@link GroupingCondition} + * + * @see Scan#setAttribute(String, byte[]) + * @see Constants#INDEX_EXPRESSION + * @see IndexUtils#toBytes(IndexExpression) + */ +public class MultiIndexExpression implements IndexExpression { + + private static final long serialVersionUID = 5322668904124942100L; + + private List indexExpressions = new ArrayList(); + + private GroupingCondition groupingCondition; + + public MultiIndexExpression(GroupingCondition condition) { + this.groupingCondition = condition; + } + + public GroupingCondition getGroupingCondition() { + return this.groupingCondition; + } + + public void addIndexExpression(IndexExpression indexExpression) { + this.indexExpressions.add(indexExpression); + } + + public List getIndexExpressions() { + return this.indexExpressions; + } +} Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/NoIndexExpression.java =================================================================== --- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/NoIndexExpression.java (revision 0) +++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/NoIndexExpression.java (working copy) @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.client; + +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.index.Constants; + +/** + * Pass this as the index expression via Scan attribute when a scan query should not use any index + * which is possible to be used + * + * @see Scan#setAttribute(String, byte[]) + * @see Constants#INDEX_EXPRESSION + * @see IndexUtils#toBytes(IndexExpression) + */ +public class NoIndexExpression implements IndexExpression { + + private static final long serialVersionUID = 5445463806972787598L; + +} Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/RangeExpression.java =================================================================== --- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/RangeExpression.java (revision 0) +++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/RangeExpression.java (working copy) @@ -0,0 +1,86 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.client; + +import java.io.Serializable; + +import org.apache.hadoop.hbase.index.Column; + +/** + * Can be used to specify a range condition on a column associated with an index. When the range is + * non closed at one end (to specific upper bound but only lower bound) pass the corresponding bound + * value as null. + */ +public class RangeExpression implements Serializable { + + private static final long serialVersionUID = 8772267632040419734L; + + private Column column; + private byte[] lowerBoundValue; + private byte[] upperBoundValue; + private boolean lowerBoundInclusive; + private boolean upperBoundInclusive; + + public Column getColumn() { + return column; + } + + public byte[] getLowerBoundValue() { + return lowerBoundValue; + } + + public byte[] getUpperBoundValue() { + return upperBoundValue; + } + + public boolean isLowerBoundInclusive() { + return lowerBoundInclusive; + } + + public boolean isUpperBoundInclusive() { + return upperBoundInclusive; + } + + /** + * When the range is non closed at one end (to specific upper bound but only lower bound) pass + * the corresponding bound value as null. + * + * @param column + * @param lowerBoundValue + * @param upperBoundValue + * @param lowerBoundInclusive + * @param upperBoundInclusive + */ + public RangeExpression(Column column, byte[] lowerBoundValue, byte[] upperBoundValue, + boolean lowerBoundInclusive, boolean upperBoundInclusive) { + if (column == null || (lowerBoundValue == null && upperBoundValue == null)) { + throw new IllegalArgumentException(); + } + this.column = column; + this.lowerBoundValue = lowerBoundValue; + this.upperBoundValue = upperBoundValue; + this.lowerBoundInclusive = lowerBoundInclusive; + this.upperBoundInclusive = upperBoundInclusive; + } + + @Override + public String toString() { + return "RangeExpression : Column[" + this.column + "], lowerBoundInclusive : " + + this.lowerBoundInclusive + ", upperBoundInclusive : " + this.upperBoundInclusive; + } +} Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/SingleIndexExpression.java =================================================================== --- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/SingleIndexExpression.java (revision 0) +++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/SingleIndexExpression.java (working copy) @@ -0,0 +1,77 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.client; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.index.Constants; + +/** + * This can specify one index details to be used. The index can be single column or multi column + * index. This index can be used for equals or range condition on columns. For a multi column index, + * there can be 0 or more equals condition associated with this index in the scan and at max one + * range condition (on a column). For an index the columns are in an order. There can not be any + * equals condition specified on columns coming after the range conditioned column as per the + * columns order in the index specification. + * + * @see Scan#setAttribute(String, byte[]) + * @see Constants#INDEX_EXPRESSION + * @see IndexUtils#toBytes(IndexExpression) + */ +public class SingleIndexExpression implements IndexExpression { + + private static final long serialVersionUID = 893160134306193043L; + + private String indexName; + + private List equalsExpressions = new ArrayList(); + + private RangeExpression rangeExpression; + + public SingleIndexExpression(String indexName) { + this.indexName = indexName; + } + + public String getIndexName() { + return indexName; + } + + /** + * This is expected to be called in the order of columns specified in the Index. + * If index is on columns cf1:c1, cf1:c2 and cf2:c3 when creating the SingleIndexExpression + * call this method in the same order as of above + * @param equalsExpression + */ + public void addEqualsExpression(EqualsExpression equalsExpression) { + this.equalsExpressions.add(equalsExpression); + } + + public List getEqualsExpressions() { + return this.equalsExpressions; + } + + public void setRangeExpression(RangeExpression rangeExpression) { + this.rangeExpression = rangeExpression; + } + + public RangeExpression getRangeExpression() { + return this.rangeExpression; + } +} Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/master/IndexMasterObserver.java =================================================================== --- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/master/IndexMasterObserver.java (revision 0) +++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/master/IndexMasterObserver.java (working copy) @@ -0,0 +1,712 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.master; + +import java.io.IOException; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; + +import javax.annotation.PreDestroy; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.DoNotRetryIOException; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableDescriptors; +import org.apache.hadoop.hbase.TableExistsException; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.catalog.MetaReader; +import org.apache.hadoop.hbase.coprocessor.BaseMasterObserver; +import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment; +import org.apache.hadoop.hbase.coprocessor.ObserverContext; +import org.apache.hadoop.hbase.index.Column; +import org.apache.hadoop.hbase.index.ColumnQualifier; +import org.apache.hadoop.hbase.index.ColumnQualifier.ValueType; +import org.apache.hadoop.hbase.index.Constants; +import org.apache.hadoop.hbase.index.IndexSpecification; +import org.apache.hadoop.hbase.index.SecIndexLoadBalancer; +import org.apache.hadoop.hbase.index.TableIndices; +import org.apache.hadoop.hbase.index.coprocessor.regionserver.IndexRegionSplitPolicy; +import org.apache.hadoop.hbase.index.manager.IndexManager; +import org.apache.hadoop.hbase.index.util.IndexUtils; +import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; +import org.apache.hadoop.hbase.master.AssignmentManager; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.master.LoadBalancer; +import org.apache.hadoop.hbase.master.MasterServices; +import org.apache.hadoop.hbase.master.RegionPlan; +import org.apache.hadoop.hbase.master.RegionState; +import org.apache.hadoop.hbase.master.RegionStates; +import org.apache.hadoop.hbase.master.handler.CreateTableHandler; +import org.apache.hadoop.hbase.master.handler.DeleteTableHandler; +import org.apache.hadoop.hbase.master.handler.DisableTableHandler; +import org.apache.hadoop.hbase.master.handler.EnableTableHandler; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; +import org.apache.hadoop.hbase.zookeeper.ZKTable; + +/** + * + * Defines of coprocessor hooks(to support secondary indexing) of operations on + * {@link org.apache.hadoop.hbase.master.HMaster} process. + * + */ + +public class IndexMasterObserver extends BaseMasterObserver { + + private static final Log LOG = LogFactory.getLog(IndexMasterObserver.class.getName()); + + IndexManager idxManager = IndexManager.getInstance(); + + /* + * (non-Javadoc) + * + * @see org.apache.hadoop.hbase.coprocessor.BaseMasterObserver#preCreateTable(org + * .apache.hadoop.hbase.coprocessor.ObserverContext, org.apache.hadoop.hbase.HTableDescriptor, + * org.apache.hadoop.hbase.HRegionInfo[]) + */ + @Override + public void preCreateTable(ObserverContext ctx, + HTableDescriptor desc, HRegionInfo[] regions) throws IOException { + LOG.info("Entered into preCreateTable."); + MasterServices master = ctx.getEnvironment().getMasterServices(); + byte[] indexBytes = desc.getValue(Constants.INDEX_SPEC_KEY); + if (indexBytes != null) { + Map> indexColDetails = + new HashMap>(); + TableName tableName = desc.getTableName(); + checkEndsWithIndexSuffix(tableName); + TableName indexTableName = TableName.valueOf(IndexUtils.getIndexTableName(tableName)); + TableIndices tableIndices = new TableIndices(); + tableIndices.readFields(indexBytes); + List indices = tableIndices.getIndices(); + // Even if indices list is empty,it will create index table also. + if (indices.isEmpty()) { + if (LOG.isDebugEnabled()) { + LOG.debug("Empty indices. Index table may not created" + + " if master goes down in between user table creation"); + } + } + LOG.trace("Checking whether column families in " + + "index specification are in actual table column familes."); + for (IndexSpecification iSpec : indices) { + checkColumnsForValidityAndConsistency(desc, iSpec, indexColDetails); + } + LOG.trace("Column families in index specifications " + "are in actual table column familes."); + + boolean isTableExists = + MetaReader.tableExists(master.getCatalogTracker(), desc.getTableName()); + boolean isIndexTableExists = + MetaReader.tableExists(master.getCatalogTracker(), indexTableName); + + if (isTableExists && isIndexTableExists) { + throw new TableExistsException(desc.getTableName()); + } else if (isIndexTableExists) { + disableAndDeleteTable(master, indexTableName); + } + idxManager.addIndexForTable(desc.getNameAsString(), indices); + } + LOG.info("Exiting from preCreateTable."); + } + + @Override + public void postModifyTableHandler(ObserverContext ctx, + TableName tableName, HTableDescriptor htd) throws IOException { + String table = tableName.getNameAsString(); + MasterServices master = ctx.getEnvironment().getMasterServices(); + List> tableRegionsAndLocations = null; + LOG.info("Entering postModifyTable for the table " + table); + byte[] indexBytes = htd.getValue(Constants.INDEX_SPEC_KEY); + if (indexBytes != null) { + TableDescriptors tableDescriptors = master.getTableDescriptors(); + Map allTableDesc = tableDescriptors.getAll(); + String indexTableName = IndexUtils.getIndexTableName(table); + if (allTableDesc.containsKey(indexTableName)) { + // Do table modification + TableIndices tableIndices = new TableIndices(); + tableIndices.readFields(indexBytes); + List indices = tableIndices.getIndices(); + if (indices.isEmpty()) { + LOG.error("Empty indices are passed to modify the table " + table); + return; + } + IndexManager idxManager = IndexManager.getInstance(); + idxManager.removeIndices(table); + idxManager.addIndexForTable(table, indices); + LOG.info("Successfully updated the indexes for the table " + table + " to " + indices); + } else { + try { + tableRegionsAndLocations = MetaReader.getTableRegionsAndLocations( + master.getCatalogTracker(), tableName, true); + } catch (InterruptedException e) { + LOG.error("Exception while trying to create index table for the existing table " + table); + return; + } + if (tableRegionsAndLocations != null) { + HRegionInfo[] regionInfo = new HRegionInfo[tableRegionsAndLocations.size()]; + for(int i = 0 ; i< tableRegionsAndLocations.size(); i++){ + regionInfo[i] = tableRegionsAndLocations.get(i).getFirst(); + } + + byte[][] splitKeys = getSplitKeys(regionInfo); + createSecondaryIndexTable(htd, splitKeys, master, true); + } + } + } + LOG.info("Exiting postModifyTable for the table " + table); + } + + private void checkColumnsForValidityAndConsistency(HTableDescriptor desc, + IndexSpecification iSpec, Map> indexColDetails) + throws IOException { + Set cqList = iSpec.getIndexColumns(); + if (cqList.isEmpty()) { + String message = + " Index " + iSpec.getName() + + " doesn't contain any columns. Each index should contain atleast one column."; + LOG.error(message); + throw new DoNotRetryIOException(new IllegalArgumentException(message)); + } + for (ColumnQualifier cq : cqList) { + if (null == desc.getFamily(cq.getColumnFamily())) { + String message = "Column family " + cq.getColumnFamilyString() + " in index specification " + + iSpec.getName() + " not in Column families of table " + desc.getNameAsString() + '.'; + LOG.error(message); + throw new DoNotRetryIOException(new IllegalArgumentException(message)); + } + Column column = new Column(cq.getColumnFamily(), cq.getQualifier(), cq.getValuePartition()); + ValueType type = cq.getType(); + int maxlength = cq.getMaxValueLength(); + Pair colDetail = indexColDetails.get(column); + if (null != colDetail) { + if (!colDetail.getFirst().equals(type) || colDetail.getSecond() != maxlength) { + throw new DoNotRetryIOException(new IllegalArgumentException( + "ValueType/max value length of column " + column + + " not consistent across the indices")); + } + } else { + indexColDetails.put(column, new Pair(type, maxlength)); + } + } + } + + private void checkEndsWithIndexSuffix(TableName tableName) throws IOException { + if (IndexUtils.isIndexTable(tableName)) { + String message = "User table name should not be ends with " + Constants.INDEX_TABLE_SUFFIX + + '.'; + LOG.error(message); + throw new DoNotRetryIOException(new IllegalArgumentException(message)); + } + } + + private void disableAndDeleteTable(MasterServices master, TableName tableName) throws IOException { + LOG.error(tableName + " already exists. Disabling and deleting table " + tableName + '.'); + boolean disabled = master.getAssignmentManager().getZKTable().isDisabledTable(tableName); + if (false == disabled) { + LOG.info("Disabling table " + tableName + '.'); + new DisableTableHandler(master, tableName, master.getCatalogTracker(), + master.getAssignmentManager(), master.getTableLockManager(), false).prepare().process(); + if (false == master.getAssignmentManager().getZKTable().isDisabledTable(tableName)) { + throw new DoNotRetryIOException("Table " + tableName + " not disabled."); + } + } + LOG.info("Disabled table " + tableName + '.'); + LOG.info("Deleting table " + tableName + '.'); + new DeleteTableHandler(tableName, master, master).prepare().process(); + if (true == MetaReader.tableExists(master.getCatalogTracker(), tableName)) { + throw new DoNotRetryIOException("Table " + tableName + " not deleted."); + } + LOG.info("Deleted table " + tableName + '.'); + } + + @Override + public void preCreateTableHandler(ObserverContext ctx, + HTableDescriptor desc, HRegionInfo[] regions) throws IOException { + if (desc.getValue(Constants.INDEX_SPEC_KEY) != null) { + LoadBalancer balancer = ctx.getEnvironment().getMasterServices().getAssignmentManager().getBalancer(); + if(balancer instanceof SecIndexLoadBalancer){ + ((SecIndexLoadBalancer) balancer).addIndexedTable(desc.getTableName()); + } + } + } + + @Override + public void postCreateTableHandler(ObserverContext ctx, + HTableDescriptor desc, HRegionInfo[] regions) throws IOException { + LOG.info("Entered into postCreateTableHandler of table " + desc.getNameAsString() + '.'); + if (idxManager.getIndicesForTable(desc.getNameAsString()) != null) { + MasterServices master = ctx.getEnvironment().getMasterServices(); + byte[][] splitKeys = getSplitKeys(regions); + // In case of post call for the index table creation, it wont be + // IndexedHTableDescriptor + createSecondaryIndexTable(desc, splitKeys, master, false); + // if there is any user scenarios + // we can add index datails to index manager + } + LOG.info("Exiting from postCreateTableHandler of table " + desc.getNameAsString() + '.'); + } + + /** + * @param HTableDescriptor + * desc + * @param HRegionInfo + * [] regions + * @param MasterServices + * master + * @throws NotAllMetaRegionsOnlineException + * @throws IOException + */ + private void createSecondaryIndexTable(HTableDescriptor desc, byte[][] splitKeys, + MasterServices master, boolean disableTable) throws NotAllMetaRegionsOnlineException, IOException { + TableName indexTableName = + TableName.valueOf(IndexUtils.getIndexTableName(desc.getNameAsString())); + LOG.info("Creating secondary index table " + indexTableName + " for table " + + desc.getNameAsString() + '.'); + HTableDescriptor indexTableDesc = new HTableDescriptor(indexTableName); + HColumnDescriptor columnDescriptor = new HColumnDescriptor(Constants.IDX_COL_FAMILY); + String dataBlockEncodingAlgo = master.getConfiguration().get("index.data.block.encoding.algo", + "NONE"); + DataBlockEncoding[] values = DataBlockEncoding.values(); + + for (DataBlockEncoding dataBlockEncoding : values) { + if(dataBlockEncoding.toString().equals(dataBlockEncodingAlgo)){ + columnDescriptor.setDataBlockEncoding(dataBlockEncoding); + } + } + + // TODO read this data from a config file?? + columnDescriptor.setBlocksize(8*1024);// 8KB + + indexTableDesc.addFamily(columnDescriptor); + indexTableDesc.setValue(HTableDescriptor.SPLIT_POLICY, IndexRegionSplitPolicy.class.getName()); + LOG.info("Setting the split policy for the Index Table " + indexTableName + " as " + + IndexRegionSplitPolicy.class.getName() + '.'); + HRegionInfo[] newRegions = getHRegionInfos(indexTableDesc, splitKeys); + CreateTableHandler tableHandler = new CreateTableHandler(master, master.getMasterFileSystem(), indexTableDesc, + master.getConfiguration(), newRegions, master); + tableHandler.prepare(); + tableHandler.process(); + // Disable the index table so that when we enable the main table both can be enabled + if (disableTable) { + new DisableTableHandler(master, indexTableName, master.getCatalogTracker(), + master.getAssignmentManager(), master.getTableLockManager(), false).prepare().process(); + } + LOG.info("Created secondary index table " + indexTableName + " for table " + + desc.getNameAsString() + '.'); + } + + private byte[][] getSplitKeys(HRegionInfo[] regions) { + byte[][] splitKeys = null; + if (null != regions && regions.length > 1) { + // for the 1st region always the start key will be empty. We no need to + // pass this as a start key item for the index table because this will + // be added by HBase any way. So if we pass empty, HBase will create one + // extra region with start and end key as empty byte[]. + splitKeys = new byte[regions.length - 1][]; + int i = 0; + for (HRegionInfo region : regions) { + byte[] startKey = region.getStartKey(); + if (startKey.length > 0) { + splitKeys[i++] = startKey; + } + } + } + return splitKeys; + } + + private HRegionInfo[] getHRegionInfos(HTableDescriptor hTableDescriptor, byte[][] splitKeys) { + HRegionInfo[] hRegionInfos = null; + if (splitKeys == null || splitKeys.length == 0) { + hRegionInfos = new HRegionInfo[] { new HRegionInfo(hTableDescriptor.getTableName()) }; + } else { + int numRegions = splitKeys.length + 1; + hRegionInfos = new HRegionInfo[numRegions]; + byte[] startKey = null; + byte[] endKey = null; + for (int i = 0; i < numRegions; i++) { + endKey = (i == splitKeys.length) ? null : splitKeys[i]; + hRegionInfos[i] = new HRegionInfo(hTableDescriptor.getTableName(), startKey, endKey); + startKey = endKey; + } + } + return hRegionInfos; + } + + @Override + public void preAssign(ObserverContext ctx, HRegionInfo hri) + throws IOException { + boolean isRegionInTransition = checkRegionInTransition(ctx, hri); + if (isRegionInTransition) { + LOG.info("Not calling assign for region " + hri.getRegionNameAsString() + + "because the region is already in transition."); + ctx.bypass(); + return; + } + } + + @Override + public void postAssign(ObserverContext ctx, HRegionInfo regionInfo) + throws IOException { + LOG.info("Entering into postAssign of region " + regionInfo.getRegionNameAsString() + '.'); + + if (!IndexUtils.isIndexTable(regionInfo.getTable().getName())) { + MasterServices master = ctx.getEnvironment().getMasterServices(); + LoadBalancer balancer = master.getAssignmentManager().getBalancer(); + AssignmentManager am = master.getAssignmentManager(); + RegionStates regionStates = am.getRegionStates(); + // waiting until user region is removed from transition. + long timeout = master.getConfiguration().getLong("hbase.bulk.assignment.waiton.empty.rit", + 5 * 60 * 1000); + try { + am.waitOnRegionToClearRegionsInTransition(regionInfo, timeout); + } catch (InterruptedException e) { + if (LOG.isDebugEnabled()) { + LOG.debug("Interrupted while region in assignment."); + } + } + ServerName sn = regionStates.getRegionServerOfRegion(regionInfo); + TableName indexTableName = + TableName.valueOf(IndexUtils.getIndexTableName(regionInfo.getTableName())); + List tableRegions = regionStates.getRegionsOfTable(indexTableName); + for (HRegionInfo hRegionInfo : tableRegions) { + if (0 == Bytes.compareTo(hRegionInfo.getStartKey(), regionInfo.getStartKey())) { + am.addPlan(hRegionInfo.getEncodedName(), new RegionPlan(hRegionInfo, null, sn)); + LOG.info("Assigning region " + hRegionInfo.getRegionNameAsString() + " to server " + sn + + '.'); + balancer.updateRegionLocation(hRegionInfo, sn); + am.assign(hRegionInfo, true, false); + break; + } + } + } + LOG.info("Exiting from postAssign " + regionInfo.getRegionNameAsString() + '.'); + } + + @Override + public void preUnassign(ObserverContext ctx, HRegionInfo hri, + boolean force) throws IOException { + boolean isRegionInTransition = checkRegionInTransition(ctx, hri); + if (isRegionInTransition) { + LOG.info("Not calling move for region because region" + hri.getRegionNameAsString() + + " is already in transition."); + ctx.bypass(); + return; + } + } + + @Override + public void preMove(ObserverContext ctx, HRegionInfo hri, + ServerName srcServer, ServerName destServer) throws IOException { + boolean isRegionInTransition = checkRegionInTransition(ctx, hri); + if (isRegionInTransition) { + LOG.info("Not calling move for region " + hri.getRegionNameAsString() + + "because the region is already in transition."); + ctx.bypass(); + return; + } + } + + // This is just an additional precaution. This cannot ensure 100% that the RIT regions + // will not be picked up. + // Because the RIT map that is taken here is the copy of original RIT map and there is + // no sync mechanism also. + private boolean checkRegionInTransition(ObserverContext ctx, + HRegionInfo hri) { + MasterServices master = ctx.getEnvironment().getMasterServices(); + RegionStates regionStates = master.getAssignmentManager().getRegionStates(); + String tableName = hri.getTable().getNameAsString(); + if (!IndexUtils.isIndexTable(tableName)) { + if (regionStates.isRegionInTransition(hri)) { + return true; + } else { + String indexTableName = IndexUtils.getIndexTableName(tableName); + for (Entry region : regionStates.getRegionsInTransition() + .entrySet()) { + HRegionInfo regionInfo = region.getValue().getRegion(); + if (indexTableName.equals(regionInfo.getTable().getNameAsString())) { + if (Bytes.compareTo(hri.getStartKey(), regionInfo.getStartKey()) == 0) { + return true; + } + } + } + } + } + return false; + } + + @Override + public void postMove(ObserverContext ctx, HRegionInfo regionInfo, + ServerName srcServer, ServerName destServer) throws IOException { + LOG.info("Entering into postMove " + regionInfo.getRegionNameAsString() + '.'); + if (!IndexUtils.isIndexTable(regionInfo.getTable().getNameAsString())) { + MasterServices master = ctx.getEnvironment().getMasterServices(); + AssignmentManager am = master.getAssignmentManager(); + // waiting until user region is removed from transition. + long timeout = master.getConfiguration().getLong("hbase.bulk.assignment.waiton.empty.rit", + 5 * 60 * 1000); + try { + am.waitOnRegionToClearRegionsInTransition(regionInfo, timeout); + destServer = am.getRegionStates().getRegionServerOfRegion(regionInfo); + am.getBalancer().updateRegionLocation(regionInfo, destServer); + } catch (InterruptedException e) { + if (LOG.isDebugEnabled()) { + LOG.debug("Interrupted while region in assignment."); + } + } + TableName indexTableName = + TableName.valueOf(IndexUtils.getIndexTableName(regionInfo.getTable().getNameAsString())); + List tableRegions = am.getRegionStates().getRegionsOfTable(indexTableName); + for (HRegionInfo indexRegionInfo : tableRegions) { + if (0 == Bytes.compareTo(indexRegionInfo.getStartKey(), regionInfo.getStartKey())) { + LOG.info("Assigning region " + indexRegionInfo.getRegionNameAsString() + "from " + srcServer + + " to server " + destServer + '.'); + am.getBalancer().updateRegionLocation(indexRegionInfo, destServer); + am.addPlan(indexRegionInfo.getEncodedName(), new RegionPlan(indexRegionInfo, null, + destServer)); + am.unassign(indexRegionInfo); + } + } + } + LOG.info("Exiting from postMove " + regionInfo.getRegionNameAsString() + '.'); + } + + @Override + public void postDisableTableHandler(ObserverContext ctx, + TableName tableName) throws IOException { + LOG.info("Entered into postDisableTableHandler of table " + tableName); + MasterServices master = ctx.getEnvironment().getMasterServices(); + AssignmentManager am = master.getAssignmentManager(); + try { + if (!IndexUtils.isIndexTable(tableName.getNameAsString())) { + TableName indexTableName = TableName.valueOf(IndexUtils.getIndexTableName(tableName)); + // Index table may not present following three cases. + // 1) Index details are not specified during table creation then index table wont be + // created. + // 2) Even we specify index details if master restarted in the middle of user table creation + // corresponding index table wont be created. But without creating index table user table + // wont + // be disabled. No need to call disable for index table at that time. + // 3) Index table may be deleted but this wont happen without deleting user table. + if (am.getZKTable().isTablePresent(indexTableName)) { + long timeout = master.getConfiguration().getLong( + "hbase.bulk.assignment.waiton.empty.rit", 5 * 60 * 1000); + // Both user table and index table should not be in enabling/disabling state at a time. + // If disable is progress for user table then index table should be in ENABLED state. + // If enable is progress for index table wait until table enabled. + if (waitUntilTableEnabled(timeout, indexTableName, am.getZKTable())) { + new DisableTableHandler(master, indexTableName, + master.getCatalogTracker(), am, master.getTableLockManager(), false).process(); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Table " + indexTableName + " not in ENABLED state to disable."); + } + } + } + } + } finally { + // clear user table region plans in secondary index load balancer. + clearRegionPlans((HMaster) master, tableName.getNamespaceAsString()); + } + LOG.info("Exiting from postDisableTableHandler of table " + tableName); + } + + private void clearRegionPlans(HMaster master, String tableName) { + AssignmentManager am = master.getAssignmentManager(); + ((SecIndexLoadBalancer) am.getBalancer()).clearTableRegionPlans(tableName); + } + + @Override + public void postEnableTableHandler(ObserverContext ctx, + TableName tableName) throws IOException { + LOG.info("Entered into postEnableTableHandler of table " + tableName); + if (!IndexUtils.isIndexTable(tableName.getNameAsString())) { + MasterServices master = ctx.getEnvironment().getMasterServices(); + AssignmentManager am = master.getAssignmentManager(); + TableName indexTableName = TableName.valueOf(IndexUtils.getIndexTableName(tableName)); + // Index table may not present in three cases + // 1) Index details are not specified during table creation then index table wont be created. + // 2) Even we specify index details if master restarted in the middle of user table creation + // corresponding index table wont be created. Then no need to call enable for index table + // because it will be created as part of preMasterInitialization and enable. + // 3) Index table may be deleted but this wont happen without deleting user table. + if (true == am.getZKTable().isTablePresent(indexTableName)) { + long timeout = master.getConfiguration().getLong("hbase.bulk.assignment.waiton.empty.rit", + 5 * 60 * 1000); + // Both user table and index table should not be in enabling/disabling state at a time. + // If enable is progress for user table then index table should be in disabled state. + // If disable is progress for index table wait until table disabled. + if (waitUntilTableDisabled(timeout, indexTableName, am.getZKTable())) { + new EnableTableHandler(master, indexTableName, master.getCatalogTracker(), + am, master.getTableLockManager(), false).prepare().process(); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Table " + indexTableName + " not in DISABLED state to enable."); + } + } + } + } + LOG.info("Exiting from postEnableTableHandler of table " + tableName); + + } + + private boolean waitUntilTableDisabled(long timeout, TableName tableName, ZKTable zk) { + long startTime = System.currentTimeMillis(); + long remaining = timeout; + boolean disabled = false; + while (!(disabled = zk.isDisabledTable(tableName)) && remaining > 0) { + try { + Thread.sleep(100); + } catch (InterruptedException e) { + if (LOG.isDebugEnabled()) { + LOG.debug("Interrupted while waiting for table" + tableName + " set to DISABLED."); + } + } + remaining = timeout - (System.currentTimeMillis() - startTime); + } + if (remaining <= 0) { + return disabled; + } else { + return true; + } + } + + private boolean waitUntilTableEnabled(long timeout, TableName tableName, ZKTable zk) { + long startTime = System.currentTimeMillis(); + long remaining = timeout; + boolean enabled = false; + while (!(enabled = zk.isEnabledTable(tableName)) && remaining > 0) { + try { + Thread.sleep(100); + } catch (InterruptedException e) { + if (LOG.isDebugEnabled()) { + LOG.debug("Interrupted while waiting for table " + tableName + "state set to ENABLED."); + } + } + remaining = timeout - (System.currentTimeMillis() - startTime); + } + if (remaining <= 0) { + return enabled; + } else { + return true; + } + } + + @Override + public void postDeleteTableHandler(ObserverContext ctx, + TableName tableName) throws IOException { + LOG.info("Entered into postDeleteTableHandler of table " + tableName + '.'); + MasterServices master = ctx.getEnvironment().getMasterServices(); + TableName indexTableName = TableName.valueOf(IndexUtils.getIndexTableName(tableName)); + boolean indexTablePresent = + master.getAssignmentManager().getZKTable().isTablePresent(indexTableName); + // Not checking for disabled state because before deleting user table both user and index table + // should be disabled. + if ((!IndexUtils.isIndexTable(tableName)) && indexTablePresent) { + LoadBalancer balancer = master.getAssignmentManager().getBalancer(); + if (balancer instanceof SecIndexLoadBalancer) { + ((SecIndexLoadBalancer) balancer).removeIndexedTable(tableName); + } + DeleteTableHandler dth = new DeleteTableHandler(indexTableName, master, master); + dth.prepare(); + dth.process(); + } + LOG.info("Exiting from postDeleteTableHandler of table " + tableName + '.'); + } + + @Override + public void preMasterInitialization(ObserverContext ctx) + throws IOException { + LOG.info("Entering into preMasterInitialization."); + MasterServices master = ctx.getEnvironment().getMasterServices(); + AssignmentManager am = master.getAssignmentManager(); + ZKTable zkTable = am.getZKTable(); + long timeout = master.getConfiguration().getLong("hbase.bulk.assignment.waiton.empty.rit", + 5 * 60 * 1000); + try { + am.waitUntilNoRegionsInTransition(timeout); + } catch (InterruptedException e) { + if (LOG.isDebugEnabled()) { + LOG.debug("Interrupted while waiting for the regions in transition to complete.", e); + } + } + + TableDescriptors tableDescriptors = master.getTableDescriptors(); + Map descMap = tableDescriptors.getAll(); + Collection htds = descMap.values(); + for (HTableDescriptor htd : htds) { + if (!IndexUtils.isIndexTable(htd.getNameAsString()) && !htd.isMetaRegion() + && !htd.getTableName().isSystemTable()) { + if (htd.getValue(Constants.INDEX_SPEC_KEY)!=null) { + TableName tableName = htd.getTableName(); + TableName indexTableName = TableName.valueOf(IndexUtils.getIndexTableName(tableName)); + // TODO: check table exists from table descriptors list. + boolean tableExists = MetaReader.tableExists(master.getCatalogTracker(), tableName); + boolean indexTableExists = MetaReader.tableExists(master.getCatalogTracker(), + indexTableName); + if ((true == tableExists) && (false == indexTableExists)) { + LOG.info("Table has index specification details but " + "no corresponding index table."); + List regions = MetaReader.getTableRegions(master.getCatalogTracker(), + tableName); + HRegionInfo[] regionsArray = new HRegionInfo[regions.size()]; + byte[][] splitKeys = getSplitKeys(regions.toArray(regionsArray)); + createSecondaryIndexTable(htd, splitKeys, master, zkTable.isDisabledTable(tableName)); + } else if (true == tableExists && true == indexTableExists) { + // If both tables are present both should be in same state in zookeeper. If tables are + // partially enabled or disabled they will be processed as part of recovery + // enabling/disabling tables. + // if user table is in ENABLED state and index table is in DISABLED state means master + // restarted as soon as user table enabled. So here we need to enable index table. + if (zkTable.isEnabledTable(tableName) && zkTable.isDisabledTable(indexTableName)) { + new EnableTableHandler(master, indexTableName, + master.getCatalogTracker(), am, master.getTableLockManager(), false).prepare().process(); + } else if (zkTable.isDisabledTable(tableName) && zkTable.isEnabledTable(indexTableName)) { + // If user table is in DISABLED state and index table is in ENABLED state means master + // restarted as soon as user table disabled. So here we need to disable index table. + new DisableTableHandler(master, indexTableName, + master.getCatalogTracker(), am, master.getTableLockManager(), false).prepare().process(); + // clear index table region plans in secondary index load balancer. + clearRegionPlans((HMaster) master, indexTableName.getNameAsString()); + } + } + } + } + } + if (LOG.isDebugEnabled()) { + LOG.debug("Balancing after master initialization."); + } + + try { + master.getAssignmentManager().waitUntilNoRegionsInTransition(timeout); + } catch (InterruptedException e) { + if (LOG.isDebugEnabled()) { + LOG.debug("Interrupted while waiting for the regions in transition to complete.", e); + } + } + ((HMaster) master).balanceInternals(); + LOG.info("Exiting from preMasterInitialization."); + } + +} Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/BackwardSeekableRegionScanner.java =================================================================== --- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/BackwardSeekableRegionScanner.java (revision 0) +++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/BackwardSeekableRegionScanner.java (working copy) @@ -0,0 +1,142 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import java.io.IOException; +import java.util.List; + +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.RegionScanner; + +public class BackwardSeekableRegionScanner implements SeekAndReadRegionScanner { + + private ReInitializableRegionScanner delegator; + + private Scan scan; + + private HRegion hRegion; + + private byte[] startRow; + + private boolean closed = false; + + public BackwardSeekableRegionScanner(ReInitializableRegionScanner delegator, + Scan scan, HRegion hRegion, byte[] startRow) { + this.delegator = delegator; + this.scan = scan; + this.hRegion = hRegion; + this.startRow = startRow; + } + + Scan getScan() { + return scan; + } + + byte[] getStartRow() { + return startRow; + } + + // For testing. + RegionScanner getDelegator() { + return delegator; + } + + @Override + public HRegionInfo getRegionInfo() { + return this.delegator.getRegionInfo(); + } + + @Override + public boolean isFilterDone() throws IOException { + return this.delegator.isFilterDone(); + } + + @Override + public synchronized void close() throws IOException { + this.delegator.close(); + closed = true; + } + + @Override + public boolean isClosed() { + return closed; + } + @Override + public synchronized boolean next(List results) throws IOException { + return next(results, this.scan.getBatch()); + } + + @Override + public boolean next(List result, int limit) throws IOException { + boolean hasNext = false; + try { + if(this.delegator.isClosed()) return false; + hasNext = this.delegator.next(result, limit); + } catch (SeekUnderValueException e) { + Scan newScan = new Scan(this.scan); + // Start from the point where we got stopped because of seek backward + newScan.setStartRow(getLatestSeekpoint()); + this.delegator.reInit(this.hRegion.getScanner(newScan)); + hasNext = next(result, limit); + } + return hasNext; + } + + @Override + public synchronized boolean reseek(byte[] row) throws IOException { + return this.delegator.reseek(row); + } + + @Override + public void addSeekPoints(List seekPoints) { + this.delegator.addSeekPoints(seekPoints); + } + + @Override + public boolean seekToNextPoint() throws IOException { + return this.delegator.seekToNextPoint(); + } + + @Override + public byte[] getLatestSeekpoint() { + return this.delegator.getLatestSeekpoint(); + } + + @Override + public long getMaxResultSize() { + return this.delegator.getMaxResultSize(); + } + + @Override + public long getMvccReadPoint() { + return this.delegator.getMvccReadPoint(); + } + + @Override + public boolean nextRaw(List result) throws IOException { + return next(result); + } + + @Override + public boolean nextRaw(List result, int limit) throws IOException { + return next(result, limit); + } +} Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/FilterColumnValueDetail.java =================================================================== --- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/FilterColumnValueDetail.java (revision 0) +++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/FilterColumnValueDetail.java (working copy) @@ -0,0 +1,99 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; +import org.apache.hadoop.hbase.index.Column; +import org.apache.hadoop.hbase.index.ColumnQualifier.ValueType; +import org.apache.hadoop.hbase.index.ValuePartition; +import org.apache.hadoop.hbase.util.Bytes; + +public class FilterColumnValueDetail { + protected byte[] cf; + protected byte[] qualifier; + protected byte[] value; + protected CompareOp compareOp; + protected Column column; + protected int maxValueLength; + protected ValueType valueType; + + public FilterColumnValueDetail(byte[] cf, byte[] qualifier, byte[] value, CompareOp compareOp) { + this.cf = cf; + this.qualifier = qualifier; + this.value = value; + this.compareOp = compareOp; + this.column = new Column(this.cf, this.qualifier); + } + + public FilterColumnValueDetail(byte[] cf, byte[] qualifier, byte[] value, + ValuePartition valuePartition, CompareOp compareOp) { + this.cf = cf; + this.qualifier = qualifier; + this.value = value; + this.compareOp = compareOp; + this.column = new Column(this.cf, this.qualifier, valuePartition); + } + + public FilterColumnValueDetail(Column column, byte[] value, CompareOp compareOp) { + this.cf = column.getFamily(); + this.qualifier = column.getQualifier(); + this.value = value; + this.compareOp = compareOp; + this.column = column; + } + + public boolean equals(Object obj) { + if (!(obj instanceof FilterColumnValueDetail)) return false; + FilterColumnValueDetail that = (FilterColumnValueDetail) obj; + if (!this.column.equals(that.column)) { + return false; + } + // Need to check. + if (this.value != null && that.value != null) { + if (!(Bytes.equals(this.value, that.value))) return false; + } else if (this.value == null && that.value == null) { + return true; + } else { + return false; + } + return true; + } + + public int hashCode() { + return this.column.hashCode(); + } + + public String toString() { + return String.format("%s (%s, %s, %s, %s, %s)", this.getClass().getSimpleName(), + Bytes.toStringBinary(this.cf), Bytes.toStringBinary(this.qualifier), this.valueType.name(), + this.compareOp.name(), Bytes.toStringBinary(this.value)); + } + + public Column getColumn() { + return this.column; + } + + public byte[] getValue(){ + return this.value; + } + + protected void setValue(byte[] value) { + this.value = value; + } + +} \ No newline at end of file Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/FilterColumnValueRange.java =================================================================== --- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/FilterColumnValueRange.java (revision 0) +++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/FilterColumnValueRange.java (working copy) @@ -0,0 +1,96 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; +import org.apache.hadoop.hbase.index.Column; +import org.apache.hadoop.hbase.index.ValuePartition; +import org.apache.hadoop.hbase.util.Bytes; + +public class FilterColumnValueRange extends FilterColumnValueDetail { + private CompareOp upperBoundCompareOp; + private byte[] upperBoundValue; + + public FilterColumnValueRange(byte[] cf, byte[] qualifier, byte[] lowerBoundValue, + CompareOp lowerBoundCompareOp, byte[] upperBoundValue, CompareOp upperBoundCompareOp) { + super(cf, qualifier, lowerBoundValue, lowerBoundCompareOp); + this.upperBoundCompareOp = upperBoundCompareOp; + this.upperBoundValue = upperBoundValue; + } + + public FilterColumnValueRange(byte[] cf, byte[] qualifier,ValuePartition vp, byte[] lowerBoundValue, + CompareOp lowerBoundCompareOp, byte[] upperBoundValue, CompareOp upperBoundCompareOp) { + super(cf, qualifier, lowerBoundValue, vp, lowerBoundCompareOp); + this.upperBoundCompareOp = upperBoundCompareOp; + this.upperBoundValue = upperBoundValue; + } + + public FilterColumnValueRange(Column column, byte[] lowerBoundValue, + CompareOp lowerBoundCompareOp, byte[] upperBoundValue, CompareOp upperBoundCompareOp) { + super(column, lowerBoundValue, lowerBoundCompareOp); + this.upperBoundCompareOp = upperBoundCompareOp; + this.upperBoundValue = upperBoundValue; + } + + // No need to have the hashCode() and equals(Object obj) implementation here. Super class + // implementation checks for the CF name and qualifier name which is sufficient. + + public String toString() { + return String.format("%s (%s, %s, %s, %s, %s, %s, %s)", this.getClass().getSimpleName(), Bytes + .toStringBinary(this.cf), Bytes.toStringBinary(this.qualifier), this.valueType.name(), + this.compareOp.name(), Bytes.toStringBinary(this.value), + this.upperBoundCompareOp == null ? "" : this.upperBoundCompareOp.name(), + this.upperBoundValue == null ? "" : Bytes.toStringBinary(this.upperBoundValue)); + } + + public CompareOp getUpperBoundCompareOp() { + return this.upperBoundCompareOp; + } + + public byte[] getUpperBoundValue() { + return this.upperBoundValue; + } + + //The equals method is a bit tricky. + // Needs one more eye on this + @Override + public boolean equals(Object other) { + if (this == other) + return true; + FilterColumnValueRange fcvr = null; + if (other instanceof FilterColumnValueRange && super.equals(other)) { + fcvr = (FilterColumnValueRange) other; + if (this.upperBoundValue != null && fcvr.getUpperBoundValue() != null) { + if (Bytes.compareTo(this.upperBoundValue, fcvr.getUpperBoundValue()) != 0) { + return false; + } + } else if (this.upperBoundValue == null && fcvr.getUpperBoundValue() == null) { + return true; + } else { + return false; + } + } + return false; + } + + @Override + public int hashCode() { + return super.hashCode(); + } + +} \ No newline at end of file Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/FilterGroupingWorker.java =================================================================== --- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/FilterGroupingWorker.java (revision 0) +++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/FilterGroupingWorker.java (working copy) @@ -0,0 +1,542 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; +import org.apache.hadoop.hbase.filter.Filter; +import org.apache.hadoop.hbase.filter.FilterList; +import org.apache.hadoop.hbase.filter.FilterList.Operator; +import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; +import org.apache.hadoop.hbase.index.Column; +import org.apache.hadoop.hbase.index.ValuePartition; +import org.apache.hadoop.hbase.index.filter.SingleColumnRangeFilter; +import org.apache.hadoop.hbase.index.filter.SingleColumnValuePartitionFilter; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; + +/** + * This class does the grouping work for different filterlists. + * Like if we i have 3 different filter list and all have MUST_PASS_ALL condition + * finally this class groups the filters lists into one filter list with all as MUST_PASS_ALL. + * Also it checks if the combination of the filters list is given correctly. + * For eg: If i have c1 > 10 and c1 < 5. This is wrong combination. + */ +public class FilterGroupingWorker { + + private static final Log LOG = LogFactory.getLog(FilterGroupingWorker.class); + + private Map> colWithOperators = + new HashMap>(); + private Map> colWithOperatorsOfOR = new HashMap>(); + + public Filter group(Filter filter) { + if (filter instanceof FilterList) { + FilterList fList = (FilterList) filter; + // We need to create a new FL here taking up only the filters of our interest + FilterList newFList = new FilterList(fList.getOperator()); + List filters = fList.getFilters(); + if (fList.getOperator() == Operator.MUST_PASS_ONE) { + for (Filter subFilter : filters) { + Filter resultFilter = handleFilterWithinOR(subFilter); + // If result filter is not SingleColumnValueFilter or filter list that means OR branch is + // having different type of filter other than SCVF. In that case we should not consider + // the OR branch for scanning. + if (resultFilter instanceof FilterList) { + newFList.addFilter(resultFilter); + } else if (resultFilter != null) { + // This means OR filter list have at least one filter other than SCVF(may be other + // child OR branches). + return null; + } + } + addORColsToFinalList(newFList); + if (newFList.getFilters().isEmpty()) { + return null; + } + return newFList; + } else { + // AND condition as long as the condition is AND in one sub tree all those can be + // grouped under one AND parent(new one). + for (Filter subFilter : filters) { + Filter group = handleFilterWithinAND(subFilter); + // group is null means, all are AND conditions and will be handled at once with the + // below createFinalFilter + if (group != null) { + newFList.addFilter(group); + } + } + addANDColsToFinalList(newFList); + if (newFList.getFilters().isEmpty()) { + return null; + } + return newFList; + } + } else if (filter instanceof SingleColumnValueFilter + || filter instanceof SingleColumnRangeFilter) { + return filter; + } + return null; + } + + private Filter handleFilterWithinAND(Filter filter) { + if (filter instanceof FilterList) { + FilterList fList = (FilterList) filter; + if (fList.getOperator() == Operator.MUST_PASS_ONE) { + return new FilterGroupingWorker().group(fList); + } else { + List filters = fList.getFilters(); + for (Filter subFilter : filters) { + handleFilterWithinAND(subFilter); + } + } + } else if (filter instanceof SingleColumnValueFilter) { + handleScvf((SingleColumnValueFilter) filter); + } // TODO when we expose SingleColumnRangeFilter to handle that also here. + return null; + } + + /** + * Since you can use Filter Lists as children of Filter Lists, you can create a hierarchy of + * filters to be evaluated. In the hierarchy if OR branch having any filter type other than + * SCVF as child then we should not consider the branch for scanning because we cannot fetch + * seek points from other type of filters without column and value details. + * + * Ex: AND AND + * __________|_______ | + * | | --> SCVF + * OR SCVF + * _______|______ + * | | + * ROWFILTER SVCF + * If the OR is root then we should skip index table scanning for this filter. + * OR + * _______|______ --> null + * | | + * ROWFILTER SVCF + * + * If the OR is child of another OR branch then parent OR branch will be excluded for scanning. + * Ex: + * AND AND + * __________|_______ | + * | | --> SCVF + * OR SCVF + * _______|______ + * | | + * OR SVCF + * _______|______ + * | | + * ROWFILTER SVCF + * + * + * @param filter + * @return if filter is filter list with AND condition then we will return AND branch after grouping. + * if filter is filter list with OR condition return null if no children is of type other + * than SCVF or filter list else return different filter. + * if filter is SCVF then return null. + * + * returning null means we are combining the filter(s) with children of parent OR filter + * to perform optimizations. + */ + private Filter handleFilterWithinOR(Filter filter) { + if (filter instanceof FilterList) { + FilterList fList = (FilterList) filter; + if (fList.getOperator() == Operator.MUST_PASS_ONE) { + List filters = fList.getFilters(); + Filter resultFilter = null; + for (Filter subFilter : filters) { + // If this OR branch in the filter list have filter type other than SCVF we should report + // it to parent by returning the other type of filter in such a way that the branch will + // be skipped from index scan. + resultFilter = handleFilterWithinOR(subFilter); + if (resultFilter == null || (resultFilter instanceof FilterList)) { + continue; + } else { + return resultFilter; + } + } + return null; + } else { + return new FilterGroupingWorker().group(fList); + } + } else if (filter instanceof SingleColumnValueFilter) { + handleScvfOfOR((SingleColumnValueFilter) filter); + return null; + }// TODO when we expose SingleColumnRangeFilter to handle that also here. + // filter other than SingleColumnValueFilter. + return filter; + } + + private void handleScvfOfOR(SingleColumnValueFilter scvf) { + ValuePartition vp = null; + if (scvf instanceof SingleColumnValuePartitionFilter) { + vp = ((SingleColumnValuePartitionFilter) scvf).getValuePartition(); + } + Column key = new Column(scvf.getFamily(), scvf.getQualifier(), vp); + if (colWithOperatorsOfOR.get(key) == null) { + List valueList = new ArrayList(1); + valueList.add(new Value(scvf.getOperator(), scvf.getComparator().getValue(), scvf)); + colWithOperatorsOfOR.put(key, valueList); + } else { + List valueList = colWithOperatorsOfOR.get(key); + Iterator valueListItr = valueList.iterator(); + CompareOp operator = scvf.getOperator(); + byte[] value = scvf.getComparator().getValue(); + Value prevValueObj = null; + while (valueListItr.hasNext()) { + prevValueObj = valueListItr.next(); + // TODO As Anoop said we may have to check the Value type also.. + // We can not compare and validate this way. btw "a" and "K". + // Only in case of Numeric col type we can have this check. + byte[] prevValue = prevValueObj.getValue(); + int result = Bytes.compareTo(prevValue, value); + CompareOp prevOperator = prevValueObj.getOperator(); + switch (operator) { + case GREATER: + if (prevOperator == CompareOp.GREATER || prevOperator == CompareOp.GREATER_OR_EQUAL) { + if (result > 0) { + valueListItr.remove(); + } else { + // Already you found less or equal value than present value means present filter will + // return subset of previous filter. No need to add it to list. + return; + } + } else if (prevOperator == CompareOp.LESS || prevOperator == CompareOp.LESS_OR_EQUAL) { + // Need to handle conditions like previous is c1<5 and current c1>2. In these cases we + // can change this condition into three parts c1<2,c1>=2 AND C1=<5 ,c1>5 and add to + // list. + } else if (prevOperator == CompareOp.EQUAL) { + if (result > 0) { + valueListItr.remove(); + } else if (result == 0) { + // remove this entry and convert GREATER to GREATER_OR_EQUAL + valueListItr.remove(); + SingleColumnValueFilter newScvf = null; + if (vp == null) { + newScvf = new SingleColumnValueFilter(scvf.getFamily(), scvf.getQualifier(), + CompareOp.GREATER_OR_EQUAL, scvf.getComparator()); + } else { + newScvf = new SingleColumnValuePartitionFilter(scvf.getFamily(), + scvf.getQualifier(), CompareOp.GREATER_OR_EQUAL, scvf.getComparator(), vp); + } + Value newValue = new Value(CompareOp.GREATER_OR_EQUAL, prevValue, newScvf); + valueList.add(newValue); + return; + } + } + break; + case GREATER_OR_EQUAL: + if (prevOperator == CompareOp.GREATER || prevOperator == CompareOp.GREATER_OR_EQUAL) { + if (result >= 0) { + valueListItr.remove(); + } else { + // Already you found less value than present value means present filter will + // return subset of previous filter. No need to add it to list. + return; + } + } else if (prevOperator == CompareOp.LESS || prevOperator == CompareOp.LESS_OR_EQUAL) { + // Need to handle conditions like previous is c1<5 and current c1>2. In these cases we + // can change this condition into three parts c1<2,c1>=2 AND C1=<5 ,c1>5 and add to + // list. + } else if (prevOperator == CompareOp.EQUAL) { + if (result >= 0) { + valueListItr.remove(); + } + } + break; + case LESS: + if (prevOperator == CompareOp.LESS || prevOperator == CompareOp.LESS_OR_EQUAL) { + if (result < 0) { + valueListItr.remove(); + } else { + // Already you found less or equal value than present value means present filter will + // return subset of previous filter. No need to add it to list. + return; + } + } else if (prevOperator == CompareOp.GREATER + || prevOperator == CompareOp.GREATER_OR_EQUAL) { + // Need to handle conditions like previous is c1<5 and current c1>2. In these cases we + // can change this condition into three parts c1<2,c1>=2 AND C1=<5 ,c1>5 and add to + // list. + } else if (prevOperator == CompareOp.EQUAL) { + if (result < 0) { + valueListItr.remove(); + } else if (result == 0) { + // remove this entry and convert LESS to LESS_OR_EQUAL + valueListItr.remove(); + SingleColumnValueFilter newScvf = null; + if (vp == null) { + newScvf = new SingleColumnValueFilter(scvf.getFamily(), scvf.getQualifier(), + CompareOp.LESS_OR_EQUAL, scvf.getComparator()); + } else { + newScvf = new SingleColumnValuePartitionFilter(scvf.getFamily(), + scvf.getQualifier(), CompareOp.LESS_OR_EQUAL, scvf.getComparator(), vp); + } + Value newValue = new Value(CompareOp.LESS_OR_EQUAL, prevValue, newScvf); + valueList.add(newValue); + return; + } + } + break; + case LESS_OR_EQUAL: + if (prevOperator == CompareOp.LESS || prevOperator == CompareOp.LESS_OR_EQUAL) { + if (result <= 0) { + valueListItr.remove(); + } else { + // Already you found less or equal value than present value means present filter will + // return subset of previous filter. No need to add it to list. + return; + } + } else if (prevOperator == CompareOp.GREATER + || prevOperator == CompareOp.GREATER_OR_EQUAL) { + // Need to handle conditions like previous is c1<5 and current c1>2. In these cases we + // can change this condition into three parts c1<2,c1>=2 AND C1=<5 ,c1>5 and add to + // list. + } else if (prevOperator == CompareOp.EQUAL) { + // If we dont want to do conversion we can add into first if condition. + if (result <= 0) { + valueListItr.remove(); + } else if (result == 0) { + // remove this entry and convert GREATER to GREATER_OR_EQUAL + } + } + break; + case EQUAL: + if (prevOperator == CompareOp.GREATER) { + if (result < 0) { + return; + } else if(result == 0) { + valueListItr.remove(); + SingleColumnValueFilter newScvf = null; + if (vp == null) { + newScvf = new SingleColumnValueFilter(scvf.getFamily(), scvf.getQualifier(), + CompareOp.GREATER_OR_EQUAL, scvf.getComparator()); + } else { + newScvf = new SingleColumnValuePartitionFilter(scvf.getFamily(), + scvf.getQualifier(), CompareOp.GREATER_OR_EQUAL, scvf.getComparator(), vp); + } + Value newValue = new Value(CompareOp.GREATER_OR_EQUAL, prevValue, newScvf); + valueList.add(newValue); + return; + } + } else if( prevOperator == CompareOp.GREATER_OR_EQUAL){ + if(result <= 0){ + return; + } + } else if (prevOperator == CompareOp.LESS ) { + if(result > 0){ + return; + } else if(result == 0) { + valueListItr.remove(); + SingleColumnValueFilter newScvf = null; + if (vp == null) { + newScvf = new SingleColumnValueFilter(scvf.getFamily(), scvf.getQualifier(), + CompareOp.LESS_OR_EQUAL, scvf.getComparator()); + } else { + newScvf = new SingleColumnValuePartitionFilter(scvf.getFamily(), + scvf.getQualifier(), CompareOp.LESS_OR_EQUAL, scvf.getComparator(), vp); + } + Value newValue = new Value(CompareOp.LESS_OR_EQUAL, prevValue, newScvf); + valueList.add(newValue); + return; + } + } else if (prevOperator == CompareOp.LESS_OR_EQUAL ) { + if(result >= 0){ + return; + } + } else if (prevOperator == CompareOp.EQUAL) { + if (result == 0) { + // Already same filter exists with same condiftion. + return; + } + } + break; + case NOT_EQUAL: + case NO_OP: + // Need to check this + break; + } + } + valueList.add(new Value(scvf.getOperator(), scvf.getComparator().getValue(), scvf)); + } + } + + private void handleScvf(SingleColumnValueFilter scvf) { + ValuePartition vp = null; + if (scvf instanceof SingleColumnValuePartitionFilter) { + vp = ((SingleColumnValuePartitionFilter) scvf).getValuePartition(); + } + Column column = new Column(scvf.getFamily(), scvf.getQualifier(), vp); + Pair pair = colWithOperators.get(column); + if (pair == null) { + pair = new Pair(); + // The first operator should be set here + pair.setFirst(new Value(scvf.getOperator(), scvf.getComparator().getValue(), scvf)); + colWithOperators.put(column, pair); + } else { + if (pair.getFirst() != null && pair.getSecond() == null) { + // TODO As Anoop said we may have to check the Value type also.. + // We can not compare and validate this way. btw "a" and "K". + // Only in case of Numeric col type we can have this check. + byte[] curBoundValue = scvf.getComparator().getValue(); + byte[] prevBoundValue = pair.getFirst().getValue(); + int result = Bytes.compareTo(prevBoundValue, curBoundValue); + CompareOp curBoundOperator = scvf.getOperator(); + CompareOp prevBoundOperator = pair.getFirst().getOperator(); + switch (curBoundOperator) { + case GREATER: + case GREATER_OR_EQUAL: + if (prevBoundOperator == CompareOp.GREATER + || prevBoundOperator == CompareOp.GREATER_OR_EQUAL) { + LOG.warn("Wrong usage. It should be < > || > <. Cannot be > >"); + if (result > 1) { + pair.setFirst(new Value(curBoundOperator, curBoundValue, scvf)); + } + pair.setSecond(null); + } else if (prevBoundOperator == CompareOp.LESS + || prevBoundOperator == CompareOp.LESS_OR_EQUAL) { + if (result < 1) { + LOG.warn("Possible wrong usage as there cannot be a value < 10 and > 20"); + pair.setFirst(null); + pair.setSecond(null); + } else { + pair.setSecond(new Value(curBoundOperator, curBoundValue, scvf)); + } + } else if (prevBoundOperator == CompareOp.EQUAL) { + LOG.warn("Use the equal operator and ignore the current one"); + pair.setSecond(null); + } + break; + case LESS: + case LESS_OR_EQUAL: + if (prevBoundOperator == CompareOp.LESS || prevBoundOperator == CompareOp.LESS_OR_EQUAL) { + LOG.warn("Wrong usage. It should be < > || > <. Cannot be > >"); + if (result < 1) { + pair.setFirst(new Value(curBoundOperator, curBoundValue, scvf)); + } + pair.setSecond(null); + } else if (prevBoundOperator == CompareOp.GREATER + || prevBoundOperator == CompareOp.GREATER_OR_EQUAL) { + if (result > 1) { + LOG.warn("Possible wrong usage as there cannot be a value < 10 and > 20"); + pair.setFirst(null); + pair.setSecond(null); + } else { + pair.setSecond(new Value(curBoundOperator, curBoundValue, scvf)); + } + } else if (prevBoundOperator == CompareOp.EQUAL) { + LOG.warn("Use the EQUAL operator only and ignore the current one."); + pair.setSecond(null); + } + break; + case EQUAL: + // For equal condition give priority to equals only.. + // If the prevOperator is also == and the current is also == + // take the second one.(Currently) + if (prevBoundOperator == CompareOp.LESS || prevBoundOperator == CompareOp.LESS_OR_EQUAL + || prevBoundOperator == CompareOp.EQUAL || prevBoundOperator == CompareOp.GREATER + || prevBoundOperator == CompareOp.GREATER_OR_EQUAL) { + pair.setFirst(new Value(curBoundOperator, curBoundValue, scvf)); + pair.setSecond(null); + } + break; + case NOT_EQUAL: + case NO_OP: + // Need to check this + break; + } + } else { + LOG.warn("Am getting an extra comparison coming for the same col family." + + "I cannot have 3 conditions on the same column"); + pair.setFirst(null); + pair.setSecond(null); + } + } + } + + private void addANDColsToFinalList(FilterList filterList) { + for (Entry> entry : colWithOperators.entrySet()) { + Pair value = entry.getValue(); + if (value.getFirst() != null && value.getSecond() != null) { + // Here we are introducing a new Filter + SingleColumnRangeFilter rangeFltr = new SingleColumnRangeFilter(entry.getKey().getFamily(), + entry.getKey().getQualifier(), entry.getKey().getValuePartition(), value.getFirst() + .getValue(), value.getFirst().getOperator(), value.getSecond().getValue(), value + .getSecond().getOperator()); + filterList.addFilter(rangeFltr); + } else if (value.getFirst() != null) { + if (value.getFirst().getOperator() == CompareOp.EQUAL) { + filterList.addFilter(value.getFirst().getFilter()); + } else { + SingleColumnRangeFilter rangeFltr = new SingleColumnRangeFilter(entry.getKey() + .getFamily(), entry.getKey().getQualifier(), entry.getKey().getValuePartition(), + value.getFirst().getValue(), value.getFirst().getOperator(), null, null); + filterList.addFilter(rangeFltr); + } + } + } + } + + private void addORColsToFinalList(FilterList filterList) { + for (Entry> entry : colWithOperatorsOfOR.entrySet()) { + List valueList = entry.getValue(); + for (Value value : valueList) { + if (value.getOperator() == CompareOp.EQUAL) { + filterList.addFilter(value.getFilter()); + } else { + SingleColumnRangeFilter rangeFltr = new SingleColumnRangeFilter(entry.getKey() + .getFamily(), entry.getKey().getQualifier(), entry.getKey().getValuePartition(), + value.getValue(), value.getOperator(), null, null); + filterList.addFilter(rangeFltr); + } + } + } + } + + private static class Value { + private CompareOp operator; + private byte[] value; + private Filter filter; + + public Value(CompareOp operator, byte[] value, Filter filter) { + this.operator = operator; + this.value = value; + this.filter = filter; + } + + public CompareOp getOperator() { + return this.operator; + } + + public byte[] getValue() { + return this.value; + } + + public Filter getFilter() { + return this.filter; + } + } +} \ No newline at end of file Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/FilterNode.java =================================================================== --- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/FilterNode.java (revision 0) +++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/FilterNode.java (working copy) @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.hbase.index.Column; +import org.apache.hadoop.hbase.index.IndexSpecification; +import org.apache.hadoop.hbase.util.Pair; + +public interface FilterNode { + Map, IndexSpecification> getIndexToUse(); + + Map>> getPossibleUseIndices(); + + Map>> getPossibleFutureUseIndices(); +} \ No newline at end of file Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/IndexFilterNode.java =================================================================== --- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/IndexFilterNode.java (revision 0) +++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/IndexFilterNode.java (working copy) @@ -0,0 +1,102 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.hbase.index.Column; +import org.apache.hadoop.hbase.index.IndexSpecification; +import org.apache.hadoop.hbase.util.Pair; + +public class IndexFilterNode implements LeafFilterNode { + + private IndexSpecification indexToUse; + // all possible indices which can be used. This includes the selected indexToUse also. + // This contains the an integer as the second item in the Pair. This is the relative overhead + // in scanning the index region. The lesser the value the lesser the overhead in scanning the + // index region. This will be set with the number of columns in the index specification. + private List> possibleUseIndices; + + private List> possibleFutureUseIndices; + + private FilterColumnValueDetail filterColumnValueDetail; + + @Override + public Map>> getPossibleFutureUseIndices() { + // TODO avoid create of Map instance all the time... + Map>> reply = + new HashMap>>(); + reply.put(filterColumnValueDetail.getColumn(), possibleFutureUseIndices); + return reply; + } + + public IndexFilterNode(IndexSpecification indexToUse, + List> possibleUseIndices, + List> possibleFutureUseIndices, + FilterColumnValueDetail filterColumnValueDetail) { + this.indexToUse = indexToUse; + this.possibleUseIndices = possibleUseIndices; + this.possibleFutureUseIndices = possibleFutureUseIndices; + this.filterColumnValueDetail = filterColumnValueDetail; + } + + /** + * all possible indices which can be used. This includes the selected indexToUse also. This + * contains the an integer as the second item in the Pair. This is the relative overhead in + * scanning the index region. The lesser the value the lesser the overhead in scanning the index + * region. This will be set with the number of columns in the index specification. + * + * @return + */ + @Override + public Map>> getPossibleUseIndices() { + // TODO avoid create of Map instance all the time... + Map>> reply = + new HashMap>>(); + reply.put(filterColumnValueDetail.getColumn(), possibleUseIndices); + return reply; + } + + @Override + public Map, IndexSpecification> getIndexToUse() { + // TODO avoid create of Map instance all the time... + Map, IndexSpecification> reply = + new HashMap, IndexSpecification>(); + List key = new ArrayList(1); + key.add(filterColumnValueDetail); + reply.put(key, indexToUse); + return reply; + } + + @Override + public IndexSpecification getBestIndex() { + return this.indexToUse; + } + + @Override + public FilterColumnValueDetail getFilterColumnValueDetail() { + return this.filterColumnValueDetail; + } + + public void setFilterColumnValueDetail(FilterColumnValueDetail filterColumnValueDetail) { + this.filterColumnValueDetail = filterColumnValueDetail; + } +} \ No newline at end of file Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/IndexRegionObserver.java =================================================================== --- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/IndexRegionObserver.java (revision 0) +++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/IndexRegionObserver.java (working copy) @@ -0,0 +1,955 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.ConcurrentHashMap; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.DoNotRetryIOException; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.KeyValueUtil; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.HConstants.OperationStatusCode; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.catalog.MetaEditor; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Mutation; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver; +import org.apache.hadoop.hbase.coprocessor.ObserverContext; +import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; +import org.apache.hadoop.hbase.index.ColumnQualifier; +import org.apache.hadoop.hbase.index.Constants; +import org.apache.hadoop.hbase.index.IndexSpecification; +import org.apache.hadoop.hbase.index.TableIndices; +import org.apache.hadoop.hbase.index.io.IndexHalfStoreFileReader; +import org.apache.hadoop.hbase.index.manager.IndexManager; +import org.apache.hadoop.hbase.index.util.ByteArrayBuilder; +import org.apache.hadoop.hbase.index.util.IndexUtils; +import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper; +import org.apache.hadoop.hbase.io.Reference; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.HRegion.Operation; +import org.apache.hadoop.hbase.regionserver.StoreFile.Reader; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.regionserver.InternalScanner; +import org.apache.hadoop.hbase.regionserver.KeyValueScanner; +import org.apache.hadoop.hbase.regionserver.MiniBatchOperationInProgress; +import org.apache.hadoop.hbase.regionserver.OperationStatus; +import org.apache.hadoop.hbase.regionserver.RegionScanner; +import org.apache.hadoop.hbase.regionserver.RegionServerServices; +import org.apache.hadoop.hbase.regionserver.ScanType; +import org.apache.hadoop.hbase.regionserver.SplitTransaction; +import org.apache.hadoop.hbase.regionserver.Store; +import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest; +import org.apache.hadoop.hbase.regionserver.wal.WALEdit; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; +import org.apache.hadoop.hbase.util.PairOfSameType; + +import com.google.common.collect.HashMultimap; +import com.google.common.collect.Multimap; + +public class IndexRegionObserver extends BaseRegionObserver { + + private static final Log LOG = LogFactory.getLog(IndexRegionObserver.class); + + // variable will be set to true in test case for testing + // All below public static fields are used for testing. + static boolean isTestingEnabled = false; + + public static boolean isSeekpointAddded = false; + + public static boolean isIndexedFlowUsed = false; + + public static List seekPoints = null; + + public static List seekPointsForMultipleIndices = null; + + private Map scannerMap = + new ConcurrentHashMap(); + + private IndexManager indexManager = IndexManager.getInstance(); + + public static final ThreadLocal threadLocal = new ThreadLocal() { + @Override + protected IndexEdits initialValue() { + return new IndexEdits(); + } + }; + + @Override + public void postOpen(ObserverContext contx) { + HTableDescriptor tableDesc = contx.getEnvironment().getRegion().getTableDesc(); + RegionServerServices rss = contx.getEnvironment().getRegionServerServices(); + TableName tableName = tableDesc.getTableName(); + if (isNotIndexedTableDescriptor(tableDesc)) { + return; + } + LOG.trace("Entering postOpen for the table " + tableName); + this.indexManager.incrementRegionCount(tableName.getNameAsString()); + List list = indexManager.getIndicesForTable(tableName.getNameAsString()); + if (null != list) { + LOG.trace("Index Manager already contains an entry for the table " + + ". Hence returning from postOpen"); + return; + } + byte[] indexBytes = tableDesc.getValue(Constants.INDEX_SPEC_KEY); + TableIndices tableIndices = new TableIndices(); + try { + tableIndices.readFields(indexBytes); + } catch (IOException e) { + rss.abort("Some unidentified scenario while reading from the " + + "table descriptor . Aborting RegionServer", e); + } + list = tableIndices.getIndices(); + if (list != null && list.size() > 0) { + indexManager.addIndexForTable(tableName.getNameAsString(), list); + LOG.trace("Added index Specification in the Manager for the " + tableName); + } + LOG.trace("Exiting postOpen for the table " + tableName); + } + + @Override + public void preBatchMutate(final ObserverContext ctx, + final MiniBatchOperationInProgress miniBatchOp) + throws IOException { + HRegionServer rs = (HRegionServer) ctx.getEnvironment().getRegionServerServices(); + HRegion userRegion = ctx.getEnvironment().getRegion(); + HTableDescriptor userTableDesc = userRegion.getTableDesc(); + String tableName = userTableDesc.getNameAsString(); + if (isNotIndexedTableDescriptor(userTableDesc)) { + return; + } + List indices = indexManager.getIndicesForTable(tableName); + if (indices == null || indices.isEmpty()) { + LOG.trace("skipping preBatchMutate for the table " + tableName + " as there are no indices"); + return; + } + LOG.trace("Entering preBatchMutate for the table " + tableName); + LOG.trace("Indices for the table " + tableName + " are: " + indices); + HRegion indexRegion = getIndexTableRegion(tableName, userRegion, rs); + // Storing this found HRegion in the index table within the thread locale. + IndexEdits indexEdits = threadLocal.get(); + indexEdits.indexRegion = indexRegion; + for (int i = 0; i < miniBatchOp.size(); i++) { + Mutation m = miniBatchOp.getOperation(i); + if (m instanceof Put) { + try { + prepareIndexMutations(indices, userRegion, m, tableName, indexRegion); + } catch (IOException e) { + miniBatchOp.setOperationStatus(i, new OperationStatus( + OperationStatusCode.SANITY_CHECK_FAILURE, e.getMessage())); + } + } else if (m instanceof Delete) { + prepareIndexMutations(indices, userRegion, m, tableName, indexRegion); + } + } + indexEdits.setUpdateLocked(); + indexRegion.updatesLock(); + LOG.trace("Exiting preBatchMutate for the table " + tableName); + } + + // TODO make this api generic and use in scanner open hook to find the index region. + private HRegion getIndexTableRegion(String tableName, HRegion userRegion, HRegionServer rs) + throws IOException { + TableName indexTableName = TableName.valueOf(IndexUtils.getIndexTableName(tableName)); + Collection idxTabRegions = rs.getOnlineRegions(indexTableName); + for (HRegion idxTabRegion : idxTabRegions) { + // TODO start key check is enough? May be we can check for the + // possibility for N-1 Mapping? + if (Bytes.equals(idxTabRegion.getStartKey(), userRegion.getStartKey())) { + return idxTabRegion; + } + } + // No corresponding index region found in the RS online regions list! + String message = "Index Region not found on the region server . " + + "So skipping the put. Need Balancing"; + LOG.warn(message); + // TODO give a proper Exception msg + throw new DoNotRetryIOException(message); + } + + private void prepareIndexMutations(List indices, HRegion userRegion, + Mutation mutation, String tableName, HRegion indexRegion) throws IOException { + IndexEdits indexEdits = threadLocal.get(); + if (mutation instanceof Put) { + for (IndexSpecification index : indices) { + // Handle each of the index + Mutation indexPut = IndexUtils.prepareIndexPut((Put) mutation, index, indexRegion); + if (null != indexPut) { + // This mutation can be null when the user table mutation is not + // containing all of the indexed col value. + indexEdits.add(indexPut); + } + } + } else if (mutation instanceof Delete) { + Collection indexDeletes = prepareIndexDeletes((Delete) mutation, + userRegion, indices, indexRegion); + indexEdits.addAll(indexDeletes); + } else { + // TODO : Log or throw exception + } + } + + Collection prepareIndexDeletes(Delete delete, HRegion userRegion, + List indexSpecs, HRegion indexRegion) throws IOException { + Collection indexDeletes = new LinkedHashSet(); + for (Entry> entry : delete.getFamilyCellMap().entrySet()) { + for (Cell cell : entry.getValue()) { + indexDeletes.addAll(getIndexDeletes(indexSpecs, userRegion, indexRegion, + KeyValueUtil.ensureKeyValue(cell))); + } + } + return indexDeletes; + } + + private static Collection getIndexDeletes(List indexSpecs, + HRegion userRegion, HRegion indexRegion, Cell deleteKV) throws IOException { + Collection indexDeletes = new LinkedHashSet(); + List indicesToUpdate = new LinkedList(); + Multimap groupedKV = doGetAndGroupByTS(indexSpecs, userRegion, deleteKV, + indicesToUpdate); + + // There can be multiple index kvs for each user kv + // So, prepare all resultant index delete kvs for this user delete kv + for (Entry> entry : groupedKV.asMap().entrySet()) { + for (IndexSpecification index : indicesToUpdate) { + ByteArrayBuilder indexRow = IndexUtils.getIndexRowKeyHeader(index, indexRegion.getStartKey(), + deleteKV.getRow()); + boolean update = false; + for (ColumnQualifier cq : index.getIndexColumns()) { + Cell kvFound = null; + for (Cell kv : entry.getValue()) { + if (Bytes.equals(cq.getColumnFamily(), kv.getFamily()) + && Bytes.equals(cq.getQualifier(), kv.getQualifier())) { + kvFound = kv; + update = true; + break; + } + } + if (kvFound == null) { + indexRow.position(indexRow.position() + cq.getMaxValueLength()); + } else { + IndexUtils.updateRowKeyForKV(cq, kvFound, indexRow); + } + } + if (update) { + // Append the actual row key at the end of the index row key. + indexRow.put(deleteKV.getRow()); + Delete idxDelete = new Delete(indexRow.array()); + if (((KeyValue) deleteKV).isDeleteType()) { + idxDelete.deleteColumn(Constants.IDX_COL_FAMILY, Constants.IDX_COL_QUAL, + entry.getKey()); + } else { + idxDelete.deleteFamily(Constants.IDX_COL_FAMILY, entry.getKey()); + } + indexDeletes.add(idxDelete); + } + } + } + return indexDeletes; + } + + private static Multimap doGetAndGroupByTS(List indexSpecs, + HRegion userRegion, Cell deleteKV, + List indicesToConsider) throws IOException { + + Get get = new Get(deleteKV.getRow()); + long maxTS = HConstants.LATEST_TIMESTAMP; + + if (deleteKV.getTimestamp() < maxTS) { + // Add +1 to make the current get includes the timestamp + maxTS = deleteKV.getTimestamp() + 1; + } + get.setTimeRange(0L, maxTS); + + for (IndexSpecification index : indexSpecs) { + // Get all indices involves this family/qualifier + if (index.contains(deleteKV.getFamily(), deleteKV.getQualifier())) { + indicesToConsider.add(index); + for (ColumnQualifier cq : index.getIndexColumns()) { + get.addColumn(cq.getColumnFamily(), cq.getQualifier()); + } + } + } + if (((KeyValue) deleteKV).isDeleteType()) { + get.setMaxVersions(1); + } else if (((KeyValue) deleteKV).isDeleteColumnOrFamily()) { + get.setMaxVersions(); + } + List userKVs = userRegion.get(get).list(); + + // Group KV based on timestamp + Multimap groupedKV = HashMultimap.create(); + + if (userKVs != null) { + for (Cell userKV : userKVs) { + groupedKV.put(userKV.getTimestamp(), userKV); + } + } + return groupedKV; + } + + // collection of edits for index table's memstore and WAL + public static class IndexEdits { + private WALEdit walEdit = new WALEdit(); + private HRegion indexRegion; + private boolean updatesLocked = false; + + /** + * Collection of mutations with locks. Locks will be null always as they not yet acquired for + * index table. + * + * @see HRegion#batchMutate(Pair[]) + */ + private List mutations = new ArrayList(); + + public WALEdit getWALEdit() { + return this.walEdit; + } + + public boolean isUpdatesLocked(){ + return this.updatesLocked; + } + + public void setUpdateLocked(){ + updatesLocked = true; + } + + public void add(Mutation mutation) { + // Check if WAL is disabled + for (List kvs : mutation.getFamilyCellMap().values()) { + for (Cell cell : kvs) { + this.walEdit.add(KeyValueUtil.ensureKeyValue(cell)); + } + } + // There is no lock acquired for index table. So, set it to null + this.mutations.add(mutation); + } + + public void addAll(Collection mutations) { + for (Mutation mutation : mutations) { + add(mutation); + } + } + + public List getIndexMutations() { + return this.mutations; + } + + public HRegion getRegion() { + return this.indexRegion; + } + } + + @Override + public void postBatchMutate(final ObserverContext ctx, + final MiniBatchOperationInProgress miniBatchOp) { + HTableDescriptor userTableDesc = ctx.getEnvironment().getRegion().getTableDesc(); + String tableName = userTableDesc.getNameAsString(); + if (isNotIndexedTableDescriptor(userTableDesc)) { + return; + } + List indices = indexManager.getIndicesForTable(tableName); + if (indices == null || indices.isEmpty()) { + LOG.trace("skipping postBatchMutate for the table " + tableName + " as there are no indices"); + return; + } + LOG.trace("Entering postBatchMutate for the table " + tableName); + IndexEdits indexEdits = threadLocal.get(); + List indexMutations = indexEdits.getIndexMutations(); + + if (indexMutations.size() == 0) { + return; + } + HRegion hr = indexEdits.getRegion(); + LOG.trace("Updating index table " + hr.getRegionInfo().getTableName()); + try { + hr.batchMutateForIndex(indexMutations + .toArray(new Mutation[indexMutations.size()])); + } catch (IOException e) { + // TODO This can come? If so we need to revert the actual put + // and make the op failed. + LOG.error("Error putting data into the index region", e); + } + LOG.trace("Exiting postBatchMutate for the table " + tableName); + } + + private boolean isNotIndexedTableDescriptor(HTableDescriptor userTableDesc) { + TableName tableName = userTableDesc.getTableName(); + return IndexUtils.isCatalogOrSystemTable(tableName) + || IndexUtils.isIndexTable(tableName) + || (!IndexUtils.isIndexTable(tableName) && userTableDesc.getValue(Constants.INDEX_SPEC_KEY) == null); + } + + @Override + public void postBatchMutateIndispensably(ObserverContext ctx, + MiniBatchOperationInProgress miniBatchOp, boolean success) throws IOException { + IndexEdits indexEdits = threadLocal.get(); + if(indexEdits != null){ + if(indexEdits.isUpdatesLocked()){ + indexEdits.getRegion().updatesUnlock(); + } + } + threadLocal.remove(); + } + + @Override + public boolean postScannerFilterRow(ObserverContext e, + InternalScanner s, byte[] currentRow, boolean hasMore) throws IOException { + HTableDescriptor tableDesc = e.getEnvironment().getRegion().getTableDesc(); + if(isNotIndexedTableDescriptor(tableDesc)){ + return hasMore; + } + SeekAndReadRegionScanner bsrs = SeekAndReadRegionScannerHolder.getRegionScanner(); + if (bsrs != null) { + while (false == bsrs.seekToNextPoint()) { + SeekPointFetcher seekPointFetcher = scannerMap.get(bsrs); + if (null != seekPointFetcher) { + List seekPoints = new ArrayList(1); + seekPointFetcher.nextSeekPoints(seekPoints, 1); + // TODO use return boolean? + if (seekPoints.isEmpty()) { + LOG.trace("No seekpoints are remaining hence returning.. "); + return false; + } + bsrs.addSeekPoints(seekPoints); + if (isTestingEnabled) { + setSeekPoints(seekPoints); + setSeekpointAdded(true); + addSeekPoints(seekPoints); + } + }else{ + // This will happen for a region with no index + break; + } + } + } + return true; + } + + public RegionScanner postScannerOpen(ObserverContext e, Scan scan, + RegionScanner s) { + HRegion region = e.getEnvironment().getRegion(); + HTableDescriptor tableDesc = region.getTableDesc(); + String tableName = tableDesc.getNameAsString(); + HRegionServer rs = (HRegionServer) e.getEnvironment().getRegionServerServices(); + if (isNotIndexedTableDescriptor(tableDesc)) { + return s; + } + // If the passed region is a region from an indexed table + SeekAndReadRegionScanner bsrs = null; + + try { + List indexlist = IndexManager.getInstance().getIndicesForTable(tableName); + if (indexlist == null || indexlist.isEmpty()) { + // Not an indexed table. Just return. + return s; + } + if (indexlist != null) { + LOG.info("Entering postScannerOpen for the table " + tableName); + Collection onlineRegions = ((HRegionServer) rs).getOnlineRegionsLocalContext(); + for (HRegion onlineIdxRegion : onlineRegions) { + if (IndexUtils.isCatalogOrSystemTable(onlineIdxRegion.getTableDesc().getTableName())) { + continue; + } + if (onlineIdxRegion.equals(region)) { + continue; + } + if (Bytes.equals(onlineIdxRegion.getStartKey(), region.getStartKey()) + && Bytes.equals(Bytes.toBytes(IndexUtils.getIndexTableName(region.getTableDesc() + .getNameAsString())), onlineIdxRegion.getTableDesc().getName())) { + ScanFilterEvaluator mapper = new ScanFilterEvaluator(); + IndexRegionScanner indexScanner = mapper.evaluate(scan, indexlist, + onlineIdxRegion.getStartKey(), onlineIdxRegion, tableName); + if(indexScanner == null ) return s; + SeekPointFetcher spf = new SeekPointFetcher(indexScanner); + ReInitializableRegionScanner reinitializeScanner = new ReInitializableRegionScannerImpl(s, scan, spf); + bsrs = new BackwardSeekableRegionScanner(reinitializeScanner, scan, region, null); + scannerMap.put(bsrs, spf); + LOG.trace("Scanner Map has " + scannerMap); + break; + } + } + LOG.trace("Exiting postScannerOpen for the table " + tableName); + } + } catch (Exception ex) { + LOG.error("Exception occured in postScannerOpen for the table " + tableName,ex); + } + if (bsrs != null) { + return bsrs; + } else { + return s; + } + } + + public boolean preScannerNext(ObserverContext e, InternalScanner s, + List results, int nbRows, boolean hasMore) throws IOException { + HRegion region = e.getEnvironment().getRegion(); + String tableName = region.getTableDesc().getNameAsString(); + try { + if (s instanceof SeekAndReadRegionScanner) { + LOG.trace("Entering preScannerNext for the table " + tableName); + BackwardSeekableRegionScanner bsrs = (BackwardSeekableRegionScanner) s; + SeekAndReadRegionScannerHolder.setRegionScanner(bsrs); + SeekPointFetcher spf = scannerMap.get(bsrs); + List seekPoints = null; + if (spf != null) { + if (isTestingEnabled) { + setIndexedFlowUsed(true); + } + seekPoints = new ArrayList(); + spf.nextSeekPoints(seekPoints, nbRows); + } + if (seekPoints == null || seekPoints.isEmpty()) { + LOG.trace("No seekpoints are remaining hence returning.. "); + SeekAndReadRegionScannerHolder.removeRegionScanner(); + e.bypass(); + return false; + } + bsrs.addSeekPoints(seekPoints); + // This setting is just for testing purpose + if (isTestingEnabled) { + setSeekPoints(seekPoints); + setSeekpointAdded(true); + addSeekPoints(seekPoints); + } + LOG.trace("Exiting preScannerNext for the table " + tableName); + } + } catch (Exception ex) { + LOG.error("Exception occured in preScannerNext for the table " + tableName + ex); + } + return true; + } + + @Override + public boolean postScannerNext(ObserverContext e, + InternalScanner s, List results, int limit, boolean hasMore) throws IOException { + if (s instanceof SeekAndReadRegionScanner) { + SeekAndReadRegionScannerHolder.removeRegionScanner(); + } + return true; + } + + @Override + public void preScannerClose(ObserverContext e, InternalScanner s) + throws IOException { + if (s instanceof BackwardSeekableRegionScanner) { + scannerMap.remove((RegionScanner) s); + } + } + + @Override + public void preSplitBeforePONR(ObserverContext e, + byte[] splitKey, List metaEntries) throws IOException { + RegionCoprocessorEnvironment environment = e.getEnvironment(); + HRegionServer rs = (HRegionServer) environment.getRegionServerServices(); + HRegion region = environment.getRegion(); + String userTableName = region.getTableDesc().getNameAsString(); + LOG.trace("Entering preSplitBeforePONR for the table " + userTableName + " for the region " + + region.getRegionInfo()); + String indexTableName = IndexUtils.getIndexTableName(userTableName); + TableIndices tableIndices = new TableIndices(); + if (indexManager.getIndicesForTable(userTableName) == null) { + byte[] indexBytes = region.getTableDesc().getValue(Constants.INDEX_SPEC_KEY); + if (indexBytes == null) { + return; + } + tableIndices.readFields(indexBytes); + indexManager.addIndexForTable(userTableName, tableIndices.getIndices()); + } + if (indexManager.getIndicesForTable(userTableName) != null) { + HRegion indexRegion = null; + SplitTransaction st = null; + try { + indexRegion = getIndexRegion(rs, region.getStartKey(), indexTableName); + if (null != indexRegion) { + LOG.info("Flushing the cache for the index table " + indexTableName + " for the region " + + indexRegion.getRegionInfo()); + indexRegion.flushcache(); + if (LOG.isInfoEnabled()) { + LOG.info("Forcing split for the index table " + indexTableName + " with split key " + + Bytes.toString(splitKey)); + } + st = new SplitTransaction(indexRegion, splitKey); + if (!st.prepare()) { + LOG.error("Prepare for the index table " + indexTableName + + " failed. So returning null. "); + e.bypass(); + return; + } + indexRegion.forceSplit(splitKey); + PairOfSameType daughterRegions = st.stepsBeforePONR(rs, rs, false); + splitThreadLocal.set(new SplitInfo(indexRegion,daughterRegions,st)); + HRegionInfo copyOfParent = new HRegionInfo(region.getRegionInfo()); + copyOfParent.setOffline(true); + copyOfParent.setSplit(true); + // Put for parent + Put putParent = MetaEditor.makePutFromRegionInfo(copyOfParent); + MetaEditor.addDaughtersToPut(putParent, daughterRegions.getFirst().getRegionInfo(), + daughterRegions.getSecond().getRegionInfo()); + metaEntries.add(putParent); + // Puts for daughters + Put putA = MetaEditor.makePutFromRegionInfo(daughterRegions.getFirst().getRegionInfo()); + Put putB = MetaEditor.makePutFromRegionInfo(daughterRegions.getSecond().getRegionInfo()); + st.addLocation(putA, rs.getServerName(), 1); + st.addLocation(putB, rs.getServerName(), 1); + metaEntries.add(putA); + metaEntries.add(putB); + LOG.info("Daughter regions created for the index table " + indexTableName+ " for the region "+indexRegion.getRegionInfo()); + return; + } + else { + LOG.error("IndexRegion for the table " + indexTableName + " is null. So returning null. "); + e.bypass(); + return; + } + } catch (Exception ex) { + LOG.error("Error while spliting the indexTabRegion or not able to get the indexTabRegion:" + + indexRegion != null ? indexRegion.getRegionName() : "", ex); + st.rollback(rs, rs); + e.bypass(); + return; + } + } + } + + @Override + public void preSplit(ObserverContext e) throws IOException { + if (splitThreadLocal != null) { + splitThreadLocal.remove(); + } + } + + private HRegion getIndexRegion(HRegionServer rs, byte[] startKey, String indexTableName) + throws IOException { + List indexTabRegions = rs.getOnlineRegions(TableName.valueOf(indexTableName)); + for (HRegion indexRegion : indexTabRegions) { + if (Bytes.equals(startKey, indexRegion.getStartKey())) { + return indexRegion; + } + } + return null; + } + + @Override + public void preSplitAfterPONR(ObserverContext e) throws IOException { + RegionCoprocessorEnvironment environment = e.getEnvironment(); + HRegionServer rs = (HRegionServer) environment.getRegionServerServices(); + HRegion region = environment.getRegion(); + String userTableName = region.getTableDesc().getNameAsString(); + String indexTableName = IndexUtils.getIndexTableName(userTableName); + if (IndexUtils.isIndexTable(userTableName)) { + return; + } + LOG.trace("Entering postSplit for the table " + userTableName + " for the region " + + region.getRegionInfo()); + IndexManager indexManager = IndexManager.getInstance(); + SplitTransaction splitTransaction = null; + if (indexManager.getIndicesForTable(userTableName) != null) { + try { + SplitInfo splitInfo = splitThreadLocal.get(); + splitTransaction = splitInfo.getSplitTransaction(); + PairOfSameType daughters = splitInfo.getDaughters(); + if (splitTransaction != null && daughters != null) { + splitTransaction.stepsAfterPONR(rs, rs, daughters); + LOG.info("Daughter regions are opened and split transaction finished for zknodes for index table " + + indexTableName + " for the region " + region.getRegionInfo()); + } + } catch (Exception ex) { + String msg = "Splitting of index region has failed in stepsAfterPONR stage so aborting the server"; + LOG.error(msg, ex); + rs.abort(msg); + } + } + } + + // A thread local variable used to get the splitted region information of the index region. + // This is needed becuase in order to do the PONR entry we need the info of the index + // region's daughter entries. + public static final ThreadLocal splitThreadLocal = new ThreadLocal() { + protected SplitInfo initialValue() { + return null; + }; + }; + + + @Override + public void preRollBackSplit(ObserverContext ctx) throws IOException { + RegionCoprocessorEnvironment environment = ctx.getEnvironment(); + HRegionServer rs = (HRegionServer) environment.getRegionServerServices(); + HRegion region = environment.getRegion(); + HTableDescriptor tableDesc = region.getTableDesc(); + String userTableName = tableDesc.getNameAsString(); + if (isNotIndexedTableDescriptor(tableDesc)) { + return; + } + LOG.trace("Entering preRollBack for the table " + userTableName + " for the region " + + region.getRegionInfo()); + SplitInfo splitInfo = splitThreadLocal.get(); + if(splitInfo == null) return; + SplitTransaction splitTransaction = splitInfo.getSplitTransaction(); + try { + if (splitTransaction != null) { + splitTransaction.rollback(rs, rs); + LOG.info("preRollBack successfully done for the table " + userTableName + + " for the region " + region.getRegionInfo()); + } + } catch (Exception e) { + LOG.error( + "Error while rolling back the split failure for index region " + + splitInfo.getParent(), e); + rs.abort("Abort; we got an error during rollback of index"); + } + } + + // For testing to check whether final step of seek point is added + public static void setSeekpointAdded(boolean isSeekpointAddded) { + IndexRegionObserver.isSeekpointAddded = isSeekpointAddded; + } + + // For testing + public static boolean getSeekpointAdded() { + return isSeekpointAddded; + } + + // For testing to ensure indexed flow is used or not + public static void setIndexedFlowUsed(boolean isIndexedFlowUsed) { + IndexRegionObserver.isIndexedFlowUsed = isIndexedFlowUsed; + } + + // For testing + public static boolean getIndexedFlowUsed() { + return isIndexedFlowUsed; + } + + // For testing + static List getSeekpoints() { + return seekPoints; + } + + // For testing to ensure cache size is returned correctly + public static void setSeekPoints(List seekPoints) { + IndexRegionObserver.seekPoints = seekPoints; + } + + public static void setIsTestingEnabled(boolean isTestingEnabled) { + IndexRegionObserver.isTestingEnabled = isTestingEnabled; + } + + public static void addSeekPoints(List seekPoints){ + if(seekPoints == null){ + IndexRegionObserver.seekPointsForMultipleIndices = null; + return; + } + if(IndexRegionObserver.seekPointsForMultipleIndices == null){ + IndexRegionObserver.seekPointsForMultipleIndices = new ArrayList(); + } + IndexRegionObserver.seekPointsForMultipleIndices.addAll(seekPoints); + } + + public static List getMultipleSeekPoints(){ + return IndexRegionObserver.seekPointsForMultipleIndices; + } + + private static class SeekAndReadRegionScannerHolder { + private static ThreadLocal holder = + new ThreadLocal(); + + public static void setRegionScanner(SeekAndReadRegionScanner scanner) { + holder.set(scanner); + } + + public static SeekAndReadRegionScanner getRegionScanner() { + return holder.get(); + } + + public static void removeRegionScanner() { + holder.remove(); + } + } + + @Override + public InternalScanner preCompactScannerOpen(ObserverContext c, + Store store, List scanners, ScanType scanType, long earliestPutTs, + InternalScanner s, CompactionRequest request) throws IOException { + HRegionServer rs = (HRegionServer) c.getEnvironment().getRegionServerServices(); + if(!IndexUtils.isIndexTable(store.getTableName())){ + // Not an index table + return null; + } + long smallestReadPoint = c.getEnvironment().getRegion().getSmallestReadPoint(); + String actualTableName = + IndexUtils.getActualTableName(store.getTableName().getNameAsString()); + TTLStoreScanner ttlStoreScanner = new TTLStoreScanner(store, smallestReadPoint, earliestPutTs, + scanType, scanners, new TTLExpiryChecker(), actualTableName, rs); + return ttlStoreScanner; + } + + @Override + public void postClose(ObserverContext e, boolean abortRequested) { + HRegion region = e.getEnvironment().getRegion(); + byte[] tableName = region.getRegionInfo().getTable().getName(); + if (IndexUtils.isCatalogOrSystemTable(region.getTableDesc().getTableName()) + || IndexUtils.isIndexTable(tableName)) { + return; + } + if (splitThreadLocal.get() == null) { + this.indexManager.decrementRegionCount(Bytes.toString(tableName), true); + }else{ + this.indexManager.decrementRegionCount(Bytes.toString(tableName), false); + } + } + + private boolean isValidIndexMutation(HTableDescriptor userTableDesc) { + String tableName = userTableDesc.getTableName().getNameAsString(); + if (IndexUtils.isCatalogOrSystemTable(userTableDesc.getTableName()) + || IndexUtils.isIndexTable(tableName)) { + return false; + } + List indices = indexManager.getIndicesForTable(tableName); + if (indices == null || indices.isEmpty()) { + LOG.trace("skipping preBatchMutate for the table " + tableName + " as there are no indices"); + return false; + } + return true; + } + + private void acquireLockOnIndexRegion(String tableName, HRegion userRegion, HRegionServer rs, + Operation op) throws IOException { + HRegion indexRegion = getIndexTableRegion(tableName, userRegion, rs); + indexRegion.checkResources(); + indexRegion.startRegionOperation(op); + } + + @Override + public void postCloseRegionOperation(ObserverContext ctx, + Operation op) throws IOException { + if (op.equals(Operation.BATCH_MUTATE)) { + HRegionServer rs = (HRegionServer) ctx.getEnvironment().getRegionServerServices(); + HRegion userRegion = ctx.getEnvironment().getRegion(); + HTableDescriptor userTableDesc = userRegion.getTableDesc(); + String tableName = userTableDesc.getNameAsString(); + if (isNotIndexedTableDescriptor(userTableDesc)) { + return; + } + if (!isValidIndexMutation(userTableDesc)) { + // Ideally need not release any lock because in the preStartRegionOperationHook we would not + // have + // acquired + // any lock on the index region + return; + } + HRegion indexRegion = getIndexTableRegion(tableName, userRegion, rs); + // This check for isClosed and isClosing is needed because we should not unlock + // when the index region lock would have already been released before throwing NSRE + + // TODO : What is the scenario that i may get an IllegalMonitorStateException + if (!indexRegion.isClosed() || !indexRegion.isClosing()) { + indexRegion.closeRegionOperation(); + } + } + } + + @Override + public void + postStartRegionOperation(ObserverContext e, Operation op) + throws IOException { + if (op.equals(Operation.BATCH_MUTATE)) { + HRegionServer rs = (HRegionServer) e.getEnvironment().getRegionServerServices(); + HRegion userRegion = e.getEnvironment().getRegion(); + HTableDescriptor userTableDesc = userRegion.getTableDesc(); + String tableName = userTableDesc.getNameAsString(); + if (isNotIndexedTableDescriptor(userTableDesc)) { + return; + } + if (!isValidIndexMutation(userTableDesc)) { + return; + } + acquireLockOnIndexRegion(tableName, userRegion, rs, op); + } + } + + @Override + public Reader preStoreFileReaderOpen(ObserverContext ctx, + FileSystem fs, Path path, FSDataInputStreamWrapper in, long size, CacheConfig cacheConf, + Reference r, Reader reader) throws IOException { + Configuration conf = ctx.getEnvironment().getConfiguration(); + if (reader == null && r != null && isIndexRegionReference(path)) { + return new IndexHalfStoreFileReader(fs, path, cacheConf, in, size, r, conf); + } + return reader; + } + + private boolean isIndexRegionReference(Path path) { + String tablePath = path.getParent().getParent().getParent().getName(); + return tablePath.endsWith(Constants.INDEX_TABLE_SUFFIX); + } + + /** + * + * Information of dependent region split processed in coprocessor hook + * {@link RegionCoprocessorHost#preSplitBeforePONR(byte[])} + * + */ + public static class SplitInfo { + private PairOfSameType daughterRegions; + private SplitTransaction st; + private HRegion parent; + + public SplitInfo(final HRegion parent, final PairOfSameType pairOfSameType, + final SplitTransaction st) { + this.parent = parent; + this.daughterRegions = pairOfSameType; + this.st = st; + } + + public PairOfSameType getDaughters() { + return this.daughterRegions; + } + + public SplitTransaction getSplitTransaction() { + return this.st; + } + + public HRegion getParent(){ + return this.parent; + } + } + +} Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/IndexRegionScanner.java =================================================================== --- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/IndexRegionScanner.java (revision 0) +++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/IndexRegionScanner.java (working copy) @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import org.apache.hadoop.hbase.regionserver.RegionScanner; + +public interface IndexRegionScanner extends RegionScanner{ + + public void advance(); + + public void setRangeFlag(boolean range); + + public boolean isRange(); + + public void setScannerIndex(int index); + + public int getScannerIndex(); + + public boolean hasChildScanners(); + +} Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/IndexRegionScannerForAND.java =================================================================== --- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/IndexRegionScannerForAND.java (revision 0) +++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/IndexRegionScannerForAND.java (working copy) @@ -0,0 +1,308 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.index.util.IndexUtils; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; + +public class IndexRegionScannerForAND implements IndexRegionScanner{ + + private static final Log LOG = LogFactory.getLog(IndexRegionScannerForAND.class); + + private List scanners = null; + + private Map,Integer>> rowCache = + new HashMap, Integer>>(); + + private int scannersCount = 0; + + boolean hasRangeScanners = false; + + private int scannerIndex = -1; + + public IndexRegionScannerForAND(List scanners) { + this.scanners = scanners; + scannersCount = scanners.size(); + } + + @Override + public void advance() { + for (IndexRegionScanner scn : this.scanners) { + scn.advance(); + } + } + + @Override + public HRegionInfo getRegionInfo() { + return null; + } + + @Override + public boolean isFilterDone() { + return false; + } + + @Override + public boolean hasChildScanners() { + return scannersCount != 0; + }; + @Override + public void setRangeFlag(boolean range) { + hasRangeScanners = range; + } + + @Override + public boolean isRange(){ + return this.hasRangeScanners; + } + + @Override + public void setScannerIndex(int index) { + scannerIndex = index; + } + + @Override + public int getScannerIndex() { + return scannerIndex; + } + + @Override + public synchronized boolean reseek(byte[] row) throws IOException { + // ideally reseek on AND may not come as AND cannot be a child of another AND. + if(this.scanners.isEmpty()) return false; + for(IndexRegionScanner scn : this.scanners){ + boolean reseek = scn.reseek(row); + if(!reseek) return false; + } + return true; + } + + @Override + public synchronized void close() throws IOException { + for(IndexRegionScanner scn : this.scanners){ + scn.close(); + } + this.scanners.clear(); + } + + @Override + public synchronized boolean next(List results) throws IOException { + if (this.scanners != null && !this.scanners.isEmpty()) { + List> valueList = new ArrayList>(); + byte[] maxRowKey = null; + while (results.size() < 1) { + List intermediateResult = new ArrayList(); + boolean haveSameRows = true; + Cell kv = null; + Iterator scnItr = this.scanners.iterator(); + while (scnItr.hasNext()) { + IndexRegionScanner scn = scnItr.next(); + if (!hasRangeScanners) { + if (checkForScanner(valueList, scn.getScannerIndex())) continue; + } + boolean hasMore = scn.next(intermediateResult); + if (!hasRangeScanners) { + if (intermediateResult != null && !intermediateResult.isEmpty()) { + byte[] rowKey = IndexUtils.getRowKeyFromKV(intermediateResult.get(0)); + if (maxRowKey == null) { + maxRowKey = rowKey; + } else { + int result = Bytes.compareTo(maxRowKey, rowKey); + if (haveSameRows) + haveSameRows = (result == 0); + maxRowKey = result > 0 ? maxRowKey : rowKey; + } + if (kv == null) kv = intermediateResult.get(0); + intermediateResult.clear(); + valueList.add(new Pair(rowKey, scn)); + } + } else { + if (!intermediateResult.isEmpty()) { + boolean matching = checkAndPutMatchingEntry(intermediateResult.get(0), + scn.getScannerIndex()); + if (matching) { + results.addAll(intermediateResult); + } + intermediateResult.clear(); + } + } + if (!hasMore) { + if (LOG.isDebugEnabled()) { + LOG.debug("Removing scanner " + scn + " from the list."); + } + scn.close(); + scnItr.remove(); + if (hasRangeScanners) { + // TODO: we can remove unnecessary rows(which never become matching entries) from + // cache on scanner close. + if (this.scanners.isEmpty()) return false; + } + if (results.size() > 0) { + break; + } + } + if (results.size() > 0) { + return !this.scanners.isEmpty(); + } + } + if (!hasRangeScanners) { + if (haveSameRows && valueList.size() == scannersCount) { + if (kv != null) results.add(kv); + return this.scanners.size() == scannersCount; + } else if (haveSameRows && valueList.size() != scannersCount) { + close(); + return false; + } else { + // In case of AND if the reseek on any one scanner returns false + // we can close the entire scanners in the AND subtree + if(!reseekTheScanners(valueList, maxRowKey)){ + close(); + return false; + } + } + } + } + if (hasRangeScanners) { + return true; + } else { + return this.scanners.size() == scannersCount; + } + } + return false; + } + + private boolean checkForScanner(List> valueList,int scnIndex) { + for (Pair value : valueList) { + if(value.getSecond().getScannerIndex()==scnIndex){ + return true; + } + } + return false; + } + + private boolean checkAndPutMatchingEntry(Cell cell, int index) { + String rowKeyFromKV = Bytes.toString(IndexUtils.getRowKeyFromKV(cell)); + Pair, Integer> countPair = rowCache.get(rowKeyFromKV); + if (countPair == null) { + // If present scanners count is not equal to actual scanners count,no need to put row key into + // cache because this will never become matching entry. + if (this.scanners.size() == scannersCount) { + List scannerFlags = new ArrayList(scannerIndex); + for (int i = 0; i < scannersCount; i++) { + scannerFlags.add(false); + } + countPair = new Pair, Integer>(scannerFlags, 0); + //TODO verify + //rowCache.put(rowKeyFromKV, countPair); + } else { + return false; + } + } + Boolean indexFlag = countPair.getFirst().get(index); + Integer countObj = countPair.getSecond(); + // If count is equal to scanner count before increment means its returned already. skip the + // result. + if (countObj == scannersCount) { + return false; + } + if (!indexFlag) { + countObj++; + countPair.getFirst().set(index, true); + countPair.setSecond(countObj); + rowCache.put(rowKeyFromKV, countPair); + } + if (countObj == scannersCount) { + return true; + } else { + // If the difference between actual scanner count(scannerCount) and number of scanners have + // this row(countObj) is more than present scanners size then remove row from cache because + // this never be maching entry. + if ((scannersCount - countObj) > this.scanners.size()) { + rowCache.remove(rowKeyFromKV); + return false; + } + } + return false; + } + + private boolean reseekTheScanners(List> valueList, byte[] maxRowKey) + throws IOException { + Iterator> itr = valueList.iterator(); + while(itr.hasNext()){ + Pair rowVsScanner = itr.next(); + IndexRegionScanner scn = rowVsScanner.getSecond(); + // We need to call reseek on OR scanner even the last returned row key is equal or more than + // max row key to set reseek flag. + if (scn instanceof IndexRegionScannerForOR) { + rowVsScanner.getSecond().reseek(maxRowKey); + itr.remove(); + continue; + } + if (Bytes.compareTo(rowVsScanner.getFirst(), maxRowKey) < 0) { + if(!scn.reseek(maxRowKey)){ + return false; + } + itr.remove(); + } + } + return true; + } + + @Override + public boolean next(List result, int limit) throws IOException { + // TODO Auto-generated method stub + return false; + } + + @Override + public long getMaxResultSize() { + // TODO Auto-generated method stub + return 0; + } + + @Override + public long getMvccReadPoint() { + // TODO Auto-generated method stub + return 0; + } + + @Override + public boolean nextRaw(List result) throws IOException { + // TODO Auto-generated method stub + return false; + } + + @Override + public boolean nextRaw(List result, int limit) throws IOException { + // TODO Auto-generated method stub + return false; + } + +} Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/IndexRegionScannerForOR.java =================================================================== --- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/IndexRegionScannerForOR.java (revision 0) +++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/IndexRegionScannerForOR.java (working copy) @@ -0,0 +1,334 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.TreeMap; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.index.util.IndexUtils; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; + + +public class IndexRegionScannerForOR implements IndexRegionScanner{ + + private static final Log LOG = LogFactory.getLog(IndexRegionScannerForOR.class); + + private List scanners = null; + + //private Pair lastReturnedValue = null; + + private byte[] lastReturnedValue = null; + + // Cache to store the values retrieved by range scans + private Map rowCache = new HashMap(); + + private Map> valueMap = + new TreeMap>(Bytes.BYTES_COMPARATOR); + + private boolean hasRangeScanners = false; + + private boolean isRootScanner = false; + + private int scannerIndex = -1; + + private boolean firstScan = true; + + private boolean reseeked = false; + + public IndexRegionScannerForOR(List scanners) { + this.scanners = scanners; + } + + @Override + public void advance() { + if(this.lastReturnedValue != null){ +// this.lastReturnedValue.getFirst().advance(); + } + this.lastReturnedValue = null; + } + + @Override + public HRegionInfo getRegionInfo() { + return null; + } + + @Override + public boolean isFilterDone() { + return false; + } + + @Override + public void setRangeFlag(boolean range) { + hasRangeScanners = range; + } + + public void setRootFlag(boolean isRootScanner) { + this.isRootScanner = isRootScanner; + } + + @Override + public boolean isRange(){ + return this.hasRangeScanners; + } + + @Override + public void setScannerIndex(int index) { + scannerIndex = index; + } + + @Override + public int getScannerIndex() { + return scannerIndex; + } + + @Override + public boolean hasChildScanners() { + return !scanners.isEmpty(); + } + @Override + public synchronized boolean reseek(byte[] row) throws IOException { + boolean success = false; + if (!valueMap.isEmpty()) { + Iterator>> itr = valueMap.entrySet() + .iterator(); + while (itr.hasNext()) { + Entry> entry = itr.next(); + IndexRegionScanner scn = entry.getValue().getFirst(); + if (Bytes.compareTo(entry.getKey(), row) < 0) { + // If the reseek does not retrieve any row then it means we have reached the end of the + // scan. So this scanner can be safely removed. + if(!scn.reseek(row)){ + removeScanner(scn.getScannerIndex()); + } + itr.remove(); + } else { + break; + } + } + } + reseeked = true; + this.lastReturnedValue = null; + return success; + } + + private void removeScanner(int scnIndex) { + Iterator itr = this.scanners.iterator(); + while(itr.hasNext()){ + if(itr.next().getScannerIndex()==scnIndex){ + itr.remove(); + break; + } + } + } + + @Override + public synchronized void close() throws IOException { + for(IndexRegionScanner scn : this.scanners){ + scn.close(); + } + this.valueMap.clear(); + this.scanners.clear(); + this.lastReturnedValue = null; + } + + @Override + public synchronized boolean next(List results) throws IOException { + OUTER: while (results.size() < 1) { + List intermediateResult = new ArrayList(); + if (hasRangeScanners || firstScan || reseeked) { + Iterator scnItr = this.scanners.iterator(); + INNER : while (scnItr.hasNext()) { + IndexRegionScanner scn = scnItr.next(); + if (reseeked) { + boolean exists = checkForScanner(scn.getScannerIndex()); + if (exists) continue; + } + boolean hasMore = scn.next(intermediateResult); + if (!hasRangeScanners) { + if (intermediateResult != null && intermediateResult.size() > 0) { + byte[] rowKeyFromKV = IndexUtils.getRowKeyFromKV(intermediateResult.get(0)); + while (valueMap.containsKey(rowKeyFromKV)) { + intermediateResult.clear(); + hasMore = scn.next(intermediateResult); + if (!intermediateResult.isEmpty()) { + rowKeyFromKV = IndexUtils.getRowKeyFromKV(intermediateResult.get(0)); + } else { + break; + } + } + if (!hasMore && intermediateResult.isEmpty()) { + // Allow other scanners to scan. Nothing to do. + } else { + valueMap.put(rowKeyFromKV, new Pair(scn, + intermediateResult.get(0))); + intermediateResult.clear(); + } + } + } + if (!hasMore) { + if (LOG.isDebugEnabled()) { + LOG.debug("Removing scanner " + scn + " from the list."); + } + scn.close(); + scnItr.remove(); + } + if (hasRangeScanners) { + if (!intermediateResult.isEmpty()) { + String rowKey = Bytes.toString(IndexUtils.getRowKeyFromKV(intermediateResult.get(0))); + if (isRootScanner && !rowCache.containsKey(rowKey)) { + rowCache.put(rowKey, false); + results.addAll(intermediateResult); + return !this.scanners.isEmpty(); + } else if (isRootScanner) { + // dont add to results because already exists and scan for other entry. + intermediateResult.clear(); + continue OUTER; + } else { + results.addAll(intermediateResult); + return !this.scanners.isEmpty(); + } + } + } + } + if (firstScan) firstScan = false; + if (reseeked) reseeked = false; + } else { + // Scan on previous scanner which returned minimum values. + Entry> minEntry = null; + if (!valueMap.isEmpty()) { + minEntry = valueMap.entrySet().iterator().next(); + IndexRegionScanner scn = minEntry.getValue().getFirst(); + if (Bytes.compareTo(lastReturnedValue, minEntry.getKey()) == 0) { + valueMap.remove(minEntry.getKey()); + boolean hasMore = scn.next(intermediateResult); + byte[] rowKeyFromKV = null; + if (!intermediateResult.isEmpty()) { + rowKeyFromKV = IndexUtils.getRowKeyFromKV(intermediateResult.get(0)); + while (valueMap.containsKey(rowKeyFromKV)) { + intermediateResult.clear(); + if (hasMore) { + hasMore = minEntry.getValue().getFirst().next(intermediateResult); + } + if (intermediateResult.isEmpty()) { + rowKeyFromKV = null; + scn.close(); + removeScanner(scn.getScannerIndex()); + break; + } + rowKeyFromKV = IndexUtils.getRowKeyFromKV(intermediateResult.get(0)); + } + } + if (rowKeyFromKV != null) { + valueMap.put(rowKeyFromKV, new Pair(scn, + intermediateResult.get(0))); + intermediateResult.clear(); + } + } + } else { + return false; + } + } + if (!valueMap.isEmpty()) { + Entry> minEntry = valueMap.entrySet().iterator() + .next(); + lastReturnedValue = minEntry.getKey(); + results.add(minEntry.getValue().getSecond()); + return true; + } else { + return false; + } + } + if (LOG.isDebugEnabled()) { + LOG.debug(results.size() + " seek points obtained. Values: " + + (!results.isEmpty() ? Bytes.toString(results.get(0).getRow()) : 0)); + } + return !results.isEmpty(); + } + + private boolean checkForScanner(int scnIndex) { + Collection> scanners = valueMap.values(); + for (Pair scn : scanners) { + if (scnIndex == scn.getFirst().getScannerIndex()) { + return true; + } + } + return false; + } + +/* private void getTheSmallerValue(KeyValue keyValue, IndexRegionScanner scn ) { + byte[] currentRow = getRowKeyFromKV(keyValue); + if(this.lastReturnedValue == null){ + this.lastReturnedValue.setFirst(new Pair(currentRow,scn.getScannerIndex())); + this.lastReturnedValue.setSecond(keyValue); + } else { + byte[] lastRow = getRowKeyFromKV(this.lastReturnedValue.getSecond()); + if(Bytes.compareTo(currentRow, lastRow) < 0){ + this.lastReturnedValue.setFirst(new Pair(currentRow,scn.getScannerIndex())); + this.lastReturnedValue.setSecond(keyValue); + } + // TODO: In case of equal need to check what to do? + if(Bytes.compareTo(currentRow, lastRow) == 0){ + scn.advance(); + } + } + } +*/ + + @Override + public boolean next(List result, int limit) throws IOException { + // TODO Auto-generated method stub + return false; + } + + @Override + public long getMaxResultSize() { + // TODO Auto-generated method stub + return 0; + } + + @Override + public long getMvccReadPoint() { + // TODO Auto-generated method stub + return 0; + } + + @Override + public boolean nextRaw(List result) throws IOException { + // TODO Auto-generated method stub + return false; + } + + @Override + public boolean nextRaw(List result, int limit) throws IOException { + // TODO Auto-generated method stub + return false; + } + +} Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/IndexRegionSplitPolicy.java =================================================================== --- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/IndexRegionSplitPolicy.java (revision 0) +++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/IndexRegionSplitPolicy.java (working copy) @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import org.apache.hadoop.hbase.regionserver.RegionSplitPolicy; + +/** + * Split policy for index regions to avoid split from external requests. + */ +public class IndexRegionSplitPolicy extends RegionSplitPolicy { + + @Override + protected boolean shouldSplit() { + return false; + } + +} Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/LeafFilterNode.java =================================================================== --- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/LeafFilterNode.java (revision 0) +++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/LeafFilterNode.java (working copy) @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import org.apache.hadoop.hbase.index.IndexSpecification; + +interface LeafFilterNode extends FilterNode { + FilterColumnValueDetail getFilterColumnValueDetail(); + + void setFilterColumnValueDetail(FilterColumnValueDetail filterColumnValueDetail); + + IndexSpecification getBestIndex(); +} \ No newline at end of file Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/LeafIndexRegionScanner.java =================================================================== --- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/LeafIndexRegionScanner.java (revision 0) +++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/LeafIndexRegionScanner.java (working copy) @@ -0,0 +1,176 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import java.io.IOException; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.index.IndexSpecification; +import org.apache.hadoop.hbase.regionserver.RegionScanner; +import org.apache.hadoop.hbase.util.Bytes; + + +public class LeafIndexRegionScanner implements IndexRegionScanner { + + private static final Log LOG = LogFactory.getLog(LeafIndexRegionScanner.class); + + private RegionScanner delegator = null; + + private Cell currentKV = null; + + private boolean hadMore = true; + + private final IndexSpecification index; + + private boolean isRangeScanner = false; + + private TTLExpiryChecker ttlExpiryChecker; + + private int scannerIndex = -1; + + public LeafIndexRegionScanner(IndexSpecification index, RegionScanner delegator, TTLExpiryChecker ttlExpiryChecker) { + this.delegator = delegator; + this.index = index; + this.ttlExpiryChecker = ttlExpiryChecker; + } + + @Override + public void advance() { + this.currentKV = null; + } + + @Override + public HRegionInfo getRegionInfo() { + return this.delegator.getRegionInfo(); + } + + @Override + public boolean isFilterDone() throws IOException { + return this.delegator.isFilterDone(); + } + + @Override + public void setRangeFlag(boolean range) { + isRangeScanner = range; + } + + @Override + public boolean isRange(){ + return this.isRangeScanner; + } + + @Override + public void setScannerIndex(int index) { + scannerIndex = index; + } + + @Override + public int getScannerIndex() { + return scannerIndex; + } + + @Override + public boolean hasChildScanners() { + return false; + } + // TODO the passing row to be the full key in the index table. + // The callee need to take care of this creation.. + @Override + public synchronized boolean reseek(byte[] row) throws IOException { + if(!hadMore) return false; + byte[] targetRowKey = createRowKeyForReseek(row); + return this.delegator.reseek(targetRowKey); + } + + private byte[] createRowKeyForReseek(byte[] targetRow) { + byte[] curRK = this.currentKV.getRow(); + byte[] curValue = this.currentKV.getValue(); + short actualTabRKOffset = Bytes.toShort(curValue, 2); + byte[] newRowKey = new byte[actualTabRKOffset + targetRow.length]; + System.arraycopy(curRK, 0, newRowKey, 0, actualTabRKOffset); + System.arraycopy(targetRow, 0, newRowKey, actualTabRKOffset, targetRow.length); + return newRowKey; + } + + @Override + public synchronized void close() throws IOException { + this.delegator.close(); + } + + @Override + public synchronized boolean next(List results) throws IOException { + boolean hasMore = false; + do { + // this check here will prevent extra next call when in the previous + // next last row was fetched and after that an advance was called on this + // scanner. So instead of making a next call again we can return from here. + if (!this.hadMore) return false; + hasMore = this.delegator.next(results); + if (results != null && results.size() > 0) { + Cell kv = results.get(0); + if (this.ttlExpiryChecker.checkIfTTLExpired(this.index.getTTL(), kv.getTimestamp())) { + results.clear(); + LOG.info("The ttl has expired for the kv " + kv); + } else { + if (!isRangeScanner) { + // This is need to reseek in case of EQUAL scanners. + this.currentKV = kv; + break; + } + } + } + } while (results.size() < 1 && hasMore); + this.hadMore = hasMore; + return hasMore; + } + + @Override + public boolean next(List result, int limit) throws IOException { + // We wont call this method at all.. As we have only CF:qualifier per row in index table. + throw new UnsupportedOperationException("Use only next(List results) method."); + } + + @Override + public long getMaxResultSize() { + // TODO Auto-generated method stub + return 0; + } + + @Override + public long getMvccReadPoint() { + // TODO Auto-generated method stub + return 0; + } + + @Override + public boolean nextRaw(List result) throws IOException { + // TODO Auto-generated method stub + return false; + } + + @Override + public boolean nextRaw(List result, int limit) throws IOException { + // TODO Auto-generated method stub + return false; + } + +} Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/NoIndexFilterNode.java =================================================================== --- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/NoIndexFilterNode.java (revision 0) +++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/NoIndexFilterNode.java (working copy) @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.hbase.index.Column; +import org.apache.hadoop.hbase.index.IndexSpecification; +import org.apache.hadoop.hbase.util.Pair; + +public class NoIndexFilterNode implements FilterNode { + + @Override + public Map, IndexSpecification> getIndexToUse() { + return null; + } + + @Override + public Map>> getPossibleUseIndices() { + return null; + } + + @Override + public Map>> getPossibleFutureUseIndices() { + return null; + } +} \ No newline at end of file Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/NonLeafFilterNode.java =================================================================== --- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/NonLeafFilterNode.java (revision 0) +++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/NonLeafFilterNode.java (working copy) @@ -0,0 +1,80 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.hbase.index.Column; +import org.apache.hadoop.hbase.index.GroupingCondition; +import org.apache.hadoop.hbase.index.IndexSpecification; +import org.apache.hadoop.hbase.util.Pair; + +public class NonLeafFilterNode implements FilterNode { + + private List filterNodes = new ArrayList(); + + private GroupingCondition groupingCondition; + + private Map, IndexSpecification> indicesToUse = + new HashMap, IndexSpecification>(); + + public NonLeafFilterNode(GroupingCondition condition) { + this.groupingCondition = condition; + } + + public GroupingCondition getGroupingCondition() { + return groupingCondition; + } + + public List getFilterNodes() { + return filterNodes; + } + + public void addFilterNode(FilterNode filterNode) { + this.filterNodes.add(filterNode); + } + + public void addIndicesToUse(FilterColumnValueDetail f, IndexSpecification i) { + List key = new ArrayList(1); + key.add(f); + this.indicesToUse.put(key, i); + } + + public void addIndicesToUse(List lf, IndexSpecification i) { + this.indicesToUse.put(lf, i); + } + + @Override + public Map, IndexSpecification> getIndexToUse() { + return this.indicesToUse; + } + + @Override + public Map>> getPossibleUseIndices() { + return null; + } + + @Override + public Map>> getPossibleFutureUseIndices() { + // There is no question of future use possible indices on a non leaf node. + return null; + } +} \ No newline at end of file Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/PossibleIndexFilterNode.java =================================================================== --- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/PossibleIndexFilterNode.java (revision 0) +++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/PossibleIndexFilterNode.java (working copy) @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.hbase.index.Column; +import org.apache.hadoop.hbase.index.IndexSpecification; +import org.apache.hadoop.hbase.util.Pair; + +public class PossibleIndexFilterNode implements LeafFilterNode { + + private List> possibleFutureUseIndices; + + private FilterColumnValueDetail filterColumnValueDetail; + + public PossibleIndexFilterNode(List> possibleFutureUseIndices, + FilterColumnValueDetail filterColumnValueDetail) { + this.possibleFutureUseIndices = possibleFutureUseIndices; + this.filterColumnValueDetail = filterColumnValueDetail; + } + + @Override + public Map>> getPossibleFutureUseIndices() { + // TODO avoid create of Map instance all the time... + Map>> reply = + new HashMap>>(); + reply.put(filterColumnValueDetail.getColumn(), possibleFutureUseIndices); + return reply; + } + + @Override + public Map, IndexSpecification> getIndexToUse() { + return null; + } + + public Map>> getPossibleUseIndices() { + return null; + } + + @Override + public FilterColumnValueDetail getFilterColumnValueDetail() { + return this.filterColumnValueDetail; + } + + @Override + public void setFilterColumnValueDetail(FilterColumnValueDetail filterColumnValueDetail) { + this.filterColumnValueDetail = filterColumnValueDetail; + } + + @Override + public IndexSpecification getBestIndex() { + return null; + } +} \ No newline at end of file Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/ReInitializableRegionScanner.java =================================================================== --- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/ReInitializableRegionScanner.java (revision 0) +++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/ReInitializableRegionScanner.java (working copy) @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import java.io.IOException; + +import org.apache.hadoop.hbase.regionserver.RegionScanner; + +public interface ReInitializableRegionScanner extends SeekAndReadRegionScanner { + + //TODO better name + void reInit(RegionScanner rs) throws IOException; + +} Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/ReInitializableRegionScannerImpl.java =================================================================== --- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/ReInitializableRegionScannerImpl.java (revision 0) +++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/ReInitializableRegionScannerImpl.java (working copy) @@ -0,0 +1,178 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Set; +import java.util.TreeSet; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.regionserver.RegionScanner; +import org.apache.hadoop.hbase.util.Bytes; + +// TODO better name? +public class ReInitializableRegionScannerImpl implements ReInitializableRegionScanner { + + private RegionScanner delegator; + + private int batch; + + private byte[] lastSeekedRowKey; + + private byte[] lastSeekAttemptedRowKey; + + private static final Log LOG = LogFactory.getLog(ReInitializableRegionScannerImpl.class); + + // Can this be queue? + // Criteria for the selection to be additional heap overhead wrt the object + // used cost in operation + private Set seekPoints; + + private SeekPointFetcher seekPointFetcher; + + private boolean closed = false; + + public ReInitializableRegionScannerImpl(RegionScanner delegator, Scan scan, + SeekPointFetcher seekPointFetcher) { + this.delegator = delegator; + this.batch = scan.getBatch(); + this.seekPoints = new TreeSet(Bytes.BYTES_COMPARATOR); + this.seekPointFetcher = seekPointFetcher; + } + + @Override + public HRegionInfo getRegionInfo() { + return this.delegator.getRegionInfo(); + } + + @Override + public boolean isFilterDone() throws IOException { + return this.delegator.isFilterDone(); + } + + @Override + public synchronized void close() throws IOException { + try { + this.delegator.close(); + } finally { + this.seekPointFetcher.close(); + } + closed = true; + } + + public boolean isClosed() { + return closed; + } + + @Override + public void addSeekPoints(List seekPoints) { + // well this add will do the sorting and remove duplicates. :) + for (byte[] seekPoint : seekPoints) { + this.seekPoints.add(seekPoint); + } + } + + @Override + public boolean seekToNextPoint() throws IOException { + // At this class level if seek is called it must be forward seek. + // call reseek() directly with the next seek point + Iterator spIterator = this.seekPoints.iterator(); + if (spIterator.hasNext()) { + this.lastSeekAttemptedRowKey = spIterator.next(); + if (null != this.lastSeekedRowKey + && Bytes.BYTES_COMPARATOR.compare(this.lastSeekedRowKey, this.lastSeekAttemptedRowKey) > 0) { + throw new SeekUnderValueException(); + } + spIterator.remove(); + LOG.trace("Next seek point " + Bytes.toString(this.lastSeekAttemptedRowKey)); + boolean reseekResult = closed ? false : this.reseek(this.lastSeekAttemptedRowKey); + if (!reseekResult) return false; + this.lastSeekedRowKey = this.lastSeekAttemptedRowKey; + return true; + } + return false; + } + + @Override + public synchronized boolean reseek(byte[] row) throws IOException { + return this.delegator.reseek(row); + } + + @Override + public void reInit(RegionScanner rs) throws IOException { + this.delegator.close(); + this.delegator = rs; + this.lastSeekedRowKey = null; + this.lastSeekAttemptedRowKey = null; + this.closed = false; + } + + @Override + public byte[] getLatestSeekpoint() { + return this.lastSeekAttemptedRowKey; + } + + @Override + public long getMvccReadPoint() { + return this.delegator.getMvccReadPoint(); + } + + @Override + public boolean nextRaw(List result) throws IOException { + return this.delegator.nextRaw(result); + } + + @Override + public boolean nextRaw(List result, int limit) throws IOException { + return this.delegator.nextRaw(result, limit); + } + + @Override + public boolean next(List results) throws IOException { + return next(results, this.batch); + } + + @Override + public boolean next(List result, int limit) throws IOException { + // Before every next call seek to the appropriate position. + if (closed) return false; + while (!seekToNextPoint()) { + List seekPoints = new ArrayList(1); + // TODO Do we need to fetch more seekpoints here? + if (!closed) this.seekPointFetcher.nextSeekPoints(seekPoints, 1); + if (seekPoints.isEmpty()) { + // nothing further to fetch from the index table. + return false; + } + this.seekPoints.addAll(seekPoints); + } + return closed ? false : this.delegator.next(result, limit); + } + + @Override + public long getMaxResultSize() { + return this.delegator.getMaxResultSize(); + } +} Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/ScanFilterEvaluator.java =================================================================== --- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/ScanFilterEvaluator.java (revision 0) +++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/ScanFilterEvaluator.java (working copy) @@ -0,0 +1,1183 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.TreeSet; +import java.util.concurrent.ConcurrentHashMap; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; +import org.apache.hadoop.hbase.filter.Filter; +import org.apache.hadoop.hbase.filter.FilterList; +import org.apache.hadoop.hbase.filter.FilterList.Operator; +import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; +import org.apache.hadoop.hbase.index.Column; +import org.apache.hadoop.hbase.index.ColumnQualifier; +import org.apache.hadoop.hbase.index.GroupingCondition; +import org.apache.hadoop.hbase.index.ValuePartition; +import org.apache.hadoop.hbase.index.ColumnQualifier.ValueType; +import org.apache.hadoop.hbase.index.Constants; +import org.apache.hadoop.hbase.index.IndexSpecification; +import org.apache.hadoop.hbase.index.client.EqualsExpression; +import org.apache.hadoop.hbase.index.client.IndexExpression; +import org.apache.hadoop.hbase.index.client.MultiIndexExpression; +import org.apache.hadoop.hbase.index.client.NoIndexExpression; +import org.apache.hadoop.hbase.index.client.RangeExpression; +import org.apache.hadoop.hbase.index.client.SingleIndexExpression; +import org.apache.hadoop.hbase.index.filter.SingleColumnRangeFilter; +import org.apache.hadoop.hbase.index.filter.SingleColumnValuePartitionFilter; +import org.apache.hadoop.hbase.index.manager.IndexManager; +import org.apache.hadoop.hbase.index.util.ByteArrayBuilder; +import org.apache.hadoop.hbase.index.util.IndexUtils; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.RegionScanner; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; + +public class ScanFilterEvaluator { + + private static final Log LOG = LogFactory.getLog(ScanFilterEvaluator.class); + + private static final Set EMPTY_INDEX_SET = new HashSet(0); + + // TODO we can make this class singleton? This cache will be shared by all regions. + // Then need to change the Map to include the table info also. A Map + private Map colFilterNodeCache = + new ConcurrentHashMap(); + + /** + * This method will evaluate the filter(s) used along with the user table scan against the indices + * available on the table and will provide a list of scans we would like to perform on the index + * table corresponding to this user scan. + * + * @param scan + * @param indices + * @param regionStartKey + * @param idxRegion + * @param tableName + * @return + * @throws IOException + */ + public IndexRegionScanner evaluate(Scan scan, List indices, + byte[] regionStartKey, HRegion idxRegion, String tableName) throws IOException { + Filter filter = scan.getFilter(); + byte[] indexExpBytes = scan.getAttribute(Constants.INDEX_EXPRESSION); + if (filter == null) { + // When the index(s) to be used is passed through scan attributes and Scan not having any + // filters it is invalid + if (indexExpBytes != null) { + LOG.warn("Passed an Index expression along with the Scan but without any filters on Scan!" + + " The index wont be used"); + } + return null; + } + FilterNode node = null; + IndexRegionScanner indexRegionScanner = null; + if (indexExpBytes != null) { + // Which index(s) to be used is already passed by the user. + try { + IndexExpression indexExpression = org.apache.hadoop.hbase.index.client.IndexUtils + .toIndexExpression(indexExpBytes); + if (indexExpression instanceof NoIndexExpression) { + // Client side says not to use any index for this Scan. + LOG.info("NoIndexExpression is passed as the index to be used for this Scan." + + " No possible index will be used."); + return null; + } + Map nameVsIndex = new HashMap(); + for (IndexSpecification index : indices) { + nameVsIndex.put(index.getName(), index); + } + node = convertIdxExpToFilterNode(indexExpression, nameVsIndex, tableName); + } catch (Exception e) { + LOG.error("There is an Exception in getting IndexExpression from Scan attribute!" + + " The index won't be used", e); + } + } else { + Filter newFilter = doFiltersRestruct(filter); + if (newFilter != null) { + node = evalFilterForIndexSelection(newFilter, indices); + } + } + if (node != null) { + indexRegionScanner = createIndexScannerScheme(node, regionStartKey, scan.getStartRow(), + scan.getStopRow(), idxRegion, tableName); + // TODO - This check and set looks ugly. Correct me.. + if (indexRegionScanner instanceof IndexRegionScannerForOR) { + ((IndexRegionScannerForOR) indexRegionScanner).setRootFlag(true); + } + if (indexRegionScanner != null) { + indexRegionScanner.setScannerIndex(0); + if (indexRegionScanner.hasChildScanners()) { + return indexRegionScanner; + } else { + return null; + } + } + } + return indexRegionScanner; + } + + private FilterNode convertIdxExpToFilterNode(IndexExpression indexExpression, + Map nameVsIndex, String tableName) { + if (indexExpression instanceof MultiIndexExpression) { + MultiIndexExpression mie = (MultiIndexExpression) indexExpression; + NonLeafFilterNode nlfn = new NonLeafFilterNode(mie.getGroupingCondition()); + for (IndexExpression ie : mie.getIndexExpressions()) { + FilterNode fn = convertIdxExpToFilterNode(ie, nameVsIndex, tableName); + nlfn.addFilterNode(fn); + } + return nlfn; + } else { + // SingleIndexExpression + SingleIndexExpression sie = (SingleIndexExpression) indexExpression; + IndexSpecification index = nameVsIndex.get(sie.getIndexName()); + if (index == null) { + throw new RuntimeException("No index:" + sie.getIndexName() + " added for table:" + + tableName); + } + Map colVsCQ = new HashMap(); + for (ColumnQualifier cq : index.getIndexColumns()) { + colVsCQ + .put(new Column(cq.getColumnFamily(), cq.getQualifier(), cq.getValuePartition()), cq); + } + // TODO -- seems some thing wrong with IndexFilterNode! Ideally I need to create that here. + NonLeafFilterNode nlfn = new NonLeafFilterNode(null); + List fcvds = new ArrayList(); + // Here by we expect that the equals expressions are given the order of columns in index. + // TODO add reordering if needed? + for (EqualsExpression ee : sie.getEqualsExpressions()) { + ColumnQualifier cq = colVsCQ.get(ee.getColumn()); + if (cq == null) { + throw new RuntimeException("The column:[" + ee.getColumn() + "] is not a part of index: " + + sie.getIndexName()); + } + FilterColumnValueDetail fcvd = new FilterColumnValueDetail(ee.getColumn(), ee.getValue(), + CompareOp.EQUAL); + fcvd.maxValueLength = cq.getMaxValueLength(); + fcvd.valueType = cq.getType(); + fcvds.add(fcvd); + } + // RangeExpression to come after the EqualsExpressions + RangeExpression re = sie.getRangeExpression(); + if (re != null) { + ColumnQualifier cq = colVsCQ.get(re.getColumn()); + if (cq == null) { + throw new RuntimeException("The column:[" + re.getColumn() + "] is not a part of index: " + + sie.getIndexName()); + } + CompareOp lowerBoundCompareOp = re.isLowerBoundInclusive() ? CompareOp.GREATER_OR_EQUAL + : CompareOp.GREATER; + CompareOp upperBoundCompareOp = re.isUpperBoundInclusive() ? CompareOp.LESS_OR_EQUAL + : CompareOp.LESS; + if (re.getLowerBoundValue() == null) { + lowerBoundCompareOp = null; + } + if (re.getUpperBoundValue() == null) { + upperBoundCompareOp = null; + } + FilterColumnValueRange fcvr = new FilterColumnValueRange(re.getColumn(), + re.getLowerBoundValue(), lowerBoundCompareOp, re.getUpperBoundValue(), + upperBoundCompareOp); + fcvr.maxValueLength = cq.getMaxValueLength(); + fcvr.valueType = cq.getType(); + fcvds.add(fcvr); + } + nlfn.addIndicesToUse(fcvds, index); + return nlfn; + } + } + + IndexRegionScanner createIndexScannerScheme(FilterNode node, byte[] regionStartKey, + byte[] startRow, byte[] stopRow, HRegion indexRegion, String userTableName) + throws IOException { + List scanners = new ArrayList(); + IndexRegionScanner idxScanner = null; + int scannerIndex = -1; + if (node instanceof NonLeafFilterNode) { + boolean hasRangeScanner = false; + NonLeafFilterNode nlfNode = (NonLeafFilterNode) node; + Map, IndexSpecification> indicesToUse = nlfNode.getIndexToUse(); + for (Entry, IndexSpecification> entry : indicesToUse.entrySet()) { + List fcvdList = entry.getKey(); + byte[] indexName = Bytes.toBytes(entry.getValue().getName()); + ByteArrayBuilder indexNameBuilder = ByteArrayBuilder.allocate(IndexUtils + .getMaxIndexNameLength()); + indexNameBuilder.put(indexName); + Scan scan = createScan(regionStartKey, indexName, fcvdList, startRow, stopRow); + boolean isRange = isHavingRangeFilters(fcvdList); + if(!hasRangeScanner && isRange) hasRangeScanner = isRange; + createRegionScanner(indexRegion, userTableName, scanners, indexNameBuilder, scan, isRange,++scannerIndex); + } + for (FilterNode fn : nlfNode.getFilterNodes()) { + IndexRegionScanner childIndexScaner = createIndexScannerScheme(fn, regionStartKey, + startRow, stopRow, indexRegion, userTableName); + childIndexScaner.setScannerIndex(++scannerIndex); + scanners.add(childIndexScaner); + if(!hasRangeScanner) hasRangeScanner = childIndexScaner.isRange(); + } + idxScanner = createScannerForNonLeafNode(scanners, nlfNode.getGroupingCondition()); + idxScanner.setRangeFlag(hasRangeScanner); + return idxScanner; + } else if (node instanceof PossibleIndexFilterNode) { + LOG.info("No index can be used for the column " + + ((PossibleIndexFilterNode) node).getFilterColumnValueDetail().getColumn()); + return null; + } else if (node instanceof IndexFilterNode) { + // node is IndexFilterNode + IndexFilterNode ifNode = (IndexFilterNode) node; + // There will be only one entry in this Map + List filterColsDetails = ifNode.getIndexToUse().keySet().iterator() + .next(); + byte[] indexName = Bytes.toBytes(ifNode.getBestIndex().getName()); + ByteArrayBuilder indexNameBuilder = ByteArrayBuilder.allocate(IndexUtils + .getMaxIndexNameLength()); + indexNameBuilder.put(indexName); + Scan scan = createScan(regionStartKey, indexName, filterColsDetails, startRow, stopRow); + boolean isRange = isHavingRangeFilters(filterColsDetails); + createRegionScanner(indexRegion, userTableName, scanners, indexNameBuilder, scan, isRange, + ++scannerIndex); + idxScanner = createScannerForNonLeafNode(scanners, null); + idxScanner.setRangeFlag(isRange); + return idxScanner; + } + return null; + } + + private boolean isHavingRangeFilters(List fcvdList) { + for (FilterColumnValueDetail fcvd : fcvdList) { + if (fcvd instanceof FilterColumnValueRange) { + return true; + } + } + return false; + } + + private IndexRegionScanner createScannerForNonLeafNode(List scanners, + GroupingCondition condition) { + IndexRegionScanner idxScanner; + if (condition == GroupingCondition.OR) { + idxScanner = new IndexRegionScannerForOR(scanners); + } else { + idxScanner = new IndexRegionScannerForAND(scanners); + } + return idxScanner; + } + + private void createRegionScanner(HRegion indexRegion, String userTableName, + List scanners, ByteArrayBuilder indexNameBuilder, Scan scan, + boolean isRange, int scannerIndex) throws IOException { + RegionScanner scannerForIndexRegion = indexRegion.getScanner(scan); + LeafIndexRegionScanner leafIndexRegionScanner = new LeafIndexRegionScanner(IndexManager + .getInstance().getIndex(userTableName, indexNameBuilder.array()), + scannerForIndexRegion, new TTLExpiryChecker()); + leafIndexRegionScanner.setScannerIndex(scannerIndex); + leafIndexRegionScanner.setRangeFlag(isRange); + scanners.add(leafIndexRegionScanner); + } + + private Scan createScan(byte[] regionStartKey, byte[] indexName, + List filterColsDetails, byte[] startRow, byte[] stopRow) { + Scan scan = new Scan(); + // common key is the regionStartKey + indexName + byte[] commonKey = createCommonKeyForIndex(regionStartKey, indexName); + scan.setStartRow(createStartOrStopKeyForIndexScan(filterColsDetails, commonKey, startRow, true)); + // Find the end key for the scan + scan.setStopRow(createStartOrStopKeyForIndexScan(filterColsDetails, commonKey, stopRow, false)); + return scan; + } + + private byte[] createCommonKeyForIndex(byte[] regionStartKey, byte[] indexName) { + // Format for index table rowkey [Startkey for the index region] + [one 0 byte]+ + // [Index name] + [Padding for the max index name] + .... + int commonKeyLength = regionStartKey.length + 1 + IndexUtils.getMaxIndexNameLength(); + ByteArrayBuilder builder = ByteArrayBuilder.allocate(commonKeyLength); + // Adding the startkey for the index region and single 0 Byte. + builder.put(regionStartKey); + builder.position(builder.position() + 1); + // Adding the index name and the padding needed + builder.put(indexName); + // No need to add the padding bytes specifically. In the array all the bytes will be 0s. + return builder.array(); + } + + // When it comes here, only the last column in the colDetails will be a range. + // Others will be exact value. EQUALS condition. + private byte[] createStartOrStopKeyForIndexScan(List colDetails, + byte[] commonKey, byte[] userTabRowKey, boolean isStartKey) { + int totalValueLen = 0; + for (FilterColumnValueDetail fcvd : colDetails) { + totalValueLen += fcvd.maxValueLength; + } + int userTabRowKeyLen = userTabRowKey == null ? 0 : userTabRowKey.length; + ByteArrayBuilder builder = ByteArrayBuilder.allocate(commonKey.length + totalValueLen + + userTabRowKeyLen); + builder.put(commonKey); + for (int i = 0; i < colDetails.size(); i++) { + FilterColumnValueDetail fcvd = colDetails.get(i); + if (i == colDetails.size() - 1) { + // This is the last column in the colDetails. Here the range check can come + if (isStartKey) { + if (fcvd.compareOp == null) { + // This can happen when the condition is a range condition and only upper bound is + // specified. ie. c1<100 + // Now the column value can be specified as 0 bytes. Just we need to advance the byte[] + assert fcvd instanceof FilterColumnValueRange; + assert fcvd.value == null; + builder.position(builder.position() + fcvd.maxValueLength); + } else if (fcvd.compareOp.equals(CompareOp.EQUAL) + || fcvd.compareOp.equals(CompareOp.GREATER_OR_EQUAL)) { + copyColumnValueToKey(builder, fcvd.value, fcvd.maxValueLength, fcvd.valueType); + } else if (fcvd.compareOp.equals(CompareOp.GREATER)) { + // We can go with the value + 1 + // For eg : if type is int and value is 45, make startkey considering value as 46 + // If type is String and value is 'ad' make startkey considering value as 'ae' + + copyColumnValueToKey(builder, fcvd.value, + fcvd.maxValueLength, fcvd.valueType); + IndexUtils.incrementValue(builder.array(), false); + } + } else { + CompareOp op = fcvd.compareOp; + byte[] value = fcvd.value; + if (fcvd instanceof FilterColumnValueRange) { + op = ((FilterColumnValueRange) fcvd).getUpperBoundCompareOp(); + value = ((FilterColumnValueRange) fcvd).getUpperBoundValue(); + } + if (op == null) { + // This can happen when the condition is a range condition and only lower bound is + // specified. ie. c1>=10 + assert fcvd instanceof FilterColumnValueRange; + assert value == null; + // Here the stop row is already partially built. As there is no upper bound all the + // possibles values for that column we need to consider. Well the max value for a + // column with maxValueLength=10 will a byte array of 10 bytes with all bytes as FF. + // But we can put this FF bytes into the stop row because the stop row will not be + // considered by the scanner. So we need to increment this by 1. + // We can increment the byte[] created until now by 1. + byte[] stopRowTillNow = builder.array(0, builder.position()); + stopRowTillNow = IndexUtils.incrementValue(stopRowTillNow, true); + // Now we need to copy back this incremented value to the builder. + builder.position(0); + builder.put(stopRowTillNow); + // Now just advance the builder pos by fcvd.maxValueLength as we need all 0 bytes + builder.position(builder.position() + fcvd.maxValueLength); + } else if (op.equals(CompareOp.EQUAL) || op.equals(CompareOp.LESS_OR_EQUAL)) { + copyColumnValueToKey(builder, value, fcvd.maxValueLength, fcvd.valueType); + IndexUtils.incrementValue(builder.array(), false); + + } else if (op.equals(CompareOp.LESS)) { + copyColumnValueToKey(builder, value, fcvd.maxValueLength, fcvd.valueType); + } + } + } else { + // For all other columns other than the last column the condition must be EQUALS + copyColumnValueToKey(builder, fcvd.value, fcvd.maxValueLength, fcvd.valueType); + } + } + if (userTabRowKeyLen > 0) { + builder.put(userTabRowKey); + } + + return builder.array(); + } + + private void copyColumnValueToKey(ByteArrayBuilder builder, byte[] colValue, int maxValueLength, + ValueType valueType) { + colValue = IndexUtils.changeValueAccToDataType(colValue, valueType); + builder.put(colValue); + int paddingLength = maxValueLength - colValue.length; + builder.position(builder.position() + paddingLength); + } + + // This method will do the reorder and restructuring of the filters so as the finding of + // suitable index for the query will be easier for the remaining steps + // This will do 2 things basically + // 1.Group the AND condition on SingleColumnValueFilter together so as to make a range condition. + // In this process it will validate the correctness of the range and will avoid the invalid + // things. + // 2.Move the adjacent AND conditions together within on filter list. If N number of adjacent + // filters or filter lists are there and all are combined using AND condition then it is same + // as all within one list. + // Default access for testability + Filter doFiltersRestruct(Filter filter) { + if (filter instanceof SingleColumnValueFilter) { + ValuePartition vp = null; + if (filter instanceof SingleColumnValuePartitionFilter) { + vp = ((SingleColumnValuePartitionFilter) filter).getValuePartition(); + } + SingleColumnValueFilter scvf = (SingleColumnValueFilter) filter; + if (scvf.getOperator().equals(CompareOp.LESS) + || scvf.getOperator().equals(CompareOp.LESS_OR_EQUAL) + || scvf.getOperator().equals(CompareOp.GREATER) + || scvf.getOperator().equals(CompareOp.GREATER_OR_EQUAL)) { + return new SingleColumnRangeFilter(scvf.getFamily(), scvf.getQualifier(), vp, scvf + .getComparator().getValue(), scvf.getOperator(), null, null); + } + } + FilterGroupingWorker groupWorker = new FilterGroupingWorker(); + return groupWorker.group(filter); + } + + // For a multi column index, the index look up with a value range can be supported only on the + // last column. The previous columns lookups to be exact value look up. + // Index idx1 on columns c1 and c2. A lookup condition with c1=x and c2 btw x and z can be + // supported using the index idx1. But if the lookup is c1 btw a and k and c2... we can better + // not support using idx1 there. (May be check later we can support this) + // In this case 2 indices idx1 and idx2 on col1 and col2 respectively can be used. + // In the above case a lookup only on c1, either it is a range lookup or single value lookup + // we can make use of the multi column index on c1 and c2. + // In this case if there is an index on c1 alone, we better use that. + // But a single column lookup on column c1 and there is an index on c2 and c1 , we can not use + // this index. It is equivalent to c1=x and c2 btw min to max. + // Default access for testability + FilterNode evalFilterForIndexSelection(Filter filter, List indices) { + if (filter instanceof FilterList) { + FilterList fList = (FilterList) filter; + GroupingCondition condition = (fList.getOperator() == Operator.MUST_PASS_ALL) ? GroupingCondition.AND + : GroupingCondition.OR; + NonLeafFilterNode nonLeafFilterNode = new NonLeafFilterNode(condition); + List filters = fList.getFilters(); + for (Filter fltr : filters) { + FilterNode node = evalFilterForIndexSelection(fltr, indices); + nonLeafFilterNode.addFilterNode(node); + } + return handleNonLeafFilterNode(nonLeafFilterNode); + } else if (filter instanceof SingleColumnValueFilter) { + // Check for the availability of index + return selectBestFitAndPossibleIndicesForSCVF(indices, (SingleColumnValueFilter) filter); + } else if (filter instanceof SingleColumnRangeFilter) { + return selectBestFitAndPossibleIndicesForSCRF(indices, (SingleColumnRangeFilter) filter); + } + return new NoIndexFilterNode(); + } + + private FilterNode selectBestFitAndPossibleIndicesForSCRF(List indices, + SingleColumnRangeFilter filter) { + FilterColumnValueRange range = new FilterColumnValueRange(filter.getFamily(), + filter.getQualifier(), filter.getValuePartition(), filter.getLowerBoundValue(), + filter.getLowerBoundOp(), filter.getUpperBoundValue(), filter.getUpperBoundOp()); + return selectBestFitIndexForColumn(indices, range); + } + + private FilterNode handleNonLeafFilterNode(NonLeafFilterNode nonLeafFilterNode) { + // check whether this node can be changed to a NoIndexFilterNode + // This we can do when the condition in nonLeafFilterNode is OR and any of the + // child node is NoIndexFilterNode + if (nonLeafFilterNode.getGroupingCondition() == GroupingCondition.OR) { + return handleORCondition(nonLeafFilterNode); + } else { + // AND condition + return handleANDCondition(nonLeafFilterNode); + } + } + + private FilterNode handleORCondition(NonLeafFilterNode nonLeafFilterNode) { + Iterator nonLeafFilterNodeItr = nonLeafFilterNode.getFilterNodes().iterator(); + while (nonLeafFilterNodeItr.hasNext()) { + FilterNode filterNode = nonLeafFilterNodeItr.next(); + if (filterNode instanceof IndexFilterNode) { + FilterColumnValueDetail filterColumnValueDetail = ((IndexFilterNode) filterNode) + .getFilterColumnValueDetail(); + IndexSpecification indexToUse = ((IndexFilterNode) filterNode).getBestIndex(); + nonLeafFilterNode.addIndicesToUse(filterColumnValueDetail, indexToUse); + nonLeafFilterNodeItr.remove(); + } else if (filterNode instanceof PossibleIndexFilterNode + || filterNode instanceof NoIndexFilterNode) { + // The moment an OR condition contains a column on which there is no index which can be + // used, the entire OR node becomes as non indexed. + return new NoIndexFilterNode(); + } + // A NonLeafFilterNode under the OR node need to be kept as it is. + } + return nonLeafFilterNode; + } + + private FilterNode handleANDCondition(NonLeafFilterNode nonLeafFilterNode) { + Map leafNodes = new HashMap(); + Iterator filterNodesIterator = nonLeafFilterNode.getFilterNodes().iterator(); + while (filterNodesIterator.hasNext()) { + FilterNode filterNode = filterNodesIterator.next(); + if (filterNode instanceof LeafFilterNode) { + LeafFilterNode lfNode = (LeafFilterNode) filterNode; + leafNodes.put(lfNode.getFilterColumnValueDetail().column, lfNode); + filterNodesIterator.remove(); + } else if (filterNode instanceof NoIndexFilterNode) { + filterNodesIterator.remove(); + } + // Any NonLeafFilterNode under this NonLeafFilterNode will be kept as it is. + // This will be a OR condition corresponding node. + } + // This below method will consider all the leafNodes just under that and will try to + // finalize one or more index to be used for those col. It will try to get the best + // combination minimizing the number of indices to be used. If I have say 5 leaf cols + // under this AND node and there is one index on these 5 cols, well I can use that one + // index. If not will try to find indices which can be applied on the subsets of these + // 5 cols, say one on 3 cols and other on 2 cols + if (!leafNodes.isEmpty()) { + Map, IndexSpecification> colVsIndex = finalizeIndexForLeafNodes(leafNodes); + if (LOG.isDebugEnabled()) { + LOG.debug("Index(s) which will be used for columns " + leafNodes.keySet() + " : " + + colVsIndex); + } + if (colVsIndex != null) { + addIndicesToNonLeafAndNode(colVsIndex, nonLeafFilterNode, leafNodes); + } + } + return nonLeafFilterNode; + } + + // Here also 2 many loops we can avoid this + @SuppressWarnings("unchecked") + private Map, IndexSpecification> finalizeIndexForLeafNodes( + Map leafNodes) { + // go with the breakups and check + // suppose there are 5 cols under the AND condition and are c1,c2,c3,c4,c5 + // There can be different break ups for the cols possible. + // [5],[4,1],[3,2],[3,1,1],[2,2,1],[2,1,1,1],[1,1,1,1,1] + // In each of these breakup also we can get many columns combinations. + // Except in first and last where either all cols in one group or 1 column only. + // For [4,1] there can be 5c1 combinations possible. + Set>>> colBreakUps = getColsBreakUps(leafNodes.keySet()); + ColBreakUpIndexDetails bestColBreakUpIndexDetails = null; + for (List>> colBreakUp : colBreakUps) { + ColBreakUpIndexDetails colBreakUpIndexDetails = findBestIndicesForColSplitsInBreakUp( + colBreakUp, leafNodes); + if (colBreakUpIndexDetails == null) { + continue; + } + if (colBreakUpIndexDetails.isBestIndex) { + // This means this is THE best index. It solves all the columns and exactly those cols only + // there as part of the indices too.. What else we need... + bestColBreakUpIndexDetails = colBreakUpIndexDetails; + break; + } else { + if (bestColBreakUpIndexDetails == null + || isIndicesGroupBetterThanCurBest(colBreakUpIndexDetails, bestColBreakUpIndexDetails)) { + bestColBreakUpIndexDetails = colBreakUpIndexDetails; + } + } + } + // TODO some more logging of the output.. + return bestColBreakUpIndexDetails == null ? null + : bestColBreakUpIndexDetails.bestIndicesForBreakUp; + } + + private void addIndicesToNonLeafAndNode(Map, IndexSpecification> colsVsIndex, + NonLeafFilterNode nonLeafFilterNode, Map leafNodes) { + for (Entry, IndexSpecification> entry : colsVsIndex.entrySet()) { + List cols = entry.getKey(); + int colsSize = cols.size(); + IndexSpecification index = entry.getValue(); + // The FilterColumnValueDetail for cols need to be in the same order as that of cols + // in the index. This order will be important for creating the start/stop keys for + // index scan. + List fcvds = new ArrayList(colsSize); + int i = 0; + for (ColumnQualifier cq : index.getIndexColumns()) { + FilterColumnValueDetail fcvd = leafNodes.get( + new Column(cq.getColumnFamily(), cq.getQualifier(), cq.getValuePartition())) + .getFilterColumnValueDetail(); + assert fcvd != null; + fcvds.add(fcvd); + i++; + if (i == colsSize) { + // The selected index might be on more cols than those we are interested in now. + // All those will be towards the end. + break; + } + } + LOG.info("Index using for the columns " + cols + " : " + index); + nonLeafFilterNode.addIndicesToUse(fcvds, index); + } + } + + private static class ColBreakUpIndexDetails { + private Map, IndexSpecification> bestIndicesForBreakUp; + private int bestIndicesCardinality = -1; + private int colsResolvedByBestIndices = -1; + private boolean isBestIndex = false; + } + + private ColBreakUpIndexDetails findBestIndicesForColSplitsInBreakUp( + List>> colBreakUpCombs, Map leafNodes) { + int totalColsInBreakup = leafNodes.size(); + ColBreakUpIndexDetails bestColBreakUpIndexDetails = null; + // List>> breakUp contains different combinations of the columns in a + // particular break up case. Say for 3 cols, a,b,c and the breakup is [2,1], this list + // contains combinations [[a,b],[c]],[[b,c],[a]],[[a,c],[b]] + for (List> colBreakUpComb : colBreakUpCombs) { + if (LOG.isDebugEnabled()) { + LOG.debug("Checking for the best index(s) for the cols combination : " + colBreakUpComb); + } + Map, IndexSpecification> colsVsIndex = + new HashMap, IndexSpecification>(); + for (List cols : colBreakUpComb) { + IndexSpecification index = findBestIndex(cols, leafNodes); + if (LOG.isDebugEnabled()) { + LOG.debug("Best index selected for the cols " + cols + " : " + index); + } + if (index != null) { + colsVsIndex.put(cols, index); + } + } + if (colsVsIndex.size() == 0) { + // For this cols breakup not even single index we are able to find which solves atleast + // one column. Just continue with the next combination. + if (LOG.isDebugEnabled()) { + LOG.debug("Not even one index found for the cols combination : " + colBreakUpComb); + } + continue; + } + int netIndicesCardinality = 0; + int colsResolved = 0; + for (Entry, IndexSpecification> entry : colsVsIndex.entrySet()) { + colsResolved += entry.getKey().size(); + netIndicesCardinality += entry.getValue().getIndexColumns().size(); + } + // Am I THE real best combination + if (colBreakUpComb.size() == colsVsIndex.size()) { + assert colsResolved == totalColsInBreakup; + // Means we have index for all the cols combinations + // Now check the net cardinality of all the indices + if (netIndicesCardinality == totalColsInBreakup) { + // This is the best + LOG.info("Got indices combination for the cols " + colsVsIndex + " which is THE BEST!"); + ColBreakUpIndexDetails bestDetails = new ColBreakUpIndexDetails(); + bestDetails.bestIndicesForBreakUp = colsVsIndex; + bestDetails.isBestIndex = true; + //colsResolved and netIndicesCardinality not needed in this case + return bestDetails; + } + } + // Checking whether this colsIndexMap combination better than the previous best. + ColBreakUpIndexDetails curDetails = new ColBreakUpIndexDetails(); + curDetails.bestIndicesForBreakUp = colsVsIndex; + curDetails.colsResolvedByBestIndices = colsResolved; + curDetails.bestIndicesCardinality = netIndicesCardinality; + if (bestColBreakUpIndexDetails == null + || isIndicesGroupBetterThanCurBest(curDetails, bestColBreakUpIndexDetails)) { + bestColBreakUpIndexDetails = curDetails; + } + } + return bestColBreakUpIndexDetails; + } + + private boolean isIndicesGroupBetterThanCurBest(ColBreakUpIndexDetails curColBreakUpIndexDetails, + ColBreakUpIndexDetails bestColBreakUpIndexDetails) { + // Conditions to decide whether current one is better than the current best + // 1. The total number of cols all the indices resolve + // 2. The number of indices which resolves the cols + // 3. The net cardinality of all the indices + // TODO later can check for the dynamic metric data about indices and decide which can be used. + int colsResolvedByCurIndices = curColBreakUpIndexDetails.colsResolvedByBestIndices; + int colsResolvedByBestIndices = bestColBreakUpIndexDetails.colsResolvedByBestIndices; + if (colsResolvedByCurIndices > colsResolvedByBestIndices) { + // The more cols it resolves the better. + return true; + } else if (colsResolvedByCurIndices == colsResolvedByBestIndices) { + int indicesInCurComb = curColBreakUpIndexDetails.bestIndicesForBreakUp.size(); + int indicesInBestComb = bestColBreakUpIndexDetails.bestIndicesForBreakUp.size(); + if (indicesInCurComb < indicesInBestComb) { + // The lesser the number of indices the better. + return true; + } else if (indicesInCurComb == indicesInBestComb) { + int curIndicesCardinality = curColBreakUpIndexDetails.bestIndicesCardinality; + int bestIndicesCardinality = bestColBreakUpIndexDetails.bestIndicesCardinality; + if (curIndicesCardinality < bestIndicesCardinality) { + // The lesser the cardinality the better. + return true; + } + } + } + return false; + } + + private IndexSpecification findBestIndex(List cols, Map leafNodes) { + if (LOG.isDebugEnabled()) { + LOG.debug("Trying to find a best index for the cols : " + cols); + } + Set indicesToUse = getPossibleIndicesForCols(cols, leafNodes); + // indicesToUse will never come as null.... + if (LOG.isDebugEnabled()) { + LOG.debug("Possible indices for cols " + cols + " : " + indicesToUse); + } + IndexSpecification bestIndex = null; + int bestIndexCardinality = -1; + for (IndexSpecification index : indicesToUse) { + if (isIndexSuitable(index, cols, leafNodes)) { + if (LOG.isDebugEnabled()) { + LOG.debug("Index " + index + " seems to be suitable for the columns " + cols); + } + if (index.getIndexColumns().size() == cols.size()) { + // Yea we got the best index. Juts return this. No need to loop through and check + // with other indices + return index; + } + // Compare this index with the current best. This will be better if its cardinality + // is better(lesser) than the current best's + // TODO pluggable interface to decide which index to be used when both this and current + // best index having same cardinality. + if (bestIndex == null || index.getIndexColumns().size() < bestIndexCardinality) { + bestIndex = index; + bestIndexCardinality = index.getIndexColumns().size(); + } + } + } + return bestIndex; + } + + private Set getPossibleIndicesForCols(List cols, + Map leafNodes) { + // When there are more than one range conditioned columns, no need to check with any of the + // index. We will not get any matched index for all cols. + int noOfRangeConds = 0; + for (Column col : cols) { + LeafFilterNode leafFilterNode = leafNodes.get(col); + if (leafFilterNode != null + && leafFilterNode.getFilterColumnValueDetail() instanceof FilterColumnValueRange) { + noOfRangeConds++; + } + if (noOfRangeConds > 1) { + return EMPTY_INDEX_SET; + } + } + // For a given column with a range condition an index can be its possible index or + // future possible index iff this column is the last column in the index + // Suppose an index on c1&c2 and the column c1 is having of range 10-20, then this index + // can not be used for this range condition. [provided some condition is there on c2 also] + // AND(c1=10,c2=20,c3 btw[10,20]) + Set indicesToUse = new HashSet(); + for (Column col : cols) { + LeafFilterNode lfn = leafNodes.get(col); + if (lfn != null) { + FilterColumnValueDetail filterColumnValueDetail = lfn.getFilterColumnValueDetail(); + IndexSpecification bestIndex = lfn.getBestIndex(); + if (bestIndex != null) { + indicesToUse.add(bestIndex); + } + Map>> possibleUseIndices = lfn + .getPossibleUseIndices(); + if (possibleUseIndices != null) { + List> possibleIndices = possibleUseIndices + .get(filterColumnValueDetail.getColumn()); + if (possibleIndices != null) { + for (Pair pair : possibleIndices) { + indicesToUse.add(pair.getFirst()); + } + } + } + Map>> possibleFutureUseIndices = lfn + .getPossibleFutureUseIndices(); + if (possibleFutureUseIndices != null) { + List> possibleIndices = possibleFutureUseIndices + .get(filterColumnValueDetail.getColumn()); + if (possibleIndices != null) { + for (Pair pair : possibleIndices) { + indicesToUse.add(pair.getFirst()); + } + } + } + } + } + return indicesToUse; + } + + // Whether the passed index is suitable to be used for the given columns. + // index is suitable when the it contains all the columns. + // Also index should not contain any other column before any of the column + // in the passed cols list + private boolean isIndexSuitable(IndexSpecification index, List cols, + Map leafNodes) { + int matchedCols = 0; + for (ColumnQualifier cq : index.getIndexColumns()) { + Column column = new Column(cq.getColumnFamily(), cq.getQualifier(), cq.getValuePartition()); + if (cols.contains(column)) { + matchedCols++; + // leafNodes.get(column) will never be null.. Don't worry + if (leafNodes.get(column).getFilterColumnValueDetail() instanceof FilterColumnValueRange) { + // When the condition on the column is a range condition, we need to ensure in this index + // 1. The column is the last column + // or + // 2. There are no columns in this index which is part of the cols list + if (matchedCols != cols.size()) { + return false; + } + } + } else { + if (matchedCols != cols.size()) { + return false; + } + } + if (matchedCols == cols.size()) { + return true; + } + } + return false; + } + + @SuppressWarnings("unchecked") + Set>>> getColsBreakUps(Set columns) { + int totalColsCount = columns.size(); + // Breakups should be sorted such that the best one come before the others. The best one is the + // one with lower number of splits. If the number of splits same, then the which contains more + // cols in one split. + // eg: When there are total 5 cols, then split [7] is better than [4,1] and [4,1] is better + // than [3,2] + Comparator>>> comparator = new Comparator>>>() { + @Override + public int compare(List>> l1, List>> l2) { + List> firstColsCombl1 = l1.get(0); + List> firstColsCombl2 = l2.get(0); + int result = firstColsCombl1.size() - firstColsCombl2.size(); + if (result != 0) + return result; + int size = Math.min(firstColsCombl1.size(), firstColsCombl2.size()); + for (int i = 0; i < size; i++) { + int colsSizeInSplit1 = firstColsCombl1.get(i).size(); + int colsSizeInSplit2 = firstColsCombl2.get(i).size(); + result = colsSizeInSplit1 - colsSizeInSplit2; + if (result != 0) + return -result; + } + return 0; + } + }; + Set>>> listOfCols = new TreeSet>>>(comparator); + ColCombination comb = new ColCombination(); + ArrayList breakUp = null; + int firstItemColsCount = totalColsCount; + while (firstItemColsCount >= 1) { + breakUp = new ArrayList(); + breakUp.add(firstItemColsCount); + int totalColsCountInBreakUp = firstItemColsCount; + while (totalColsCountInBreakUp < totalColsCount) { + // Fill with 1... + breakUp.add(1); + totalColsCountInBreakUp++; + } + listOfCols.add(makeColCombination(columns, breakUp, comb)); + // now try to combine the after 1st entry 1s into bigger items. + // But this combined value never should be more than the 1st value. + // group last 2 items and create a new breakup.. + while (true) { + // In order to do this group there should be atleast 3 elements in the breakUp list. + if (breakUp.size() < 3) { + break; + } + breakUp = (ArrayList) breakUp.clone(); + int lastElem = breakUp.remove(breakUp.size() - 1); + int secondLastElem = breakUp.remove(breakUp.size() - 1); + // Add this element to the current last. ie. the old second last element + // Well this element must be 1 only. + int newLastElem = lastElem + secondLastElem; + if (newLastElem > firstItemColsCount) { + break; + } + breakUp.add(newLastElem); + listOfCols.add(makeColCombination(columns, breakUp, comb)); + } + firstItemColsCount--; + } + return listOfCols; + } + + private List>> makeColCombination(Set columns, List breakUp, + ColCombination comb) { + int size = breakUp.size(); + return comb.formColCombinations(size, columns, breakUp); + } + + + /** + * Generates the column combination + */ + private static class ColCombination { + + public List>> formColCombinations(int breakUpSize, Set columns, + List breakUp) { + // If no of cols and break up is same it will be all 1s combination. So just return the list + List>> finalCols = new ArrayList>>(); + ArrayList> possibleCols = new ArrayList>(); + LinkedList copyOfColumns = new LinkedList(columns); + int lastIndex = 0; + boolean allOnes = false; + // for every break up find the combination + for (Integer r : breakUp) { + if (possibleCols.size() == 0) { + possibleCols = (ArrayList>) createCombination(copyOfColumns, r, possibleCols); + copyOfColumns = new LinkedList(columns); + if (columns.size() == breakUpSize) { + // for [1 1 1 1 1] + allOnes = true; + break; + } + } else { + int colSize = possibleCols.size(); + List> cloneOfPossibleCols = new ArrayList>(possibleCols); + + for (int i = lastIndex; i < colSize; i++) { + + List> possibleCols1 = new ArrayList>(); + List col = clone(cloneOfPossibleCols, i); + copyOfColumns.removeAll(cloneOfPossibleCols.get(i)); + possibleCols1 = createCombination(copyOfColumns, r, possibleCols); + + copyOfColumns = new LinkedList(columns); + boolean cloned = true; + for (List set : possibleCols1) { + if (cloned == false) { + col = clone(cloneOfPossibleCols, i); + } + col.addAll(set); + possibleCols.add(col); + + cloned = false; + } + } + lastIndex = colSize; + } + } + // Optimize here + if (!allOnes) { + for (int i = lastIndex; i < possibleCols.size(); i++) { + List> subList = new ArrayList>(); + int prev = 0; + for (int j = 0; j < breakUp.size(); j++) { + int index = breakUp.get(j); + subList.add(possibleCols.get(i).subList(prev, prev + index)); + prev = prev + index; + + } + finalCols.add(subList); + } + } else { + finalCols.add(possibleCols); + } + return finalCols; + } + + private List clone(List> cloneOfPossibleCols, int i) { + return (List) ((ArrayList) cloneOfPossibleCols.get(i)).clone(); + } + + private List> createCombination(LinkedList copyOfColumns, int r, + ArrayList> possibleCols2) { + int j; + int k = 0; + List> possibleCols = new ArrayList>(); + ArrayList colList = new ArrayList(); + int size = copyOfColumns.size(); + for (int i = 0; i < (1 << size); i++) { + if (r != findNoOfBitsSet(i)) { + continue; + } + j = i; + k = 0; + while (j != 0) { + if (j % 2 == 1) { + colList.add(copyOfColumns.get(k)); + } + j /= 2; + k++; + } + possibleCols.add((List) colList.clone()); + colList.clear(); + } + return possibleCols; + } + + private int findNoOfBitsSet(int n) { + int count = 0; + while (n != 0) { + n &= n - 1; + count++; + } + return count; + } + } + + // This is at the ColumnValueFilter level. That means value for one cf:qualifier(let me call + // this column as col1) is provided in the condition + // Any index on the table which contain the column col1 as the 1st index column in it can be + // selected for the usage. If the table having an index on col1 & col2 we can consider this + // index as col1 is the 1st column in the index columns list. But if the index is on col2 & col1 + // ie. 1st col in the index cols list is not col1, we wont be able to use that index. + // Now comes the question of best fit index. When there are more than one possible index to be + // used, we need to go with one index. Others are only possible candidates to be considered. + // Suppose there is an index on col1&col2 and another on col1 alone. Here to consider the 2nd + // index will be better as that will be having lesser data in the index table, that we need to + // scan. So exact match is always better. + // Suppose there is an index on col1&col2 and another on col1&col2&col3, it this case it would + // be better to go with index on col1&col2 as this will be giving lesser data to be scanned. + // To be general, if there are more possible indices which can be used, we can select the one + // which is having lesser number of index cols in it. + private FilterNode selectBestFitAndPossibleIndicesForSCVF(List indices, + SingleColumnValueFilter filter) { + if (CompareOp.NOT_EQUAL == filter.getOperator() || CompareOp.NO_OP == filter.getOperator()) { + return new NoIndexFilterNode(); + } + FilterColumnValueDetail detail = null; + if (filter instanceof SingleColumnValuePartitionFilter) { + SingleColumnValuePartitionFilter escvf = (SingleColumnValuePartitionFilter) filter; + detail = new FilterColumnValueDetail(escvf.getFamily(), escvf.getQualifier(), escvf + .getComparator().getValue(), escvf.getValuePartition(), escvf.getOperator()); + } else { + detail = new FilterColumnValueDetail(filter.getFamily(), + filter.getQualifier(), filter.getComparator().getValue(), filter.getOperator()); + } + return selectBestFitIndexForColumn(indices, detail); + } + + private FilterNode selectBestFitIndexForColumn(List indices, + FilterColumnValueDetail detail) { + FilterNode filterNode = this.colFilterNodeCache.get(new ColumnWithValue(detail.getColumn(), + detail.getValue())); + if (filterNode != null) { + // This column is being already evaluated to find the best an possible indices + // No need to work on this + return filterNode; + } + List> possibleUseIndices = + new ArrayList>(); + List> possibleFutureUseIndices = + new ArrayList>(); + IndexSpecification bestFitIndex = null; + int bestFitIndexColsSize = Integer.MAX_VALUE; + for (IndexSpecification index : indices) { + Set indexColumns = index.getIndexColumns(); + // Don't worry . This Set is LinkedHashSet. So this will maintain the order. + Iterator indexColumnsIterator = indexColumns.iterator(); + ColumnQualifier firstCQInIndex = indexColumnsIterator.next(); + Column firstColInIndex = new Column(firstCQInIndex.getColumnFamily(), + firstCQInIndex.getQualifier(), firstCQInIndex.getValuePartition()); + if (firstColInIndex.equals(detail.column)) { + possibleUseIndices.add(new Pair(index, indexColumns.size())); + // When we have condition on col1 and we have indices on col1&Col2 and col1&col3 + // which one we should select? We dont know the data related details of every index. + // So select any one. May be the 1st come selected. + // TODO later we can add more intelligence and then add index level data details + if (indexColumns.size() < bestFitIndexColsSize) { + bestFitIndex = index; + bestFitIndexColsSize = indexColumns.size(); + } + detail.maxValueLength = firstCQInIndex.getMaxValueLength(); + detail.valueType = firstCQInIndex.getType(); + } + // This index might be useful at a topper level.... + // I will explain in detail + // When the filter from customer is coming this way as shown below + // & + // ___|___ + // | | + // c1=10 c2=5 + // Suppose we have an index on c2&c1 only on the table, when we check for c1=10 node + // we will not find any index for that as index is not having c1 as the 1st column. + // For c2=5 node, the index is a possible option. Now when we consider the & node, + // the index seems perfect match. That is why even for the c1=10 node also, we can not + // sat it is a NoIndexFilterNode, if there are any indices on the table which contain c1 + // as one column in the index columns. + else { + ColumnQualifier cq = null; + Column column = null; + while (indexColumnsIterator.hasNext()) { + cq = indexColumnsIterator.next(); + column = new Column(cq.getColumnFamily(), cq.getQualifier(), cq.getValuePartition()); + if (column.equals(detail.column)) { + possibleFutureUseIndices.add(new Pair(index, indexColumns + .size())); + detail.maxValueLength = firstCQInIndex.getMaxValueLength(); + detail.valueType = firstCQInIndex.getType(); + break; + } + } + } + } + if (null != bestFitIndex) { + filterNode = new IndexFilterNode(bestFitIndex, possibleUseIndices, possibleFutureUseIndices, + detail); + } else if (!possibleFutureUseIndices.isEmpty()) { + // when bestFitIndex is null possibleUseIndices will be empty for sure + filterNode = new PossibleIndexFilterNode(possibleFutureUseIndices, detail); + } else { + filterNode = new NoIndexFilterNode(); + } + byte[] val = null; + if (detail instanceof FilterColumnValueRange) { + // Probably we have LESS condition + if (detail.getValue() == null) { + val = ((FilterColumnValueRange) detail).getUpperBoundValue(); + } else { + val = detail.getValue(); + } + } + this.colFilterNodeCache.put(new ColumnWithValue(detail.getColumn(), val), filterNode); + return filterNode; + } + // TODO toString() in the VO classes + + public static class ColumnWithValue { + private Column column; + private byte[] value; + + public ColumnWithValue(Column column, byte[] value) { + this.column = column; + this.value = value; + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof ColumnWithValue)) + return false; + ColumnWithValue that = (ColumnWithValue) obj; + Column col = that.getColumn(); + boolean equals = this.column.equals(col); + if (equals && Bytes.equals(this.value, that.getValue())) { + return true; + } else { + return false; + } + } + + public Column getColumn() { + return this.column; + } + + public byte[] getValue() { + return this.value; + } + + @Override + public int hashCode() { + return this.column.hashCode(); + } + } +} \ No newline at end of file Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/SeekAndReadRegionScanner.java =================================================================== --- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/SeekAndReadRegionScanner.java (revision 0) +++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/SeekAndReadRegionScanner.java (working copy) @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import java.io.IOException; +import java.util.List; + +import org.apache.hadoop.hbase.regionserver.RegionScanner; + +public interface SeekAndReadRegionScanner extends RegionScanner { + + void addSeekPoints(List seekPoints); + + byte[] getLatestSeekpoint(); + + /** + * This will make the scanner to reseek to the next seek point. + * @return True when seeked to the next point. False when there is no further seek points. + * @throws IOException + */ + boolean seekToNextPoint() throws IOException; + + + + public boolean isClosed(); + +} Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/SeekPointFetcher.java =================================================================== --- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/SeekPointFetcher.java (revision 0) +++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/SeekPointFetcher.java (working copy) @@ -0,0 +1,84 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.regionserver.RegionScanner; +import org.apache.hadoop.hbase.util.Bytes; + +// TODO better name +public class SeekPointFetcher { + + private static final Log LOG = LogFactory.getLog(SeekPointFetcher.class); + private RegionScanner indexRegionScanner; + + public SeekPointFetcher(RegionScanner indexRegionScanner){ + this.indexRegionScanner = indexRegionScanner; + } + + /** + * Fetches the next N seek points for the scan. + * @param seekPoints + * @param noOfSeekPoints + * @return false when the scan on the index table for having no more rows remaining. + * @throws IOException + */ + public synchronized boolean nextSeekPoints(List seekPoints, int noOfSeekPoints) + throws IOException { + boolean hasMore = true; + List indexScanResult = new ArrayList(); + for (int i = 0; i < noOfSeekPoints; i++) { + hasMore = indexRegionScanner.next(indexScanResult); + if (indexScanResult.size() > 0) { + populateSeekPointsWithTableRowKey(seekPoints, indexScanResult.get(0)); + } + indexScanResult.clear(); + if (hasMore == false) break; + } + // TODO log the seekpoints INFO level. + return hasMore; + } + + private void populateSeekPointsWithTableRowKey(List seekPoints, Cell cell) { + byte[] row = cell.getRow(); + // Row key of the index table entry = region startkey + index name + column value(s) + // + actual table rowkey. + // Every row in the index table will have exactly one KV in that. The value will be + // 4 bytes. First 2 bytes specify length of the region start key bytes part in the + // rowkey. Last 2 bytes specify the offset to the actual table rowkey part within the + // index table rowkey. + byte[] value = cell.getValue(); + short actualRowKeyOffset = Bytes.toShort(value, 2); + if (LOG.isTraceEnabled()) { + LOG.trace("row value for the index table " + Bytes.toString(row)); + } + byte[] actualTableRowKey = new byte[row.length - actualRowKeyOffset]; + System.arraycopy(row, actualRowKeyOffset, actualTableRowKey, 0, actualTableRowKey.length); + seekPoints.add(actualTableRowKey); + } + + public synchronized void close() throws IOException{ + this.indexRegionScanner.close(); + } +} Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/SeekUnderValueException.java =================================================================== --- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/SeekUnderValueException.java (revision 0) +++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/SeekUnderValueException.java (working copy) @@ -0,0 +1,27 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import java.io.IOException; + +//TODO better name? +public class SeekUnderValueException extends IOException { + + private static final long serialVersionUID = -4319585454456479286L; + +} Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TTLExpiryChecker.java =================================================================== --- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TTLExpiryChecker.java (revision 0) +++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TTLExpiryChecker.java (working copy) @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import org.apache.hadoop.hbase.HConstants; + +public class TTLExpiryChecker { + + public boolean checkIfTTLExpired(long ttl, long timestamp) { + if (ttl == HConstants.FOREVER) { + return false; + } else if (ttl == -1) { + return false; + } else { + // second -> ms adjust for user data + ttl *= 1000; + } + if ((System.currentTimeMillis() - timestamp) > ttl) { + return true; + } + return false; + } +} Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TTLStoreScanner.java =================================================================== --- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TTLStoreScanner.java (revision 0) +++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TTLStoreScanner.java (working copy) @@ -0,0 +1,150 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import java.io.IOException; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.index.IndexSpecification; +import org.apache.hadoop.hbase.index.manager.IndexManager; +import org.apache.hadoop.hbase.index.util.ByteArrayBuilder; +import org.apache.hadoop.hbase.index.util.IndexUtils; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.regionserver.InternalScanner; +import org.apache.hadoop.hbase.regionserver.KeyValueScanner; +import org.apache.hadoop.hbase.regionserver.ScanType; +import org.apache.hadoop.hbase.regionserver.Store; +import org.apache.hadoop.hbase.regionserver.StoreScanner; +import org.apache.hadoop.hbase.util.Bytes; + +public class TTLStoreScanner implements InternalScanner { + private InternalScanner delegate; + private Store store; + private long smallestReadPoint; + private long earliestTS; + private ScanType type; // (This should be scan type) + private static final Log LOG = LogFactory.getLog(TTLStoreScanner.class); + + private TTLExpiryChecker ttlExpiryChecker; + private String actualTableName; + private HRegionServer rs; + private Boolean userRegionAvailable = null; + + public TTLStoreScanner(Store store, long smallestReadPoint, long earliestTS, ScanType type, + List scanners, TTLExpiryChecker ttlExpiryChecker, + String actualTableName, HRegionServer rs) + throws IOException { + this.store = store; + this.smallestReadPoint = smallestReadPoint; + this.earliestTS = earliestTS; + this.type = type; + Scan scan = new Scan(); + scan.setMaxVersions(store.getFamily().getMaxVersions()); + delegate = new StoreScanner(store, store.getScanInfo(), scan, scanners, + type, this.smallestReadPoint, this.earliestTS); + this.ttlExpiryChecker = ttlExpiryChecker; + this.actualTableName = actualTableName; + this.rs = rs; + } + + @Override + public boolean next(List results) throws IOException { + return this.next(results, 1); + } + + @Override + public boolean next(List result, int limit) throws IOException { + boolean next = this.delegate.next(result, limit); + // Ideally here i should get only one result(i.e) only one kv + for (Iterator iterator = result.iterator(); iterator.hasNext();) { + KeyValue kv = (KeyValue) iterator.next(); + byte[] indexNameInBytes = formIndexNameFromKV(kv); + // From the indexname get the TTL + IndexSpecification index = IndexManager.getInstance().getIndex(this.actualTableName, + indexNameInBytes); + HRegionInfo hri = store.getRegionInfo(); + if (this.type == ScanType.COMPACT_DROP_DELETES) { + if (this.userRegionAvailable == null) { + this.userRegionAvailable = isUserTableRegionAvailable(hri, this.rs); + } + // If index is null probably index is been dropped through drop index + // If user region not available it may be due to the reason that the user region has not yet + // opened but + // the index region has opened. + // Its better not to avoid the kv here, and write it during this current compaction. + // Anyway later compaction will avoid it. May lead to false positives but better than + // data loss + if (null == index && userRegionAvailable) { + // Remove the dropped index from the results + LOG.info("The index has been removed for the kv " + kv); + iterator.remove(); + continue; + } + } + if (index != null) { + boolean ttlExpired = this.ttlExpiryChecker.checkIfTTLExpired(index.getTTL(), + kv.getTimestamp()); + if (ttlExpired) { + result.clear(); + LOG.info("The ttl has expired for the kv " + kv); + return false; + } + } + } + return next; + } + + @Override + public void close() throws IOException { + this.delegate.close(); + } + + private byte[] formIndexNameFromKV(KeyValue kv) { + byte[] rowKey = kv.getRow(); + // First two bytes are going to be the + ByteArrayBuilder keyBuilder = ByteArrayBuilder.allocate(rowKey.length); + // Start from 2nd offset because the first 2 bytes corresponds to the rowkeylength + keyBuilder.put(rowKey, 0, rowKey.length); + String replacedKey = Bytes.toString(keyBuilder.array()); + String emptyByte = Bytes.toString(new byte[1]); + int indexOf = replacedKey.indexOf(emptyByte); + return keyBuilder.array(indexOf+1, IndexUtils.getMaxIndexNameLength()); + } + + private boolean isUserTableRegionAvailable(HRegionInfo indexHri, HRegionServer rs) { + Collection userRegions = rs.getOnlineRegions(TableName.valueOf(this.actualTableName)); + for (HRegion userRegion : userRegions) { + // TODO start key check is enough? May be we can check for the + // possibility for N-1 Mapping? + if (Bytes.equals(userRegion.getStartKey(), indexHri.getStartKey())) { + return true; + } + } + return false; + } +} Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/wal/IndexWALObserver.java =================================================================== --- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/wal/IndexWALObserver.java (revision 0) +++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/wal/IndexWALObserver.java (working copy) @@ -0,0 +1,126 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.wal; + +import java.io.IOException; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.CoprocessorEnvironment; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.coprocessor.ObserverContext; +import org.apache.hadoop.hbase.coprocessor.WALCoprocessorEnvironment; +import org.apache.hadoop.hbase.coprocessor.WALObserver; +import org.apache.hadoop.hbase.index.IndexSpecification; +import org.apache.hadoop.hbase.index.coprocessor.regionserver.IndexRegionObserver; +import org.apache.hadoop.hbase.index.coprocessor.regionserver.IndexRegionObserver.IndexEdits; +import org.apache.hadoop.hbase.index.manager.IndexManager; +import org.apache.hadoop.hbase.index.util.IndexUtils; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.wal.HLogKey; +import org.apache.hadoop.hbase.regionserver.wal.WALEdit; + +public class IndexWALObserver implements WALObserver { + + private static final Log LOG = LogFactory.getLog(IndexWALObserver.class); + + private IndexManager indexManager = IndexManager.getInstance(); + + @Override + public boolean preWALWrite(ObserverContext ctx, HRegionInfo info, + HLogKey logKey, WALEdit logEdit) throws IOException { + TableName tableName = info.getTable(); + if (IndexUtils.isCatalogOrSystemTable(tableName) || IndexUtils.isIndexTable(tableName)) { + return true; + } + List indices = indexManager.getIndicesForTable(tableName.getNameAsString()); + if (indices != null && !indices.isEmpty()) { + LOG.trace("Entering preWALWrite for the table " + tableName); + String indexTableName = IndexUtils.getIndexTableName(tableName); + IndexEdits iEdits = IndexRegionObserver.threadLocal.get(); + WALEdit indexWALEdit = iEdits.getWALEdit(); + // This size will be 0 when none of the Mutations to the user table to be indexed. + // or write to WAL is disabled for the Mutations + if (indexWALEdit.getKeyValues().size() == 0) { + return true; + } + LOG.trace("Adding indexWALEdits into WAL for table " + tableName); + HRegion indexRegion = iEdits.getRegion(); + // TS in all KVs within WALEdit will be the same. So considering the 1st one. + Long time = indexWALEdit.getKeyValues().get(0).getTimestamp(); + indexRegion.getLog().appendNoSync(indexRegion.getRegionInfo(), + TableName.valueOf(indexTableName), indexWALEdit, logKey.getClusterIds(), time, + indexRegion.getTableDesc(), indexRegion.getSequenceId(), true, HConstants.NO_NONCE, + HConstants.NO_NONCE); + LOG.trace("Exiting preWALWrite for the table " + tableName); + } + return true; + } + + @Override + public void start(CoprocessorEnvironment env) throws IOException { + } + + @Override + public void stop(CoprocessorEnvironment env) throws IOException { + } + + @Override + public void postWALWrite(ObserverContext ctx, HRegionInfo info, + HLogKey logKey, WALEdit logEdit) throws IOException { + } + + /* + * HLog log =ctx.getEnvironment().getWAL(); List kvList = logEdit.getKeyValues(); String + * userTableName = info.getEncodedName(); List indices = + * IndexManager.getInstance().getIndicesForTable(userTableName); if(null != indices){ + * handleWalEdit(userTableName, log, kvList, indices, info); } return false; } + * + * private void handleWalEdit(String userTableName, HLog log, List kvList, + * List indices, HRegionInfo info) { HashMap>> myMap = new HashMap>>(); + * + * for (KeyValue kv : kvList) { for (IndexSpecification idx : indices) { Set + * colSet = idx.getIndexColumns(); for (ColumnQualifier col : colSet) { if + * (Bytes.equals(kv.getFamily(), col.getColumnFamily()) && Bytes.equals(kv.getQualifier(), + * col.getQualifier())) { Map> mapOfCfToListOfKV = myMap.get(kv.getRow()); + * if (mapOfCfToListOfKV == null) { mapOfCfToListOfKV = new HashMap>(); + * myMap.put(kv.getRow(), mapOfCfToListOfKV); } // listOfKVs.add(kv); List listOfKV = + * mapOfCfToListOfKV.get(idx.getName().getBytes()); if(listOfKV == null ){ listOfKV = new + * ArrayList(); mapOfCfToListOfKV.put(idx.getName().getBytes(), listOfKV); } + * listOfKV.add(kv.clone()); } } } } + * + * createIndexWalEdit(myMap, indices, info); } + * + * private void createIndexWalEdit( HashMap>> myMap, + * List indices, HRegionInfo info) { int totalValueLength = 0; byte[] + * primaryRowKey = null; byte[] prStartKey = info.getStartKey(); for (Map.Entry>> mainEntry : myMap .entrySet()) { + * + * + * Map> idxMap = mainEntry.getValue(); for(IndexSpecification index : + * indices){ byte[] name = index.getName().getBytes(); if(null != idxMap.get(name)){ + * Set colSet = index.getIndexColumns(); for (ColumnQualifier c : colSet) { + * totalValueLength = totalValueLength + c.getMaxValueLength(); } primaryRowKey = + * mainEntry.getKey(); int rowLength = prStartKey.length + name.length; rowLength += + * totalValueLength; rowLength += primaryRowKey.length; } } } } + */ +} Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/filter/SingleColumnRangeFilter.java =================================================================== --- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/filter/SingleColumnRangeFilter.java (revision 0) +++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/filter/SingleColumnRangeFilter.java (working copy) @@ -0,0 +1,128 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.filter; + +import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; +import org.apache.hadoop.hbase.filter.FilterBase; +import org.apache.hadoop.hbase.index.ValuePartition; +import org.apache.hadoop.hbase.util.Bytes; + +// TODO make it a full fledged Filter implementation +/** + * This is internal class. Not exposed as a Filter implementation + */ +public class SingleColumnRangeFilter extends FilterBase { + + private byte[] family; + + private byte[] qualifier; + + private CompareOp lowerCompareOp; + + private CompareOp upperCompareop; + + private byte[] upperBoundValue; + + private byte[] lowerBoundValue; + + private ValuePartition valuePartition = null; + + /** + * When the range is having only one bound pass that value and condition as the boundValue1 + * and boundOp1. Here boundValue2 and boundOp2 can be passed as null + * + * @param cf + * @param qualifier + * @param boundValue1 + * @param boundOp1 + * @param boundValue2 + * @param boundOp2 + */ + public SingleColumnRangeFilter(byte[] cf, byte[] qualifier, byte[] boundValue1, + CompareOp boundOp1, byte[] boundValue2, CompareOp boundOp2) { + this.family = cf; + this.qualifier = qualifier; + // CompareOp.LESS or CompareOp.LESS_OR_EQUAL will be the upper bound + if (boundOp1.equals(CompareOp.LESS) || boundOp1.equals(CompareOp.LESS_OR_EQUAL)) { + this.upperCompareop = boundOp1; + this.upperBoundValue = boundValue1; + this.lowerCompareOp = boundOp2; + this.lowerBoundValue = boundValue2; + } else { + this.upperCompareop = boundOp2; + this.upperBoundValue = boundValue2; + this.lowerCompareOp = boundOp1; + this.lowerBoundValue = boundValue1; + } + } + + public SingleColumnRangeFilter(byte[] cf, byte[] qualifier,ValuePartition vp, byte[] boundValue1, + CompareOp boundOp1, byte[] boundValue2, CompareOp boundOp2) { + this(cf,qualifier,boundValue1,boundOp1,boundValue2,boundOp2); + this.valuePartition = vp; + } + + public void setUpperBoundValue(byte[] upperBoundValue, CompareOp upperOp) { + this.upperBoundValue = upperBoundValue; + this.upperCompareop = upperOp; + } + + public void setLowerBoundValue(byte[] lowerBoundValue, CompareOp lowerOp) { + this.lowerBoundValue = lowerBoundValue; + this.lowerCompareOp = lowerOp; + } + + public byte[] getFamily() { + return this.family; + } + + public byte[] getQualifier() { + return this.qualifier; + } + + public CompareOp getUpperBoundOp() { + return this.upperCompareop; + } + + public CompareOp getLowerBoundOp() { + return this.lowerCompareOp; + } + + public byte[] getLowerBoundValue() { + return this.lowerBoundValue; + } + + public byte[] getUpperBoundValue() { + return this.upperBoundValue; + } + + public ValuePartition getValuePartition() { + return valuePartition; + } + + public void setValuePartition(ValuePartition valuePartition) { + this.valuePartition = valuePartition; + } + + public String toString() { + return String.format("%s (%s, %s, %s, %s, %s, %s)", this.getClass().getSimpleName(), + Bytes.toStringBinary(this.family), Bytes.toStringBinary(this.qualifier), + this.lowerCompareOp == null ? "" : this.lowerCompareOp.name(), this.lowerBoundValue == null ? "" :Bytes.toStringBinary(this.lowerBoundValue), + this.upperCompareop == null ? "" : this.upperCompareop.name(), this.upperBoundValue == null ? "" : Bytes.toStringBinary(this.upperBoundValue)); + } +} Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/filter/SingleColumnValuePartitionFilter.java =================================================================== --- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/filter/SingleColumnValuePartitionFilter.java (revision 0) +++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/filter/SingleColumnValuePartitionFilter.java (working copy) @@ -0,0 +1,214 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.filter; + +import java.io.IOException; + +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValueUtil; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.exceptions.DeserializationException; +import org.apache.hadoop.hbase.filter.BinaryComparator; +import org.apache.hadoop.hbase.filter.ByteArrayComparable; +import org.apache.hadoop.hbase.filter.CompareFilter; +import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; +import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; +import org.apache.hadoop.hbase.index.SeparatorPartition; +import org.apache.hadoop.hbase.index.SpatialPartition; +import org.apache.hadoop.hbase.index.ValuePartition; +import org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos; +import org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition.PartitionType; +import org.apache.hadoop.hbase.protobuf.ProtobufUtil; + +import com.google.protobuf.ExtensionRegistry; +import com.google.protobuf.InvalidProtocolBufferException; + +/** + * This filter is used to filter cells based on a part of it's value. + * It takes a {@link CompareFilter.CompareOp} operator (equal, greater, not equal, etc), and either + * a byte [] value or a WritableByteArrayComparable. + *

+ * You must also specify a family and qualifier. Only the value of this column + * will be tested. When using this filter on a {@link Scan} with specified + * inputs, the column to be tested should also be added as input (otherwise + * the filter will regard the column as missing). + *

+ * To prevent the entire row from being emitted if the column is not found + * on a row, use {@link #setFilterIfMissing}. + * Otherwise, if the column is found, the entire row will be emitted only if + * the value passes. If the value fails, the row will be filtered out. + *

+ * In order to test values of previous versions (timestamps), set + * {@link #setLatestVersionOnly} to false. The default is true, meaning that + * only the latest version's value is tested and all previous versions are ignored. + */ +public class SingleColumnValuePartitionFilter extends SingleColumnValueFilter { + + private boolean foundColumn = false; + private boolean matchedColumn = false; + private ValuePartition valuePartition = null; + + + public SingleColumnValuePartitionFilter(final byte[] family, final byte[] qualifier, + final CompareOp compareOp, final byte[] value, ValuePartition vp) { + this(family, qualifier, compareOp, new BinaryComparator(value), vp); + } + + public SingleColumnValuePartitionFilter(final byte[] family, final byte[] qualifier, + final CompareOp compareOp, final ByteArrayComparable comparator, ValuePartition vp) { + super(family, qualifier, compareOp, comparator); + this.valuePartition = vp; + } + + public ValuePartition getValuePartition() { + return valuePartition; + } + + public boolean filterRow() { + // If column was found, return false if it was matched, true if it was not + // If column not found, return true if we filter if missing, false if not + return this.foundColumn? !this.matchedColumn: this.getFilterIfMissing(); + } + + public void reset() { + foundColumn = false; + matchedColumn = false; + } + + @Override + public ReturnCode filterKeyValue(Cell cell) { + // TODO get rid of this. + KeyValue keyValue = KeyValueUtil.ensureKeyValue(cell); + if (this.matchedColumn) { + // We already found and matched the single column, all keys now pass + return ReturnCode.INCLUDE; + } else if (this.getLatestVersionOnly() && this.foundColumn) { + // We found but did not match the single column, skip to next row + return ReturnCode.NEXT_ROW; + } + if (!keyValue.matchingColumn(this.columnFamily, this.columnQualifier)) { + return ReturnCode.INCLUDE; + } + foundColumn = true; + byte[] value = valuePartition.getPartOfValue(keyValue.getValue()); + if (filterColumnValue(value, 0, value.length)) { + return this.getLatestVersionOnly()? ReturnCode.NEXT_ROW: ReturnCode.INCLUDE; + } + this.matchedColumn = true; + return ReturnCode.INCLUDE; + } + + private boolean filterColumnValue(final byte [] data, final int offset, + final int length) { + int compareResult = this.getComparator().compareTo(data, offset, length); + switch (this.getOperator()) { + case LESS: + return compareResult <= 0; + case LESS_OR_EQUAL: + return compareResult < 0; + case EQUAL: + return compareResult != 0; + case NOT_EQUAL: + return compareResult == 0; + case GREATER_OR_EQUAL: + return compareResult > 0; + case GREATER: + return compareResult >= 0; + default: + throw new RuntimeException("Unknown Compare op " + this.getOperator().name()); + } + } + + @Override + public byte[] toByteArray() { + ValuePartitionProtos.SingleColumnValuePartitionFilter.Builder builder = + ValuePartitionProtos.SingleColumnValuePartitionFilter.newBuilder(); + builder.setSingleColumnValueFilter(super.convert()); + builder.setValuePartition(valuePartition.convert()); + return builder.build().toByteArray(); + } + + public static SingleColumnValuePartitionFilter parseFrom(final byte[] pbBytes) + throws DeserializationException { + ValuePartitionProtos.SingleColumnValuePartitionFilter builder = null; + ExtensionRegistry registry; + try { + registry= ExtensionRegistry.newInstance(); + ValuePartitionProtos.registerAllExtensions(registry); + builder = ValuePartitionProtos.SingleColumnValuePartitionFilter.newBuilder().mergeFrom(pbBytes,registry).build(); + } catch (InvalidProtocolBufferException e) { + throw new DeserializationException(e); + } + org.apache.hadoop.hbase.protobuf.generated.FilterProtos.SingleColumnValueFilter scvf = + builder.getSingleColumnValueFilter(); + org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition vpProto = + builder.getValuePartition(); + final CompareOp compareOp = CompareOp.valueOf(scvf.getCompareOp().name()); + final ByteArrayComparable comparator; + try { + comparator = ProtobufUtil.toComparator(scvf.getComparator()); + } catch (IOException ioe) { + throw new DeserializationException(ioe); + } + ValuePartition vp = null; + if (vpProto.getPartitionType().equals(PartitionType.SPATIAL)) { + vp = + new SpatialPartition( + vpProto + .getExtension(org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SpatialPartition.offset), + vpProto + .getExtension(org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SpatialPartition.length)); + } else if (vpProto.getPartitionType().equals(PartitionType.SEPARATOR)) { + vp = + new SeparatorPartition( + vpProto + .getExtension( + org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SeparatorPartition.separator) + .toByteArray(), + vpProto + .getExtension(org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SeparatorPartition.position)); + } + SingleColumnValuePartitionFilter scvpf = new SingleColumnValuePartitionFilter(scvf.getColumnFamily().toByteArray(), scvf + .getColumnQualifier().toByteArray(), compareOp, comparator, vp); + scvpf.setFilterIfMissing(scvf.getFilterIfMissing()); + scvpf.setLatestVersionOnly(scvf.getLatestVersionOnly()); + return scvpf; + } + +/* @Override + public void write(DataOutput out) throws IOException { + super.write(out); + out.writeUTF(valuePartition.getPartitionType().name()); + valuePartition.write(out); + } + + @Override + public void readFields(DataInput in) throws IOException { + super.readFields(in); + PartitionType p = PartitionType.valueOf(in.readUTF()); + if (p.equals(PartitionType.SEPARATOR)) { + valuePartition = new SeparatorPartition(); + } else if (p.equals(PartitionType.SPATIAL)) { + valuePartition = new SpatialPartition(); + } + if (valuePartition != null) { + valuePartition.readFields(in); + } + }*/ +} Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/io/IndexHalfStoreFileReader.java =================================================================== --- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/io/IndexHalfStoreFileReader.java (revision 0) +++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/io/IndexHalfStoreFileReader.java (working copy) @@ -0,0 +1,452 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.io; + +import java.io.IOException; +import java.nio.ByteBuffer; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValue.Type; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper; +import org.apache.hadoop.hbase.io.Reference; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; +import org.apache.hadoop.hbase.io.hfile.HFileScanner; +import org.apache.hadoop.hbase.regionserver.StoreFile; +import org.apache.hadoop.hbase.util.Bytes; + +/** + * A facade for a {@link org.apache.hadoop.hbase.io.hfile.HFile.Reader} that serves up either the + * top or bottom half of a HFile where 'bottom' is the first half of the file containing the keys + * that sort lowest and 'top' is the second half of the file with keys that sort greater than those + * of the bottom half. The top includes the split files midkey, of the key that follows if it does + * not exist in the file. + * + *

+ * This type works in tandem with the {@link Reference} type. This class is used reading while + * Reference is used writing. + * + *

+ * This file is not splitable. Calls to {@link #midkey()} return null. + */ + +public class IndexHalfStoreFileReader extends StoreFile.Reader { + private static final int ROW_KEY_LENGTH = 2; + private final boolean top; + // This is the key we split around. Its the first possible entry on a row: + // i.e. empty column and a timestamp of LATEST_TIMESTAMP. + private final byte[] splitkey; + private final byte[] splitRow; + + /** + * @param p + * @param cacheConf + * @param r + * @throws IOException + */ + public IndexHalfStoreFileReader(final FileSystem fs, final Path p, final CacheConfig cacheConf, + final Reference r, final Configuration conf) throws IOException { + super(fs, p, cacheConf, conf); + this.splitkey = r.getSplitKey(); + // Is it top or bottom half? + this.top = Reference.isTopFileRegion(r.getFileRegion()); + this.splitRow = KeyValue.createKeyValueFromKey(splitkey).getRow(); + } + + /** + * @param p + * @param cacheConf + * @param r + * @throws IOException + */ + public IndexHalfStoreFileReader(final FileSystem fs, final Path p, final CacheConfig cacheConf, + final FSDataInputStreamWrapper in, long size, final Reference r, final Configuration conf) + throws IOException { + super(fs, p, in, size, cacheConf, conf); + this.splitkey = r.getSplitKey(); + // Is it top or bottom half? + this.top = Reference.isTopFileRegion(r.getFileRegion()); + this.splitRow = KeyValue.createKeyValueFromKey(splitkey).getRow(); + } + + protected boolean isTop() { + return this.top; + } + + @Override + public HFileScanner getScanner(final boolean cacheBlocks, final boolean pread, + final boolean isCompaction) { + final HFileScanner s = super.getScanner(cacheBlocks, pread, isCompaction); + return new HFileScanner() { + final HFileScanner delegate = s; + public boolean atEnd = false; + + public ByteBuffer getKey() { + if (atEnd) { + return null; + } + if (!top) { + return delegate.getKey(); + } + // If it is top store file replace the StartKey of the Key with SplitKey + return getChangedKey(delegate.getKeyValue()); + } + + private ByteBuffer getChangedKey(KeyValue kv) { + // new KeyValue(row, family, qualifier, timestamp, type, value) + byte[] newRowkey = getNewRowkeyByRegionStartKeyReplacedWithSplitKey(kv); + KeyValue newKv = new KeyValue(newRowkey, kv.getFamily(), kv.getQualifier(), + kv.getTimestamp(), Type.codeToType(kv.getType()), null); + ByteBuffer keyBuffer = ByteBuffer.wrap(newKv.getKey()); + return keyBuffer; + } + + private byte[] getNewRowkeyByRegionStartKeyReplacedWithSplitKey(KeyValue kv) { + // TODO any other way when Delete type? + if (KeyValue.isDelete(kv.getType()) && kv.getValue().length == 0) { + return replaceDeleteKeyWithSplitKey(kv.getRow()); + } + byte[] original = kv.getRow(); + byte[] value = kv.getValue(); + int lenOfRegionStartKey = Bytes.toShort(value, 0); // 1st 2 bytes length of the region + int lenOfRemainingKey = original.length - lenOfRegionStartKey; + byte[] keyReplacedStartKey = new byte[lenOfRemainingKey + splitRow.length]; + System.arraycopy(splitRow, 0, keyReplacedStartKey, 0, splitRow.length); + System.arraycopy(original, lenOfRegionStartKey, keyReplacedStartKey, splitRow.length, + lenOfRemainingKey); + return keyReplacedStartKey; + } + + public String getKeyString() { + if (atEnd) { + return null; + } + return Bytes.toStringBinary(getKey()); + } + + public ByteBuffer getValue() { + if (atEnd) { + return null; + } + if (!top) { + return delegate.getValue(); + } + // If it is top store file change the value corresponding to the changed key like + // [first 2 bytes]StartKey length replace with SplitKey length + // [last 2 bytes]ActualRowKey offset add with difference of SplitKey & StartKey + byte[] changedValue = getChangedValue(delegate.getKeyValue().getValue()); + return ByteBuffer.wrap(changedValue); + } + + private byte[] getChangedValue(byte[] value) { + if (value.length == 0) return value; // The value can be empty when the KV type is DELETE. + int lenghtOfTheStartKey = Bytes.toShort(value, 0); + int offsetOfActualKey = Bytes.toShort(value, 2); + offsetOfActualKey = offsetOfActualKey + (splitRow.length - lenghtOfTheStartKey); + byte[] changedValue = new byte[4]; + System + .arraycopy(Bytes.toBytes((short) splitRow.length), 0, changedValue, 0, ROW_KEY_LENGTH); + System.arraycopy(Bytes.toBytes((short) offsetOfActualKey), 0, changedValue, ROW_KEY_LENGTH, + ROW_KEY_LENGTH); + return changedValue; + } + + public String getValueString() { + if (atEnd) { + return null; + } + return Bytes.toStringBinary(getValue()); + } + + public KeyValue getKeyValue() { + if (atEnd) { + return null; + } + KeyValue kv = delegate.getKeyValue(); + if (!top) { + return kv; + } + // If it is a top store file change the StartKey with SplitKey in Key + // and produce the new value corresponding to the change in key + byte[] changedKey = getNewRowkeyByRegionStartKeyReplacedWithSplitKey(kv); + byte[] changedValue = getChangedValue(kv.getValue()); + KeyValue changedKv = new KeyValue(changedKey, kv.getFamily(), kv.getQualifier(), + kv.getTimestamp(), Type.codeToType(kv.getType()), changedValue); + return changedKv; + } + + public boolean next() throws IOException { + if (atEnd) { + return false; + } + // TODO check what will be returned when next moves the cursor to the last entry + // in the file + while (true) { + boolean b = delegate.next(); + if (!b) { + atEnd = true; + return b; + } + // We need to check whether the current KV pointed by this reader is corresponding to + // this split or not. + // In case of top store file if the ActualRowKey >= SplitKey + // In case of bottom store file if the ActualRowKey < Splitkey + if (isSatisfiedMidKeyCondition(delegate.getKeyValue())) { + return true; + } + } + } + + public boolean seekBefore(byte[] key) throws IOException { + return seekBefore(key, 0, key.length); + } + + public boolean seekBefore(byte[] key, int offset, int length) throws IOException { + if (top) { + byte[] fk = getFirstKey(); + // This will be null when the file is empty in which we can not seekBefore to any key + if (fk == null) { + return false; + } + if (getComparator().compare(key, offset, length, fk, 0, fk.length) <= 0) { + return false; + } + KeyValue replacedKey = getKeyPresentInHFiles(key); + return this.delegate.seekBefore(replacedKey.getBuffer(), replacedKey.getKeyOffset(), + replacedKey.getKeyLength()); + } else { + // The equals sign isn't strictly necessary just here to be consistent with seekTo + if (getComparator().compare(key, offset, length, splitkey, 0, splitkey.length) >= 0) { + return this.delegate.seekBefore(splitkey, 0, splitkey.length); + } + } + return this.delegate.seekBefore(key, offset, length); + } + + public boolean seekTo() throws IOException { + boolean b = delegate.seekTo(); + if (!b) { + atEnd = true; + return b; + } + while (true) { + // We need to check the first occurrence of satisfying the condition + // In case of top store file if the ActualRowKey >= SplitKey + // In case of bottom store file if the ActualRowKey < Splitkey + if (isSatisfiedMidKeyCondition(delegate.getKeyValue())) { + return true; + } + b = delegate.next(); + if (!b) { + return b; + } + } + } + + public int seekTo(byte[] key) throws IOException { + return seekTo(key, 0, key.length); + } + + public int seekTo(byte[] key, int offset, int length) throws IOException { + if (top) { + if (getComparator().compare(key, offset, length, splitkey, 0, splitkey.length) < 0) { + return -1; + } + KeyValue replacedKey = getKeyPresentInHFiles(key); + + int seekTo = delegate.seekTo(replacedKey.getBuffer(), replacedKey.getKeyOffset(), + replacedKey.getKeyLength()); + return seekTo; + /* + * if (seekTo == 0 || seekTo == -1) { return seekTo; } else if (seekTo == 1) { boolean + * next = this.next(); } + */ + } else { + if (getComparator().compare(key, offset, length, splitkey, 0, splitkey.length) >= 0) { + // we would place the scanner in the second half. + // it might be an error to return false here ever... + boolean res = delegate.seekBefore(splitkey, 0, splitkey.length); + if (!res) { + throw new IOException( + "Seeking for a key in bottom of file, but key exists in top of file, failed on seekBefore(midkey)"); + } + return 1; + } + } + return delegate.seekTo(key, offset, length); + } + + public int reseekTo(byte[] key) throws IOException { + return reseekTo(key, 0, key.length); + } + + public int reseekTo(byte[] key, int offset, int length) throws IOException { + if (top) { + if (getComparator().compare(key, offset, length, splitkey, 0, splitkey.length) < 0) { + return -1; + } + KeyValue replacedKey = getKeyPresentInHFiles(key); + return delegate.reseekTo(replacedKey.getBuffer(), replacedKey.getKeyOffset(), + replacedKey.getKeyLength()); + } else { + if (getComparator().compare(key, offset, length, splitkey, 0, splitkey.length) >= 0) { + // we would place the scanner in the second half. + // it might be an error to return false here ever... + boolean res = delegate.seekBefore(splitkey, 0, splitkey.length); + if (!res) { + throw new IOException( + "Seeking for a key in bottom of file, but key exists in top of file, failed on seekBefore(midkey)"); + } + return 1; + } + } + return delegate.reseekTo(key, offset, length); + } + + public org.apache.hadoop.hbase.io.hfile.HFile.Reader getReader() { + return this.delegate.getReader(); + } + + // TODO: Need to change as per IndexHalfStoreFileReader + public boolean isSeeked() { + return this.delegate.isSeeked(); + } + }; + } + + private boolean isSatisfiedMidKeyCondition(KeyValue kv) { + if (KeyValue.isDelete(kv.getType()) && kv.getValue().length == 0) { + // In case of a Delete type KV, let it be going to both the daughter regions. + // No problems in doing so. In the correct daughter region where it belongs to, this delete + // tomb will really delete a KV. In the other it will just hang around there with no actual + // kv coming for which this is a delete tomb. :) + return true; + } + byte[] row = kv.getRow(); + int offsetToActuRowKey = Bytes.toShort(kv.getValue(), ROW_KEY_LENGTH); + int actuRowKeyLength = (row.length - offsetToActuRowKey); + byte[] actuRowKey = new byte[actuRowKeyLength]; + System.arraycopy(row, offsetToActuRowKey, actuRowKey, 0, actuRowKeyLength); + int compareResult = Bytes.compareTo(actuRowKey, splitRow); + if (top) { + if (compareResult >= 0) { + return true; + } + } else { + if (compareResult < 0) { + return true; + } + } + return false; + } + + // In case of top half store, the passed key will be with the start key of the daughter region. + // But in the actual HFiles, the key will be with the start key of the old parent region. + // In order to make the real seek in the HFiles, we need to build the old key. + private KeyValue getKeyPresentInHFiles(byte[] key) { + KeyValue keyValue = new KeyValue(key); + KeyValue keyValCopy = keyValue.shallowCopy(); + int rowLength = keyValCopy.getRowLength(); + int rowOffset = keyValCopy.getRowOffset(); + byte[] row = keyValCopy.getRow(); + String oneByteStr = Bytes.toString(new byte[1]); + int rowIndex = Bytes.toString(row).indexOf(oneByteStr); + byte[] firstKey=getFirstKey(); + // In firstkey first 2 bytes will reperesent the key length so don't consider it + byte[] actualFirstKey = new byte[firstKey.length - ROW_KEY_LENGTH]; + // copy from 2nd position of firstkey + System.arraycopy(firstKey, ROW_KEY_LENGTH, actualFirstKey, 0, firstKey.length - ROW_KEY_LENGTH); + // Get the main table start key using the one byte as separator + int firstindex = Bytes.toString(actualFirstKey).indexOf(oneByteStr); + byte[] startRow = new byte[firstindex]; + System.arraycopy(actualFirstKey, 0, startRow, 0, firstindex); + + // This comes incase of deletefamily + if (top && 0 == keyValCopy.getValueLength() + && keyValCopy.getTimestamp() == HConstants.LATEST_TIMESTAMP + && Bytes.compareTo(row, splitRow) == 0 && keyValCopy.isDeleteFamily()) { + KeyValue createFirstDeleteFamilyOnRow = KeyValue.createFirstDeleteFamilyOnRow(startRow, + keyValCopy.getFamily()); + return createFirstDeleteFamilyOnRow; + } + + byte[] rowAfterSplitKey = new byte[row.length - rowIndex]; + byte[] afterRow = new byte[key.length - (rowOffset + rowLength)]; + byte[] replacedKey = new byte[rowAfterSplitKey.length + afterRow.length + firstindex + + ROW_KEY_LENGTH]; + + // copy the bytes after split key til the row end + System.arraycopy(row, rowIndex, rowAfterSplitKey, 0, row.length - rowIndex); + // Copy the bytes after row till end + System.arraycopy(key, rowOffset + rowLength, afterRow, 0, + (key.length - (rowOffset + rowLength))); + + short length = (short) (rowAfterSplitKey.length + firstindex); + byte[] rowKeyLengthBytes = Bytes.toBytes(length); + // This is for padding the row length to the first 2 byte positions + System.arraycopy(rowKeyLengthBytes, 0, replacedKey, 0, rowKeyLengthBytes.length); + // Copy the actualFirstKey till firstIndex to replacedKey.. This will be the start key of main + // table + System.arraycopy(actualFirstKey, 0, replacedKey, ROW_KEY_LENGTH, firstindex); + // Now copy the rowAfterSplitKey + System.arraycopy(rowAfterSplitKey, 0, replacedKey, firstindex + rowKeyLengthBytes.length, + rowAfterSplitKey.length); + // Now copy the afterRow part + System.arraycopy(afterRow, 0, replacedKey, firstindex + rowAfterSplitKey.length + rowKeyLengthBytes.length, + afterRow.length); + return KeyValue.createKeyValueFromKey(replacedKey); + } + + @Override + public byte[] getLastKey() { + // This method wont get used for the index region. There is no need to call getClosestRowBefore + // on the index table. Also this is a split region. Can not be further split + throw new UnsupportedOperationException("Method is not implemented!"); + } + + private byte[] replaceDeleteKeyWithSplitKey(byte[] key) { + String oneByteStr = Bytes.toString(new byte[1]); + byte[] firstKey = getFirstKey(); + int lenOfRegionStartKeyPart = Bytes.toString(firstKey).indexOf(oneByteStr); + int remainingKeyLen = key.length - lenOfRegionStartKeyPart; + byte[] replacedKey = new byte[remainingKeyLen + splitRow.length]; + System.arraycopy(splitRow, 0, replacedKey, 0, splitRow.length); + System.arraycopy(key, lenOfRegionStartKeyPart, replacedKey, splitRow.length, remainingKeyLen); + return replacedKey; + } + + @Override + public byte[] midkey() throws IOException { + // Returns null to indicate file is not splitable. + return null; + } + + @Override + public byte[] getFirstKey() { + return super.getFirstKey(); + } + + @Override + public boolean passesKeyRangeFilter(Scan scan) { + return true; + } +} \ No newline at end of file Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/manager/IndexManager.java =================================================================== --- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/manager/IndexManager.java (revision 0) +++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/manager/IndexManager.java (working copy) @@ -0,0 +1,149 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.manager; + +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.hadoop.hbase.index.IndexSpecification; +import org.apache.hadoop.hbase.index.util.ByteArrayBuilder; +import org.apache.hadoop.hbase.index.util.IndexUtils; +import org.apache.hadoop.hbase.util.Bytes; + +/** + * IndexManager manages the index details of each table. + */ +public class IndexManager{ + + // manager is Singleton object + private static IndexManager manager = new IndexManager(); + + private Map> tableVsIndices = + new ConcurrentHashMap>(); + + private ConcurrentHashMap tableVsNumberOfRegions = + new ConcurrentHashMap(); + + // TODO one DS is enough + private Map> tableIndexMap = + new ConcurrentHashMap>(); + + private IndexManager() { + + } + + /** + * @return IndexManager instance + */ + public static IndexManager getInstance() { + return manager; + } + + /** + * @param table name on which index applying + * @param IndexSpecification list of table + */ + public void addIndexForTable(String tableName, List indexList) { + this.tableVsIndices.put(tableName, indexList); + // TODO the inner map needs to be thread safe when we support dynamic index add/remove + Map indexMap = + new TreeMap(Bytes.BYTES_COMPARATOR); + for (IndexSpecification index : indexList) { + ByteArrayBuilder keyBuilder = ByteArrayBuilder.allocate(IndexUtils.getMaxIndexNameLength()); + keyBuilder.put(Bytes.toBytes(index.getName())); + indexMap.put(keyBuilder.array(), index); + } + this.tableIndexMap.put(tableName, indexMap); + } + + /** + * @param table + * name on which index applying + * @return IndexSpecification list for the table or return null if no index + * for the table + */ + public List getIndicesForTable(String tableName) { + return this.tableVsIndices.get(tableName); + } + + /** + * @param tableName on which index applying + */ + public void removeIndices(String tableName) { + this.tableVsIndices.remove(tableName); + this.tableIndexMap.remove(tableName); + } + + /** + * @param tableName + * @param indexName + * @return + */ + public IndexSpecification getIndex(String tableName, byte[] indexName) { + Map indices = this.tableIndexMap.get(tableName); + if (indices != null) { + return indices.get(indexName); + } + return null; + } + + public void incrementRegionCount(String tableName) { + AtomicInteger count = this.tableVsNumberOfRegions.get(tableName); + // Here synchronization is needed for the first time count operation to be + // initialized + if (null == count) { + synchronized (tableVsNumberOfRegions) { + count = this.tableVsNumberOfRegions.get(tableName); + if (null == count) { + count = new AtomicInteger(0); + this.tableVsNumberOfRegions.put(tableName, count); + } + } + } + count.incrementAndGet(); + + } + + public void decrementRegionCount(String tableName, boolean removeIndices) { + // Need not be synchronized here because anyway the decrement operator + // will work atomically. Ultimately atleast one thread will see the count + // to be 0 which should be sufficient to remove the indices + AtomicInteger count = this.tableVsNumberOfRegions.get(tableName); + if (null != count) { + int next = count.decrementAndGet(); + if (next == 0) { + this.tableVsNumberOfRegions.remove(tableName); + if (removeIndices) { + this.removeIndices(tableName); + } + } + } + } + + // API needed for test cases. + public int getTableRegionCount(String tableName){ + AtomicInteger count = this.tableVsNumberOfRegions.get(tableName); + if(count != null){ + return count.get(); + } + return 0; + } +} Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/mapreduce/IndexCreationMapper.java =================================================================== --- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/mapreduce/IndexCreationMapper.java (revision 0) +++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/mapreduce/IndexCreationMapper.java (working copy) @@ -0,0 +1,90 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.mapreduce; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Mutation; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.index.Constants; +import org.apache.hadoop.hbase.index.IndexSpecification; +import org.apache.hadoop.hbase.index.TableIndices; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.mapreduce.TableMapper; + +public class IndexCreationMapper extends TableMapper { + + private List indices = new ArrayList(); + private static byte[][] startKeys = null; + + @Override + protected void setup(Context context) throws IOException, InterruptedException { + String tableNameToIndex = context.getConfiguration().get(TableIndexer.TABLE_NAME_TO_INDEX); + HTable hTable = null; + try { + hTable = new HTable(context.getConfiguration(), tableNameToIndex); + this.startKeys = hTable.getStartKeys(); + byte[] indexBytes = hTable.getTableDescriptor().getValue(Constants.INDEX_SPEC_KEY); + if (indexBytes != null) { + TableIndices tableIndices = new TableIndices(); + tableIndices.readFields(indexBytes); + this.indices = tableIndices.getIndices(); + } + } finally { + hTable.close(); + } + }; + + public void map(ImmutableBytesWritable row, Result value, Context context) throws IOException, + InterruptedException { + Configuration conf = context.getConfiguration(); + resultToPut(context, row,value, conf); + } + + private void resultToPut(Context context, + ImmutableBytesWritable key, Result result, Configuration conf) throws IOException, + InterruptedException { + byte[] row = key.get(); + Put put = null; + for (Cell kv : result.raw()) { + if(((KeyValue) kv).isDelete()){ + // Skipping delete records as any way the deletes will mask the actual + // puts + continue; + }else{ + if (put == null) { + put = new Put(row); + } + put.add(kv); + } + } + if (put != null) { + List indexPuts = IndexMapReduceUtil.getIndexPut(put, this.indices, this.startKeys, conf); + for (Put idxPut : indexPuts) { + context.write(new ImmutableBytesWritable(idxPut.getRow()), idxPut); + } + } + } +} Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/mapreduce/IndexHFileOutputFormat.java =================================================================== --- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/mapreduce/IndexHFileOutputFormat.java (revision 0) +++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/mapreduce/IndexHFileOutputFormat.java (working copy) @@ -0,0 +1,256 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.mapreduce; + +import java.io.IOException; +import java.util.Map; +import java.util.TreeMap; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.index.Constants; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.io.compress.Compression; +import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; +import org.apache.hadoop.hbase.io.hfile.AbstractHFileWriter; +import org.apache.hadoop.hbase.io.hfile.HFile; +import org.apache.hadoop.hbase.io.hfile.HFileContext; +import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder; +import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoder; +import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoderImpl; +import org.apache.hadoop.hbase.io.hfile.NoOpDataBlockEncoder; +import org.apache.hadoop.hbase.mapreduce.HFileOutputFormat; +import org.apache.hadoop.hbase.regionserver.HStore; +import org.apache.hadoop.hbase.regionserver.StoreFile; +import org.apache.hadoop.hbase.regionserver.TimeRangeTracker; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.io.WritableUtils; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.RecordWriter; +import org.apache.hadoop.mapreduce.TaskAttemptContext; +import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter; +import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; + +public class IndexHFileOutputFormat extends FileOutputFormat { + + static Log LOG = LogFactory.getLog(IndexHFileOutputFormat.class); + private static final String DATABLOCK_ENCODING_CONF_KEY = "hbase.mapreduce.hfileoutputformat.datablock.encoding"; + TimeRangeTracker trt = new TimeRangeTracker(); + + public static void configureIncrementalLoad(Job job, HTable table) throws IOException { + HFileOutputFormat.configureIncrementalLoad(job, table); + // Override OutputFormatClass + job.setOutputFormatClass(IndexHFileOutputFormat.class); + } + + public RecordWriter getRecordWriter( + final TaskAttemptContext context) throws IOException, InterruptedException { + + // Get the path of the temporary output file + final Path outputPath = FileOutputFormat.getOutputPath(context); + final Path outputdir = new FileOutputCommitter(outputPath, context).getWorkPath(); + + final Configuration conf = context.getConfiguration(); + final FileSystem fs = outputdir.getFileSystem(conf); + + // These configs. are from hbase-*.xml + final long maxsize = conf.getLong(HConstants.HREGION_MAX_FILESIZE, + HConstants.DEFAULT_MAX_FILE_SIZE); + final Map blockSizeMap = HFileOutputFormat.createFamilyBlockSizeMap(conf); + // Invented config. Add to hbase-*.xml if other than default compression. + final String defaultCompression = conf.get("hfile.compression", + Compression.Algorithm.NONE.getName()); + final boolean compactionExclude = conf.getBoolean( + "hbase.mapreduce.hfileoutputformat.compaction.exclude", false); + + final boolean indexedTable = conf.getBoolean(IndexMapReduceUtil.IS_INDEXED_TABLE, false); + + final Path indexDir = new Path(outputdir, IndexMapReduceUtil.INDEX_DATA_DIR); + + final FileSystem indexFs = indexDir.getFileSystem(conf); + + if (indexedTable) { + if (!indexFs.exists(indexDir)) { + indexFs.mkdirs(indexDir); + } + } + + // create a map from column family to the compression algorithm + final Map compressionMap = HFileOutputFormat.createFamilyCompressionMap(conf); + + final String dataBlockEncodingStr = conf.get(DATABLOCK_ENCODING_CONF_KEY); + + return new RecordWriter() { + // Map of families to writers and how much has been output on the writer. + private final Map writers = new TreeMap( + Bytes.BYTES_COMPARATOR); + private byte[] previousRow = HConstants.EMPTY_BYTE_ARRAY; + private final byte[] now = Bytes.toBytes(System.currentTimeMillis()); + private boolean rollRequested = false; + + public void write(ImmutableBytesWritable row, KeyValue kv) throws IOException { + // null input == user explicitly wants to flush + if (row == null && kv == null) { + rollWriters(); + return; + } + boolean indexed = false; + + byte[] rowKey = kv.getRow(); + long length = kv.getLength(); + byte[] family = kv.getFamily(); + byte[] qualifier = kv.getQualifier(); + + if (Bytes.equals(family, Constants.IDX_COL_FAMILY) + && Bytes.equals(qualifier, Constants.IDX_COL_QUAL)) { + indexed = true; + } + + WriterLength wl = null; + if (indexed) { + wl = this.writers.get(Bytes.toBytes(IndexMapReduceUtil.INDEX_DATA_DIR)); + } else { + wl = this.writers.get(family); + } + + // If this is a new column family, verify that the directory exists + if (wl == null) { + if (indexed) { + indexFs.mkdirs(new Path(indexDir, Bytes.toString(family))); + } else { + fs.mkdirs(new Path(outputdir, Bytes.toString(family))); + } + } + + // If any of the HFiles for the column families has reached + // maxsize, we need to roll all the writers + if (wl != null && wl.written + length >= maxsize) { + this.rollRequested = true; + } + + // This can only happen once a row is finished though + if (rollRequested && Bytes.compareTo(this.previousRow, rowKey) != 0) { + rollWriters(); + } + + // create a new HLog writer, if necessary + if (wl == null || wl.writer == null) { + wl = getNewWriter(family, conf, indexed); + } + + // we now have the proper HLog writer. full steam ahead + kv.updateLatestStamp(this.now); + trt.includeTimestamp(kv); + wl.writer.append(kv); + wl.written += length; + + // Copy the row so we know when a row transition. + this.previousRow = rowKey; + } + + private void rollWriters() throws IOException { + for (WriterLength wl : this.writers.values()) { + if (wl.writer != null) { + LOG.info("Writer=" + wl.writer.getPath() + + ((wl.written == 0) ? "" : ", wrote=" + wl.written)); + close(wl.writer); + } + wl.writer = null; + wl.written = 0; + } + this.rollRequested = false; + } + + /* + * Create a new HFile.Writer. + * + * @param family + * + * @return A WriterLength, containing a new HFile.Writer. + * + * @throws IOException + */ + private WriterLength getNewWriter(byte[] family, Configuration conf, boolean indexData) + throws IOException { + WriterLength wl = new WriterLength(); + + Path familydir = null; + + String compression = compressionMap.get(family); + compression = compression == null ? defaultCompression : compression; + String blockSizeString = blockSizeMap.get(family); + int blockSize = blockSizeString == null ? HConstants.DEFAULT_BLOCKSIZE + : Integer.parseInt(blockSizeString); + HFileContext hFileContext = + new HFileContextBuilder().withBlockSize(blockSize) + .withCompression(AbstractHFileWriter.compressionByName(compression)) + .withDataBlockEncoding(DataBlockEncoding.valueOf(dataBlockEncodingStr)) + .withChecksumType(HStore.getChecksumType(conf)) + .withBytesPerCheckSum(HStore.getBytesPerChecksum(conf)).build(); + if (indexData) { + familydir = new Path(indexDir, Bytes.toString(family)); + wl.writer = HFile.getWriterFactoryNoCache(conf) + .withPath(indexFs, StoreFile.getUniqueFile(indexFs, familydir)) + .withFileContext(hFileContext) + .withComparator(KeyValue.COMPARATOR).create(); + this.writers.put(Bytes.toBytes(IndexMapReduceUtil.INDEX_DATA_DIR), wl); + } else { + familydir = new Path(outputdir, Bytes.toString(family)); + wl.writer = HFile.getWriterFactoryNoCache(conf) + .withPath(fs, StoreFile.getUniqueFile(fs, familydir)) + .withFileContext(hFileContext) + .withComparator(KeyValue.COMPARATOR).create(); + this.writers.put(family, wl); + } + + return wl; + } + + private void close(final HFile.Writer w) throws IOException { + if (w != null) { + w.appendFileInfo(StoreFile.BULKLOAD_TIME_KEY, Bytes.toBytes(System.currentTimeMillis())); + w.appendFileInfo(StoreFile.BULKLOAD_TASK_KEY, + Bytes.toBytes(context.getTaskAttemptID().toString())); + w.appendFileInfo(StoreFile.MAJOR_COMPACTION_KEY, Bytes.toBytes(true)); + w.appendFileInfo(StoreFile.EXCLUDE_FROM_MINOR_COMPACTION_KEY, + Bytes.toBytes(compactionExclude)); + w.appendFileInfo(StoreFile.TIMERANGE_KEY, WritableUtils.toByteArray(trt)); + w.close(); + } + } + + public void close(TaskAttemptContext c) throws IOException, InterruptedException { + for (WriterLength wl : this.writers.values()) { + close(wl.writer); + } + } + }; + + } + + static class WriterLength { + long written = 0; + HFile.Writer writer = null; + } +} Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/mapreduce/IndexImportTsv.java =================================================================== --- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/mapreduce/IndexImportTsv.java (revision 0) +++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/mapreduce/IndexImportTsv.java (working copy) @@ -0,0 +1,280 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.mapreduce; + +import java.io.IOException; +import java.util.HashSet; +import java.util.Set; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.index.client.IndexAdmin; +import org.apache.hadoop.hbase.index.util.IndexUtils; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.mapreduce.ImportTsv; +import org.apache.hadoop.hbase.mapreduce.PutSortReducer; +import org.apache.hadoop.hbase.mapreduce.TableInputFormat; +import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; +import org.apache.hadoop.hbase.util.Base64; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; +import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; +import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; +import org.apache.hadoop.util.GenericOptionsParser; + +public class IndexImportTsv extends ImportTsv { + + final static String NAME = "indeximporttsv"; + final static Class DEFAULT_MAPPER = IndexTsvImporterMapper.class; + + private static HBaseAdmin hbaseAdmin; + + /** + * Sets up the actual job. + * + * @param conf + * The current configuration. + * @param args + * The command line parameters. + * @return The newly created job. + * @throws IOException + * When setting up the job fails. + * @throws InterruptedException + */ + public static Job createSubmittableJob(Configuration conf, String[] args) throws IOException, + ClassNotFoundException { + + // Support non-XML supported characters + // by re-encoding the passed separator as a Base64 string. + String actualSeparator = conf.get(ImportTsv.SEPARATOR_CONF_KEY); + if (actualSeparator != null) { + conf.set(ImportTsv.SEPARATOR_CONF_KEY, Base64.encodeBytes(actualSeparator.getBytes())); + } + + // See if a non-default Mapper was set + String mapperClassName = conf.get(ImportTsv.MAPPER_CONF_KEY); + Class mapperClass = mapperClassName != null ? Class.forName(mapperClassName) : DEFAULT_MAPPER; + + String tableName = args[0]; + Path inputDir = new Path(args[1]); + + String input = conf.get(IndexUtils.TABLE_INPUT_COLS); + HTableDescriptor htd = null; + if (!doesTableExist(tableName)) { + htd = getTableDesc(conf, tableName); + if (input != null) { + htd = IndexUtils.parse(tableName, htd, input, null); + hbaseAdmin.createTable(htd); + } + } + + conf.set(TableInputFormat.INPUT_TABLE, tableName); + conf.setBoolean(IndexMapReduceUtil.IS_INDEXED_TABLE, input != null); + Job job = new Job(conf, NAME + "_" + tableName); + job.setJarByClass(mapperClass); + FileInputFormat.setInputPaths(job, inputDir); + job.setInputFormatClass(TextInputFormat.class); + job.setMapperClass(mapperClass); + + String hfileOutPath = conf.get(ImportTsv.BULK_OUTPUT_CONF_KEY); + if (hfileOutPath != null) { + HTable table = new HTable(conf, tableName); + job.setReducerClass(PutSortReducer.class); + Path outputDir = new Path(hfileOutPath); + FileOutputFormat.setOutputPath(job, outputDir); + job.setMapOutputKeyClass(ImmutableBytesWritable.class); + job.setMapOutputValueClass(Put.class); + IndexHFileOutputFormat.configureIncrementalLoad(job, table); + } else { + // No reducers. Just write straight to table. Call initTableReducerJob + // to set up the TableOutputFormat. + TableMapReduceUtil.initTableReducerJob(tableName, null, job); + job.setNumReduceTasks(0); + } + + TableMapReduceUtil.addDependencyJars(job); + TableMapReduceUtil.addDependencyJars(job.getConfiguration(), + com.google.common.base.Function.class /* Guava used by TsvParser */); + return job; + } + + static boolean doesTableExist(String tableName) throws IOException { + return hbaseAdmin.tableExists(Bytes.toBytes(tableName)); + } + + static HTableDescriptor getTableDesc(Configuration conf, String tableName) throws IOException { + HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName)); + String columns[] = conf.getStrings(ImportTsv.COLUMNS_CONF_KEY); + Set cfSet = new HashSet(); + for (String aColumn : columns) { + if (TsvParser.ROWKEY_COLUMN_SPEC.equals(aColumn)) + continue; + // we are only concerned with the first one (in case this is a cf:cq) + cfSet.add(aColumn.split(":", 2)[0]); + } + for (String cf : cfSet) { + HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toBytes(cf)); + htd.addFamily(hcd); + } + return htd; + } + + /* + * @param errorMsg Error message. Can be null. + */ + private static void usage(final String errorMsg) { + if (errorMsg != null && errorMsg.length() > 0) { + System.err.println("ERROR: " + errorMsg); + } + String usage = "Usage: " + + NAME + + " -Dimporttsv.columns=a,b,c \n" + + "\n" + + "Imports the given input directory of TSV data into the specified table.\n" + + "\n" + + "The column names of the TSV data must be specified using the -Dimporttsv.columns\n" + + "option. This option takes the form of comma-separated column names, where each\n" + + "column name is either a simple column family, or a columnfamily:qualifier. The special\n" + + "column name HBASE_ROW_KEY is used to designate that this column should be used\n" + + "as the row key for each imported record. You must specify exactly one column\n" + + "to be the row key, and you must specify a column name for every column that exists in the\n" + + "input data. Another special column HBASE_TS_KEY designates that this column should be\n" + + "used as timestamp for each record. Unlike HBASE_ROW_KEY, HBASE_TS_KEY is optional.\n" + + "You must specify atmost one column as timestamp key for each imported record.\n" + + "Record with invalid timestamps (blank, non-numeric) will be treated as bad record.\n" + + "Note: if you use this option, then 'importtsv.timestamp' option will be ignored.\n" + + "\n" + + "By default importtsv will load data directly into HBase. To instead generate\n" + + "HFiles of data to prepare for a bulk data load, pass the option:\n" + + " -D" + + ImportTsv.BULK_OUTPUT_CONF_KEY + + "=/path/for/output\n" + + " Note: if you do not use this option, then the target table must already exist in HBase\n" + + "\n" + + "Other options that may be specified with -D include:\n" + + " -D" + + ImportTsv.SKIP_LINES_CONF_KEY + + "=false - fail if encountering an invalid line\n" + + " '-D" + + ImportTsv.SEPARATOR_CONF_KEY + + "=|' - eg separate on pipes instead of tabs\n" + + " -D" + + ImportTsv.TIMESTAMP_CONF_KEY + + "=currentTimeAsLong - use the specified timestamp for the import\n" + + " -D" + + ImportTsv.MAPPER_CONF_KEY + + "=my.Mapper - A user-defined Mapper to use instead of " + + DEFAULT_MAPPER.getName() + + "\n" + + "For performance consider the following options:\n" + + " -Dmapred.map.tasks.speculative.execution=false\n" + + " -Dmapred.reduce.tasks.speculative.execution=false\n" + + " -Dtable.columns.index='IDX1=>cf1:[q1->datatype& length],[q2]," + + "[q3];cf2:[q1->datatype&length],[q2->datatype&length],[q3->datatype& lenght]#IDX2=>cf1:q5,q5'" + + " The format used here is: \n" + " IDX1 - Index name\n" + + " cf1 - Columnfamilyname\n" + " q1 - qualifier\n" + + " datatype - datatype (Int, String, Double, Float)\n" + + " length - length of the value\n" + + " The columnfamily should be seperated by ';'\n" + + " The qualifier and the datatype and its length should be enclosed in '[]'.\n" + + " The qualifier details are specified using '->' following qualifer name and the" + + " details are seperated by '&'\n" + + " If the qualifier details are not specified default values are used.\n" + + " # is used to seperate between two index details"; + System.err.println(usage); + } + + /** + * Used only by test method + * + * @param conf + */ + static void createHbaseAdmin(Configuration conf) throws IOException { + hbaseAdmin = new IndexAdmin(conf); + } + + /** + * Main entry point. + * + * @param args + * The command line parameters. + * @throws Exception + * When running the job fails. + */ + public static void main(String[] args) throws Exception { + Configuration conf = HBaseConfiguration.create(); + String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs(); + if (otherArgs.length < 2) { + usage("Wrong number of arguments: " + otherArgs.length); + System.exit(-1); + } + + // Make sure columns are specified + String columns[] = conf.getStrings(ImportTsv.COLUMNS_CONF_KEY); + if (columns == null) { + usage("No columns specified. Please specify with -D" + ImportTsv.COLUMNS_CONF_KEY + "=..."); + System.exit(-1); + } + + // Make sure they specify exactly one column as the row key + int rowkeysFound = 0; + for (String col : columns) { + if (col.equals(TsvParser.ROWKEY_COLUMN_SPEC)) + rowkeysFound++; + } + if (rowkeysFound != 1) { + usage("Must specify exactly one column as " + TsvParser.ROWKEY_COLUMN_SPEC); + System.exit(-1); + } + + // Make sure we have at most one column as the timestamp key + int tskeysFound = 0; + for (String col : columns) { + if (col.equals(TsvParser.TIMESTAMPKEY_COLUMN_SPEC)) + tskeysFound++; + } + if (tskeysFound > 1) { + usage("Must specify at most one column as " + TsvParser.TIMESTAMPKEY_COLUMN_SPEC); + System.exit(-1); + } + + // Make sure one or more columns are specified excluding rowkey and timestamp key + if (columns.length - (rowkeysFound + tskeysFound) < 1) { + usage("One or more columns in addition to the row key and timestamp(optional) are required"); + System.exit(-1); + } + + // If timestamp option is not specified, use current system time. + long timstamp = conf.getLong(ImportTsv.TIMESTAMP_CONF_KEY, System.currentTimeMillis()); + + // Set it back to replace invalid timestamp (non-numeric) with current system time + conf.setLong(ImportTsv.TIMESTAMP_CONF_KEY, timstamp); + + hbaseAdmin = new IndexAdmin(conf); + Job job = createSubmittableJob(conf, otherArgs); + System.exit(job.waitForCompletion(true) ? 0 : 1); + } +} Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/mapreduce/IndexLoadIncrementalHFile.java =================================================================== --- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/mapreduce/IndexLoadIncrementalHFile.java (revision 0) +++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/mapreduce/IndexLoadIncrementalHFile.java (working copy) @@ -0,0 +1,758 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.mapreduce; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Deque; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.TreeMap; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.TableNotFoundException; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HConnection; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.RegionServerCallable; +import org.apache.hadoop.hbase.client.RpcRetryingCallerFactory; +import org.apache.hadoop.hbase.client.coprocessor.SecureBulkLoadClient; +import org.apache.hadoop.hbase.index.util.IndexUtils; +import org.apache.hadoop.hbase.io.HalfStoreFileReader; +import org.apache.hadoop.hbase.io.Reference; +import org.apache.hadoop.hbase.io.Reference.Range; +import org.apache.hadoop.hbase.io.compress.Compression.Algorithm; +import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; +import org.apache.hadoop.hbase.io.hfile.AbstractHFileWriter; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; +import org.apache.hadoop.hbase.io.hfile.HFileContext; +import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder; +import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoder; +import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoderImpl; +import org.apache.hadoop.hbase.io.hfile.HFile; +import org.apache.hadoop.hbase.io.hfile.HFileScanner; +import org.apache.hadoop.hbase.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.regionserver.BloomType; +import org.apache.hadoop.hbase.regionserver.HStore; +import org.apache.hadoop.hbase.regionserver.StoreFile; +import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; + +import com.google.common.collect.HashMultimap; +import com.google.common.collect.Multimap; +import com.google.common.collect.Multimaps; +import com.google.common.util.concurrent.ThreadFactoryBuilder; + +/** + * Tool to load the output of HFileOutputFormat into an existing table. + * + * @see #usage() + */ +public class IndexLoadIncrementalHFile extends Configured implements Tool { + + private static Log LOG = LogFactory.getLog(IndexLoadIncrementalHFile.class); + static AtomicLong regionCount = new AtomicLong(0); + private HBaseAdmin hbAdmin; + private Configuration cfg; + + public static String NAME = "completebulkload"; + private static final String ASSIGN_SEQ_IDS = "hbase.mapreduce.bulkload.assign.sequenceNumbers"; + private boolean assignSeqIds; + + private boolean useSecure; + private Token userToken; + private String bulkToken; + + + public IndexLoadIncrementalHFile(Configuration conf) throws Exception { + super(conf); + this.cfg = conf; + this.hbAdmin = new HBaseAdmin(conf); + this.useSecure = User.isHBaseSecurityEnabled(conf); + assignSeqIds = conf.getBoolean(ASSIGN_SEQ_IDS, true); + } + + private void usage() { + System.err.println("usage: " + NAME + " /path/to/hfileoutputformat-output " + "tablename"); + } + + /** + * Represents an HFile waiting to be loaded. An queue is used in this class in order to support + * the case where a region has split during the process of the load. When this happens, the HFile + * is split into two physical parts across the new region boundary, and each part is added back + * into the queue. The import process finishes when the queue is empty. + */ + static class LoadQueueItem { + final byte[] family; + final Path hfilePath; + + public LoadQueueItem(byte[] family, Path hfilePath) { + this.family = family; + this.hfilePath = hfilePath; + } + + public String toString() { + return "family:" + Bytes.toString(family) + " path:" + hfilePath.toString(); + } + } + + /** + * Walk the given directory for all HFiles, and return a Queue containing all such files. + */ + private void discoverLoadQueue(Deque ret, Path hfofDir) throws IOException { + FileSystem fs = hfofDir.getFileSystem(getConf()); + + if (!fs.exists(hfofDir)) { + throw new FileNotFoundException("HFileOutputFormat dir " + hfofDir + " not found"); + } + + FileStatus[] familyDirStatuses = fs.listStatus(hfofDir); + if (familyDirStatuses == null) { + throw new FileNotFoundException("No families found in " + hfofDir); + } + + for (FileStatus stat : familyDirStatuses) { + if (!stat.isDir()) { + LOG.warn("Skipping non-directory " + stat.getPath()); + continue; + } + Path familyDir = stat.getPath(); + // Skip _logs and .index, etc + if (familyDir.getName().startsWith("_") + || familyDir.getName().startsWith(IndexMapReduceUtil.INDEX_DATA_DIR)) + continue; + byte[] family = Bytes.toBytes(familyDir.getName()); + Path[] hfiles = FileUtil.stat2Paths(fs.listStatus(familyDir)); + for (Path hfile : hfiles) { + if (hfile.getName().startsWith("_")) + continue; + ret.add(new LoadQueueItem(family, hfile)); + } + } + } + + /** + * Perform a bulk load of the given directory into the given pre-existing table. This method is + * not threadsafe. + * + * @param hfofDir + * the directory that was provided as the output path of a job using HFileOutputFormat + * @param table + * the table to load into + * @throws TableNotFoundException + * if table does not yet exist + */ + public void doBulkLoad(Path hfofDir, final HTable table) throws TableNotFoundException, + IOException { + final HConnection conn = table.getConnection(); + + if (!conn.isTableAvailable(table.getTableName())) { + throw new TableNotFoundException("Table " + Bytes.toStringBinary(table.getTableName()) + + "is not currently available."); + } + + // initialize thread pools + int nrThreads = cfg.getInt("hbase.loadincremental.threads.max", Runtime.getRuntime() + .availableProcessors()); + ThreadFactoryBuilder builder = new ThreadFactoryBuilder(); + builder.setNameFormat("LoadIncrementalHFiles-%1$d"); + ExecutorService pool = new ThreadPoolExecutor(nrThreads, nrThreads, 60, TimeUnit.SECONDS, + new LinkedBlockingQueue(), builder.build()); + ((ThreadPoolExecutor) pool).allowCoreThreadTimeOut(true); + + // LQI queue does not need to be threadsafe -- all operations on this queue + // happen in this thread + Deque queue = new LinkedList(); + try { + discoverLoadQueue(queue, hfofDir); + int count = 0; + + if (queue.isEmpty()) { + LOG.warn("Bulk load operation did not find any files to load in " + "directory " + + hfofDir.toUri() + ". Does it contain files in " + + "subdirectories that correspond to column family names?"); + return; + } + + if (queue.isEmpty()) { + LOG.warn("Bulk load operation did not find any files to load in " + "directory " + + hfofDir.toUri() + ". Does it contain files in " + + "subdirectories that correspond to column family names?"); + } + + // Assumes that region splits can happen while this occurs. + while (!queue.isEmpty()) { + // need to reload split keys each iteration. + final Pair startEndKeys = table.getStartEndKeys(); + if (count != 0) { + LOG.info("Split occured while grouping HFiles, retry attempt " + +count + " with " + + queue.size() + " files remaining to group or split"); + } + + int maxRetries = cfg.getInt("hbase.bulkload.retries.number", 0); + if (maxRetries != 0 && count >= maxRetries) { + LOG.error("Retry attempted " + count + " times without completing, bailing out"); + return; + } + count++; + + // Using ByteBuffer for byte[] equality semantics + Multimap regionGroups = groupOrSplitPhase(table, pool, queue, + startEndKeys); + + bulkLoadPhase(table, conn, pool, queue, regionGroups); + + // NOTE: The next iteration's split / group could happen in parallel to + // atomic bulkloads assuming that there are splits and no merges, and + // that we can atomically pull out the groups we want to retry. + } + + } finally { + pool.shutdown(); + if (queue != null && !queue.isEmpty()) { + StringBuilder err = new StringBuilder(); + err.append("-------------------------------------------------\n"); + err.append("Bulk load aborted with some files not yet loaded:\n"); + err.append("-------------------------------------------------\n"); + for (LoadQueueItem q : queue) { + err.append(" ").append(q.hfilePath).append('\n'); + } + LOG.error(err); + } + } + } + + /** + * This takes the LQI's grouped by likely regions and attempts to bulk load them. Any failures are + * re-queued for another pass with the groupOrSplitPhase. + */ + protected void bulkLoadPhase(final HTable table, final HConnection conn, ExecutorService pool, + Deque queue, final Multimap regionGroups) + throws IOException { + // atomically bulk load the groups. + Set>> loadingFutures = new HashSet>>(); + for (Entry> e : regionGroups.asMap().entrySet()) { + final byte[] first = e.getKey().array(); + final Collection lqis = e.getValue(); + + final Callable> call = new Callable>() { + public List call() throws Exception { + List toRetry = tryAtomicRegionLoad(conn, table.getName(), first, lqis); + return toRetry; + } + }; + loadingFutures.add(pool.submit(call)); + } + + // get all the results. + for (Future> future : loadingFutures) { + try { + List toRetry = future.get(); + + // LQIs that are requeued to be regrouped. + queue.addAll(toRetry); + + } catch (ExecutionException e1) { + Throwable t = e1.getCause(); + if (t instanceof IOException) { + // At this point something unrecoverable has happened. + // TODO Implement bulk load recovery + throw new IOException("BulkLoad encountered an unrecoverable problem", t); + } + LOG.error("Unexpected execution exception during bulk load", e1); + throw new IllegalStateException(t); + } catch (InterruptedException e1) { + LOG.error("Unexpected interrupted exception during bulk load", e1); + throw new IllegalStateException(e1); + } + } + } + + /** + * @return A Multimap that groups LQI by likely bulk load region targets. + */ + private Multimap groupOrSplitPhase(final HTable table, + ExecutorService pool, Deque queue, final Pair startEndKeys) + throws IOException { + // need synchronized only within this scope of this + // phase because of the puts that happen in futures. + Multimap rgs = HashMultimap.create(); + final Multimap regionGroups = Multimaps.synchronizedMultimap(rgs); + + // drain LQIs and figure out bulk load groups + Set>> splittingFutures = new HashSet>>(); + while (!queue.isEmpty()) { + final LoadQueueItem item = queue.remove(); + + final Callable> call = new Callable>() { + public List call() throws Exception { + List splits = groupOrSplit(regionGroups, item, table, startEndKeys); + return splits; + } + }; + splittingFutures.add(pool.submit(call)); + } + // get all the results. All grouping and splitting must finish before + // we can attempt the atomic loads. + for (Future> lqis : splittingFutures) { + try { + List splits = lqis.get(); + if (splits != null) { + queue.addAll(splits); + } + } catch (ExecutionException e1) { + Throwable t = e1.getCause(); + if (t instanceof IOException) { + LOG.error("IOException during splitting", e1); + throw (IOException) t; // would have been thrown if not parallelized, + } + LOG.error("Unexpected execution exception during splitting", e1); + throw new IllegalStateException(t); + } catch (InterruptedException e1) { + LOG.error("Unexpected interrupted exception during splitting", e1); + throw new IllegalStateException(e1); + } + } + return regionGroups; + } + + // unique file name for the table + String getUniqueName(byte[] tableName) { + String name = Bytes.toStringBinary(tableName) + "," + regionCount.incrementAndGet(); + return name; + } + + protected List splitStoreFile(final LoadQueueItem item, final HTable table, + byte[] startKey, byte[] splitKey) throws IOException { + final Path hfilePath = item.hfilePath; + + // We use a '_' prefix which is ignored when walking directory trees + // above. + final Path tmpDir = new Path(item.hfilePath.getParent(), "_tmp"); + + LOG.info("HFile at " + hfilePath + " no longer fits inside a single " + "region. Splitting..."); + + String uniqueName = getUniqueName(table.getTableName()); + HColumnDescriptor familyDesc = table.getTableDescriptor().getFamily(item.family); + Path botOut = new Path(tmpDir, uniqueName + ".bottom"); + Path topOut = new Path(tmpDir, uniqueName + ".top"); + splitStoreFile(getConf(), hfilePath, familyDesc, splitKey, botOut, topOut); + + // Add these back at the *front* of the queue, so there's a lower + // chance that the region will just split again before we get there. + List lqis = new ArrayList(2); + lqis.add(new LoadQueueItem(item.family, botOut)); + lqis.add(new LoadQueueItem(item.family, topOut)); + + LOG.info("Successfully split into new HFiles " + botOut + " and " + topOut); + return lqis; + } + + /** + * Attempt to assign the given load queue item into its target region group. If the hfile boundary + * no longer fits into a region, physically splits the hfile such that the new bottom half will + * fit and returns the list of LQI's corresponding to the resultant hfiles. + * + * protected for testing + */ + protected List groupOrSplit(Multimap regionGroups, + final LoadQueueItem item, final HTable table, final Pair startEndKeys) + throws IOException { + final Path hfilePath = item.hfilePath; + final FileSystem fs = hfilePath.getFileSystem(getConf()); + HFile.Reader hfr = HFile.createReader(fs, hfilePath, new CacheConfig(getConf()), getConf()); + final byte[] first, last; + try { + hfr.loadFileInfo(); + first = hfr.getFirstRowKey(); + last = hfr.getLastRowKey(); + } finally { + hfr.close(); + } + + LOG.info("Trying to load hfile=" + hfilePath + " first=" + Bytes.toStringBinary(first) + + " last=" + Bytes.toStringBinary(last)); + if (first == null || last == null) { + assert first == null && last == null; + // TODO what if this is due to a bad HFile? + LOG.info("hfile " + hfilePath + " has no entries, skipping"); + return null; + } + if (Bytes.compareTo(first, last) > 0) { + throw new IllegalArgumentException("Invalid range: " + Bytes.toStringBinary(first) + " > " + + Bytes.toStringBinary(last)); + } + int idx = Arrays.binarySearch(startEndKeys.getFirst(), first, Bytes.BYTES_COMPARATOR); + if (idx < 0) { + // not on boundary, returns -(insertion index). Calculate region it + // would be in. + idx = -(idx + 1) - 1; + } + final int indexForCallable = idx; + boolean lastKeyInRange = Bytes.compareTo(last, startEndKeys.getSecond()[idx]) < 0 + || Bytes.equals(startEndKeys.getSecond()[idx], HConstants.EMPTY_BYTE_ARRAY); + if (!lastKeyInRange) { + List lqis = splitStoreFile(item, table, + startEndKeys.getFirst()[indexForCallable], startEndKeys.getSecond()[indexForCallable]); + return lqis; + } + + // group regions. + regionGroups.put(ByteBuffer.wrap(startEndKeys.getFirst()[idx]), item); + return null; + } + + /** + * Attempts to do an atomic load of many hfiles into a region. If it fails, + * it returns a list of hfiles that need to be retried. If it is successful + * it will return an empty list. + * + * NOTE: To maintain row atomicity guarantees, region server callable should + * succeed atomically and fails atomically. + * + * Protected for testing. + * + * @return empty list if success, list of items to retry on recoverable + * failure + */ + protected List tryAtomicRegionLoad(final HConnection conn, + final TableName tableName, final byte[] first, Collection lqis) throws IOException { + + final List> famPaths = + new ArrayList>(lqis.size()); + for (LoadQueueItem lqi : lqis) { + famPaths.add(Pair.newPair(lqi.family, lqi.hfilePath.toString())); + } + + final RegionServerCallable svrCallable = + new RegionServerCallable(conn, tableName, first) { + @Override + public Boolean call() throws Exception { + SecureBulkLoadClient secureClient = null; + boolean success = false; + + try { + LOG.debug("Going to connect to server " + getLocation() + " for row " + + Bytes.toStringBinary(getRow())); + byte[] regionName = getLocation().getRegionInfo().getRegionName(); + if(!useSecure) { + success = ProtobufUtil.bulkLoadHFile(getStub(), famPaths, regionName, assignSeqIds); + } else { + HTable table = new HTable(conn.getConfiguration(), getTableName()); + secureClient = new SecureBulkLoadClient(table); + success = secureClient.bulkLoadHFiles(famPaths, userToken, bulkToken, + getLocation().getRegionInfo().getStartKey()); + } + return success; + } finally { + //Best effort copying of files that might not have been imported + //from the staging directory back to original location + //in user directory + if(secureClient != null && !success) { + FileSystem fs = FileSystem.get(cfg); + for(Pair el : famPaths) { + Path hfileStagingPath = null; + Path hfileOrigPath = new Path(el.getSecond()); + try { + hfileStagingPath= new Path(secureClient.getStagingPath(bulkToken, el.getFirst()), + hfileOrigPath.getName()); + if(fs.rename(hfileStagingPath, hfileOrigPath)) { + LOG.debug("Moved back file " + hfileOrigPath + " from " + + hfileStagingPath); + } else if(fs.exists(hfileStagingPath)){ + LOG.debug("Unable to move back file " + hfileOrigPath + " from " + + hfileStagingPath); + } + } catch(Exception ex) { + LOG.debug("Unable to move back file " + hfileOrigPath + " from " + + hfileStagingPath, ex); + } + } + } + } + } + }; + + try { + List toRetry = new ArrayList(); + Configuration conf = getConf(); + boolean success = RpcRetryingCallerFactory.instantiate(conf). newCaller() + .callWithRetries(svrCallable); + if (!success) { + LOG.warn("Attempt to bulk load region containing " + + Bytes.toStringBinary(first) + " into table " + + tableName + " with files " + lqis + + " failed. This is recoverable and they will be retried."); + toRetry.addAll(lqis); // return lqi's to retry + } + // success + return toRetry; + } catch (IOException e) { + LOG.error("Encountered unrecoverable error from region server", e); + throw e; + } + } + + /** + * Split a storefile into a top and bottom half, maintaining the metadata, recreating bloom + * filters, etc. + */ + static void splitStoreFile(Configuration conf, Path inFile, HColumnDescriptor familyDesc, + byte[] splitKey, Path bottomOut, Path topOut) throws IOException { + // Open reader with no block cache, and not in-memory + Reference topReference = new Reference(splitKey, Range.top); + Reference bottomReference = new Reference(splitKey, Range.bottom); + + copyHFileHalf(conf, inFile, topOut, topReference, familyDesc); + copyHFileHalf(conf, inFile, bottomOut, bottomReference, familyDesc); + } + + /** + * Copy half of an HFile into a new HFile. + */ + private static void copyHFileHalf(Configuration conf, Path inFile, Path outFile, + Reference reference, HColumnDescriptor familyDescriptor) throws IOException { + FileSystem fs = inFile.getFileSystem(conf); + CacheConfig cacheConf = new CacheConfig(conf); + HalfStoreFileReader halfReader = null; + StoreFile.Writer halfWriter = null; + + try { + halfReader = new HalfStoreFileReader(fs, inFile, cacheConf, reference, conf); + Map fileInfo = halfReader.loadFileInfo(); + + int blocksize = familyDescriptor.getBlocksize(); + Algorithm compression = familyDescriptor.getCompression(); + BloomType bloomFilterType = familyDescriptor.getBloomFilterType(); + + HFileContext hFileContext = + new HFileContextBuilder().withBlockSize(blocksize).withCompression(compression) + .withDataBlockEncoding(familyDescriptor.getDataBlockEncoding()) + .withChecksumType(HStore.getChecksumType(conf)) + .withBytesPerCheckSum(HStore.getBytesPerChecksum(conf)).build(); + + halfWriter = + new StoreFile.WriterBuilder(conf, cacheConf, fs) + .withFilePath(outFile) + .withBloomType(bloomFilterType) + .withFileContext(hFileContext).build(); + HFileScanner scanner = halfReader.getScanner(false, false, false); + scanner.seekTo(); + do { + KeyValue kv = scanner.getKeyValue(); + halfWriter.append(kv); + } while (scanner.next()); + + for (Map.Entry entry : fileInfo.entrySet()) { + if (shouldCopyHFileMetaKey(entry.getKey())) { + halfWriter.appendFileInfo(entry.getKey(), entry.getValue()); + } + } + } finally { + if (halfWriter != null) + halfWriter.close(); + if (halfReader != null) + halfReader.close(cacheConf.shouldEvictOnClose()); + } + } + + private static boolean shouldCopyHFileMetaKey(byte[] key) { + return !HFile.isReservedFileInfoKey(key); + } + + private boolean doesTableExist(String tableName) throws Exception { + return hbAdmin.tableExists(tableName); + } + + /* + * Infers region boundaries for a new table. Parameter: bdryMap is a map between keys to an + * integer belonging to {+1, -1} If a key is a start key of a file, then it maps to +1 If a key is + * an end key of a file, then it maps to -1 Algo: 1) Poll on the keys in order: a) Keep adding the + * mapped values to these keys (runningSum) b) Each time runningSum reaches 0, add the start Key + * from when the runningSum had started to a boundary list. 2) Return the boundary list. + */ + public static byte[][] inferBoundaries(TreeMap bdryMap) { + ArrayList keysArray = new ArrayList(); + int runningValue = 0; + byte[] currStartKey = null; + boolean firstBoundary = true; + + for (Map.Entry item : bdryMap.entrySet()) { + if (runningValue == 0) + currStartKey = item.getKey(); + runningValue += item.getValue(); + if (runningValue == 0) { + if (!firstBoundary) + keysArray.add(currStartKey); + firstBoundary = false; + } + } + + return keysArray.toArray(new byte[0][0]); + } + + /* + * If the table is created for the first time, then "completebulkload" reads the files twice. More + * modifications necessary if we want to avoid doing it. + */ + private void createTable(String tableName, String dirPath) throws Exception { + Path hfofDir = new Path(dirPath); + FileSystem fs = hfofDir.getFileSystem(getConf()); + + if (!fs.exists(hfofDir)) { + throw new FileNotFoundException("HFileOutputFormat dir " + hfofDir + " not found"); + } + + FileStatus[] familyDirStatuses = fs.listStatus(hfofDir); + if (familyDirStatuses == null) { + throw new FileNotFoundException("No families found in " + hfofDir); + } + + HTableDescriptor htd = new HTableDescriptor(tableName); + HColumnDescriptor hcd = null; + + // Add column families + // Build a set of keys + byte[][] keys = null; + TreeMap map = new TreeMap(Bytes.BYTES_COMPARATOR); + + for (FileStatus stat : familyDirStatuses) { + if (!stat.isDir()) { + LOG.warn("Skipping non-directory " + stat.getPath()); + continue; + } + Path familyDir = stat.getPath(); + // Skip _logs & .index etc + if (familyDir.getName().startsWith("_")) + continue; + + if (familyDir.getName().startsWith(IndexMapReduceUtil.INDEX_DATA_DIR)) { + LOG.warn("Ignoring all the HFile specific to " + tableName + " indexed data."); + continue; + } + + byte[] family = Bytes.toBytes(familyDir.getName()); + + hcd = new HColumnDescriptor(family); + htd.addFamily(hcd); + + Path[] hfiles = FileUtil.stat2Paths(fs.listStatus(familyDir)); + for (Path hfile : hfiles) { + if (hfile.getName().startsWith("_")) + continue; + HFile.Reader reader = HFile.createReader(fs, hfile, new CacheConfig(getConf()), getConf()); + final byte[] first, last; + try { + if (hcd.getCompressionType() != reader.getCompressionAlgorithm()) { + hcd.setCompressionType(reader.getCompressionAlgorithm()); + LOG.info("Setting compression " + hcd.getCompressionType().name() + " for family " + + hcd.toString()); + } + reader.loadFileInfo(); + first = reader.getFirstRowKey(); + last = reader.getLastRowKey(); + + LOG.info("Trying to figure out region boundaries hfile=" + hfile + " first=" + + Bytes.toStringBinary(first) + " last=" + Bytes.toStringBinary(last)); + + // To eventually infer start key-end key boundaries + Integer value = map.containsKey(first) ? (Integer) map.get(first) : 0; + map.put(first, value + 1); + + value = map.containsKey(last) ? (Integer) map.get(last) : 0; + map.put(last, value - 1); + } finally { + reader.close(); + } + } + } + + keys = IndexLoadIncrementalHFile.inferBoundaries(map); + this.hbAdmin.createTable(htd, keys); + + LOG.info("Table " + tableName + " is available!!"); + } + + @Override + public int run(String[] args) throws Exception { + if (args.length != 2) { + usage(); + return -1; + } + + String dirPath = args[0]; + String tableName = args[1]; + + boolean tableExists = this.doesTableExist(tableName); + if (!tableExists) + this.createTable(tableName, dirPath); + + String indexTableName = IndexUtils.getIndexTableName(tableName); + boolean indexedTable = this.hbAdmin.isTableAvailable(indexTableName); + if (indexedTable) { + // load the index data to the indextable + Path indxhfDir = new Path(dirPath, IndexMapReduceUtil.INDEX_DATA_DIR); + HTable idxTable = new HTable(this.cfg, indexTableName); + doBulkLoad(indxhfDir, idxTable); + } + + Path hfofDir = new Path(dirPath); + HTable table = new HTable(this.cfg, tableName); + doBulkLoad(hfofDir, table); + + return 0; + } + + public static void main(String[] args) throws Exception { + int ret = ToolRunner.run(new IndexLoadIncrementalHFile(HBaseConfiguration.create()), args); + System.exit(ret); + } +} Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/mapreduce/IndexMapReduceUtil.java =================================================================== --- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/mapreduce/IndexMapReduceUtil.java (revision 0) +++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/mapreduce/IndexMapReduceUtil.java (working copy) @@ -0,0 +1,88 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.mapreduce; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.index.Constants; +import org.apache.hadoop.hbase.index.IndexSpecification; +import org.apache.hadoop.hbase.index.util.IndexUtils; +import org.apache.hadoop.hbase.util.Bytes; + +public class IndexMapReduceUtil { + + public static final String INDEX_DATA_DIR = ".index"; + + public static final String IS_INDEXED_TABLE = "indeximporttsv.isindexedtable"; + + static Log LOG = LogFactory.getLog(IndexMapReduceUtil.class); + + public static HTableDescriptor getTableDescriptor(String tableName, Configuration conf) + throws IOException { + HBaseAdmin admin = null; + try { + admin = new HBaseAdmin(conf); + return admin.getTableDescriptor(TableName.valueOf(tableName)); + } finally { + if (admin != null) { + admin.close(); + } + } + } + + public static boolean isIndexedTable(String tableName, Configuration conf) throws IOException { + return getTableDescriptor(tableName, conf).getValue(Constants.INDEX_SPEC_KEY) != null; + } + + public static List getIndexPut(Put userPut, List indices, + byte[][] startKeys, Configuration conf) throws IOException { + List indexPuts = new ArrayList(); + for (IndexSpecification index : indices) { + byte[] startkey = getStartKey(conf, startKeys, userPut.getRow()); + Put indexPut = IndexUtils.prepareIndexPut(userPut, index, startkey); + if (indexPut != null) { + indexPuts.add(indexPut); + } + } + return indexPuts; + } + + public static byte[] getStartKey(Configuration conf, byte[][] startKeys, byte[] row) + throws IOException { + if (startKeys != null && startKeys.length != 0) { + for (int i = 0; i < (startKeys.length - 1); i++) { + int diff = Bytes.compareTo(row, startKeys[i]); + if (diff == 0 || (diff > 0 && Bytes.compareTo(row, startKeys[i + 1]) < 0)) { + return startKeys[i]; + } + } + return startKeys[startKeys.length - 1]; + } + return null; + } + +} Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/mapreduce/IndexTsvImporterMapper.java =================================================================== --- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/mapreduce/IndexTsvImporterMapper.java (revision 0) +++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/mapreduce/IndexTsvImporterMapper.java (working copy) @@ -0,0 +1,206 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.mapreduce; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.index.Constants; +import org.apache.hadoop.hbase.index.IndexSpecification; +import org.apache.hadoop.hbase.index.TableIndices; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.mapreduce.ImportTsv; +import org.apache.hadoop.hbase.mapreduce.ImportTsv.TsvParser; +import org.apache.hadoop.hbase.mapreduce.TableInputFormat; +import org.apache.hadoop.hbase.util.Base64; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapreduce.Counter; +import org.apache.hadoop.mapreduce.Mapper; + +public class IndexTsvImporterMapper extends Mapper { + + /** Timestamp for all inserted rows */ + private long ts; + + /** Column seperator */ + private String separator; + + /** Should skip bad lines */ + private boolean skipBadLines; + private Counter badLineCount; + + private TsvParser parser; + + private String hfileOutPath; + + private boolean indexedTable; + + private List indices = new ArrayList(); + + private byte[][] startKeys = null; + + public long getTs() { + return ts; + } + + public boolean getSkipBadLines() { + return skipBadLines; + } + + public Counter getBadLineCount() { + return badLineCount; + } + + public void incrementBadLineCount(int count) { + this.badLineCount.increment(count); + } + + /** + * Handles initializing this class with objects specific to it (i.e., the parser). Common + * initialization that might be leveraged by a subsclass is done in doSetup. Hence a + * subclass may choose to override this method and call doSetup as well before + * handling it's own custom params. + * + * @param context + */ + @Override + protected void setup(Context context) throws IOException { + doSetup(context); + + Configuration conf = context.getConfiguration(); + + parser = new TsvParser(conf.get(ImportTsv.COLUMNS_CONF_KEY), separator); + if (parser.getRowKeyColumnIndex() == -1) { + throw new RuntimeException("No row key column specified"); + } + String tableName = conf.get(TableInputFormat.INPUT_TABLE); + HTable hTable = null; + try { + hTable = new HTable(conf, tableName); + this.startKeys = hTable.getStartKeys(); + byte[] indexBytes = hTable.getTableDescriptor().getValue(Constants.INDEX_SPEC_KEY); + if (indexBytes != null) { + TableIndices tableIndices = new TableIndices(); + tableIndices.readFields(indexBytes); + this.indices = tableIndices.getIndices(); + } + } finally { + hTable.close(); + } + } + + /** + * Handles common parameter initialization that a subclass might want to leverage. + * + * @param context + */ + protected void doSetup(Context context) { + Configuration conf = context.getConfiguration(); + + // If a custom separator has been used, + // decode it back from Base64 encoding. + separator = conf.get(ImportTsv.SEPARATOR_CONF_KEY); + if (separator == null) { + separator = ImportTsv.DEFAULT_SEPARATOR; + } else { + separator = new String(Base64.decode(separator)); + } + + // Should never get 0 as we are setting this to a valid value in job configuration. + ts = conf.getLong(ImportTsv.TIMESTAMP_CONF_KEY, 0); + + skipBadLines = context.getConfiguration().getBoolean(ImportTsv.SKIP_LINES_CONF_KEY, true); + badLineCount = context.getCounter("ImportTsv", "Bad Lines"); + hfileOutPath = conf.get(ImportTsv.BULK_OUTPUT_CONF_KEY); + indexedTable = conf.getBoolean(IndexMapReduceUtil.IS_INDEXED_TABLE, false); + } + + /** + * Convert a line of TSV text into an HBase table row. + */ + @Override + public void map(LongWritable offset, Text value, Context context) throws IOException { + byte[] lineBytes = value.getBytes(); + + try { + ImportTsv.TsvParser.ParsedLine parsed = parser.parse(lineBytes, value.getLength()); + ImmutableBytesWritable rowKey = new ImmutableBytesWritable(lineBytes, + parsed.getRowKeyOffset(), parsed.getRowKeyLength()); + // Retrieve timestamp if exists + ts = parsed.getTimestamp(ts); + Configuration conf = context.getConfiguration(); + Put put = new Put(rowKey.copyBytes()); + for (int i = 0; i < parsed.getColumnCount(); i++) { + if (i == parser.getRowKeyColumnIndex() || i == parser.getTimestampKeyColumnIndex()) { + continue; + } + KeyValue kv = new KeyValue(lineBytes, parsed.getRowKeyOffset(), parsed.getRowKeyLength(), + parser.getFamily(i), 0, parser.getFamily(i).length, parser.getQualifier(i), 0, + parser.getQualifier(i).length, ts, KeyValue.Type.Put, lineBytes, + parsed.getColumnOffset(i), parsed.getColumnLength(i)); + put.add(kv); + } + + if (indexedTable && hfileOutPath != null) { + List indexPuts = null; + try { + // Genrate Index entry for the index put + indexPuts = IndexMapReduceUtil.getIndexPut(put, indices, startKeys, conf); + } catch (IOException e) { + if (skipBadLines) { + System.err.println("Bad line at offset: " + offset.get() + ":\n" + e.getMessage()); + incrementBadLineCount(1); + return; + } else { + throw e; + } + } + for (Put usrPut : indexPuts) { + context.write(new ImmutableBytesWritable(usrPut.getRow()), usrPut); + } + } + // Write user table put + context.write(rowKey, put); + + } catch (TsvParser.BadTsvLineException badLine) { + if (skipBadLines) { + System.err.println("Bad line at offset: " + offset.get() + ":\n" + badLine.getMessage()); + incrementBadLineCount(1); + return; + } else { + throw new IOException(badLine); + } + } catch (IllegalArgumentException e) { + if (skipBadLines) { + System.err.println("Bad line at offset: " + offset.get() + ":\n" + e.getMessage()); + incrementBadLineCount(1); + return; + } else { + throw new IOException(e); + } + } catch (InterruptedException e) { + e.printStackTrace(); + } + } +} \ No newline at end of file Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/mapreduce/TableIndexer.java =================================================================== --- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/mapreduce/TableIndexer.java (revision 0) +++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/mapreduce/TableIndexer.java (working copy) @@ -0,0 +1,155 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.mapreduce; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.index.util.IndexUtils; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.mapreduce.HFileOutputFormat; +import org.apache.hadoop.hbase.mapreduce.KeyValueSortReducer; +import org.apache.hadoop.hbase.mapreduce.TableInputFormat; +import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; +import org.apache.hadoop.util.GenericOptionsParser; + +public class TableIndexer { + + public static final String TABLE_NAME_TO_INDEX = "tablename.to.index"; + final static String BULK_OUTPUT_CONF_KEY = "import.bulk.output"; + + private final static int DEFAULT_CACHING = 500; + private final static int DEFAULT_VERSIONS = 1; + + // This can be a comma seperated list + // We can pass like + // IDX1=>cf1:[q1->datatype& + // length],[q2],[q3];cf2:[q1->datatype&length],[q2->datatype&length],[q3->datatype& + // lenght]#IDX2=>cf1:q5,q5 + + private static Map> cfs = new HashMap>(); + + public static void main(String[] args) throws Exception { + Configuration conf = HBaseConfiguration.create(); + String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs(); + if (otherArgs.length < 2) { + System.out.println("Caching and Versions not specified"); + System.exit(-1); + } + int caching = -1; + int versions = -1; + try { + caching = Integer.parseInt(otherArgs[0]); + } catch (NumberFormatException nfe) { + caching = DEFAULT_CACHING; + } + try { + versions = Integer.parseInt(otherArgs[1]); + } catch (NumberFormatException nfe) { + versions = DEFAULT_VERSIONS; + } + + String tableNameToIndex = conf.get(TABLE_NAME_TO_INDEX); + if (tableNameToIndex == null) { + System.out + .println("Wrong usage. Usage is pass the table -Dtablename.to.index=table1 " + + "-Dtable.columns.index='IDX1=>cf1:[q1->datatype& length],[q2]," + + "[q3];cf2:[q1->datatype&length],[q2->datatype&length],[q3->datatype& lenght]#IDX2=>cf1:q5,q5'"); + System.out.println("The format used here is: "); + System.out.println("IDX1 - Index name"); + System.out.println("cf1 - Columnfamilyname"); + System.out.println("q1 - qualifier"); + System.out.println("datatype - datatype (Int, String, Double, Float)"); + System.out.println("length - length of the value"); + System.out.println("The columnfamily should be seperated by ';'"); + System.out + .println("The qualifier and the datatype and its length should be enclosed in '[]'." + + " The qualifier details are specified using '->' following qualifer name and the details are seperated by '&'"); + System.out.println("If the qualifier details are not specified default values are used."); + System.out.println("# is used to seperate between two index details"); + System.out.println("Pass the scanner caching and maxversions as arguments."); + System.exit(-1); + } + IndexUtils.createIndexTable(tableNameToIndex, conf, cfs); + createMapReduceJob(tableNameToIndex, conf, caching, versions); + } + + private static void createMapReduceJob(String tableNameToIndex, Configuration conf, int caching, + int versions) throws IOException, InterruptedException, ClassNotFoundException { + // Set the details to TableInputFormat + Scan s = new Scan(); + s.setCaching(caching); + s.setMaxVersions(versions); + conf.set(TableInputFormat.INPUT_TABLE, tableNameToIndex); + + Set>> entrySet = cfs.entrySet(); + for (Entry> entry : entrySet) { + List quals = entry.getValue(); + addColumn(quals, Bytes.toBytes(entry.getKey()), s); + } + Job job = new Job(conf, "CreateIndex"); + String hfileOutPath = conf.get(BULK_OUTPUT_CONF_KEY); + + TableMapReduceUtil.initTableMapperJob(tableNameToIndex, // input table + s, // Scan instance to control CF and attribute selection + IndexCreationMapper.class, // mapper class + ImmutableBytesWritable.class, // mapper output key + Put.class, // mapper output value + job); + + TableMapReduceUtil.initTableReducerJob(IndexUtils.getIndexTableName(tableNameToIndex), // output + // table + null, // reducer class + job); + + if (hfileOutPath != null) { + HTable table = new HTable(conf, tableNameToIndex); + job.setReducerClass(KeyValueSortReducer.class); + Path outputDir = new Path(hfileOutPath); + FileOutputFormat.setOutputPath(job, outputDir); + HFileOutputFormat.configureIncrementalLoad(job, table); + } else { + job.setNumReduceTasks(0); + } + + TableMapReduceUtil.addDependencyJars(job.getConfiguration(), + com.google.common.base.Preconditions.class); + job.waitForCompletion(true); + assert job.isComplete() == true; + } + + private static void addColumn(List quals, byte[] cf, Scan s) { + for (String q : quals) { + s.addColumn(cf, Bytes.toBytes(q)); + } + } + +} Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/protobuf/generated/ValuePartitionProtos.java =================================================================== --- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/protobuf/generated/ValuePartitionProtos.java (revision 0) +++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/protobuf/generated/ValuePartitionProtos.java (working copy) @@ -0,0 +1,2158 @@ +// Generated by the protocol buffer compiler. DO NOT EDIT! +// source: ValuePartition.proto + +package org.apache.hadoop.hbase.index.protobuf.generated; + +public final class ValuePartitionProtos { + private ValuePartitionProtos() {} + public static void registerAllExtensions( + com.google.protobuf.ExtensionRegistry registry) { + registry.add(org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SpatialPartition.offset); + registry.add(org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SpatialPartition.length); + registry.add(org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SeparatorPartition.separator); + registry.add(org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SeparatorPartition.position); + } + public interface ValuePartitionOrBuilder extends + com.google.protobuf.GeneratedMessage. + ExtendableMessageOrBuilder { + + // optional .ValuePartition.PartitionType partition_type = 1 [default = NONE]; + /** + * optional .ValuePartition.PartitionType partition_type = 1 [default = NONE]; + */ + boolean hasPartitionType(); + /** + * optional .ValuePartition.PartitionType partition_type = 1 [default = NONE]; + */ + org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition.PartitionType getPartitionType(); + } + /** + * Protobuf type {@code ValuePartition} + */ + public static final class ValuePartition extends + com.google.protobuf.GeneratedMessage.ExtendableMessage< + ValuePartition> implements ValuePartitionOrBuilder { + // Use ValuePartition.newBuilder() to construct. + private ValuePartition(com.google.protobuf.GeneratedMessage.ExtendableBuilder builder) { + super(builder); + this.unknownFields = builder.getUnknownFields(); + } + private ValuePartition(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); } + + private static final ValuePartition defaultInstance; + public static ValuePartition getDefaultInstance() { + return defaultInstance; + } + + public ValuePartition getDefaultInstanceForType() { + return defaultInstance; + } + + private final com.google.protobuf.UnknownFieldSet unknownFields; + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet + getUnknownFields() { + return this.unknownFields; + } + private ValuePartition( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + initFields(); + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!parseUnknownField(input, unknownFields, + extensionRegistry, tag)) { + done = true; + } + break; + } + case 8: { + int rawValue = input.readEnum(); + org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition.PartitionType value = org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition.PartitionType.valueOf(rawValue); + if (value == null) { + unknownFields.mergeVarintField(1, rawValue); + } else { + bitField0_ |= 0x00000001; + partitionType_ = value; + } + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException( + e.getMessage()).setUnfinishedMessage(this); + } finally { + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.internal_static_ValuePartition_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.internal_static_ValuePartition_fieldAccessorTable + .ensureFieldAccessorsInitialized( + org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition.class, org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition.Builder.class); + } + + public static com.google.protobuf.Parser PARSER = + new com.google.protobuf.AbstractParser() { + public ValuePartition parsePartialFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new ValuePartition(input, extensionRegistry); + } + }; + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + /** + * Protobuf enum {@code ValuePartition.PartitionType} + */ + public enum PartitionType + implements com.google.protobuf.ProtocolMessageEnum { + /** + * SEPARATOR = 0; + */ + SEPARATOR(0, 0), + /** + * SPATIAL = 1; + */ + SPATIAL(1, 1), + /** + * NONE = 2; + */ + NONE(2, 2), + ; + + /** + * SEPARATOR = 0; + */ + public static final int SEPARATOR_VALUE = 0; + /** + * SPATIAL = 1; + */ + public static final int SPATIAL_VALUE = 1; + /** + * NONE = 2; + */ + public static final int NONE_VALUE = 2; + + + public final int getNumber() { return value; } + + public static PartitionType valueOf(int value) { + switch (value) { + case 0: return SEPARATOR; + case 1: return SPATIAL; + case 2: return NONE; + default: return null; + } + } + + public static com.google.protobuf.Internal.EnumLiteMap + internalGetValueMap() { + return internalValueMap; + } + private static com.google.protobuf.Internal.EnumLiteMap + internalValueMap = + new com.google.protobuf.Internal.EnumLiteMap() { + public PartitionType findValueByNumber(int number) { + return PartitionType.valueOf(number); + } + }; + + public final com.google.protobuf.Descriptors.EnumValueDescriptor + getValueDescriptor() { + return getDescriptor().getValues().get(index); + } + public final com.google.protobuf.Descriptors.EnumDescriptor + getDescriptorForType() { + return getDescriptor(); + } + public static final com.google.protobuf.Descriptors.EnumDescriptor + getDescriptor() { + return org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition.getDescriptor().getEnumTypes().get(0); + } + + private static final PartitionType[] VALUES = values(); + + public static PartitionType valueOf( + com.google.protobuf.Descriptors.EnumValueDescriptor desc) { + if (desc.getType() != getDescriptor()) { + throw new java.lang.IllegalArgumentException( + "EnumValueDescriptor is not for this type."); + } + return VALUES[desc.getIndex()]; + } + + private final int index; + private final int value; + + private PartitionType(int index, int value) { + this.index = index; + this.value = value; + } + + // @@protoc_insertion_point(enum_scope:ValuePartition.PartitionType) + } + + private int bitField0_; + // optional .ValuePartition.PartitionType partition_type = 1 [default = NONE]; + public static final int PARTITION_TYPE_FIELD_NUMBER = 1; + private org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition.PartitionType partitionType_; + /** + * optional .ValuePartition.PartitionType partition_type = 1 [default = NONE]; + */ + public boolean hasPartitionType() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + /** + * optional .ValuePartition.PartitionType partition_type = 1 [default = NONE]; + */ + public org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition.PartitionType getPartitionType() { + return partitionType_; + } + + private void initFields() { + partitionType_ = org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition.PartitionType.NONE; + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + if (!extensionsAreInitialized()) { + memoizedIsInitialized = 0; + return false; + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + com.google.protobuf.GeneratedMessage + .ExtendableMessage.ExtensionWriter extensionWriter = + newExtensionWriter(); + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeEnum(1, partitionType_.getNumber()); + } + extensionWriter.writeUntil(200, output); + getUnknownFields().writeTo(output); + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream + .computeEnumSize(1, partitionType_.getNumber()); + } + size += extensionsSerializedSize(); + size += getUnknownFields().getSerializedSize(); + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + @java.lang.Override + public boolean equals(final java.lang.Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition)) { + return super.equals(obj); + } + org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition other = (org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition) obj; + + boolean result = true; + result = result && (hasPartitionType() == other.hasPartitionType()); + if (hasPartitionType()) { + result = result && + (getPartitionType() == other.getPartitionType()); + } + result = result && + getUnknownFields().equals(other.getUnknownFields()); + result = result && + getExtensionFields().equals(other.getExtensionFields()); + return result; + } + + private int memoizedHashCode = 0; + @java.lang.Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptorForType().hashCode(); + if (hasPartitionType()) { + hash = (37 * hash) + PARTITION_TYPE_FIELD_NUMBER; + hash = (53 * hash) + hashEnum(getPartitionType()); + } + hash = hashFields(hash, getExtensionFields()); + hash = (29 * hash) + getUnknownFields().hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition parseFrom(java.io.InputStream input) + throws java.io.IOException { + return PARSER.parseFrom(input); + } + public static org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return PARSER.parseFrom(input, extensionRegistry); + } + public static org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return PARSER.parseDelimitedFrom(input); + } + public static org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return PARSER.parseDelimitedFrom(input, extensionRegistry); + } + public static org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return PARSER.parseFrom(input); + } + public static org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return PARSER.parseFrom(input, extensionRegistry); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + /** + * Protobuf type {@code ValuePartition} + */ + public static final class Builder extends + com.google.protobuf.GeneratedMessage.ExtendableBuilder< + org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition, Builder> implements org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartitionOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.internal_static_ValuePartition_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.internal_static_ValuePartition_fieldAccessorTable + .ensureFieldAccessorsInitialized( + org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition.class, org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition.Builder.class); + } + + // Construct using org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { + } + } + private static Builder create() { + return new Builder(); + } + + public Builder clear() { + super.clear(); + partitionType_ = org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition.PartitionType.NONE; + bitField0_ = (bitField0_ & ~0x00000001); + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.internal_static_ValuePartition_descriptor; + } + + public org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition getDefaultInstanceForType() { + return org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition.getDefaultInstance(); + } + + public org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition build() { + org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition buildPartial() { + org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition result = new org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition(this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.partitionType_ = partitionType_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition) { + return mergeFrom((org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition other) { + if (other == org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition.getDefaultInstance()) return this; + if (other.hasPartitionType()) { + setPartitionType(other.getPartitionType()); + } + this.mergeExtensionFields(other); + this.mergeUnknownFields(other.getUnknownFields()); + return this; + } + + public final boolean isInitialized() { + if (!extensionsAreInitialized()) { + + return false; + } + return true; + } + + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition) e.getUnfinishedMessage(); + throw e; + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + private int bitField0_; + + // optional .ValuePartition.PartitionType partition_type = 1 [default = NONE]; + private org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition.PartitionType partitionType_ = org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition.PartitionType.NONE; + /** + * optional .ValuePartition.PartitionType partition_type = 1 [default = NONE]; + */ + public boolean hasPartitionType() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + /** + * optional .ValuePartition.PartitionType partition_type = 1 [default = NONE]; + */ + public org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition.PartitionType getPartitionType() { + return partitionType_; + } + /** + * optional .ValuePartition.PartitionType partition_type = 1 [default = NONE]; + */ + public Builder setPartitionType(org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition.PartitionType value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + partitionType_ = value; + onChanged(); + return this; + } + /** + * optional .ValuePartition.PartitionType partition_type = 1 [default = NONE]; + */ + public Builder clearPartitionType() { + bitField0_ = (bitField0_ & ~0x00000001); + partitionType_ = org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition.PartitionType.NONE; + onChanged(); + return this; + } + + // @@protoc_insertion_point(builder_scope:ValuePartition) + } + + static { + defaultInstance = new ValuePartition(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:ValuePartition) + } + + public interface SpatialPartitionOrBuilder + extends com.google.protobuf.MessageOrBuilder { + } + /** + * Protobuf type {@code SpatialPartition} + */ + public static final class SpatialPartition extends + com.google.protobuf.GeneratedMessage + implements SpatialPartitionOrBuilder { + // Use SpatialPartition.newBuilder() to construct. + private SpatialPartition(com.google.protobuf.GeneratedMessage.Builder builder) { + super(builder); + this.unknownFields = builder.getUnknownFields(); + } + private SpatialPartition(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); } + + private static final SpatialPartition defaultInstance; + public static SpatialPartition getDefaultInstance() { + return defaultInstance; + } + + public SpatialPartition getDefaultInstanceForType() { + return defaultInstance; + } + + private final com.google.protobuf.UnknownFieldSet unknownFields; + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet + getUnknownFields() { + return this.unknownFields; + } + private SpatialPartition( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + initFields(); + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!parseUnknownField(input, unknownFields, + extensionRegistry, tag)) { + done = true; + } + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException( + e.getMessage()).setUnfinishedMessage(this); + } finally { + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.internal_static_SpatialPartition_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.internal_static_SpatialPartition_fieldAccessorTable + .ensureFieldAccessorsInitialized( + org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SpatialPartition.class, org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SpatialPartition.Builder.class); + } + + public static com.google.protobuf.Parser PARSER = + new com.google.protobuf.AbstractParser() { + public SpatialPartition parsePartialFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new SpatialPartition(input, extensionRegistry); + } + }; + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + private void initFields() { + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + getUnknownFields().writeTo(output); + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + size += getUnknownFields().getSerializedSize(); + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + @java.lang.Override + public boolean equals(final java.lang.Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SpatialPartition)) { + return super.equals(obj); + } + org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SpatialPartition other = (org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SpatialPartition) obj; + + boolean result = true; + result = result && + getUnknownFields().equals(other.getUnknownFields()); + return result; + } + + private int memoizedHashCode = 0; + @java.lang.Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptorForType().hashCode(); + hash = (29 * hash) + getUnknownFields().hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SpatialPartition parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SpatialPartition parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SpatialPartition parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SpatialPartition parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SpatialPartition parseFrom(java.io.InputStream input) + throws java.io.IOException { + return PARSER.parseFrom(input); + } + public static org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SpatialPartition parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return PARSER.parseFrom(input, extensionRegistry); + } + public static org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SpatialPartition parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return PARSER.parseDelimitedFrom(input); + } + public static org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SpatialPartition parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return PARSER.parseDelimitedFrom(input, extensionRegistry); + } + public static org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SpatialPartition parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return PARSER.parseFrom(input); + } + public static org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SpatialPartition parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return PARSER.parseFrom(input, extensionRegistry); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SpatialPartition prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + /** + * Protobuf type {@code SpatialPartition} + */ + public static final class Builder extends + com.google.protobuf.GeneratedMessage.Builder + implements org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SpatialPartitionOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.internal_static_SpatialPartition_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.internal_static_SpatialPartition_fieldAccessorTable + .ensureFieldAccessorsInitialized( + org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SpatialPartition.class, org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SpatialPartition.Builder.class); + } + + // Construct using org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SpatialPartition.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { + } + } + private static Builder create() { + return new Builder(); + } + + public Builder clear() { + super.clear(); + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.internal_static_SpatialPartition_descriptor; + } + + public org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SpatialPartition getDefaultInstanceForType() { + return org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SpatialPartition.getDefaultInstance(); + } + + public org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SpatialPartition build() { + org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SpatialPartition result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SpatialPartition buildPartial() { + org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SpatialPartition result = new org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SpatialPartition(this); + onBuilt(); + return result; + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SpatialPartition) { + return mergeFrom((org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SpatialPartition)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SpatialPartition other) { + if (other == org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SpatialPartition.getDefaultInstance()) return this; + this.mergeUnknownFields(other.getUnknownFields()); + return this; + } + + public final boolean isInitialized() { + return true; + } + + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SpatialPartition parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SpatialPartition) e.getUnfinishedMessage(); + throw e; + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + + // @@protoc_insertion_point(builder_scope:SpatialPartition) + } + + static { + defaultInstance = new SpatialPartition(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:SpatialPartition) + public static final int OFFSET_FIELD_NUMBER = 100; + /** + * extend .ValuePartition { ... } + */ + public static final + com.google.protobuf.GeneratedMessage.GeneratedExtension< + org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition, + java.lang.Integer> offset = com.google.protobuf.GeneratedMessage + .newMessageScopedGeneratedExtension( + org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SpatialPartition.getDefaultInstance(), + 0, + java.lang.Integer.class, + null); + public static final int LENGTH_FIELD_NUMBER = 101; + /** + * extend .ValuePartition { ... } + */ + public static final + com.google.protobuf.GeneratedMessage.GeneratedExtension< + org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition, + java.lang.Integer> length = com.google.protobuf.GeneratedMessage + .newMessageScopedGeneratedExtension( + org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SpatialPartition.getDefaultInstance(), + 1, + java.lang.Integer.class, + null); + } + + public interface SeparatorPartitionOrBuilder + extends com.google.protobuf.MessageOrBuilder { + } + /** + * Protobuf type {@code SeparatorPartition} + */ + public static final class SeparatorPartition extends + com.google.protobuf.GeneratedMessage + implements SeparatorPartitionOrBuilder { + // Use SeparatorPartition.newBuilder() to construct. + private SeparatorPartition(com.google.protobuf.GeneratedMessage.Builder builder) { + super(builder); + this.unknownFields = builder.getUnknownFields(); + } + private SeparatorPartition(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); } + + private static final SeparatorPartition defaultInstance; + public static SeparatorPartition getDefaultInstance() { + return defaultInstance; + } + + public SeparatorPartition getDefaultInstanceForType() { + return defaultInstance; + } + + private final com.google.protobuf.UnknownFieldSet unknownFields; + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet + getUnknownFields() { + return this.unknownFields; + } + private SeparatorPartition( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + initFields(); + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!parseUnknownField(input, unknownFields, + extensionRegistry, tag)) { + done = true; + } + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException( + e.getMessage()).setUnfinishedMessage(this); + } finally { + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.internal_static_SeparatorPartition_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.internal_static_SeparatorPartition_fieldAccessorTable + .ensureFieldAccessorsInitialized( + org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SeparatorPartition.class, org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SeparatorPartition.Builder.class); + } + + public static com.google.protobuf.Parser PARSER = + new com.google.protobuf.AbstractParser() { + public SeparatorPartition parsePartialFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new SeparatorPartition(input, extensionRegistry); + } + }; + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + private void initFields() { + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + getUnknownFields().writeTo(output); + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + size += getUnknownFields().getSerializedSize(); + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + @java.lang.Override + public boolean equals(final java.lang.Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SeparatorPartition)) { + return super.equals(obj); + } + org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SeparatorPartition other = (org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SeparatorPartition) obj; + + boolean result = true; + result = result && + getUnknownFields().equals(other.getUnknownFields()); + return result; + } + + private int memoizedHashCode = 0; + @java.lang.Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptorForType().hashCode(); + hash = (29 * hash) + getUnknownFields().hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SeparatorPartition parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SeparatorPartition parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SeparatorPartition parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SeparatorPartition parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SeparatorPartition parseFrom(java.io.InputStream input) + throws java.io.IOException { + return PARSER.parseFrom(input); + } + public static org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SeparatorPartition parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return PARSER.parseFrom(input, extensionRegistry); + } + public static org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SeparatorPartition parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return PARSER.parseDelimitedFrom(input); + } + public static org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SeparatorPartition parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return PARSER.parseDelimitedFrom(input, extensionRegistry); + } + public static org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SeparatorPartition parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return PARSER.parseFrom(input); + } + public static org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SeparatorPartition parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return PARSER.parseFrom(input, extensionRegistry); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SeparatorPartition prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + /** + * Protobuf type {@code SeparatorPartition} + */ + public static final class Builder extends + com.google.protobuf.GeneratedMessage.Builder + implements org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SeparatorPartitionOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.internal_static_SeparatorPartition_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.internal_static_SeparatorPartition_fieldAccessorTable + .ensureFieldAccessorsInitialized( + org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SeparatorPartition.class, org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SeparatorPartition.Builder.class); + } + + // Construct using org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SeparatorPartition.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { + } + } + private static Builder create() { + return new Builder(); + } + + public Builder clear() { + super.clear(); + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.internal_static_SeparatorPartition_descriptor; + } + + public org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SeparatorPartition getDefaultInstanceForType() { + return org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SeparatorPartition.getDefaultInstance(); + } + + public org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SeparatorPartition build() { + org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SeparatorPartition result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SeparatorPartition buildPartial() { + org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SeparatorPartition result = new org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SeparatorPartition(this); + onBuilt(); + return result; + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SeparatorPartition) { + return mergeFrom((org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SeparatorPartition)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SeparatorPartition other) { + if (other == org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SeparatorPartition.getDefaultInstance()) return this; + this.mergeUnknownFields(other.getUnknownFields()); + return this; + } + + public final boolean isInitialized() { + return true; + } + + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SeparatorPartition parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SeparatorPartition) e.getUnfinishedMessage(); + throw e; + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + + // @@protoc_insertion_point(builder_scope:SeparatorPartition) + } + + static { + defaultInstance = new SeparatorPartition(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:SeparatorPartition) + public static final int SEPARATOR_FIELD_NUMBER = 102; + /** + * extend .ValuePartition { ... } + */ + public static final + com.google.protobuf.GeneratedMessage.GeneratedExtension< + org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition, + com.google.protobuf.ByteString> separator = com.google.protobuf.GeneratedMessage + .newMessageScopedGeneratedExtension( + org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SeparatorPartition.getDefaultInstance(), + 0, + com.google.protobuf.ByteString.class, + null); + public static final int POSITION_FIELD_NUMBER = 103; + /** + * extend .ValuePartition { ... } + */ + public static final + com.google.protobuf.GeneratedMessage.GeneratedExtension< + org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition, + java.lang.Integer> position = com.google.protobuf.GeneratedMessage + .newMessageScopedGeneratedExtension( + org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SeparatorPartition.getDefaultInstance(), + 1, + java.lang.Integer.class, + null); + } + + public interface SingleColumnValuePartitionFilterOrBuilder + extends com.google.protobuf.MessageOrBuilder { + + // required .SingleColumnValueFilter single_column_value_filter = 1; + /** + * required .SingleColumnValueFilter single_column_value_filter = 1; + */ + boolean hasSingleColumnValueFilter(); + /** + * required .SingleColumnValueFilter single_column_value_filter = 1; + */ + org.apache.hadoop.hbase.protobuf.generated.FilterProtos.SingleColumnValueFilter getSingleColumnValueFilter(); + /** + * required .SingleColumnValueFilter single_column_value_filter = 1; + */ + org.apache.hadoop.hbase.protobuf.generated.FilterProtos.SingleColumnValueFilterOrBuilder getSingleColumnValueFilterOrBuilder(); + + // required .ValuePartition value_partition = 2; + /** + * required .ValuePartition value_partition = 2; + */ + boolean hasValuePartition(); + /** + * required .ValuePartition value_partition = 2; + */ + org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition getValuePartition(); + /** + * required .ValuePartition value_partition = 2; + */ + org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartitionOrBuilder getValuePartitionOrBuilder(); + } + /** + * Protobuf type {@code SingleColumnValuePartitionFilter} + */ + public static final class SingleColumnValuePartitionFilter extends + com.google.protobuf.GeneratedMessage + implements SingleColumnValuePartitionFilterOrBuilder { + // Use SingleColumnValuePartitionFilter.newBuilder() to construct. + private SingleColumnValuePartitionFilter(com.google.protobuf.GeneratedMessage.Builder builder) { + super(builder); + this.unknownFields = builder.getUnknownFields(); + } + private SingleColumnValuePartitionFilter(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); } + + private static final SingleColumnValuePartitionFilter defaultInstance; + public static SingleColumnValuePartitionFilter getDefaultInstance() { + return defaultInstance; + } + + public SingleColumnValuePartitionFilter getDefaultInstanceForType() { + return defaultInstance; + } + + private final com.google.protobuf.UnknownFieldSet unknownFields; + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet + getUnknownFields() { + return this.unknownFields; + } + private SingleColumnValuePartitionFilter( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + initFields(); + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!parseUnknownField(input, unknownFields, + extensionRegistry, tag)) { + done = true; + } + break; + } + case 10: { + org.apache.hadoop.hbase.protobuf.generated.FilterProtos.SingleColumnValueFilter.Builder subBuilder = null; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + subBuilder = singleColumnValueFilter_.toBuilder(); + } + singleColumnValueFilter_ = input.readMessage(org.apache.hadoop.hbase.protobuf.generated.FilterProtos.SingleColumnValueFilter.PARSER, extensionRegistry); + if (subBuilder != null) { + subBuilder.mergeFrom(singleColumnValueFilter_); + singleColumnValueFilter_ = subBuilder.buildPartial(); + } + bitField0_ |= 0x00000001; + break; + } + case 18: { + org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition.Builder subBuilder = null; + if (((bitField0_ & 0x00000002) == 0x00000002)) { + subBuilder = valuePartition_.toBuilder(); + } + valuePartition_ = input.readMessage(org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition.PARSER, extensionRegistry); + if (subBuilder != null) { + subBuilder.mergeFrom(valuePartition_); + valuePartition_ = subBuilder.buildPartial(); + } + bitField0_ |= 0x00000002; + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException( + e.getMessage()).setUnfinishedMessage(this); + } finally { + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.internal_static_SingleColumnValuePartitionFilter_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.internal_static_SingleColumnValuePartitionFilter_fieldAccessorTable + .ensureFieldAccessorsInitialized( + org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SingleColumnValuePartitionFilter.class, org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SingleColumnValuePartitionFilter.Builder.class); + } + + public static com.google.protobuf.Parser PARSER = + new com.google.protobuf.AbstractParser() { + public SingleColumnValuePartitionFilter parsePartialFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new SingleColumnValuePartitionFilter(input, extensionRegistry); + } + }; + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + private int bitField0_; + // required .SingleColumnValueFilter single_column_value_filter = 1; + public static final int SINGLE_COLUMN_VALUE_FILTER_FIELD_NUMBER = 1; + private org.apache.hadoop.hbase.protobuf.generated.FilterProtos.SingleColumnValueFilter singleColumnValueFilter_; + /** + * required .SingleColumnValueFilter single_column_value_filter = 1; + */ + public boolean hasSingleColumnValueFilter() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + /** + * required .SingleColumnValueFilter single_column_value_filter = 1; + */ + public org.apache.hadoop.hbase.protobuf.generated.FilterProtos.SingleColumnValueFilter getSingleColumnValueFilter() { + return singleColumnValueFilter_; + } + /** + * required .SingleColumnValueFilter single_column_value_filter = 1; + */ + public org.apache.hadoop.hbase.protobuf.generated.FilterProtos.SingleColumnValueFilterOrBuilder getSingleColumnValueFilterOrBuilder() { + return singleColumnValueFilter_; + } + + // required .ValuePartition value_partition = 2; + public static final int VALUE_PARTITION_FIELD_NUMBER = 2; + private org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition valuePartition_; + /** + * required .ValuePartition value_partition = 2; + */ + public boolean hasValuePartition() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + /** + * required .ValuePartition value_partition = 2; + */ + public org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition getValuePartition() { + return valuePartition_; + } + /** + * required .ValuePartition value_partition = 2; + */ + public org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartitionOrBuilder getValuePartitionOrBuilder() { + return valuePartition_; + } + + private void initFields() { + singleColumnValueFilter_ = org.apache.hadoop.hbase.protobuf.generated.FilterProtos.SingleColumnValueFilter.getDefaultInstance(); + valuePartition_ = org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition.getDefaultInstance(); + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + if (!hasSingleColumnValueFilter()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasValuePartition()) { + memoizedIsInitialized = 0; + return false; + } + if (!getSingleColumnValueFilter().isInitialized()) { + memoizedIsInitialized = 0; + return false; + } + if (!getValuePartition().isInitialized()) { + memoizedIsInitialized = 0; + return false; + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeMessage(1, singleColumnValueFilter_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + output.writeMessage(2, valuePartition_); + } + getUnknownFields().writeTo(output); + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream + .computeMessageSize(1, singleColumnValueFilter_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.CodedOutputStream + .computeMessageSize(2, valuePartition_); + } + size += getUnknownFields().getSerializedSize(); + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + @java.lang.Override + public boolean equals(final java.lang.Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SingleColumnValuePartitionFilter)) { + return super.equals(obj); + } + org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SingleColumnValuePartitionFilter other = (org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SingleColumnValuePartitionFilter) obj; + + boolean result = true; + result = result && (hasSingleColumnValueFilter() == other.hasSingleColumnValueFilter()); + if (hasSingleColumnValueFilter()) { + result = result && getSingleColumnValueFilter() + .equals(other.getSingleColumnValueFilter()); + } + result = result && (hasValuePartition() == other.hasValuePartition()); + if (hasValuePartition()) { + result = result && getValuePartition() + .equals(other.getValuePartition()); + } + result = result && + getUnknownFields().equals(other.getUnknownFields()); + return result; + } + + private int memoizedHashCode = 0; + @java.lang.Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptorForType().hashCode(); + if (hasSingleColumnValueFilter()) { + hash = (37 * hash) + SINGLE_COLUMN_VALUE_FILTER_FIELD_NUMBER; + hash = (53 * hash) + getSingleColumnValueFilter().hashCode(); + } + if (hasValuePartition()) { + hash = (37 * hash) + VALUE_PARTITION_FIELD_NUMBER; + hash = (53 * hash) + getValuePartition().hashCode(); + } + hash = (29 * hash) + getUnknownFields().hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SingleColumnValuePartitionFilter parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SingleColumnValuePartitionFilter parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SingleColumnValuePartitionFilter parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SingleColumnValuePartitionFilter parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SingleColumnValuePartitionFilter parseFrom(java.io.InputStream input) + throws java.io.IOException { + return PARSER.parseFrom(input); + } + public static org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SingleColumnValuePartitionFilter parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return PARSER.parseFrom(input, extensionRegistry); + } + public static org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SingleColumnValuePartitionFilter parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return PARSER.parseDelimitedFrom(input); + } + public static org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SingleColumnValuePartitionFilter parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return PARSER.parseDelimitedFrom(input, extensionRegistry); + } + public static org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SingleColumnValuePartitionFilter parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return PARSER.parseFrom(input); + } + public static org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SingleColumnValuePartitionFilter parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return PARSER.parseFrom(input, extensionRegistry); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SingleColumnValuePartitionFilter prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + /** + * Protobuf type {@code SingleColumnValuePartitionFilter} + */ + public static final class Builder extends + com.google.protobuf.GeneratedMessage.Builder + implements org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SingleColumnValuePartitionFilterOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.internal_static_SingleColumnValuePartitionFilter_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.internal_static_SingleColumnValuePartitionFilter_fieldAccessorTable + .ensureFieldAccessorsInitialized( + org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SingleColumnValuePartitionFilter.class, org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SingleColumnValuePartitionFilter.Builder.class); + } + + // Construct using org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SingleColumnValuePartitionFilter.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { + getSingleColumnValueFilterFieldBuilder(); + getValuePartitionFieldBuilder(); + } + } + private static Builder create() { + return new Builder(); + } + + public Builder clear() { + super.clear(); + if (singleColumnValueFilterBuilder_ == null) { + singleColumnValueFilter_ = org.apache.hadoop.hbase.protobuf.generated.FilterProtos.SingleColumnValueFilter.getDefaultInstance(); + } else { + singleColumnValueFilterBuilder_.clear(); + } + bitField0_ = (bitField0_ & ~0x00000001); + if (valuePartitionBuilder_ == null) { + valuePartition_ = org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition.getDefaultInstance(); + } else { + valuePartitionBuilder_.clear(); + } + bitField0_ = (bitField0_ & ~0x00000002); + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.internal_static_SingleColumnValuePartitionFilter_descriptor; + } + + public org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SingleColumnValuePartitionFilter getDefaultInstanceForType() { + return org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SingleColumnValuePartitionFilter.getDefaultInstance(); + } + + public org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SingleColumnValuePartitionFilter build() { + org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SingleColumnValuePartitionFilter result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SingleColumnValuePartitionFilter buildPartial() { + org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SingleColumnValuePartitionFilter result = new org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SingleColumnValuePartitionFilter(this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + if (singleColumnValueFilterBuilder_ == null) { + result.singleColumnValueFilter_ = singleColumnValueFilter_; + } else { + result.singleColumnValueFilter_ = singleColumnValueFilterBuilder_.build(); + } + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + if (valuePartitionBuilder_ == null) { + result.valuePartition_ = valuePartition_; + } else { + result.valuePartition_ = valuePartitionBuilder_.build(); + } + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SingleColumnValuePartitionFilter) { + return mergeFrom((org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SingleColumnValuePartitionFilter)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SingleColumnValuePartitionFilter other) { + if (other == org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SingleColumnValuePartitionFilter.getDefaultInstance()) return this; + if (other.hasSingleColumnValueFilter()) { + mergeSingleColumnValueFilter(other.getSingleColumnValueFilter()); + } + if (other.hasValuePartition()) { + mergeValuePartition(other.getValuePartition()); + } + this.mergeUnknownFields(other.getUnknownFields()); + return this; + } + + public final boolean isInitialized() { + if (!hasSingleColumnValueFilter()) { + + return false; + } + if (!hasValuePartition()) { + + return false; + } + if (!getSingleColumnValueFilter().isInitialized()) { + + return false; + } + if (!getValuePartition().isInitialized()) { + + return false; + } + return true; + } + + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SingleColumnValuePartitionFilter parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.SingleColumnValuePartitionFilter) e.getUnfinishedMessage(); + throw e; + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + private int bitField0_; + + // required .SingleColumnValueFilter single_column_value_filter = 1; + private org.apache.hadoop.hbase.protobuf.generated.FilterProtos.SingleColumnValueFilter singleColumnValueFilter_ = org.apache.hadoop.hbase.protobuf.generated.FilterProtos.SingleColumnValueFilter.getDefaultInstance(); + private com.google.protobuf.SingleFieldBuilder< + org.apache.hadoop.hbase.protobuf.generated.FilterProtos.SingleColumnValueFilter, org.apache.hadoop.hbase.protobuf.generated.FilterProtos.SingleColumnValueFilter.Builder, org.apache.hadoop.hbase.protobuf.generated.FilterProtos.SingleColumnValueFilterOrBuilder> singleColumnValueFilterBuilder_; + /** + * required .SingleColumnValueFilter single_column_value_filter = 1; + */ + public boolean hasSingleColumnValueFilter() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + /** + * required .SingleColumnValueFilter single_column_value_filter = 1; + */ + public org.apache.hadoop.hbase.protobuf.generated.FilterProtos.SingleColumnValueFilter getSingleColumnValueFilter() { + if (singleColumnValueFilterBuilder_ == null) { + return singleColumnValueFilter_; + } else { + return singleColumnValueFilterBuilder_.getMessage(); + } + } + /** + * required .SingleColumnValueFilter single_column_value_filter = 1; + */ + public Builder setSingleColumnValueFilter(org.apache.hadoop.hbase.protobuf.generated.FilterProtos.SingleColumnValueFilter value) { + if (singleColumnValueFilterBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + singleColumnValueFilter_ = value; + onChanged(); + } else { + singleColumnValueFilterBuilder_.setMessage(value); + } + bitField0_ |= 0x00000001; + return this; + } + /** + * required .SingleColumnValueFilter single_column_value_filter = 1; + */ + public Builder setSingleColumnValueFilter( + org.apache.hadoop.hbase.protobuf.generated.FilterProtos.SingleColumnValueFilter.Builder builderForValue) { + if (singleColumnValueFilterBuilder_ == null) { + singleColumnValueFilter_ = builderForValue.build(); + onChanged(); + } else { + singleColumnValueFilterBuilder_.setMessage(builderForValue.build()); + } + bitField0_ |= 0x00000001; + return this; + } + /** + * required .SingleColumnValueFilter single_column_value_filter = 1; + */ + public Builder mergeSingleColumnValueFilter(org.apache.hadoop.hbase.protobuf.generated.FilterProtos.SingleColumnValueFilter value) { + if (singleColumnValueFilterBuilder_ == null) { + if (((bitField0_ & 0x00000001) == 0x00000001) && + singleColumnValueFilter_ != org.apache.hadoop.hbase.protobuf.generated.FilterProtos.SingleColumnValueFilter.getDefaultInstance()) { + singleColumnValueFilter_ = + org.apache.hadoop.hbase.protobuf.generated.FilterProtos.SingleColumnValueFilter.newBuilder(singleColumnValueFilter_).mergeFrom(value).buildPartial(); + } else { + singleColumnValueFilter_ = value; + } + onChanged(); + } else { + singleColumnValueFilterBuilder_.mergeFrom(value); + } + bitField0_ |= 0x00000001; + return this; + } + /** + * required .SingleColumnValueFilter single_column_value_filter = 1; + */ + public Builder clearSingleColumnValueFilter() { + if (singleColumnValueFilterBuilder_ == null) { + singleColumnValueFilter_ = org.apache.hadoop.hbase.protobuf.generated.FilterProtos.SingleColumnValueFilter.getDefaultInstance(); + onChanged(); + } else { + singleColumnValueFilterBuilder_.clear(); + } + bitField0_ = (bitField0_ & ~0x00000001); + return this; + } + /** + * required .SingleColumnValueFilter single_column_value_filter = 1; + */ + public org.apache.hadoop.hbase.protobuf.generated.FilterProtos.SingleColumnValueFilter.Builder getSingleColumnValueFilterBuilder() { + bitField0_ |= 0x00000001; + onChanged(); + return getSingleColumnValueFilterFieldBuilder().getBuilder(); + } + /** + * required .SingleColumnValueFilter single_column_value_filter = 1; + */ + public org.apache.hadoop.hbase.protobuf.generated.FilterProtos.SingleColumnValueFilterOrBuilder getSingleColumnValueFilterOrBuilder() { + if (singleColumnValueFilterBuilder_ != null) { + return singleColumnValueFilterBuilder_.getMessageOrBuilder(); + } else { + return singleColumnValueFilter_; + } + } + /** + * required .SingleColumnValueFilter single_column_value_filter = 1; + */ + private com.google.protobuf.SingleFieldBuilder< + org.apache.hadoop.hbase.protobuf.generated.FilterProtos.SingleColumnValueFilter, org.apache.hadoop.hbase.protobuf.generated.FilterProtos.SingleColumnValueFilter.Builder, org.apache.hadoop.hbase.protobuf.generated.FilterProtos.SingleColumnValueFilterOrBuilder> + getSingleColumnValueFilterFieldBuilder() { + if (singleColumnValueFilterBuilder_ == null) { + singleColumnValueFilterBuilder_ = new com.google.protobuf.SingleFieldBuilder< + org.apache.hadoop.hbase.protobuf.generated.FilterProtos.SingleColumnValueFilter, org.apache.hadoop.hbase.protobuf.generated.FilterProtos.SingleColumnValueFilter.Builder, org.apache.hadoop.hbase.protobuf.generated.FilterProtos.SingleColumnValueFilterOrBuilder>( + singleColumnValueFilter_, + getParentForChildren(), + isClean()); + singleColumnValueFilter_ = null; + } + return singleColumnValueFilterBuilder_; + } + + // required .ValuePartition value_partition = 2; + private org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition valuePartition_ = org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition.getDefaultInstance(); + private com.google.protobuf.SingleFieldBuilder< + org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition, org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition.Builder, org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartitionOrBuilder> valuePartitionBuilder_; + /** + * required .ValuePartition value_partition = 2; + */ + public boolean hasValuePartition() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + /** + * required .ValuePartition value_partition = 2; + */ + public org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition getValuePartition() { + if (valuePartitionBuilder_ == null) { + return valuePartition_; + } else { + return valuePartitionBuilder_.getMessage(); + } + } + /** + * required .ValuePartition value_partition = 2; + */ + public Builder setValuePartition(org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition value) { + if (valuePartitionBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + valuePartition_ = value; + onChanged(); + } else { + valuePartitionBuilder_.setMessage(value); + } + bitField0_ |= 0x00000002; + return this; + } + /** + * required .ValuePartition value_partition = 2; + */ + public Builder setValuePartition( + org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition.Builder builderForValue) { + if (valuePartitionBuilder_ == null) { + valuePartition_ = builderForValue.build(); + onChanged(); + } else { + valuePartitionBuilder_.setMessage(builderForValue.build()); + } + bitField0_ |= 0x00000002; + return this; + } + /** + * required .ValuePartition value_partition = 2; + */ + public Builder mergeValuePartition(org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition value) { + if (valuePartitionBuilder_ == null) { + if (((bitField0_ & 0x00000002) == 0x00000002) && + valuePartition_ != org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition.getDefaultInstance()) { + valuePartition_ = + org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition.newBuilder(valuePartition_).mergeFrom(value).buildPartial(); + } else { + valuePartition_ = value; + } + onChanged(); + } else { + valuePartitionBuilder_.mergeFrom(value); + } + bitField0_ |= 0x00000002; + return this; + } + /** + * required .ValuePartition value_partition = 2; + */ + public Builder clearValuePartition() { + if (valuePartitionBuilder_ == null) { + valuePartition_ = org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition.getDefaultInstance(); + onChanged(); + } else { + valuePartitionBuilder_.clear(); + } + bitField0_ = (bitField0_ & ~0x00000002); + return this; + } + /** + * required .ValuePartition value_partition = 2; + */ + public org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition.Builder getValuePartitionBuilder() { + bitField0_ |= 0x00000002; + onChanged(); + return getValuePartitionFieldBuilder().getBuilder(); + } + /** + * required .ValuePartition value_partition = 2; + */ + public org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartitionOrBuilder getValuePartitionOrBuilder() { + if (valuePartitionBuilder_ != null) { + return valuePartitionBuilder_.getMessageOrBuilder(); + } else { + return valuePartition_; + } + } + /** + * required .ValuePartition value_partition = 2; + */ + private com.google.protobuf.SingleFieldBuilder< + org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition, org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition.Builder, org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartitionOrBuilder> + getValuePartitionFieldBuilder() { + if (valuePartitionBuilder_ == null) { + valuePartitionBuilder_ = new com.google.protobuf.SingleFieldBuilder< + org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition, org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition.Builder, org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartitionOrBuilder>( + valuePartition_, + getParentForChildren(), + isClean()); + valuePartition_ = null; + } + return valuePartitionBuilder_; + } + + // @@protoc_insertion_point(builder_scope:SingleColumnValuePartitionFilter) + } + + static { + defaultInstance = new SingleColumnValuePartitionFilter(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:SingleColumnValuePartitionFilter) + } + + private static com.google.protobuf.Descriptors.Descriptor + internal_static_ValuePartition_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_ValuePartition_fieldAccessorTable; + private static com.google.protobuf.Descriptors.Descriptor + internal_static_SpatialPartition_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_SpatialPartition_fieldAccessorTable; + private static com.google.protobuf.Descriptors.Descriptor + internal_static_SeparatorPartition_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_SeparatorPartition_fieldAccessorTable; + private static com.google.protobuf.Descriptors.Descriptor + internal_static_SingleColumnValuePartitionFilter_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_SingleColumnValuePartitionFilter_fieldAccessorTable; + + public static com.google.protobuf.Descriptors.FileDescriptor + getDescriptor() { + return descriptor; + } + private static com.google.protobuf.Descriptors.FileDescriptor + descriptor; + static { + java.lang.String[] descriptorData = { + "\n\024ValuePartition.proto\032\014Filter.proto\"\213\001\n" + + "\016ValuePartition\022;\n\016partition_type\030\001 \001(\0162" + + "\035.ValuePartition.PartitionType:\004NONE\"5\n\r" + + "PartitionType\022\r\n\tSEPARATOR\020\000\022\013\n\007SPATIAL\020" + + "\001\022\010\n\004NONE\020\002*\005\010d\020\310\001\"T\n\020SpatialPartition2\037" + + "\n\006offset\022\017.ValuePartition\030d \002(\0052\037\n\006lengt" + + "h\022\017.ValuePartition\030e \002(\005\"[\n\022SeparatorPar" + + "tition2\"\n\tseparator\022\017.ValuePartition\030f \002" + + "(\0142!\n\010position\022\017.ValuePartition\030g \002(\005\"\212\001" + + "\n SingleColumnValuePartitionFilter\022<\n\032si", + "ngle_column_value_filter\030\001 \002(\0132\030.SingleC" + + "olumnValueFilter\022(\n\017value_partition\030\002 \002(" + + "\0132\017.ValuePartitionBP\n0org.apache.hadoop." + + "hbase.index.protobuf.generatedB\024ValuePar" + + "titionProtosH\001\210\001\001\240\001\001" + }; + com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = + new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { + public com.google.protobuf.ExtensionRegistry assignDescriptors( + com.google.protobuf.Descriptors.FileDescriptor root) { + descriptor = root; + internal_static_ValuePartition_descriptor = + getDescriptor().getMessageTypes().get(0); + internal_static_ValuePartition_fieldAccessorTable = new + com.google.protobuf.GeneratedMessage.FieldAccessorTable( + internal_static_ValuePartition_descriptor, + new java.lang.String[] { "PartitionType", }); + internal_static_SpatialPartition_descriptor = + getDescriptor().getMessageTypes().get(1); + internal_static_SpatialPartition_fieldAccessorTable = new + com.google.protobuf.GeneratedMessage.FieldAccessorTable( + internal_static_SpatialPartition_descriptor, + new java.lang.String[] { }); + internal_static_SeparatorPartition_descriptor = + getDescriptor().getMessageTypes().get(2); + internal_static_SeparatorPartition_fieldAccessorTable = new + com.google.protobuf.GeneratedMessage.FieldAccessorTable( + internal_static_SeparatorPartition_descriptor, + new java.lang.String[] { }); + internal_static_SingleColumnValuePartitionFilter_descriptor = + getDescriptor().getMessageTypes().get(3); + internal_static_SingleColumnValuePartitionFilter_fieldAccessorTable = new + com.google.protobuf.GeneratedMessage.FieldAccessorTable( + internal_static_SingleColumnValuePartitionFilter_descriptor, + new java.lang.String[] { "SingleColumnValueFilter", "ValuePartition", }); + return null; + } + }; + com.google.protobuf.Descriptors.FileDescriptor + .internalBuildGeneratedFileFrom(descriptorData, + new com.google.protobuf.Descriptors.FileDescriptor[] { + org.apache.hadoop.hbase.protobuf.generated.FilterProtos.getDescriptor(), + }, assigner); + } + + // @@protoc_insertion_point(outer_class_scope) +} Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/util/ByteArrayBuilder.java =================================================================== --- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/util/ByteArrayBuilder.java (revision 0) +++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/util/ByteArrayBuilder.java (working copy) @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.util; + +// Custom ByteArrayBuilder class to avoid overhead. +public class ByteArrayBuilder { + private short pos; + private byte[] store; + + public ByteArrayBuilder(int size) { + store = new byte[size]; + pos = 0; + } + + public void put(byte[] src) { + System.arraycopy(src, 0, store, pos, src.length); + pos += src.length; + } + + public void put(byte[] src, int offset, int length) { + System.arraycopy(src, offset, store, pos, length); + pos += length; + } + + public static ByteArrayBuilder allocate(int size) { + return new ByteArrayBuilder(size); + } + + public short position() { + return pos; + } + + public void position(int newPosition) { + pos = (short) newPosition; + } + + // Be careful calling this method. This method exposes the underlying byte[] + // Any changes to this returned object will result in changes in the builder. + public byte[] array() { + return store; + } + + // This method creates a new byte[] and copy the bytes from the underlying byte[] + // Any changes to the returned object will not affect the builder. + public byte[] array(int offset, int length) { + byte[] subArray = new byte[length]; + System.arraycopy(store, offset, subArray, 0, length); + return subArray; + } +} Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/util/DecimalComparator.java =================================================================== --- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/util/DecimalComparator.java (revision 0) +++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/util/DecimalComparator.java (working copy) @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.util; + + +import org.apache.hadoop.hbase.filter.BinaryComparator; + +public class DecimalComparator extends BinaryComparator { + + protected byte[] msb = new byte[1]; + protected byte[] temp; + + /** + * Constructor + * @param value value + */ + public DecimalComparator(byte[] value) { + super(value); + byte b = value[0]; + msb[0] = (byte) ((b >> 7) & 1); + temp = new byte[value.length]; + System.arraycopy(value, 0, temp, 0, value.length); + } + + @Override + public int compareTo(byte[] value, int offset, int length) { + return super.compareTo(value, offset, length); + } + +} Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/util/DoubleComparator.java =================================================================== --- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/util/DoubleComparator.java (revision 0) +++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/util/DoubleComparator.java (working copy) @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.util; + +import org.apache.hadoop.hbase.exceptions.DeserializationException; +import org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos; + +import com.google.protobuf.InvalidProtocolBufferException; + +public class DoubleComparator extends DecimalComparator { + + public DoubleComparator(byte[] value) { + super(value); + } + + @Override + public int compareTo(byte[] actualValue, int offset, int length) { + ByteArrayBuilder val = new ByteArrayBuilder(length); + val.put(actualValue, offset, length); + byte[] array = val.array(); + if (msb[0] == 0) { + value[0] ^= (1 << 7); + array[0] ^= (1 << 7); + } else { + for (int i = 0; i < 8; i++) { + value[i] ^= 0xff; + } + + for (int i = 0; i < 8; i++) { + array[i] ^= 0xff; + } + } + int compareTo = super.compareTo(array, 0, length); + System.arraycopy(temp, 0, value, 0, value.length); + return compareTo; + } + + /** + * @param pbBytes A pb serialized {@link DoubleComparator} instance + * @return An instance of {@link DoubleComparator} made from bytes + * @throws DeserializationException + * @see #toByteArray + */ + public static DoubleComparator parseFrom(final byte [] pbBytes) + throws DeserializationException { + ComparatorProtos.BinaryComparator proto; + try { + proto = ComparatorProtos.BinaryComparator.parseFrom(pbBytes); + } catch (InvalidProtocolBufferException e) { + throw new DeserializationException(e); + } + return new DoubleComparator(proto.getComparable().getValue().toByteArray()); + } +} Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/util/FloatComparator.java =================================================================== --- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/util/FloatComparator.java (revision 0) +++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/util/FloatComparator.java (working copy) @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.util; + +import org.apache.hadoop.hbase.exceptions.DeserializationException; +import org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos; + +import com.google.protobuf.InvalidProtocolBufferException; + +public class FloatComparator extends DecimalComparator { + + public FloatComparator(byte[] value) { + super(value); + } + + @Override + public int compareTo(byte[] actualValue, int offset, int length) { + ByteArrayBuilder val = new ByteArrayBuilder(length); + val.put(actualValue, offset, length); + byte[] array = val.array(); + if (msb[0] == 0) { + value[0] ^= (1 << 7); + array[0] ^= (1 << 7); + } else { + value[0] ^= 0xff; + value[1] ^= 0xff; + value[2] ^= 0xff; + value[3] ^= 0xff; + + array[0] ^= 0xff; + array[1] ^= 0xff; + array[2] ^= 0xff; + array[3] ^= 0xff; + } + int compareTo = super.compareTo(array, 0, length); + System.arraycopy(temp, 0, value, 0, value.length); + return compareTo; + } + + /** + * @param pbBytes A pb serialized {@link FloatComparator} instance + * @return An instance of {@link FloatComparator} made from bytes + * @throws DeserializationException + * @see #toByteArray + */ + public static FloatComparator parseFrom(final byte [] pbBytes) + throws DeserializationException { + ComparatorProtos.BinaryComparator proto; + try { + proto = ComparatorProtos.BinaryComparator.parseFrom(pbBytes); + } catch (InvalidProtocolBufferException e) { + throw new DeserializationException(e); + } + return new FloatComparator(proto.getComparable().getValue().toByteArray()); + } +} Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/util/IndexUtils.java =================================================================== --- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/util/IndexUtils.java (revision 0) +++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/util/IndexUtils.java (working copy) @@ -0,0 +1,551 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.util; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValueUtil; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Durability; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.index.ColumnQualifier; +import org.apache.hadoop.hbase.index.ColumnQualifier.ValueType; +import org.apache.hadoop.hbase.index.client.IndexAdmin; +import org.apache.hadoop.hbase.index.Constants; +import org.apache.hadoop.hbase.index.IndexSpecification; +import org.apache.hadoop.hbase.index.TableIndices; +import org.apache.hadoop.hbase.index.ValuePartition; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.util.Bytes; + +public class IndexUtils { + + private static final Log LOG = LogFactory.getLog(IndexUtils.class); + + public static final String TABLE_INPUT_COLS = "table.columns.index"; + + /** + * Utility method to get the name of the index table when given the name of the actual table. + * + * @param tableName + * @return index table name + */ + public static String getIndexTableName(String tableName) { + // TODO The suffix for the index table is fixed now. Do we allow to make this configurable? + // We can handle things in byte[] way? + return tableName + Constants.INDEX_TABLE_SUFFIX; + } + + /** + * @param tableName + * @return index table name + */ + public static String getIndexTableName(TableName tableName) { + return getIndexTableName(tableName.getNameAsString()); + } + /** + * Utility method to get the name of the index table when given the name of the actual table. + * + * @param tableName + * @return index table name + */ + public static String getIndexTableName(byte[] tableName) { + return getIndexTableName(Bytes.toString(tableName)); + } + + /** + * Tells whether the passed table is a secondary index table or a normal table. + * + * @param tableName + * @return + */ + public static boolean isIndexTable(String tableName) { + return tableName.endsWith(Constants.INDEX_TABLE_SUFFIX); + } + + public static boolean isIndexTable(TableName tableName) { + return tableName.getQualifierAsString().endsWith(Constants.INDEX_TABLE_SUFFIX); + } + + public static byte [] isLegalIndexName(final byte [] indexName) { + if (indexName == null || indexName.length <= 0) { + throw new IllegalArgumentException("Name is null or empty"); + } + if (indexName[0] == '.' || indexName[0] == '-') { + throw new IllegalArgumentException("Illegal first character <" + indexName[0] + + "> at 0. Index names can only start with 'word " + + "characters': i.e. [a-zA-Z_0-9]: " + Bytes.toString(indexName)); + } + for (int i = 0; i < indexName.length; i++) { + if (Character.isLetterOrDigit(indexName[i]) || indexName[i] == '_' || + indexName[i] == '-' || indexName[i] == '.') { + continue; + } + throw new IllegalArgumentException("Illegal character <" + indexName[i] + + "> at " + i + ". Index names can only contain " + + "'word characters': i.e. [a-zA-Z_0-9-.]: " + Bytes.toString(indexName)); + } + return indexName; + } + /** + * Tells whether the passed table is a secondary index table or a normal table. + * + * @param tableName + * @return + */ + public static boolean isIndexTable(byte[] tableName) { + return isIndexTable(Bytes.toString(tableName)); + } + + /** + * Checks whether the passed table is a catalog table or not + * + * @param tableName + * @return true when the passed table is a catalog table. + */ + public static boolean isCatalogOrSystemTable(TableName tableName) { + return tableName.equals(TableName.META_TABLE_NAME) + || tableName.equals(TableName.NAMESPACE_TABLE_NAME) || tableName.isSystemTable(); + } + + /** + * Returns the max length allowed for the index name. + * + * @return + */ + public static int getMaxIndexNameLength() { + // TODO we need to allow customers to configure this value. + return Constants.DEF_MAX_INDEX_NAME_LENGTH; + } + + /** + * Returns the main table name. + * + * @param index + * table name + * @return + */ + public static String extractActualTableName(String indexTableName) { + int endIndex = indexTableName.length() - Constants.INDEX_TABLE_SUFFIX.length(); + return indexTableName.substring(0, endIndex); + } + + public static byte[] changeValueAccToDataType(byte[] value, ValueType valueType) { + byte[] valueArr = new byte[value.length]; + System.arraycopy(value, 0, valueArr, 0, value.length); + + if (valueArr.length == 0) + return valueArr; + switch (valueType) { + case String: + case Char: + break; + case Float: + float f = Bytes.toFloat(valueArr); + if (f > 0) { + valueArr[0] ^= (1 << 7); + } + else { + valueArr[0] ^= 0xff; + valueArr[1] ^= 0xff; + valueArr[2] ^= 0xff; + valueArr[3] ^= 0xff; + } + break; + case Double: + double d = Bytes.toDouble(valueArr); + if (d > 0) { + valueArr[0] ^= (1 << 7); + } + else { + for (int i = 0; i < 8; i++) { + valueArr[i] ^= 0xff; + } + } + break; + case Int: + case Long: + case Short: + case Byte: + valueArr[0] ^= (1 << 7); + break; + } + return valueArr; + } + + // TODO check this... Is this ok with all cases? + // No.. for -ve issues... Will see later.. + public static byte[] incrementValue(byte[] value, boolean copy) { + byte[] newValue = new byte[value.length]; + if (copy) { + System.arraycopy(value, 0, newValue, 0, newValue.length); + } else { + newValue = value; + } + for (int i = newValue.length - 1; i >= 0; i--) { + byte b = newValue[i]; + b = (byte) (b + 1); + if (b == 0) { + newValue[i] = 0; + } else { + newValue[i] = b; + break; + } + } + return newValue; + } + + public static String getActualTableName(String indexTableName) { + String split[] = indexTableName.split(Constants.INDEX_TABLE_SUFFIX); + return split[0]; + } + + public static TableName getActualTableName(TableName indexTable) { + return TableName.valueOf(getActualTableName(indexTable.getNameAsString())); + } + + /** + * Compare {@link FileStatus} instances by {@link Path#getName()}. Returns in reverse order. + */ + static class FileStatusFileNameComparator implements Comparator { + @Override + public int compare(FileStatus left, FileStatus right) { + return -left.compareTo(right); + } + } + + public static Put prepareIndexPut(Put userPut, IndexSpecification index, HRegion indexRegion) + throws IOException { + byte[] indexRegionStartKey = indexRegion.getStartKey(); + return prepareIndexPut(userPut, index, indexRegionStartKey); + } + + public static Delete prepareIndexDelete(Delete userDelete, IndexSpecification index, + byte[] indexRegionStartKey) throws IOException { + ByteArrayBuilder indexRow = IndexUtils.getIndexRowKeyHeader(index, indexRegionStartKey, + userDelete.getRow()); + boolean update = false; + for (ColumnQualifier cq : index.getIndexColumns()) { + Cell kvFound = null; + for (Entry> entry : userDelete.getFamilyCellMap().entrySet()) { + for (Cell cell : entry.getValue()) { + Cell kv = KeyValueUtil.ensureKeyValue(cell); + if (Bytes.equals(cq.getColumnFamily(), kv.getFamily()) + && Bytes.equals(cq.getQualifier(), kv.getQualifier())) { + kvFound = kv; + update = true; + break; + } + } + } + if (kvFound == null) { + indexRow.position(indexRow.position() + cq.getMaxValueLength()); + } else { + IndexUtils.updateRowKeyForKV(cq, kvFound, indexRow); + } + } + if (update) { + // Append the actual row key at the end of the index row key. + indexRow.put(userDelete.getRow()); + Delete idxDelete = new Delete(indexRow.array()); + idxDelete.deleteColumn(Constants.IDX_COL_FAMILY, Constants.IDX_COL_QUAL, + userDelete.getTimeStamp()); + idxDelete.setDurability(Durability.SKIP_WAL); + return idxDelete; + } + return null; + } + + // Default access specifier for the UT + public static Put prepareIndexPut(Put userPut, IndexSpecification index, + byte[] indexRegionStartKey) throws IOException { + long tsForIndexTabPut = 0; + + boolean bypass = true; + for (ColumnQualifier c : index.getIndexColumns()) { + List values = userPut.get(c.getColumnFamily(), c.getQualifier()); + if (null != values && values.size() > 0) { + bypass = false; + break; + } + } + if (bypass) { + // When this Put having no values for all the column in this index just skip this Put + // from adding corresponding entry in the index table. + return null; + } + byte[] primaryRowKey = userPut.getRow(); + ByteArrayBuilder indexRowKey = getIndexRowKeyHeader(index, indexRegionStartKey, primaryRowKey); + + // STEP 3 : Adding the column value + padding for each of the columns in + // the index. + for (ColumnQualifier indexCQ : index.getIndexColumns()) { + List values = userPut.get(indexCQ.getColumnFamily(), indexCQ.getQualifier()); + if (values == null || values.isEmpty()) { + // There is no value provided for the column. Going with the padding + // All the bytes in the byte[] 'indexRowKey' will be 0s already. + // No need to put a 0 padding bytes. Just need to advance the position by col max value + // length. + indexRowKey.position(indexRowKey.position() + indexCQ.getMaxValueLength()); + } else { + // A put can contains diff version values for the same column. + // We can consider the latest value only for the indexing. This needs to be documented. + // TODO + Cell kv = selectKVForIndexing(values); + updateRowKeyForKV(indexCQ, kv, indexRowKey); + if (tsForIndexTabPut < kv.getTimestamp()) { + tsForIndexTabPut = kv.getTimestamp(); + } + } + } + // Remember the offset of rowkey and store it as value + short rowKeyOffset = indexRowKey.position(); + + // STEP 4 : Adding the user table rowkey. + indexRowKey.put(primaryRowKey); + + // Creating the value to be put into the index column + // Last portion of index row key = [region start key length (2 bytes), offset of primary rowkey + // in index rowkey (2 bytes)] + ByteArrayBuilder indexColVal = ByteArrayBuilder.allocate(4); + indexColVal.put(Bytes.toBytes((short) indexRegionStartKey.length)); + indexColVal.put(Bytes.toBytes(rowKeyOffset)); + Put idxPut = new Put(indexRowKey.array()); + idxPut.add(Constants.IDX_COL_FAMILY, Constants.IDX_COL_QUAL, tsForIndexTabPut, + indexColVal.array()); + idxPut.setDurability(Durability.SKIP_WAL); + return idxPut; + } + + private static Cell selectKVForIndexing(List values) { + Cell kv = null; + long ts = HConstants.OLDEST_TIMESTAMP; + for (Cell value : values) { + // When the TS is same, then we need to consider the last KV + // appearing in the KVList + // as this will be added to the memstore with highest memstore TS. + if (value.getTimestamp() >= ts) { + kv = value; + ts = value.getTimestamp(); + } + } + return kv; + } + + public static ByteArrayBuilder getIndexRowKeyHeader(IndexSpecification index, + byte[] indexRegionStartKey, byte[] primaryRowKey) { + /* + * Format for the rowkey for index table [Startkey for the index region] + [one 0 byte] + [Index + * name] + [Padding for the max index name] + [[index col value]+[padding for the max col value] + * for each of the index col] + [user table row key] + * + * To know the reason for adding empty byte array refert to HDP-1666 + */ + byte[] indexName = Bytes.toBytes(index.getName()); + int totalValueLength = index.getTotalValueLength(); + int maxIndexNameLength = IndexUtils.getMaxIndexNameLength(); + int rowLength = indexRegionStartKey.length + maxIndexNameLength + totalValueLength + + primaryRowKey.length + 1; + ByteArrayBuilder row = ByteArrayBuilder.allocate(rowLength); + + // STEP 1 : Adding the startkey for the index region and single empty Byte. + row.put(indexRegionStartKey); + // one byte [0] to be added after the index region startkey. This is for the case of + // entries added to the 1st region.Here the startkey of the region will be empty byte[] + // So the 1st byte(s) which comes will be the index name and it might not fit into the + // 1st region [As per the end key of that region] + // Well all the bytes in the byte[] 'row' will be 0s already. No need to put a 0 byte + // Just need to advance the position by 1 + row.position(row.position() + 1); + + // STEP 2 : Adding the index name and the padding needed + row.put(indexName); + int padLength = maxIndexNameLength - indexName.length; + // Well all the bytes in the byte[] 'row' will be 0s already. No need to put a 0 padding bytes + // Just need to advance the position by padLength + row.position(row.position() + padLength); + return row; + } + + public static void updateRowKeyForKV(ColumnQualifier indexCQ, Cell kv, + ByteArrayBuilder indexRowKey) throws IOException { + byte[] value = getValueFromKV(kv, indexCQ); + int valuePadLength = indexCQ.getMaxValueLength() - value.length; + if (valuePadLength < 0) { + String errMsg = "The value length for the column " + indexCQ.getColumnFamilyString() + ":" + + indexCQ.getQualifierString() + " is greater than the cofigured max value length : " + + indexCQ.getMaxValueLength(); + LOG.warn(errMsg); + throw new IOException(errMsg); + } + indexRowKey.put(value); + indexRowKey.position(indexRowKey.position() + valuePadLength); + } + + private static byte[] getValueFromKV(Cell kv, ColumnQualifier indexCQ) { + ValuePartition vp = indexCQ.getValuePartition(); + byte value[] = null; + if (vp != null) { + value = vp.getPartOfValue(kv.getValue()); + if (value != null) { + value = IndexUtils.changeValueAccToDataType(value, indexCQ.getType()); + } + } else { + LOG.trace("No offset or separator is mentioned. So just returning the value fetched from kv"); + value = kv.getValue(); + value = IndexUtils.changeValueAccToDataType(value, indexCQ.getType()); + } + return value; + } + + public static byte[] getRowKeyFromKV(Cell kv) { + byte[] row = kv.getRow(); + // Row key of the index table entry = region startkey + index name + column value(s) + // + actual table rowkey. + // Every row in the index table will have exactly one KV in that. The value will be + // 4 bytes. First 2 bytes specify length of the region start key bytes part in the + // rowkey. Last 2 bytes specify the offset to the actual table rowkey part within the + // index table rowkey. + byte[] value = kv.getValue(); + short actualRowKeyOffset = Bytes.toShort(value, 2); + byte[] actualTableRowKey = new byte[row.length - actualRowKeyOffset]; + System.arraycopy(row, actualRowKeyOffset, actualTableRowKey, 0, actualTableRowKey.length); + return actualTableRowKey; + } + + public static void createIndexTable(String userTable, Configuration conf, + Map> indexColumnFamily) throws IOException, InterruptedException, + ClassNotFoundException { + HBaseAdmin hbaseAdmin = new IndexAdmin(conf); + + try { + HTableDescriptor tableDescriptor = hbaseAdmin.getTableDescriptor(Bytes.toBytes(userTable)); + + + String input = conf.get(TABLE_INPUT_COLS); + + HTableDescriptor ihtd = parse(userTable, tableDescriptor, input, + indexColumnFamily); + + // disable the table + hbaseAdmin.disableTable(userTable); + // This will create the index table. Also modifies the existing table htable descriptor. + hbaseAdmin.modifyTable(Bytes.toBytes(userTable), ihtd); + hbaseAdmin.enableTable(Bytes.toBytes(userTable)); + } finally { + if (hbaseAdmin != null) { + hbaseAdmin.close(); + } + } + } + + // This can be a comma seperated list + // We can pass like + // IDX1=>cf1:[q1->datatype& + // length],[q2],[q3];cf2:[q1->datatype&length],[q2->datatype&length],[q3->datatype& + // lenght]#IDX2=>cf1:q5,q5 + public static HTableDescriptor parse(String tableNameToIndex, + HTableDescriptor tableDesc, String input, + Map> cfs) throws IOException { + List colFamilyList = new ArrayList(); + + for (HColumnDescriptor hColumnDescriptor : tableDesc.getColumnFamilies()) { + colFamilyList.add(hColumnDescriptor.getNameAsString()); + } + TableIndices indices = new TableIndices(); + if (input != null) { + String[] indexSplits = input.split("#"); + for (String index : indexSplits) { + String[] indexName = index.split("=>"); + if (indexName.length < 2) { + System.out.println("Invalid entry."); + System.exit(-1); + } + IndexSpecification iSpec = new IndexSpecification(indexName[0]); + + String[] cfSplits = indexName[1].split(";"); + if (cfSplits.length < 1) { + System.exit(-1); + } else { + for (String cf : cfSplits) { + String[] qualSplits = cf.split(":"); + if (qualSplits.length < 2) { + System.out.println("The qualifiers are not given"); + System.exit(-1); + } + if (!colFamilyList.contains(qualSplits[0])) { + System.out.println("Valid CF not found"); + System.exit(-1); + } + String[] qualDetails = qualSplits[1].split(","); + for (String details : qualDetails) { + String substring = details.substring(1, details.lastIndexOf("]")); + if (substring != null) { + String[] splitQual = substring.split("->"); + if (splitQual.length < 2) { + System.out.println("Default value length and data type will be take"); + iSpec.addIndexColumn(new HColumnDescriptor(qualSplits[0]), splitQual[0], + ValueType.String, Constants.DEF_MAX_INDEX_NAME_LENGTH); + } else { + String[] valueType = splitQual[1].split("&"); + iSpec.addIndexColumn(new HColumnDescriptor(qualSplits[0]), splitQual[0], + ValueType.valueOf(valueType[0]), Integer.parseInt(valueType[1])); + } + if (cfs != null) { + addToMap(cfs, qualSplits, splitQual); + } + } + } + } + } + indices.addIndex(iSpec); + } + } + tableDesc.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + return tableDesc; + } + + private static void addToMap(Map> cfs, String[] qualSplits, + String[] splitQual) { + if (cfs.get(qualSplits[0]) == null) { + List qual = new ArrayList(); + qual.add(splitQual[0]); + cfs.put(qualSplits[0], qual); + } else { + List list = cfs.get(qualSplits[0]); + list.add(splitQual[0]); + } + } + +} Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/util/IntComparator.java =================================================================== --- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/util/IntComparator.java (revision 0) +++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/util/IntComparator.java (working copy) @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.util; + +import org.apache.hadoop.hbase.exceptions.DeserializationException; +import org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos; + +import com.google.protobuf.InvalidProtocolBufferException; + +public class IntComparator extends DecimalComparator { + + public IntComparator(byte[] value) { + super(value); + } + + @Override + public int compareTo(byte[] actualValue, int offset, int length) { + ByteArrayBuilder val = new ByteArrayBuilder(length); + val.put(actualValue, offset, length); + value[0] ^= (1 << 7); + byte[] array = val.array(); + array[0] ^= (1 << 7); + int compareTo = super.compareTo(array, 0, length); + System.arraycopy(temp, 0, value, 0, value.length); + return compareTo; + } + + + /** + * @param pbBytes A pb serialized {@link IntComparator} instance + * @return An instance of {@link IntComparator} made from bytes + * @throws DeserializationException + * @see #toByteArray + */ + public static IntComparator parseFrom(final byte [] pbBytes) + throws DeserializationException { + ComparatorProtos.BinaryComparator proto; + try { + proto = ComparatorProtos.BinaryComparator.parseFrom(pbBytes); + } catch (InvalidProtocolBufferException e) { + throw new DeserializationException(e); + } + return new IntComparator(proto.getComparable().getValue().toByteArray()); + } + +} Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/util/SecondaryIndexColocator.java =================================================================== --- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/util/SecondaryIndexColocator.java (revision 0) +++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/util/SecondaryIndexColocator.java (working copy) @@ -0,0 +1,545 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.util; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.Map.Entry; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.Abortable; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.ClusterStatus; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HBaseIOException; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.MasterNotRunningException; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.UnknownRegionException; +import org.apache.hadoop.hbase.ZooKeeperConnectionException; +import org.apache.hadoop.hbase.client.HConnection; +import org.apache.hadoop.hbase.client.HConnectionManager; +import org.apache.hadoop.hbase.client.MetaScanner; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.HConnectable; +import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor; +import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitorBase; +import org.apache.hadoop.hbase.index.client.IndexAdmin; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.HBaseFsckRepair; +import org.apache.hadoop.hbase.util.Pair; +import org.apache.hadoop.hbase.zookeeper.ZKTable; +import org.apache.hadoop.hbase.zookeeper.ZKTableReadOnly; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.hadoop.hbase.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos; +import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService.BlockingInterface; +import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.Table.State; + +import org.apache.zookeeper.KeeperException; + +public class SecondaryIndexColocator { + + public static boolean testingEnabled = false; + + public static final Log LOG = LogFactory.getLog(SecondaryIndexColocator.class); + + public Configuration conf = null; + + private Map> tableMap = new HashMap>(); + + private List regionsToMove = new ArrayList(); + + private Map> rsToRegionMap = new HashMap>(); + + private Map disabledandDisablingTables = new TreeMap(Bytes.BYTES_COMPARATOR); + + private Map enabledOrEnablingTables = new TreeMap(Bytes.BYTES_COMPARATOR); + + //private Set staleMetaEntries = new HashSet(); + + private List> tablesToBeSetInZK = new ArrayList>(); + + //List rit = new ArrayList(); + + private IndexAdmin admin; + private ClusterStatus status; + private HConnection connection; + private ZooKeeperWatcher watcher; + + public SecondaryIndexColocator(Configuration conf){ + this.conf = conf; + } + + public void setUp() throws IOException, ZooKeeperConnectionException{ + admin = new IndexAdmin(conf); + status = admin.getClusterStatus(); + connection = admin.getConnection(); + watcher = createZooKeeperWatcher(); + } + + public static void main(String args[]) throws Exception { + Configuration config = HBaseConfiguration.create(); + SecondaryIndexColocator secHbck = null; + try { + secHbck = new SecondaryIndexColocator(config); + secHbck.setUp(); + secHbck.admin.setBalancerRunning(false, true); + boolean inconsistent = secHbck.checkForCoLocationInconsistency(); + if (inconsistent) { + secHbck.fixCoLocationInconsistency(); + } + secHbck.admin.setBalancerRunning(true, true); + } finally { + if (secHbck != null) { + if (secHbck.admin != null) secHbck.admin.close(); + if (secHbck.watcher != null) secHbck.watcher.close(); + } + } + } + + public boolean checkForCoLocationInconsistency() throws IOException, KeeperException, InterruptedException { + getMetaInfo(); + // Do we need to complete the partial disable table + loadDisabledTables(); + //handleRIT(); + checkMetaInfoCosistency(); + setTablesInZK(); + // in the former steps there may have been movement of regions. So need to update + // in memory map of tables. + getMetaInfo(); + checkCoLocationAndGetRegionsToBeMoved(); + this.watcher.close(); + if (regionsToMove == null || regionsToMove.isEmpty()) { + return false; + } + return true; + } + +/* private void handleRIT() { + if(rit != null && !rit.isEmpty()){ + for(String s : rit){ + RegionTransitionData data = ZKAssign.getDataNoWatch(zkw, pathOrRegionName, stat) + } + } + }*/ + + private void checkMetaInfoCosistency() throws IOException, KeeperException, InterruptedException { + if (status == null) { + throw new IOException("Cluster status is not available."); + } + Collection regionServers = status.getServers(); + for (ServerName serverName : regionServers) { + BlockingInterface server = connection.getAdmin(serverName); + Set onlineRegions = new HashSet(); + List regions = ProtobufUtil.getOnlineRegions(server); + if (regions == null) + continue; + onlineRegions.addAll(regions); + if (rsToRegionMap == null) { + rsToRegionMap = new HashMap>(); + } + rsToRegionMap.put(serverName, onlineRegions); + } + if (tableMap != null && !tableMap.isEmpty()) { + for (Map.Entry> e : tableMap.entrySet()) { + if (isDisabledOrDisablingTable(Bytes.toBytes(e.getKey()))) { + // It should be disabled...But we can check this + if (disabledandDisablingTables.get(Bytes.toBytes(e.getKey())) == State.DISABLED) { + continue; + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Table " + e.getKey() + + " is in DISABLING state. Trying to close all the regions for this table."); + } + // If the table is in DISABLING state , then there might be some regions which are + // still online. Close the regions and set the table as DISABLED. + for (MetaInfo metaInfo : e.getValue()) { + List sn = new ArrayList(); + for (Map.Entry> entry : rsToRegionMap.entrySet()) { + if (entry.getValue().contains(metaInfo.getRegionInfo())) { + sn.add(entry.getKey()); + } + } + if (sn.isEmpty()) { + // region is not assigned anywhere ,so continue. + continue; + } else { + HBaseFsckRepair.fixMultiAssignment(this.admin, metaInfo.getRegionInfo(), sn); + } + } + Pair p = new Pair(); + p.setFirst(e.getKey()); + p.setSecond(State.DISABLED); + tablesToBeSetInZK.add(p); + } + } else { + // first we are checking here for the tables to be enabled which + // we left in disabled stage in the previous step. + if (!disabledandDisablingTables.containsKey(Bytes.toBytes(e.getKey())) && + !enabledOrEnablingTables.containsKey(Bytes.toBytes(e.getKey()))) { + // if reached here then this table, which is disabled + // and still not present in our in-memory map of disabledTables , + // should + // be enabled. + this.admin.enableTable(e.getKey()); + this.enabledOrEnablingTables.put(Bytes.toBytes(e.getKey()), State.ENABLED); + continue; + } + boolean movedRegions = false; + for (MetaInfo metaInfo : e.getValue()) { + List sn = new ArrayList(); + for (Map.Entry> entry : rsToRegionMap.entrySet()) { + if (entry.getValue().contains(metaInfo.getRegionInfo())) { + sn.add(entry.getKey()); + } + } + if (sn.size() == 1 && sn.get(0).equals(metaInfo.getServerName())) { + // this means region is deployed on correct rs according to META. + if (LOG.isDebugEnabled()) { + LOG.debug("Info in META for region " + + metaInfo.getRegionInfo().getRegionNameAsString() + " is correct."); + } + continue; + } + // if it reaches here , it means that the region is deployed and + // in some other rs. Need to find it and call unassign. + if (sn.isEmpty()) { + if (LOG.isDebugEnabled()) { + LOG.debug("Region " + metaInfo.getRegionInfo().getRegionNameAsString() + + " not deployed on any rs.Trying to assign"); + } + HBaseFsckRepair.fixUnassigned(this.admin, metaInfo.getRegionInfo()); + HBaseFsckRepair.waitUntilAssigned(this.admin, metaInfo.getRegionInfo()); + movedRegions = true; + } else { + if(LOG.isDebugEnabled()){ + LOG.debug("Region " + metaInfo.getRegionInfo().getRegionNameAsString() + + " is not deployed on the rs as mentioned in META. Re-assigning."); + } + HBaseFsckRepair.fixMultiAssignment(this.admin, metaInfo.getRegionInfo(), sn); + HBaseFsckRepair.waitUntilAssigned(this.admin, metaInfo.getRegionInfo()); + movedRegions = true; + } + } + if(movedRegions) { + Pair p = new Pair(); + p.setFirst(e.getKey()); + p.setSecond(State.ENABLED); + tablesToBeSetInZK.add(p); + } + } + } + } + } + + private void setTablesInZK() throws IOException, KeeperException { + if (tablesToBeSetInZK != null && !tablesToBeSetInZK.isEmpty()) { + ZKTable zkTable = new ZKTable(this.watcher); + for (Pair p : tablesToBeSetInZK) { + setStateInZK(zkTable, p.getFirst(), p.getSecond()); + } + } + } + + private void setStateInZK(ZKTable zkTable, String tableName, State state) throws IOException, + KeeperException { + if (state == State.ENABLED) { + zkTable.setEnabledTable(TableName.valueOf(tableName)); + } + if (state == State.DISABLED) { + zkTable.setDisabledTable(TableName.valueOf(tableName)); + } + } + + private boolean isDisabledOrDisablingTable(byte[] tableName) { + if(disabledandDisablingTables != null && !disabledandDisablingTables.isEmpty()){ + if(disabledandDisablingTables.containsKey(tableName)) return true; + } + return false; + } + + public void fixCoLocationInconsistency() throws HBaseIOException { + if (regionsToMove != null && !regionsToMove.isEmpty()) { + Iterator itr = regionsToMove.iterator(); + while (itr.hasNext()) { + HRegionInfo hri = itr.next(); + try { + if (LOG.isDebugEnabled()) { + LOG.debug("Moving region " + hri.getRegionNameAsString() + " to server "); + } + admin.move(hri.getEncodedNameAsBytes(),null); + itr.remove(); + } catch (UnknownRegionException e) { + LOG.error("Unnkown region exception.",e); + } catch (MasterNotRunningException e) { + LOG.error("Master not running.",e); + } catch (ZooKeeperConnectionException e) { + LOG.error("Zookeeper connection exception.",e); + } + } + } + } + + private void checkCoLocationAndGetRegionsToBeMoved() { + if (tableMap != null && !tableMap.isEmpty()) { + Iterator>> itr = tableMap.entrySet() + .iterator(); + while (itr.hasNext()) { + Map.Entry> e = itr.next(); + if (!IndexUtils.isIndexTable(e.getKey()) + && !IndexUtils.isCatalogOrSystemTable(TableName.valueOf(e.getKey()))) { + if(isDisabledOrDisablingTable(Bytes.toBytes(e.getKey()))) continue; + String indexTableName = IndexUtils.getIndexTableName(e.getKey()); + List idxRegionList = tableMap.get(indexTableName); + if (idxRegionList == null || idxRegionList.isEmpty()) { + itr.remove(); + continue; + } else { + getRegionsToMove(e.getValue(), idxRegionList); + itr.remove(); + } + } + } + } + } + + private void getRegionsToMove(List userRegions, List idxRegionList) { + Iterator userRegionItr = userRegions.iterator(); + while (userRegionItr.hasNext()) { + MetaInfo userRegionMetaInfo = userRegionItr.next(); + for (MetaInfo indexRegionMetaInfo : idxRegionList) { + if (Bytes.equals(userRegionMetaInfo.getRegionInfo().getStartKey(), indexRegionMetaInfo + .getRegionInfo().getStartKey())) { + if (!userRegionMetaInfo.getServerName().equals(indexRegionMetaInfo.getServerName())) { + if (regionsToMove == null) { + regionsToMove = new ArrayList(); + } + regionsToMove.add(userRegionMetaInfo.getRegionInfo()); + if (LOG.isDebugEnabled()) { + LOG.debug("Adding region " + + userRegionMetaInfo.getRegionInfo().getRegionNameAsString() + + " to regions to be moved list."); + } + } + break; + } + } + } + } + + private void getMetaInfo() throws IOException { + + MetaScannerVisitor visitor = new MetaScannerVisitorBase() { + + // comparator to sort KeyValues with latest timestamp + final Comparator comp = new Comparator() { + public int compare(Cell k1, Cell k2) { + return (int) (k1.getTimestamp() - k2.getTimestamp()); + } + }; + + public boolean processRow(Result result) throws IOException { + try { + + long ts = Collections.max(result.list(), comp).getTimestamp(); + // record the latest modification of this META record + HRegionInfo info = HRegionInfo.getHRegionInfo(result); + if (info == null) { + LOG.warn("No serialized HRegionInfo in " + result); + return true; + } + + Pair pair = + new Pair(info, HRegionInfo.getServerName(result)); + if (pair != null) { + String tableName = pair.getFirst().getTable().getNameAsString(); + if (tableMap == null) { + tableMap = new HashMap>(); + } + List regionsOfTable = tableMap.get(tableName); + if (regionsOfTable == null) { + regionsOfTable = new ArrayList(); + tableMap.put(tableName, regionsOfTable); + } + Iterator itr = regionsOfTable.iterator(); + while(itr.hasNext()){ + MetaInfo m = itr.next(); + if(m.getRegionInfo().equals(pair.getFirst())){ + itr.remove(); + break; + } + } + MetaInfo m = new MetaInfo(pair.getFirst(),pair.getSecond(),ts); + regionsOfTable.add(m); + } + return true; + } catch (RuntimeException e) { + LOG.error("Result=" + result); + throw e; + } + } + }; + // Scan -ROOT- to pick up META regions + MetaScanner.metaScan(conf, visitor, null, null, Integer.MAX_VALUE); + + // Scan .META. to pick up user regions + MetaScanner.metaScan(conf, visitor); + + } + /** + * Stores the regioninfo entries scanned from META + */ + static class MetaInfo { + private HRegionInfo hri; + private ServerName regionServer; + private long timeStamp; + + public MetaInfo(HRegionInfo hri, ServerName regionServer, long modTime) { + this.hri = hri; + this.regionServer = regionServer; + this.timeStamp = modTime; + } + + public HRegionInfo getRegionInfo(){ + return this.hri; + } + + public ServerName getServerName(){ + return this.regionServer; + } + + public long getTimeStamp(){ + return this.timeStamp; + } + } + + public void setAdmin(IndexAdmin admin, HConnection conn, ClusterStatus status) { + if (testingEnabled) { + this.admin = admin; + this.connection = conn; + this.status = status; + } + } + + private void loadDisabledTables() + throws ZooKeeperConnectionException, IOException, KeeperException { + HConnectionManager.execute(new HConnectable(conf) { + @Override + public Void connect(HConnection connection) throws IOException { + try { + for (Entry> e: ZKTableReadOnly.getTableNamesByState(watcher).entrySet()) { + if (e.getKey() == ZooKeeperProtos.Table.State.DISABLED + || e.getKey() == ZooKeeperProtos.Table.State.DISABLING) { + for (TableName tableName : e.getValue()) { + disabledandDisablingTables.put(tableName.getName(),e.getKey()); + } + } else if (e.getKey() == ZooKeeperProtos.Table.State.ENABLED + || e.getKey() == ZooKeeperProtos.Table.State.ENABLING) { + for (TableName tableName : e.getValue()) { + enabledOrEnablingTables.put(tableName.getName(),e.getKey()); + } + } + } + } catch (KeeperException ke) { + throw new IOException(ke); + } + return null; + } + }); + checkDisabledAndEnabledTables(); + } + + private void checkDisabledAndEnabledTables() throws IOException, KeeperException { + if (disabledandDisablingTables != null && !disabledandDisablingTables.isEmpty()) { + Map disabledHere = new TreeMap(Bytes.BYTES_COMPARATOR); + Iterator> itr = disabledandDisablingTables.entrySet().iterator(); + while (itr.hasNext()) { + Entry tableEntry = itr.next(); + if (!IndexUtils.isIndexTable(tableEntry.getKey())) { + byte[] indexTableName = Bytes.toBytes(IndexUtils.getIndexTableName(tableEntry.getKey())); + if(null == tableMap.get(Bytes.toString(indexTableName))){ + continue; + } + boolean present = disabledandDisablingTables.containsKey(indexTableName); + if (!present && (enabledOrEnablingTables.get(indexTableName) == State.ENABLED)) { + // TODO How to handle ENABLING state(if it could happen). If try to disable ENABLING table + // it throws. + if (LOG.isDebugEnabled()) { + LOG.debug("Table " + Bytes.toString(tableEntry.getKey()) + + " is disabled but corresponding index table is " + "enabled. So disabling " + + Bytes.toString(indexTableName)); + } + this.admin.disableTable(indexTableName); + disabledHere.put(indexTableName,State.DISABLED); + } + } else { + if (tableEntry.getValue() != State.DISABLED) { + continue; + } + byte[] userTableName = Bytes.toBytes(IndexUtils.getActualTableName(Bytes.toString(tableEntry.getKey()))); + if(!disabledandDisablingTables.containsKey(userTableName)){ + if (LOG.isDebugEnabled()) { + LOG.debug("Index Table " + Bytes.toString(tableEntry.getKey()) + + " is disabled but corresponding user table is enabled. So Enabling " + + Bytes.toString(tableEntry.getKey())); + } + // Here we are not enabling the table. We will do it in the next step checkMetaInfoCosistency(). + // Because if we do here, META will be updated and our in-memory map will have old entries. + // So it will surely cause unnecessary unassignments and assignments in the next step. In the next + // step anyway we are moving regions. So no problem doing it there. + // this.admin.enableTable(tableName); + itr.remove(); + } + } + } + disabledandDisablingTables.putAll(disabledHere); + } + } + + + private ZooKeeperWatcher createZooKeeperWatcher() throws IOException { + return new ZooKeeperWatcher(this.conf, "sec index colocation", new Abortable() { + @Override + public void abort(String why, Throwable e) { + LOG.error(why, e); + System.exit(1); + } + + @Override + public boolean isAborted() { + return false; + } + + }); + } + +} \ No newline at end of file Index: hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/TestColumnQualifier.java =================================================================== --- hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/TestColumnQualifier.java (revision 0) +++ hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/TestColumnQualifier.java (working copy) @@ -0,0 +1,101 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; + +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.index.ColumnQualifier.ValueType; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test setting values in ColumnQuafilier. + */ +@Category(MediumTests.class) +public class TestColumnQualifier { + + @Test + public void testColumnQualifier() throws Exception { + ColumnQualifier cq = new ColumnQualifier("cf", "cq"); + assertEquals("Column family not match with acutal value.", "cf", cq.getColumnFamilyString()); + assertEquals("Column qualifier not match with actual value.", "cq", cq.getQualifierString()); + assertEquals("Column family bytes not match with actual value.", 0, + Bytes.compareTo(Bytes.toBytes("cf"), cq.getColumnFamily())); + assertEquals("Column qualifier bytes not match with actual value.", 0, + Bytes.compareTo(Bytes.toBytes("cq"), cq.getQualifier())); + } + + public void testColumQualifierEquals() { + ColumnQualifier cq = new ColumnQualifier(Bytes.toBytes("cf"), Bytes.toBytes("cq")); + assertTrue("ColumnQualifier state mismatch.", + cq.equals(new ColumnQualifier(Bytes.toBytes("cf"), Bytes.toBytes("cq")))); + assertTrue("ColumnQualifier state mismatch.", cq.equals(new ColumnQualifier("cf", "cq"))); + } + + @Test + public void testColumnQualifierSerialization() throws Exception { + ByteArrayOutputStream bos = null; + DataOutputStream dos = null; + ByteArrayInputStream bis = null; + DataInputStream dis = null; + try { + bos = new ByteArrayOutputStream(); + dos = new DataOutputStream(bos); + ColumnQualifier cq = + new ColumnQualifier("cf", "cq", ValueType.String, 10, new SeparatorPartition("--", 10)); + cq.write(dos); + dos.flush(); + byte[] byteArray = bos.toByteArray(); + bis = new ByteArrayInputStream(byteArray); + dis = new DataInputStream(bis); + ColumnQualifier c = new ColumnQualifier(); + c.readFields(dis); + ValuePartition vp = c.getValuePartition(); + if (vp instanceof SeparatorPartition) { + + } else { + fail("value partition details no read properly."); + } + assertTrue("ColumnQualifier state mismatch.", c.equals(cq)); + } finally { + if (null != bos) { + bos.close(); + } + if (null != dos) { + dos.close(); + } + if (null != bis) { + bis.close(); + } + if (null != dis) { + dis.close(); + } + } + + } + +} Index: hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/TestIndexSpecification.java =================================================================== --- hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/TestIndexSpecification.java (revision 0) +++ hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/TestIndexSpecification.java (working copy) @@ -0,0 +1,283 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.util.Set; + +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.index.ColumnQualifier.ValueType; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test whether {@link ColumnQualifier} details provided for {@link IndexSpecification} are valid or + * not. + */ +@Category(MediumTests.class) +public class TestIndexSpecification { + + @Test + public void testInvalidIndexSpecName() throws Exception { + // Contains => + try { + new IndexSpecification("index=>name"); + fail("IllegalArgexception should be thrown."); + } catch (IllegalArgumentException e) { + } + // Contains , + try { + new IndexSpecification("index,name"); + fail("IllegalArgexception should be thrown."); + } catch (IllegalArgumentException e) { + } + // Contains ? + try { + new IndexSpecification("index?name"); + fail("IllegalArgexception should be thrown."); + } catch (IllegalArgumentException e) { + } + // Contains numbers + try { + new IndexSpecification("index0name"); + } catch (IllegalArgumentException e) { + fail("Illegal Arg should not be thrown."); + } + // Contains numbers at beginning and end + try { + new IndexSpecification("0index0name0"); + } catch (IllegalArgumentException e) { + fail("Illegal Arg should not be thrown."); + } + // Contains '.' at the beginning + try { + new IndexSpecification(".indexname"); + fail("IllegalArgexception should be thrown."); + } catch (IllegalArgumentException e) { + } + // Contains "-" at the beginnning + try { + new IndexSpecification("-indexname"); + fail("IllegalArgexception should be thrown."); + } catch (IllegalArgumentException e) { + } + // Contains '.' in the middle + try { + new IndexSpecification("index.name"); + } catch (IllegalArgumentException e) { + fail("Illegal Arg should not be thrown."); + } + // Contains '-' in the middle + try { + new IndexSpecification("index-name"); + } catch (IllegalArgumentException e) { + fail("Illegal Arg should not be thrown."); + } + } + + @Test + public void testGetName() throws Exception { + IndexSpecification iSpec = new IndexSpecification("index_name"); + assertEquals("Index name not match with acutal index name.", "index_name", iSpec.getName()); + iSpec = new IndexSpecification("index_name"); + assertEquals("Index name not match with acutal index name.", "index_name", iSpec.getName()); + + } + + @Test + public void testAddIndexColumnWithNotNull() throws Exception { + IndexSpecification iSpec = new IndexSpecification("index_name"); + Set indexColumns = iSpec.getIndexColumns(); + assertEquals("Size of column qualifiers list should be zero.", 0, indexColumns.size()); + iSpec.addIndexColumn(new HColumnDescriptor("cf"), "cq", ValueType.String, 10); + indexColumns = iSpec.getIndexColumns(); + assertTrue("ColumnQualifier state mismatch.", + indexColumns.contains(new ColumnQualifier("cf", "cq", ValueType.String, 10))); + } + + @Test + public void testAddIndexColumnWithNullCF() throws Exception { + IndexSpecification iSpec = new IndexSpecification("index_name"); + try { + iSpec.addIndexColumn(null, "cq", ValueType.String, 10); + fail("Column family name should not be null in index specification."); + } catch (IllegalArgumentException e) { + + } + } + + @Test + public void testAddIndexColumnWithNullQualifier() throws Exception { + IndexSpecification iSpec = new IndexSpecification("index_name"); + try { + iSpec.addIndexColumn(new HColumnDescriptor("cf"), null, ValueType.String, 10); + fail("Column qualifier name should not be null in index specification."); + } catch (IllegalArgumentException e) { + + } + } + + @Test + public void testAddIndexColumnWithBlankCForQualifier() throws Exception { + IndexSpecification iSpec = new IndexSpecification("index_name"); + try { + iSpec.addIndexColumn(new HColumnDescriptor(" "), " ", ValueType.String, 10); + fail("Column family name and qualifier should not be blank."); + } catch (IllegalArgumentException e) { + + } + } + + @Test + public void testAddIndexColumnWithBlankCF() throws Exception { + IndexSpecification iSpec = new IndexSpecification("index_name"); + try { + iSpec.addIndexColumn(new HColumnDescriptor(" "), "cq", ValueType.String, 10); + fail("Column family name should not be blank."); + } catch (IllegalArgumentException e) { + + } + } + + @Test + public void testAddIndexColumnWithBlankQualifier() throws Exception { + IndexSpecification iSpec = new IndexSpecification("index_name"); + try { + iSpec.addIndexColumn(new HColumnDescriptor("cf"), " ", ValueType.String, 10); + fail("Column qualifier name should not be blank."); + } catch (IllegalArgumentException e) { + + } + } + + @Test + public void testAddIndexColumnWithControlCharactersOrColonsInCF() throws Exception { + IndexSpecification iSpec = new IndexSpecification("index_name"); + try { + iSpec.addIndexColumn(new HColumnDescriptor("cf:"), "cq", ValueType.String, 10); + fail("Family names should not contain control characters or colons."); + } catch (IllegalArgumentException e) { + + } + } + + @Test + public void testAddIndexColumnWithDuplicaCFandQualifier() throws Exception { + IndexSpecification iSpec = new IndexSpecification("index_name"); + try { + iSpec.addIndexColumn(new HColumnDescriptor("cf"), "cq", ValueType.String, 10); + iSpec.addIndexColumn(new HColumnDescriptor("cf"), "cq", ValueType.String, 10); + fail("Column familily and qualifier combination should not be" + + " repeated in Index Specification."); + } catch (IllegalArgumentException e) { + + } + } + + @Test + public void testMinTTLCalculation() throws Exception { + IndexSpecification iSpec = new IndexSpecification("index_name"); + iSpec + .addIndexColumn(new HColumnDescriptor("cf").setTimeToLive(100), "cq", ValueType.String, 10); + iSpec + .addIndexColumn(new HColumnDescriptor("cf1").setTimeToLive(50), "cq", ValueType.String, 10); + assertEquals("The ttl should be 50", 50, iSpec.getTTL()); + } + + @Test + public void testMaxVersionsCalculation() throws Exception { + IndexSpecification iSpec = new IndexSpecification("index_name"); + iSpec.addIndexColumn(new HColumnDescriptor("cf").setMaxVersions(100), "cq", ValueType.String, + 10); + iSpec.addIndexColumn(new HColumnDescriptor("cf1").setMaxVersions(50), "cq", ValueType.String, + 10); + assertEquals("The max versions should be 50", 50, iSpec.getMaxVersions()); + } + + @Test + public void testMinTTLWhenDefault() throws Exception { + IndexSpecification iSpec = new IndexSpecification("index_name"); + iSpec.addIndexColumn(new HColumnDescriptor("cf"), "cq", ValueType.String, 10); + iSpec.addIndexColumn(new HColumnDescriptor("cf1"), "cq", ValueType.String, 10); + assertEquals("The ttl should be " + Integer.MAX_VALUE, HConstants.FOREVER, iSpec.getTTL()); + } + + @Test + public void testMinTTLWhenCombinationOfDefaultAndGivenValue() throws Exception { + IndexSpecification iSpec = new IndexSpecification("index_name"); + iSpec.addIndexColumn(new HColumnDescriptor("cf"), "cq", ValueType.String, 10); + iSpec + .addIndexColumn(new HColumnDescriptor("cf1").setTimeToLive(50), "cq", ValueType.String, 10); + assertEquals("The ttl should be 50", 50, iSpec.getTTL()); + } + + @Test + public void testMaxVersionsWhenDefault() throws Exception { + IndexSpecification iSpec = new IndexSpecification("index_name"); + iSpec.addIndexColumn(new HColumnDescriptor("cf"), "cq", ValueType.String, 10); + iSpec.addIndexColumn(new HColumnDescriptor("cf1"), "cq", ValueType.String, 10); + assertEquals("default max versions should be 1", 1, iSpec.getMaxVersions()); + } + + @Test + public void testIndexSpecificationSerialization() throws Exception { + ByteArrayOutputStream bos = null; + DataOutputStream dos = null; + ByteArrayInputStream bis = null; + DataInputStream dis = null; + try { + bos = new ByteArrayOutputStream(); + dos = new DataOutputStream(bos); + IndexSpecification iSpec = new IndexSpecification("index_name"); + iSpec.addIndexColumn(new HColumnDescriptor("cf"), "cq", ValueType.String, 10); + iSpec.write(dos); + dos.flush(); + byte[] byteArray = bos.toByteArray(); + bis = new ByteArrayInputStream(byteArray); + dis = new DataInputStream(bis); + IndexSpecification indexSpecification = new IndexSpecification(); + indexSpecification.readFields(dis); + assertEquals("Index name mismatch.", indexSpecification.getName(), iSpec.getName()); + assertTrue("ColumnQualifier state mismatch.", + iSpec.getIndexColumns().contains(new ColumnQualifier("cf", "cq", ValueType.String, 10))); + } finally { + if (null != bos) { + bos.close(); + } + if (null != dos) { + dos.close(); + } + if (null != bis) { + bis.close(); + } + if (null != dis) { + dis.close(); + } + } + + } +} Index: hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/TestSecIndexLoadBalancer.java =================================================================== --- hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/TestSecIndexLoadBalancer.java (revision 0) +++ hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/TestSecIndexLoadBalancer.java (working copy) @@ -0,0 +1,505 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.MiniHBaseCluster; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.catalog.MetaReader; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.apache.hadoop.hbase.index.ColumnQualifier.ValueType; +import org.apache.hadoop.hbase.index.client.IndexAdmin; +import org.apache.hadoop.hbase.index.coprocessor.master.IndexMasterObserver; +import org.apache.hadoop.hbase.index.coprocessor.regionserver.IndexRegionObserver; +import org.apache.hadoop.hbase.index.coprocessor.wal.IndexWALObserver; +import org.apache.hadoop.hbase.index.util.IndexUtils; +import org.apache.hadoop.hbase.master.AssignmentManager; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.master.LoadBalancer; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; +import org.apache.hadoop.hbase.zookeeper.ZKAssign; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(LargeTests.class) +public class TestSecIndexLoadBalancer { + + private static HBaseTestingUtility UTIL = new HBaseTestingUtility(); + + private static HBaseAdmin admin = null; + + @BeforeClass + public static void setupBeforeClass() throws Exception { + final int NUM_MASTERS = 1; + final int NUM_RS = 4; + Configuration conf = UTIL.getConfiguration(); + conf.setBoolean(HConstants.REGIONSERVER_INFO_PORT_AUTO,true); + conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, IndexMasterObserver.class.getName()); + conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, IndexRegionObserver.class.getName()); + conf.set(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY, IndexWALObserver.class.getName()); + conf.setBoolean("hbase.use.secondary.index", true); + conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, SecIndexLoadBalancer.class, + LoadBalancer.class); + UTIL.startMiniCluster(NUM_MASTERS, NUM_RS); + admin = new IndexAdmin(UTIL.getConfiguration()); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + if(admin != null) admin.close(); + UTIL.shutdownMiniCluster(); + } + + @Test(timeout = 180000) + public void testRoundRobinAssignmentDuringIndexTableCreation() throws Exception { + MiniHBaseCluster cluster = UTIL.getHBaseCluster(); + HMaster master = cluster.getMaster(); + String tableName = "testRoundRobinAssignmentDuringIndexTableCreation"; + String indexTableName = IndexUtils.getIndexTableName(tableName); + HTableDescriptor iHtd = + TestUtils.createIndexedHTableDescriptor(tableName, "cf", "index_name", "cf", "cq"); + char c = 'A'; + byte[][] split = new byte[20][]; + for (int i = 0; i < 20; i++) { + byte[] b = { (byte) c }; + split[i] = b; + c++; + } + admin.createTable(iHtd, split); + boolean isRegionColocated = TestUtils.checkForColocation(master, tableName, indexTableName); + assertTrue("User regions and index regions should colocate.", isRegionColocated); + } + + @Test(timeout = 180000) + public void testColocationWhenNormalTableModifiedToIndexedTable() throws Exception { + MiniHBaseCluster cluster = UTIL.getHBaseCluster(); + String tableName = "testColocationWhenNormalTableModifiedToIndexedTable"; + HMaster master = cluster.getMaster(); + byte[][] split = new byte[20][]; + char c = 'A'; + for (int i = 0; i < 20; i++) { + byte[] b = { (byte) c }; + split[i] = b; + c++; + } + HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName)); + HColumnDescriptor hcd = new HColumnDescriptor("cf"); + htd.addFamily(hcd); + admin.createTable(htd, split); + IndexSpecification iSpec = new IndexSpecification("index_name"); + TableIndices tableIndices = new TableIndices(); + tableIndices.addIndex(iSpec); + iSpec.addIndexColumn(hcd, "cq", ValueType.String, 10); + htd.setValue(Constants.INDEX_SPEC_KEY, tableIndices.toByteArray()); + String indexTableName = IndexUtils.getIndexTableName(tableName); + admin.disableTable(tableName); + admin.modifyTable(tableName, htd); + admin.enableTable(tableName); + boolean isRegionColocated = TestUtils.checkForColocation(master, tableName, indexTableName); + assertTrue("User regions and index regions should colocate.", isRegionColocated); + } + + @Test(timeout = 180000) + public void testColocationAfterSplit() throws Exception { + String userTableName = "testCompactionOnIndexTableShouldNotRetrieveTTLExpiredData"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + + HColumnDescriptor hcd = new HColumnDescriptor("col").setMaxVersions(Integer.MAX_VALUE); + + HColumnDescriptor hcd1 = new HColumnDescriptor("col1").setMaxVersions(Integer.MAX_VALUE); + ihtd.addFamily(hcd); + ihtd.addFamily(hcd1); + IndexSpecification iSpec = new IndexSpecification("ScanIndexf"); + iSpec.addIndexColumn(hcd, "q1", ValueType.String, 10); + iSpec.addIndexColumn(hcd1, "q2", ValueType.String, 10); + TableIndices tableIndices = new TableIndices(); + tableIndices.addIndex(iSpec); + ihtd.setValue(Constants.INDEX_SPEC_KEY, tableIndices.toByteArray()); + admin.createTable(ihtd); + TableName indexTableName = TableName.valueOf(IndexUtils.getIndexTableName(userTableName)); + Configuration conf = UTIL.getConfiguration(); + HTable table = new HTable(conf, userTableName); + + // test put with the indexed column + + Put p = new Put("row11".getBytes()); + p.add(Bytes.toBytes("col"), Bytes.toBytes("q1"), Bytes.toBytes("Val")); + p.add("col1".getBytes(), "q2".getBytes(), Bytes.toBytes("ValForCF2")); + table.put(p); + + Put p1 = new Put("row21".getBytes()); + p1.add(Bytes.toBytes("col"), Bytes.toBytes("q1"), Bytes.toBytes("Val")); + p1.add("col1".getBytes(), "q2".getBytes(), Bytes.toBytes("ValForCF2")); + table.put(p1); + + Put p2 = new Put("row31".getBytes()); + p2.add(Bytes.toBytes("col"), Bytes.toBytes("q1"), Bytes.toBytes("Val")); + p2.add("col1".getBytes(), "q2".getBytes(), Bytes.toBytes("ValForCF2")); + table.put(p2); + + Put p3 = new Put("row41".getBytes()); + p3.add(Bytes.toBytes("col"), Bytes.toBytes("q1"), Bytes.toBytes("Val")); + p3.add("col1".getBytes(), "q2".getBytes(), Bytes.toBytes("ValForCF2")); + table.put(p3); + + Put p4 = new Put("row51".getBytes()); + p4.add(Bytes.toBytes("col"), Bytes.toBytes("q1"), Bytes.toBytes("Val")); + p4.add("col1".getBytes(), "q2".getBytes(), Bytes.toBytes("ValForCF2")); + table.put(p4); + + admin.flush(userTableName); + admin.flush(indexTableName.getNameAsString()); + + admin.split(userTableName, "row31"); + MiniHBaseCluster cluster = UTIL.getHBaseCluster(); + HMaster master = cluster.getMaster(); + List regionsOfUserTable = + master.getAssignmentManager().getRegionStates().getRegionsOfTable(TableName.valueOf(userTableName)); + + while (regionsOfUserTable.size() != 2) { + Thread.sleep(100); + regionsOfUserTable = + master.getAssignmentManager().getRegionStates().getRegionsOfTable(TableName.valueOf(userTableName)); + } + + List regionsOfIndexTable = + master.getAssignmentManager().getRegionStates().getRegionsOfTable(indexTableName); + + while (regionsOfIndexTable.size() != 2) { + Thread.sleep(100); + regionsOfIndexTable = + master.getAssignmentManager().getRegionStates().getRegionsOfTable(indexTableName); + } + + for (HRegionInfo hRegionInfo : regionsOfUserTable) { + for (HRegionInfo indexRegionInfo : regionsOfIndexTable) { + if (Bytes.equals(hRegionInfo.getStartKey(), indexRegionInfo.getStartKey())) { + assertEquals("The regions should be colocated", master.getAssignmentManager() + .getRegionStates().getRegionServerOfRegion(hRegionInfo), master.getAssignmentManager() + .getRegionStates().getRegionServerOfRegion(indexRegionInfo)); + } + } + } + + } + + @Test(timeout = 180000) + public void testRandomAssignmentDuringIndexTableEnable() throws Exception { + MiniHBaseCluster cluster = UTIL.getHBaseCluster(); + HMaster master = cluster.getMaster(); + master.getConfiguration().setBoolean("hbase.master.enabletable.roundrobin", false); + HTableDescriptor iHtd = + TestUtils.createIndexedHTableDescriptor("testRandomAssignmentDuringIndexTableEnable", "cf", + "index_name", "cf", "cq"); + char c = 'A'; + byte[][] split = new byte[3][]; + for (int i = 0; i < 3; i++) { + byte[] b = { (byte) c }; + split[i] = b; + c++; + } + admin.createTable(iHtd, split); + + String tableName = "testRandomAssignmentDuringIndexTableEnable"; + String indexTableName = + "testRandomAssignmentDuringIndexTableEnable" + Constants.INDEX_TABLE_SUFFIX; + admin.disableTable(tableName); + admin.enableTable(tableName); + boolean isRegionColocated = TestUtils.checkForColocation(master, tableName, indexTableName); + assertTrue("User regions and index regions should colocate.", isRegionColocated); + + } + + @Test(timeout = 180000) + public void testBalanceCluster() throws Exception { + MiniHBaseCluster cluster = UTIL.getHBaseCluster(); + HMaster master = cluster.getMaster(); + master.getConfiguration().setBoolean("hbase.master.enabletable.roundrobin", false); + master.getConfiguration().setBoolean("hbase.master.startup.retainassign", false); + master.getConfiguration().setBoolean("hbase.master.loadbalance.bytable", false); + String tableName = "testBalanceCluster"; + + HTableDescriptor iHtd = + TestUtils.createIndexedHTableDescriptor(tableName, "cf", "index_name", "cf", "cq"); + HTableDescriptor htd = + new HTableDescriptor(TableName.valueOf("testBalanceCluster1")); + htd.addFamily(new HColumnDescriptor("fam1")); + + char c = 'A'; + byte[][] split = new byte[20][]; + for (int i = 0; i < 20; i++) { + byte[] b = { (byte) c }; + split[i] = b; + c++; + } + admin.createTable(iHtd, split); + c = 'A'; + byte[][] split1 = new byte[12][]; + for (int i = 0; i < 12; i++) { + byte[] b = { (byte) c }; + split1[i] = b; + c++; + } + admin.setBalancerRunning(false, false); + admin.createTable(htd, split1); + admin.disableTable(tableName); + admin.enableTable(tableName); + admin.setBalancerRunning(true, false); + admin.balancer(); + boolean isRegionsColocated = + TestUtils.checkForColocation(master, tableName, IndexUtils.getIndexTableName(tableName)); + assertTrue("User regions and index regions should colocate.", isRegionsColocated); + } + + @Test//(timeout = 180000) + public void testBalanceByTable() throws Exception { + ZooKeeperWatcher zkw = UTIL.getZooKeeperWatcher(UTIL); + MiniHBaseCluster cluster = UTIL.getHBaseCluster(); + HMaster master = cluster.getMaster(); + master.getConfiguration().setBoolean("hbase.master.loadbalance.bytable", true); + admin.setBalancerRunning(false, false); + HTableDescriptor iHtd = + TestUtils.createIndexedHTableDescriptor("testBalanceByTable", "cf", + "index_name", "cf", "cq"); + HTableDescriptor htd = + new HTableDescriptor(TableName.valueOf("testBalanceByTable1")); + htd.addFamily(new HColumnDescriptor("fam1")); + + char c = 'A'; + byte[][] split = new byte[20][]; + for (int i = 0; i < 20; i++) { + byte[] b = { (byte) c }; + split[i] = b; + c++; + } + admin.createTable(iHtd, split); + c = 'A'; + byte[][] split1 = new byte[12][]; + for (int i = 0; i < 12; i++) { + byte[] b = { (byte) c }; + split1[i] = b; + c++; + } + admin.createTable(htd, split1); + String tableName = "testBalanceByTable"; + String indexTableName = IndexUtils.getIndexTableName(tableName); + admin.disableTable(tableName); + admin.enableTable(tableName); + admin.setBalancerRunning(true, false); + boolean isRegionColocated = TestUtils.checkForColocation(master, tableName, indexTableName); + assertTrue("User regions and index regions should colocate.", isRegionColocated); + admin.balancer(); + Thread.sleep(10000); + ZKAssign.blockUntilNoRIT(zkw); + master.getAssignmentManager().waitUntilNoRegionsInTransition(10000); + isRegionColocated = TestUtils.checkForColocation(master, tableName, indexTableName); + assertTrue("User regions and index regions should colocate.", isRegionColocated); + } + + @Test(timeout = 180000) + public void testRoundRobinAssignmentAfterRegionServerDown() throws Exception { + ZooKeeperWatcher zkw = UTIL.getZooKeeperWatcher(UTIL); + MiniHBaseCluster cluster = UTIL.getHBaseCluster(); + HMaster master = cluster.getMaster(); + HTableDescriptor iHtd = + TestUtils.createIndexedHTableDescriptor("testRoundRobinAssignmentAfterRegionServerDown", "cf", + "index_name", "cf", "cq"); + char c = 'A'; + byte[][] split = new byte[20][]; + for (int i = 0; i < 20; i++) { + byte[] b = { (byte) c }; + split[i] = b; + c++; + } + admin.createTable(iHtd, split); + + String tableName = "testRoundRobinAssignmentAfterRegionServerDown"; + String indexTableName = + "testRoundRobinAssignmentAfterRegionServerDown" + Constants.INDEX_TABLE_SUFFIX; + HRegionServer regionServer = cluster.getRegionServer(1); + regionServer.abort("Aborting to test random assignment after region server down"); + while (master.getServerManager().areDeadServersInProgress()) { + Thread.sleep(1000); + } + int totalNumRegions = 21; + List> iTableStartKeysAndLocations = null; + do { + iTableStartKeysAndLocations = getStartKeysAndLocations(master, indexTableName); + System.out.println("wait until all regions comeup " + iTableStartKeysAndLocations.size()); + Thread.sleep(1000); + } while (totalNumRegions > iTableStartKeysAndLocations.size()); + ZKAssign.blockUntilNoRIT(zkw); + boolean isRegionColocated = TestUtils.checkForColocation(master, tableName, indexTableName); + assertTrue("User regions and index regions should colocate.", isRegionColocated); + + } + + @Test(timeout = 180000) + public void testRegionMove() throws Exception { + MiniHBaseCluster cluster = UTIL.getHBaseCluster(); + HMaster master = cluster.getMaster(); + ZooKeeperWatcher zkw = UTIL.getZooKeeperWatcher(UTIL); + TableName tableName = TableName.valueOf("testRegionMove"); + HTableDescriptor iHtd = new HTableDescriptor(tableName); + HColumnDescriptor hcd = new HColumnDescriptor("col").setMaxVersions(Integer.MAX_VALUE); + IndexSpecification iSpec = new IndexSpecification("ScanIndexf"); + iSpec.addIndexColumn(hcd, "q1", ValueType.String, 10); + TableIndices tableIndices = new TableIndices(); + tableIndices.addIndex(iSpec); + iHtd.addFamily(hcd); + iHtd.setValue(Constants.INDEX_SPEC_KEY, tableIndices.toByteArray()); + char c = 'A'; + byte[][] split = new byte[4][]; + for (int i = 0; i < 4; i++) { + byte[] b = { (byte) c }; + split[i] = b; + c++; + } + admin.createTable(iHtd, split); + List tableRegions = + MetaReader.getTableRegions(master.getCatalogTracker(), + iHtd.getTableName()); + int numRegions = cluster.getRegionServerThreads().size(); + cluster.getRegionServer(1).getServerName(); + Random random = new Random(); + for (HRegionInfo hRegionInfo : tableRegions) { + int regionNumber = random.nextInt(numRegions); + ServerName serverName = cluster.getRegionServer(regionNumber).getServerName(); + admin.move(hRegionInfo.getEncodedNameAsBytes(), Bytes.toBytes(serverName.getServerName())); + } + ZKAssign.blockUntilNoRIT(zkw); + AssignmentManager am = UTIL.getHBaseCluster().getMaster().getAssignmentManager(); + while(!am.waitUntilNoRegionsInTransition(1000)); + boolean isRegionColocated = + TestUtils.checkForColocation(master, tableName.getNameAsString(), + IndexUtils.getIndexTableName(tableName)); + assertTrue("User regions and index regions should colocate.", isRegionColocated); + + } + + @Test//(timeout = 180000) + public void testRetainAssignmentDuringMasterStartUp() throws Exception { + ZooKeeperWatcher zkw = UTIL.getZooKeeperWatcher(UTIL); + MiniHBaseCluster cluster = UTIL.getHBaseCluster(); + HMaster master = cluster.getMaster(); + master.getConfiguration().setBoolean("hbase.master.startup.retainassign", true); + HTableDescriptor iHtd = + TestUtils.createIndexedHTableDescriptor("testRetainAssignmentDuringMasterStartUp", "cf", + "index_name", "cf", "cq"); + char c = 'A'; + byte[][] split = new byte[20][]; + for (int i = 0; i < 20; i++) { + byte[] b = { (byte) c }; + split[i] = b; + c++; + } + admin.createTable(iHtd, split); + String tableName = "testRetainAssignmentDuringMasterStartUp"; + String indexTableName = + "testRetainAssignmentDuringMasterStartUp" + Constants.INDEX_TABLE_SUFFIX; + UTIL.shutdownMiniHBaseCluster(); + UTIL.startMiniHBaseCluster(1, 4); + cluster = UTIL.getHBaseCluster(); + master = cluster.getMaster(); + ZKAssign.blockUntilNoRIT(zkw); + if (admin != null) { + admin.close(); + admin = new IndexAdmin(cluster.getMaster().getConfiguration()); + } + master.getAssignmentManager().waitUntilNoRegionsInTransition(1000); + boolean isRegionColocated = TestUtils.checkForColocation(master, tableName, indexTableName); + assertTrue("User regions and index regions should colocate.", isRegionColocated); + + } + + @Test(timeout = 180000) + public void testRoundRobinAssignmentDuringMasterStartUp() throws Exception { + MiniHBaseCluster cluster = UTIL.getHBaseCluster(); + HMaster master = cluster.getMaster(); + master.getConfiguration().setBoolean("hbase.master.startup.retainassign", false); + + HTableDescriptor iHtd = + TestUtils.createIndexedHTableDescriptor("testRoundRobinAssignmentDuringMasterStartUp", + "cf", "index_name", "cf", "cq"); + char c = 'A'; + byte[][] split = new byte[20][]; + for (int i = 0; i < 20; i++) { + byte[] b = { (byte) c }; + split[i] = b; + c++; + } + admin.createTable(iHtd, split); + String tableName = "testRoundRobinAssignmentDuringMasterStartUp"; + String indexTableName = + "testRoundRobinAssignmentDuringMasterStartUp" + Constants.INDEX_TABLE_SUFFIX; + UTIL.shutdownMiniHBaseCluster(); + UTIL.startMiniHBaseCluster(1, 4); + cluster = UTIL.getHBaseCluster(); + if (admin != null) { + admin.close(); + admin = new IndexAdmin(cluster.getMaster().getConfiguration()); + } + master = cluster.getMaster(); + boolean isRegionColocated = TestUtils.checkForColocation(master, tableName, indexTableName); + assertTrue("User regions and index regions should colocate.", isRegionColocated); + + } + + private List> getStartKeysAndLocations(HMaster master, String tableName) + throws IOException, InterruptedException { + + List> tableRegionsAndLocations = + MetaReader.getTableRegionsAndLocations(master.getCatalogTracker(), + TableName.valueOf(tableName)); + List> startKeyAndLocationPairs = + new ArrayList>(tableRegionsAndLocations.size()); + Pair startKeyAndLocation = null; + for (Pair regionAndLocation : tableRegionsAndLocations) { + startKeyAndLocation = + new Pair(regionAndLocation.getFirst().getStartKey(), + regionAndLocation.getSecond()); + startKeyAndLocationPairs.add(startKeyAndLocation); + } + return startKeyAndLocationPairs; + + } + +} Index: hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/TestTableIndices.java =================================================================== --- hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/TestTableIndices.java (revision 0) +++ hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/TestTableIndices.java (working copy) @@ -0,0 +1,104 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.index.ColumnQualifier.ValueType; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test whether index specification details valid or not + */ +@Category(MediumTests.class) +public class TestTableIndices { + + @Test + public void testAddIndex() throws Exception { + TableIndices indices = new TableIndices(); + IndexSpecification iSpec = new IndexSpecification("index_name"); + iSpec.addIndexColumn(new HColumnDescriptor("cf"), "cq", ValueType.String, 10); + indices.addIndex(iSpec); + assertEquals("Index name should be equal with actual value.", "index_name", indices.getIndices().get(0) + .getName()); + assertTrue( + "Column qualifier state mismatch.", + indices.getIndices().get(0).getIndexColumns() + .contains(new ColumnQualifier("cf", "cq", ValueType.String, 10))); + + } + + @Test + public void testAddIndexWithDuplicaIndexNames() throws Exception { + TableIndices indices = new TableIndices(); + IndexSpecification iSpec = null; + try { + iSpec = new IndexSpecification("index_name"); + iSpec.addIndexColumn(new HColumnDescriptor("cf"), "cq", ValueType.String, 10); + indices.addIndex(iSpec); + iSpec = new IndexSpecification("index_name"); + iSpec.addIndexColumn(new HColumnDescriptor("cf"), "cq", ValueType.String, 10); + indices.addIndex(iSpec); + fail("Duplicate index names should not present for same table."); + } catch (IllegalArgumentException e) { + + } + + } + + @Test + public void testAddIndexWithIndexNameLengthGreaterThanMaxLength() throws Exception { + TableIndices indices = new TableIndices(); + IndexSpecification iSpec = null; + try { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < 7; i++) { + sb.append("index_name"); + } + iSpec = new IndexSpecification(new String(sb)); + iSpec.addIndexColumn(new HColumnDescriptor("cf"), "cq", ValueType.String, 10); + indices.addIndex(iSpec); + fail("Index name length should not be more than maximum length " + + Constants.DEF_MAX_INDEX_NAME_LENGTH + '.'); + } catch (IllegalArgumentException e) { + + } + } + + @Test + public void testAddIndexWithBlankIndexName() throws Exception { + TableIndices indices = new TableIndices(); + + IndexSpecification iSpec = null; + try { + iSpec = new IndexSpecification(" "); + iSpec.addIndexColumn(new HColumnDescriptor("cf"), "cq", null, 10); + indices.addIndex(iSpec); + fail("Index name should not be blank in Index Specification"); + } catch (IllegalArgumentException e) { + + } + + } + +} Index: hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/TestUtils.java =================================================================== --- hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/TestUtils.java (revision 0) +++ hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/TestUtils.java (working copy) @@ -0,0 +1,111 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.catalog.MetaReader; +import org.apache.hadoop.hbase.index.ColumnQualifier.ValueType; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; +import org.junit.experimental.categories.Category; + +@Category(LargeTests.class) +public class TestUtils { + + public static HTableDescriptor createIndexedHTableDescriptor(String tableName, + String columnFamily, String indexName, String indexColumnFamily, String indexColumnQualifier) + throws IOException { + HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName)); + IndexSpecification iSpec = new IndexSpecification(indexName); + HColumnDescriptor hcd = new HColumnDescriptor(columnFamily); + TableIndices tableIndices = new TableIndices(); + tableIndices.addIndex(iSpec); + iSpec.addIndexColumn(hcd, indexColumnQualifier, ValueType.String, 10); + htd.addFamily(hcd); + htd.setValue(Constants.INDEX_SPEC_KEY, tableIndices.toByteArray()); + return htd; + } + + public static void waitUntilIndexTableCreated(HMaster master, String tableName) + throws IOException, InterruptedException { + boolean isEnabled = false; + boolean isExist = false; + do { + isExist = MetaReader.tableExists(master.getCatalogTracker(), TableName.valueOf(tableName)); + isEnabled = + master.getAssignmentManager().getZKTable().isEnabledTable(TableName.valueOf(tableName)); + Thread.sleep(1000); + } while ((false == isExist) && (false == isEnabled)); + } + + public static List> getStartKeysAndLocations(HMaster master, + String tableName) throws IOException, InterruptedException { + + List> tableRegionsAndLocations = + MetaReader.getTableRegionsAndLocations(master.getCatalogTracker(), + TableName.valueOf(tableName)); + List> startKeyAndLocationPairs = + new ArrayList>(tableRegionsAndLocations.size()); + Pair startKeyAndLocation = null; + for (Pair regionAndLocation : tableRegionsAndLocations) { + startKeyAndLocation = + new Pair(regionAndLocation.getFirst().getStartKey(), + regionAndLocation.getSecond()); + startKeyAndLocationPairs.add(startKeyAndLocation); + } + return startKeyAndLocationPairs; + + } + + public static boolean checkForColocation(HMaster master, String tableName, String indexTableName) + throws IOException, InterruptedException { + List> uTableStartKeysAndLocations = + getStartKeysAndLocations(master, tableName); + List> iTableStartKeysAndLocations = + getStartKeysAndLocations(master, indexTableName); + + boolean regionsColocated = true; + if (uTableStartKeysAndLocations.size() != iTableStartKeysAndLocations.size()) { + regionsColocated = false; + } else { + for (int i = 0; i < uTableStartKeysAndLocations.size(); i++) { + Pair uStartKeyAndLocation = uTableStartKeysAndLocations.get(i); + Pair iStartKeyAndLocation = iTableStartKeysAndLocations.get(i); + + if (Bytes.compareTo(uStartKeyAndLocation.getFirst(), iStartKeyAndLocation.getFirst()) == 0) { + if (uStartKeyAndLocation.getSecond().equals(iStartKeyAndLocation.getSecond())) { + continue; + } + } + regionsColocated = false; + } + } + return regionsColocated; + } + +} Index: hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/TestValuePartitionInScan.java =================================================================== --- hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/TestValuePartitionInScan.java (revision 0) +++ hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/TestValuePartitionInScan.java (working copy) @@ -0,0 +1,600 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; +import org.apache.hadoop.hbase.filter.FilterList; +import org.apache.hadoop.hbase.filter.FilterList.Operator; +import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; +import org.apache.hadoop.hbase.index.ColumnQualifier.ValueType; +import org.apache.hadoop.hbase.index.client.EqualsExpression; +import org.apache.hadoop.hbase.index.client.IndexAdmin; +import org.apache.hadoop.hbase.index.client.IndexUtils; +import org.apache.hadoop.hbase.index.client.SingleIndexExpression; +import org.apache.hadoop.hbase.index.coprocessor.master.IndexMasterObserver; +import org.apache.hadoop.hbase.index.coprocessor.regionserver.IndexRegionObserver; +import org.apache.hadoop.hbase.index.coprocessor.wal.IndexWALObserver; +import org.apache.hadoop.hbase.index.filter.SingleColumnValuePartitionFilter; +import org.apache.hadoop.hbase.master.LoadBalancer; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(LargeTests.class) +public class TestValuePartitionInScan { + + private static HBaseTestingUtility UTIL = new HBaseTestingUtility(); + private static HBaseAdmin admin = null; + + @BeforeClass + public static void setupBeforeClass() throws Exception { + Configuration conf = UTIL.getConfiguration(); + conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, IndexMasterObserver.class.getName()); + conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, IndexRegionObserver.class.getName()); + conf.set(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY, IndexWALObserver.class.getName()); + conf.setBoolean("hbase.hregion.scan.loadColumnFamiliesOnDemand", false); + conf.setBoolean("hbase.use.secondary.index", true); + conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, SecIndexLoadBalancer.class, + LoadBalancer.class); + UTIL.startMiniCluster(1); + admin = new IndexAdmin(conf); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + if (admin != null) admin.close(); + UTIL.shutdownMiniCluster(); + } + + @After + public void tearDown() throws Exception { + IndexRegionObserver.setIsTestingEnabled(false); + } + + @Before + public void setUp() throws Exception { + IndexRegionObserver.setIndexedFlowUsed(false); + IndexRegionObserver.setSeekpointAdded(false); + IndexRegionObserver.setSeekPoints(null); + IndexRegionObserver.setIsTestingEnabled(true); + IndexRegionObserver.addSeekPoints(null); + } + + @Test(timeout = 180000) + public void testSeparatorPartition() throws Exception { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testSeparatorPartition"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + ValuePartition vp = new SeparatorPartition("_", 3); + IndexSpecification iSpec = new IndexSpecification("idx1"); + iSpec.addIndexColumn(hcd, "cq", vp, ValueType.String, 200); + TableIndices indices = new TableIndices(); + indices.addIndex(iSpec); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + HTable table = new HTable(conf, "testSeparatorPartition"); + byte[] value1 = "2ndFloor_solitaire_huawei_bangalore_karnataka".getBytes(); + Put p = new Put("row".getBytes()); + p.add("cf1".getBytes(), "cq".getBytes(), value1); + table.put(p); + p = new Put("row2".getBytes()); + p.add("cf1".getBytes(), "cq".getBytes(), + "7thFloor_solitaire_huawei_bangalore_karnataka".getBytes()); + table.put(p); + + p = new Put("row3".getBytes()); + p.add("cf1".getBytes(), "cq".getBytes(), "rrr_sss_hhh_bangalore_karnataka".getBytes()); + table.put(p); + + Scan scan = new Scan(); + scan.setCaching(1); + scan.setFilter(new SingleColumnValuePartitionFilter(hcd.getName(), "cq".getBytes(), + CompareOp.EQUAL, "huawei".getBytes(), vp)); + int i = 0; + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + assertTrue("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); + assertTrue("Seekpoints should get added by index scanner", + IndexRegionObserver.getSeekpointAdded()); + assertEquals("It should get two seek points from index scanner.", 2, IndexRegionObserver + .getMultipleSeekPoints().size()); + assertEquals("Overall result should have only 2 rows", 2, testRes.size()); + } + + @Test(timeout = 180000) + public void testSpatialPartition() throws Exception { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testSpatialPartition"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + ValuePartition vp = new SpatialPartition(2, 3); + IndexSpecification iSpec = new IndexSpecification("idx1"); + iSpec.addIndexColumn(hcd, "cq", vp, ValueType.String, 200); + TableIndices indices = new TableIndices(); + indices.addIndex(iSpec); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + HTable table = new HTable(conf, "testSpatialPartition"); + byte[] value1 = "helloworld".getBytes(); + Put p = new Put("row".getBytes()); + p.add("cf1".getBytes(), "cq".getBytes(), value1); + table.put(p); + p = new Put("row2".getBytes()); + p.add("cf1".getBytes(), "cq".getBytes(), "spatial".getBytes()); + table.put(p); + + p = new Put("row3".getBytes()); + p.add("cf1".getBytes(), "cq".getBytes(), "partition".getBytes()); + table.put(p); + + Scan scan = new Scan(); + scan.setFilter(new SingleColumnValuePartitionFilter(hcd.getName(), "cq".getBytes(), + CompareOp.EQUAL, "rti".getBytes(), vp)); + int i = 0; + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + assertTrue("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); + assertTrue("Seekpoints should get added by index scanner", + IndexRegionObserver.getSeekpointAdded()); + assertEquals("It should get 1 seek point from index scanner.", 1, IndexRegionObserver + .getMultipleSeekPoints().size()); + assertEquals("Overall result should have only 1 rows", 1, testRes.size()); + } + + @Test//(timeout = 180000) + public void testSpatialPartitionIfMulitplePartsOfValueAreIndexedByDifferentIndicesOnSameColumn() + throws Exception { + Configuration conf = UTIL.getConfiguration(); + String userTableName = + "testSpatialPartitionIfMulitplePartsOfValueAreIndexedByDifferentIndicesOnSameColumn"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + TableIndices indices = new TableIndices(); + ValuePartition vp = new SpatialPartition(2, 3); + IndexSpecification iSpec = new IndexSpecification("idx1"); + iSpec.addIndexColumn(hcd, "cq", vp, ValueType.String, 10); + indices.addIndex(iSpec); + ValuePartition vp2 = new SpatialPartition(5, 2); + iSpec = new IndexSpecification("idx2"); + iSpec.addIndexColumn(hcd, "cq", vp2, ValueType.String, 10); + indices.addIndex(iSpec); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + byte[] value1 = "helloworldmultiple".getBytes(); + Put p = new Put("row".getBytes()); + p.add("cf1".getBytes(), "cq".getBytes(), value1); + table.put(p); + p = new Put("row2".getBytes()); + p.add("cf1".getBytes(), "cq".getBytes(), "spatialmultiple".getBytes()); + table.put(p); + + p = new Put("row3".getBytes()); + p.add("cf1".getBytes(), "cq".getBytes(), "partitionmultiple".getBytes()); + table.put(p); + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + masterFilter.addFilter(new SingleColumnValuePartitionFilter(hcd.getName(), "cq".getBytes(), + CompareOp.EQUAL, "rti".getBytes(), vp)); + masterFilter.addFilter(new SingleColumnValuePartitionFilter(hcd.getName(), "cq".getBytes(), + CompareOp.GREATER_OR_EQUAL, "ti".getBytes(), vp2)); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + int i = 0; + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + assertTrue("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); + assertTrue("Seekpoints should get added by index scanner", + IndexRegionObserver.getSeekpointAdded()); + assertEquals("It should get two seek points from index scanner.", 1, IndexRegionObserver + .getMultipleSeekPoints().size()); + assertEquals("Overall result should have only 1 rows", 1, testRes.size()); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + masterFilter.addFilter(new SingleColumnValuePartitionFilter(hcd.getName(), "cq".getBytes(), + CompareOp.LESS, "rti".getBytes(), vp)); + masterFilter.addFilter(new SingleColumnValuePartitionFilter(hcd.getName(), "cq".getBytes(), + CompareOp.GREATER, "ti".getBytes(), vp2)); + scan = new Scan(); + scan.setFilter(masterFilter); + i = 0; + scanner = table.getScanner(scan); + testRes = new ArrayList(); + result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + assertTrue("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); + assertTrue("Seekpoints should get added by index scanner", + IndexRegionObserver.getSeekpointAdded()); + assertEquals("It should get two seek points from index scanner.", 3, IndexRegionObserver + .getMultipleSeekPoints().size()); + assertTrue("Overall result should have only 2 rows", testRes.size() == 2); + + } + + @Test(timeout = 180000) + public void + testSeparatorPartitionIfMulitplePartsOfValueAreIndexedByDifferentIndicesOnSameColumn() + throws Exception { + Configuration conf = UTIL.getConfiguration(); + String userTableName = + "testSeparatorPartitionIfMulitplePartsOfValueAreIndexedByDifferentIndicesOnSameColumn"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + TableIndices indices = new TableIndices(); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + ValuePartition vp = new SeparatorPartition("--", 3); + IndexSpecification iSpec = new IndexSpecification("idx1"); + iSpec.addIndexColumn(hcd, "cq", vp, ValueType.String, 200); + indices.addIndex(iSpec); + ValuePartition vp2 = new SeparatorPartition("--", 2); + iSpec = new IndexSpecification("idx2"); + iSpec.addIndexColumn(hcd, "cq", vp2, ValueType.String, 200); + indices.addIndex(iSpec); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + byte[] value1 = "hello--world--multiple--1".getBytes(); + Put p = new Put("row".getBytes()); + p.add("cf1".getBytes(), "cq".getBytes(), value1); + table.put(p); + p = new Put("row2".getBytes()); + p.add("cf1".getBytes(), "cq".getBytes(), "spatial--partition--multiple".getBytes()); + table.put(p); + + p = new Put("row3".getBytes()); + p.add("cf1".getBytes(), "cq".getBytes(), "partition--by--separator--multiple".getBytes()); + table.put(p); + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + masterFilter.addFilter(new SingleColumnValuePartitionFilter(hcd.getName(), "cq".getBytes(), + CompareOp.EQUAL, "multiple".getBytes(), vp)); + masterFilter.addFilter(new SingleColumnValuePartitionFilter(hcd.getName(), "cq".getBytes(), + CompareOp.GREATER_OR_EQUAL, "by".getBytes(), vp2)); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + int i = 0; + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + assertTrue("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); + assertTrue("Seekpoints should get added by index scanner", + IndexRegionObserver.getSeekpointAdded()); + assertEquals("It should get two seek points from index scanner.", 2, IndexRegionObserver + .getMultipleSeekPoints().size()); + assertTrue("Overall result should have only 1 rows", testRes.size() == 2); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + masterFilter.addFilter(new SingleColumnValuePartitionFilter(hcd.getName(), "cq".getBytes(), + CompareOp.GREATER_OR_EQUAL, "person".getBytes(), vp)); + masterFilter.addFilter(new SingleColumnValuePartitionFilter(hcd.getName(), "cq".getBytes(), + CompareOp.LESS, "multiple".getBytes(), vp2)); + scan = new Scan(); + scan.setFilter(masterFilter); + i = 0; + scanner = table.getScanner(scan); + testRes = new ArrayList(); + result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + assertTrue("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); + assertTrue("Seekpoints should get added by index scanner", + IndexRegionObserver.getSeekpointAdded()); + assertEquals("It should get two seek points from index scanner.", 3, IndexRegionObserver + .getMultipleSeekPoints().size()); + assertTrue("Overall result should have only 1 rows", testRes.size() == 1); + + } + + @Test(timeout = 180000) + public void testCombinationOfPartitionFiltersWithSCVF() throws Exception { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testCombinationOfPartitionFiltersWithSCVF"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + TableIndices indices = new TableIndices(); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + ValuePartition vp = new SeparatorPartition("--", 3); + IndexSpecification iSpec = new IndexSpecification("idx1"); + iSpec.addIndexColumn(hcd, "cq", vp, ValueType.String, 200); + indices.addIndex(iSpec); + ValuePartition vp2 = new SeparatorPartition("--", 2); + iSpec = new IndexSpecification("idx2"); + iSpec.addIndexColumn(hcd, "cq", vp2, ValueType.String, 200); + indices.addIndex(iSpec); + iSpec = new IndexSpecification("idx3"); + iSpec.addIndexColumn(hcd, "cq", ValueType.String, 200); + indices.addIndex(iSpec); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + byte[] value1 = "hello--world--multiple--1".getBytes(); + Put p = new Put("row".getBytes()); + p.add("cf1".getBytes(), "cq".getBytes(), value1); + table.put(p); + p = new Put("row2".getBytes()); + p.add("cf1".getBytes(), "cq".getBytes(), "spatial--partition--multiple".getBytes()); + table.put(p); + + p = new Put("row3".getBytes()); + p.add("cf1".getBytes(), "cq".getBytes(), "partition--by--separator--multiple".getBytes()); + table.put(p); + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + masterFilter.addFilter(new SingleColumnValuePartitionFilter(hcd.getName(), "cq".getBytes(), + CompareOp.EQUAL, "multiple".getBytes(), vp)); + masterFilter.addFilter(new SingleColumnValuePartitionFilter(hcd.getName(), "cq".getBytes(), + CompareOp.GREATER_OR_EQUAL, "by".getBytes(), vp2)); + masterFilter.addFilter(new SingleColumnValueFilter(hcd.getName(), "cq".getBytes(), + CompareOp.EQUAL, "spatial--partition--multiple".getBytes())); + + Scan scan = new Scan(); + scan.setFilter(masterFilter); + int i = 0; + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + assertTrue("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); + assertTrue("Seekpoints should get added by index scanner", + IndexRegionObserver.getSeekpointAdded()); + assertEquals("It should get two seek points from index scanner.", 1, IndexRegionObserver + .getMultipleSeekPoints().size()); + assertTrue("Overall result should have only 1 rows", testRes.size() == 1); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + masterFilter.addFilter(new SingleColumnValueFilter(hcd.getName(), "cq".getBytes(), + CompareOp.GREATER_OR_EQUAL, "partition--by--separator--multiple".getBytes())); + masterFilter.addFilter(new SingleColumnValuePartitionFilter(hcd.getName(), "cq".getBytes(), + CompareOp.GREATER_OR_EQUAL, "person".getBytes(), vp)); + masterFilter.addFilter(new SingleColumnValuePartitionFilter(hcd.getName(), "cq".getBytes(), + CompareOp.LESS, "multiple".getBytes(), vp2)); + scan = new Scan(); + scan.setFilter(masterFilter); + i = 0; + scanner = table.getScanner(scan); + testRes = new ArrayList(); + result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + assertTrue("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); + assertTrue("Seekpoints should get added by index scanner", + IndexRegionObserver.getSeekpointAdded()); + assertEquals("It should get two seek points from index scanner.", 3, IndexRegionObserver + .getMultipleSeekPoints().size()); + assertTrue("Overall result should have only 1 rows", testRes.size() == 2); + + } + + @Test(timeout = 180000) + public void testCombinationOfPartitionFiltersWithSCVFPart2() throws Exception { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testCombinationOfPartitionFiltersWithSCVFPart2"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + TableIndices indices = new TableIndices(); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + ValuePartition vp = new SeparatorPartition("--", 3); + IndexSpecification iSpec = new IndexSpecification("idx1"); + iSpec.addIndexColumn(hcd, "cq", vp, ValueType.String, 100); + iSpec.addIndexColumn(hcd, "cq1", ValueType.String, 100); + indices.addIndex(iSpec); + ValuePartition vp2 = new SeparatorPartition("--", 2); + iSpec = new IndexSpecification("idx2"); + iSpec.addIndexColumn(hcd, "cq", vp2, ValueType.String, 100); + indices.addIndex(iSpec); + iSpec = new IndexSpecification("idx3"); + iSpec.addIndexColumn(hcd, "cq1", ValueType.String, 100); + indices.addIndex(iSpec); + iSpec = new IndexSpecification("idx4"); + iSpec.addIndexColumn(hcd, "cq", ValueType.String, 100); + iSpec.addIndexColumn(hcd, "cq1", ValueType.String, 100); + indices.addIndex(iSpec); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + byte[] value1 = "hello--world--multiple--1".getBytes(); + Put p = new Put("row".getBytes()); + p.add("cf1".getBytes(), "cq".getBytes(), value1); + table.put(p); + p = new Put("row2".getBytes()); + p.add("cf1".getBytes(), "cq".getBytes(), "spatial--partition--multiple".getBytes()); + p.add("cf1".getBytes(), "cq1".getBytes(), "spatialPartition".getBytes()); + table.put(p); + p = new Put("row3".getBytes()); + p.add("cf1".getBytes(), "cq".getBytes(), "partition--by--multiple--multiple".getBytes()); + p.add("cf1".getBytes(), "cq1".getBytes(), "partitionValue".getBytes()); + table.put(p); + p = new Put("row4".getBytes()); + p.add("cf1".getBytes(), "cq".getBytes(), "partition--multiple--multiple--multiple".getBytes()); + p.add("cf1".getBytes(), "cq1".getBytes(), "multiple".getBytes()); + table.put(p); + p = new Put("row5".getBytes()); + p.add("cf1".getBytes(), "cq1".getBytes(), "abcd".getBytes()); + table.put(p); + p = new Put("row6".getBytes()); + p.add("cf1".getBytes(), "cq".getBytes(), "1234".getBytes()); + table.put(p); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + + FilterList filter1 = new FilterList(Operator.MUST_PASS_ALL); + filter1.addFilter(new SingleColumnValuePartitionFilter(hcd.getName(), "cq".getBytes(), + CompareOp.EQUAL, "multiple".getBytes(), vp)); + filter1.addFilter(new SingleColumnValuePartitionFilter(hcd.getName(), "cq".getBytes(), + CompareOp.GREATER_OR_EQUAL, "by".getBytes(), vp2)); + filter1.addFilter(new SingleColumnValueFilter(hcd.getName(), "cq".getBytes(), CompareOp.EQUAL, + "partition--multiple--multiple--multiple".getBytes())); + + FilterList filter2 = new FilterList(Operator.MUST_PASS_ONE); + filter2.addFilter(new SingleColumnValuePartitionFilter(hcd.getName(), "cq".getBytes(), + CompareOp.EQUAL, "multiple".getBytes(), vp)); + filter2.addFilter(new SingleColumnValuePartitionFilter(hcd.getName(), "cq".getBytes(), + CompareOp.EQUAL, "multiple".getBytes(), vp2)); + + FilterList filter3 = new FilterList(Operator.MUST_PASS_ALL); + filter3.addFilter(new SingleColumnValuePartitionFilter(hcd.getName(), "cq".getBytes(), + CompareOp.EQUAL, "multiple".getBytes(), vp)); + filter3.addFilter(new SingleColumnValuePartitionFilter(hcd.getName(), "cq".getBytes(), + CompareOp.EQUAL, "multiple".getBytes(), vp2)); + + FilterList filter4 = new FilterList(Operator.MUST_PASS_ALL); + filter3.addFilter(new SingleColumnValueFilter(hcd.getName(), "cq1".getBytes(), + CompareOp.GREATER_OR_EQUAL, "1234".getBytes())); + filter3.addFilter(new SingleColumnValuePartitionFilter(hcd.getName(), "cq".getBytes(), + CompareOp.GREATER_OR_EQUAL, "multiple".getBytes(), vp2)); + masterFilter.addFilter(filter1); + masterFilter.addFilter(filter2); + masterFilter.addFilter(filter3); + masterFilter.addFilter(filter4); + + Scan scan = new Scan(); + scan.setFilter(masterFilter); + int i = 0; + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + assertTrue("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); + assertTrue("Seekpoints should get added by index scanner", + IndexRegionObserver.getSeekpointAdded()); + assertEquals("It should get two seek points from index scanner.", 1, IndexRegionObserver + .getMultipleSeekPoints().size()); + assertTrue("Overall result should have only 1 rows", testRes.size() == 1); + } + + @Test(timeout = 180000) + public void testSingleColumnValuePartitionFilterBySettingAsAttributeToScan() throws Exception { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testSingleColumnValuePartitionFilterBySettingAsAttributeToScan"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + TableIndices indices = new TableIndices(); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + ValuePartition vp = new SeparatorPartition("_", 3); + IndexSpecification iSpec = new IndexSpecification("idx1"); + iSpec.addIndexColumn(hcd, "cq", vp, ValueType.String, 200); + indices.addIndex(iSpec); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + byte[] value1 = "2ndFloor_solitaire_huawei_bangalore_karnataka".getBytes(); + Put p = new Put("row".getBytes()); + p.add("cf1".getBytes(), "cq".getBytes(), value1); + table.put(p); + p = new Put("row2".getBytes()); + p.add("cf1".getBytes(), "cq".getBytes(), + "7thFloor_solitaire_huawei_bangalore_karnataka".getBytes()); + table.put(p); + + p = new Put("row3".getBytes()); + p.add("cf1".getBytes(), "cq".getBytes(), "rrr_sss_hhh_bangalore_karnataka".getBytes()); + table.put(p); + + Scan scan = new Scan(); + SingleIndexExpression singleIndexExpression = new SingleIndexExpression("idx1"); + byte[] value = "huawei".getBytes(); + Column column = new Column("cf1".getBytes(), "cq".getBytes(), vp); + EqualsExpression equalsExpression = new EqualsExpression(column, value); + singleIndexExpression.addEqualsExpression(equalsExpression); + scan.setAttribute(Constants.INDEX_EXPRESSION, IndexUtils.toBytes(singleIndexExpression)); + scan.setFilter(new SingleColumnValuePartitionFilter(hcd.getName(), "cq".getBytes(), + CompareOp.EQUAL, "huawei".getBytes(), vp)); + int i = 0; + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + assertTrue("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); + assertTrue("Seekpoints should get added by index scanner", + IndexRegionObserver.getSeekpointAdded()); + assertEquals("It should get two seek points from index scanner.", 2, IndexRegionObserver + .getMultipleSeekPoints().size()); + assertTrue("Overall result should have only 2 rows", testRes.size() == 2); + + } + +} Index: hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/client/TestIndexUtils.java =================================================================== --- hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/client/TestIndexUtils.java (revision 0) +++ hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/client/TestIndexUtils.java (working copy) @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.client; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.ByteArrayInputStream; +import java.io.ObjectInputStream; + +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.index.Column; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(MediumTests.class) +public class TestIndexUtils { + + private static final byte[] FAMILY1 = Bytes.toBytes("cf1"); + private static final byte[] QUALIFIER1 = Bytes.toBytes("c1"); + + @Test + public void testConvertingSimpleIndexExpressionToByteArray() throws Exception { + SingleIndexExpression singleIndexExpression = new SingleIndexExpression("idx1"); + Column column = new Column(FAMILY1, QUALIFIER1); + byte[] value = "1".getBytes(); + EqualsExpression equalsExpression = new EqualsExpression(column, value); + singleIndexExpression.addEqualsExpression(equalsExpression); + + byte[] bytes = IndexUtils.toBytes(singleIndexExpression); + ByteArrayInputStream bis = new ByteArrayInputStream(bytes); + ObjectInputStream ois = new ObjectInputStream(bis); + SingleIndexExpression readExp = (SingleIndexExpression) ois.readObject(); + assertEquals("idx1", readExp.getIndexName()); + assertEquals(1, readExp.getEqualsExpressions().size()); + assertTrue(Bytes.equals(value, readExp.getEqualsExpressions().get(0).getValue())); + assertEquals(column, readExp.getEqualsExpressions().get(0).getColumn()); + } + + @Test + public void testConvertingBytesIntoIndexExpression() throws Exception { + SingleIndexExpression singleIndexExpression = new SingleIndexExpression("idx1"); + Column column = new Column(FAMILY1, QUALIFIER1); + byte[] value = "1".getBytes(); + EqualsExpression equalsExpression = new EqualsExpression(column, value); + singleIndexExpression.addEqualsExpression(equalsExpression); + + } +} Index: hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/master/TestIndexMasterObserver.java =================================================================== --- hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/master/TestIndexMasterObserver.java (revision 0) +++ hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/master/TestIndexMasterObserver.java (working copy) @@ -0,0 +1,657 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.master; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.IOException; +import java.util.Comparator; +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.MiniHBaseCluster; +import org.apache.hadoop.hbase.TableExistsException; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.catalog.MetaReader; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment; +import org.apache.hadoop.hbase.coprocessor.ObserverContext; +import org.apache.hadoop.hbase.index.ColumnQualifier.ValueType; +import org.apache.hadoop.hbase.index.Constants; +import org.apache.hadoop.hbase.index.IndexSpecification; +import org.apache.hadoop.hbase.index.SecIndexLoadBalancer; +import org.apache.hadoop.hbase.index.TableIndices; +import org.apache.hadoop.hbase.index.TestUtils; +import org.apache.hadoop.hbase.index.client.IndexAdmin; +import org.apache.hadoop.hbase.index.coprocessor.regionserver.IndexRegionObserver; +import org.apache.hadoop.hbase.index.util.IndexUtils; +import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.master.LoadBalancer; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.zookeeper.ZKAssign; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.zookeeper.KeeperException; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.mockito.Mockito; + +/** + * Tests invocation of the {@link IndexMasterObserver} coprocessor hooks at all appropriate times + * during normal HMaster operations. + */ + +@Category(LargeTests.class) +public class TestIndexMasterObserver { + + private static HBaseTestingUtility UTIL = new HBaseTestingUtility(); + + private static HBaseAdmin admin; + @BeforeClass + public static void setupBeforeClass() throws Exception { + Configuration conf = UTIL.getConfiguration(); + conf.setBoolean(HConstants.REGIONSERVER_INFO_PORT_AUTO,true); + conf.setBoolean("hbase.use.secondary.index", true); + conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, IndexMasterObserver.class.getName()); + conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, IndexRegionObserver.class.getName()); + conf.set("index.data.block.encoding.algo", "PREFIX"); + conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, SecIndexLoadBalancer.class, + LoadBalancer.class); + UTIL.startMiniCluster(1); + admin = new IndexAdmin(conf); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + UTIL.shutdownMiniCluster(); + } + + // Check this test case why its failing. + @Test(timeout = 180000) + public void testCreateTableWithIndexTableSuffix() throws Exception { + HTableDescriptor htd = + TestUtils.createIndexedHTableDescriptor("testCreateTableWithIndexTableSuffix" + + Constants.INDEX_TABLE_SUFFIX, "cf", "index_name", "cf", "cq"); + try { + admin.createTable(htd); + fail("User table should not ends with " + Constants.INDEX_TABLE_SUFFIX); + } catch (IOException e) { + + } + } + + @Test(timeout = 180000) + public void testCreateIndexTableWhenIndexTableAlreadyExist() throws Exception { + HTableDescriptor iHtd = + TestUtils.createIndexedHTableDescriptor("testCreateIndexTableWhenIndexTableAlreadyExist", "cf", + "index_name", "cf", "cq"); + admin.createTable(iHtd); + + admin.disableTable("testCreateIndexTableWhenIndexTableAlreadyExist"); + admin.deleteTable("testCreateIndexTableWhenIndexTableAlreadyExist"); + + admin.createTable(iHtd); + + assertTrue("Table is not created.", + admin.isTableAvailable("testCreateIndexTableWhenIndexTableAlreadyExist")); + + String indexTableName = + "testCreateIndexTableWhenIndexTableAlreadyExist" + Constants.INDEX_TABLE_SUFFIX; + + assertTrue("Index table is not created.", admin.isTableAvailable(indexTableName)); + } + + @Test(timeout = 180000) + public void testCreateIndexTableWhenBothIndexAndUserTableExist() throws Exception { + HTableDescriptor iHtd = + TestUtils.createIndexedHTableDescriptor("testCreateIndexTableWhenBothIndexAndUserTableExist", "cf", + "index_name", "cf", "cq"); + admin.createTable(iHtd); + + try { + admin.createTable(iHtd); + fail("Should throw TableExistsException " + "if both user table and index table exist."); + } catch (TableExistsException t) { + + } + } + + @Test(timeout = 180000) + public void testCreateIndexTableWithOutIndexDetails() throws Exception { + HTableDescriptor iHtd = + new HTableDescriptor(TableName.valueOf("testCreateIndexTableWithOutIndexDetails")); + TableIndices indices = new TableIndices(); + iHtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(iHtd); + assertTrue("Table is not created.", + admin.isTableAvailable("testCreateIndexTableWithOutIndexDetails")); + + String indexTableName = + "testCreateIndexTableWithOutIndexDetails" + Constants.INDEX_TABLE_SUFFIX; + + assertTrue("Index tables is not created.", admin.isTableAvailable(indexTableName)); + } + + @Test(timeout = 180000) + public void testCreateIndexTableWhenExistedIndexTableDisabled() throws Exception { + String tableName = "testCreateIndexTableWhenExistedIndexTableDisabled"; + HTableDescriptor iHtd = + TestUtils.createIndexedHTableDescriptor(tableName, "cf", + "index_name", "cf", "cq"); + char c = 'A'; + byte[][] split = new byte[20][]; + for (int i = 0; i < 20; i++) { + byte[] b = { (byte) c }; + split[i] = b; + c++; + } + admin.createTable(iHtd, split); + admin.disableTable(tableName); + admin.deleteTable(tableName); + + iHtd = + TestUtils.createIndexedHTableDescriptor(tableName, "cf", + "index_name", "cf", "cq"); + admin.createTable(iHtd); + assertTrue("Table is not created.", + admin.isTableAvailable(tableName)); + + String indexTableName = IndexUtils.getIndexTableName(tableName); + assertTrue("Index tables is not created.", admin.isTableAvailable(indexTableName)); + } + + @Test(timeout = 180000) + public void testCreateIndexTableWhenExistedTableDisableFailed() throws Exception { + MiniHBaseCluster cluster = UTIL.getMiniHBaseCluster(); + HMaster master = cluster.getMaster(); + HTableDescriptor htd = + new HTableDescriptor(TableName.valueOf("testCreateIndexTableWhenExistedTableDisableFailed")); + admin.createTable(htd); + + HTableDescriptor iHtd = + TestUtils.createIndexedHTableDescriptor("testCreateIndexTableWhenExistedTableDisableFailed", "cf", + "index_name", "cf", "cq"); + IndexMasterObserver ms = Mockito.mock(IndexMasterObserver.class); + Mockito + .doThrow(new RuntimeException()) + .when(ms) + .preCreateTable((ObserverContext) Mockito.anyObject(), + (HTableDescriptor) Mockito.anyObject(), (HRegionInfo[]) Mockito.anyObject()); + + try { + char c = 'A'; + byte[][] split = new byte[20][]; + for (int i = 0; i < 20; i++) { + byte[] b = { (byte) c }; + split[i] = b; + c++; + } + admin.createTable(iHtd, split); + assertFalse(master.getAssignmentManager().getZKTable() + .isDisabledTable(TableName.valueOf("testCreateIndexTableWhenExistedTableDisableFailed"))); + // fail("Should throw RegionException."); + } catch (IOException r) { + } + } + + @Test(timeout = 180000) + public void testCreateIndexTableWhenExistedTableDeleteFailed() throws Exception { + HTableDescriptor iHtd = + TestUtils.createIndexedHTableDescriptor("testCreateIndexTableWhenExistedTableDeleteFailed", "cf", + "index_name", "cf", "cq"); + char c = 'A'; + byte[][] split = new byte[20][]; + for (int i = 0; i < 20; i++) { + byte[] b = { (byte) c }; + split[i] = b; + c++; + } + admin.createTable(iHtd, split); + admin.disableTable("testCreateIndexTableWhenExistedTableDeleteFailed"); + admin.deleteTable("testCreateIndexTableWhenExistedTableDeleteFailed"); + IndexMasterObserver ms = Mockito.mock(IndexMasterObserver.class); + Mockito + .doThrow(new RuntimeException()) + .when(ms) + .preCreateTable((ObserverContext) Mockito.anyObject(), + (HTableDescriptor) Mockito.anyObject(), (HRegionInfo[]) Mockito.anyObject()); + try { + admin.createTable(iHtd); + } catch (IOException e) { + + } + } + + @Test(timeout = 180000) + public void testIndexTableCreationAfterMasterRestart() throws Exception { + HTableDescriptor iHtd = + TestUtils.createIndexedHTableDescriptor("testIndexTableCreationAfterMasterRestart", "cf", + "index_name", "cf", "cq"); + admin.createTable(iHtd); + admin.disableTable("testIndexTableCreationAfterMasterRestart" + Constants.INDEX_TABLE_SUFFIX); + admin.deleteTable("testIndexTableCreationAfterMasterRestart" + Constants.INDEX_TABLE_SUFFIX); + MiniHBaseCluster cluster = UTIL.getHBaseCluster(); + cluster.abortMaster(0); + cluster.waitOnMaster(0); + // start up a new master + cluster.startMaster(); + assertTrue(cluster.waitForActiveAndReadyMaster()); + String indexTableName = + "testIndexTableCreationAfterMasterRestart" + Constants.INDEX_TABLE_SUFFIX; + + assertTrue("Index tables is not created.", admin.isTableAvailable(indexTableName)); + } + + @Test(timeout = 180000) + public void testIndexTableCreationAlongWithNormalTablesAfterMasterRestart() throws Exception { + TableName tableName = + TableName.valueOf("testIndexTableCreationAlongWithNormalTablesAfterMasterRestart"); + HTableDescriptor htd = new HTableDescriptor(tableName); + admin.createTable(htd); + MiniHBaseCluster cluster = UTIL.getHBaseCluster(); + cluster.abortMaster(0); + cluster.waitOnMaster(0); + HMaster master = cluster.startMaster().getMaster(); + cluster.waitForActiveAndReadyMaster(); + + boolean tableExist = + MetaReader.tableExists(master.getCatalogTracker(), + TableName.valueOf(IndexUtils.getIndexTableName(tableName))); + assertFalse("Index table should be not created after master start up.", tableExist); + } + + // Check this test case. + @Test(timeout = 180000) + public void testPreCreateShouldNotBeSuccessfulIfIndicesAreNotSame() throws IOException, + KeeperException, InterruptedException { + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + String userTableName = "testNotConsisIndex1"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + IndexSpecification iSpec1 = new IndexSpecification("Index1"); + iSpec1.addIndexColumn(hcd, "q1", ValueType.String, 10); + ihtd.addFamily(hcd); + TableIndices indices = new TableIndices(); + indices.addIndex(iSpec1); + IndexSpecification iSpec2 = new IndexSpecification("Index2"); + iSpec2.addIndexColumn(hcd, "q1", ValueType.Int, 10); + indices.addIndex(iSpec2); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + boolean returnVal = false; + try { + admin.createTable(ihtd); + fail("Exception should be thrown"); + } catch (IOException e) { + + returnVal = true; + } + Assert.assertTrue(returnVal); + ZKAssign.blockUntilNoRIT(zkw); + } + + @Test(timeout = 180000) + public void testPreCreateShouldNotBeSuccessfulIfIndicesAreNotSameAtLength() throws IOException, + KeeperException, InterruptedException { + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + String userTableName = "testNotConsisIndex2"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + IndexSpecification iSpec1 = new IndexSpecification("Index1"); + iSpec1.addIndexColumn(hcd, "q1", ValueType.String, 10); + iSpec1.addIndexColumn(hcd, "q2", ValueType.String, 4); + ihtd.addFamily(hcd); + TableIndices indices = new TableIndices(); + indices.addIndex(iSpec1); + IndexSpecification iSpec2 = new IndexSpecification("Index2"); + iSpec2.addIndexColumn(hcd, "q3", ValueType.String, 10); + iSpec2.addIndexColumn(hcd, "q2", ValueType.String, 10); + indices.addIndex(iSpec2); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + boolean returnVal = false; + try { + admin.createTable(ihtd); + fail("Exception should be thrown"); + } catch (IOException e) { + returnVal = true; + } + Assert.assertTrue(returnVal); + ZKAssign.blockUntilNoRIT(zkw); + } + + @Test(timeout = 180000) + public void testPreCreateShouldNotBeSuccessfulIfIndicesAreNotSameAtType() throws IOException, + KeeperException, InterruptedException { + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + String userTableName = "testNotConsisIndex3"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + IndexSpecification iSpec1 = new IndexSpecification("Index1"); + iSpec1.addIndexColumn(hcd, "q1", ValueType.String, 10); + iSpec1.addIndexColumn(hcd, "q2", ValueType.String, 10); + ihtd.addFamily(hcd); + IndexSpecification iSpec2 = new IndexSpecification("Index2"); + iSpec2.addIndexColumn(hcd, "q1", ValueType.Int, 10); + iSpec2.addIndexColumn(hcd, "q3", ValueType.String, 10); + TableIndices indices = new TableIndices(); + indices.addIndex(iSpec1); + indices.addIndex(iSpec2); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + boolean returnVal = false; + try { + admin.createTable(ihtd); + fail("Exception should be thrown"); + } catch (IOException e) { + returnVal = true; + } + Assert.assertTrue(returnVal); + ZKAssign.blockUntilNoRIT(zkw); + } + + @Test(timeout = 180000) + public void testPreCreateShouldNotBeSuccessfulIfIndicesAreNotSameAtBothTypeAndLength() + throws IOException, KeeperException, InterruptedException { + String userTableName = "testNotConsisIndex4"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + IndexSpecification iSpec1 = new IndexSpecification("Index1"); + iSpec1.addIndexColumn(hcd, "q1", ValueType.String, 10); + iSpec1.addIndexColumn(hcd, "q2", ValueType.String, 10); + ihtd.addFamily(hcd); + IndexSpecification iSpec2 = new IndexSpecification("Index2"); + iSpec2.addIndexColumn(hcd, "q1", ValueType.Int, 10); + iSpec2.addIndexColumn(hcd, "q2", ValueType.String, 7); + TableIndices indices = new TableIndices(); + indices.addIndex(iSpec1); + indices.addIndex(iSpec2); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + boolean returnVal = false; + try { + admin.createTable(ihtd); + fail("IOException should be thrown"); + } catch (IOException e) { + returnVal = true; + } + Assert.assertTrue(returnVal); + } + + @Test(timeout = 180000) + public void testPreCreateShouldBeSuccessfulIfIndicesAreSame() throws IOException, + KeeperException, InterruptedException { + String userTableName = "testConsistIndex"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + IndexSpecification iSpec1 = new IndexSpecification("Index1"); + iSpec1.addIndexColumn(hcd, "q1", ValueType.String, 10); + ihtd.addFamily(hcd); + IndexSpecification iSpec2 = new IndexSpecification("Index2"); + iSpec2.addIndexColumn(hcd, "q1", ValueType.String, 10); + TableIndices indices = new TableIndices(); + indices.addIndex(iSpec1); + indices.addIndex(iSpec2); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + try { + admin.createTable(ihtd); + } catch (IOException e) { + fail("Exception should not be thrown"); + } + } + + @Test(timeout = 180000) + public void testIndexTableShouldBeDisabledIfUserTableDisabled() throws Exception { + String tableName = "testIndexTableDisabledIfUserTableDisabled"; + String indexTableName = IndexUtils.getIndexTableName(tableName); + HTableDescriptor iHtd = + TestUtils.createIndexedHTableDescriptor(tableName, "cf", "index_name", "cf", "cq"); + admin.createTable(iHtd); + admin.disableTable(tableName); + assertTrue("User table should be disabled.", admin.isTableDisabled(tableName)); + assertTrue("Index table should be disabled.", admin.isTableDisabled(indexTableName)); + } + + @Test(timeout = 180000) + public void testIndexTableShouldBeEnabledIfUserTableEnabled() throws Exception { + String tableName = "testIndexTableEnabledIfUserTableEnabled"; + String indexTableName = IndexUtils.getIndexTableName(tableName); + HTableDescriptor iHtd = + TestUtils.createIndexedHTableDescriptor(tableName, "cf", "index_name", "cf", "cq"); + admin.createTable(iHtd); + admin.disableTable(tableName); + admin.enableTable(tableName); + assertTrue("User table should be enabled.", admin.isTableEnabled(tableName)); + assertTrue("Index table should be enabled.", admin.isTableEnabled(indexTableName)); + } + + @Test(timeout = 180000) + public void testDisabledIndexTableShouldBeEnabledIfUserTableEnabledAndMasterRestarted() + throws Exception { + String tableName = "testDisabledIndexTableEnabledIfUserTableEnabledAndMasterRestarted"; + String indexTableName = IndexUtils.getIndexTableName(tableName); + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + HTableDescriptor iHtd = + TestUtils.createIndexedHTableDescriptor(tableName, "cf", "index_name", "cf", "cq"); + admin.createTable(iHtd); + admin.disableTable(indexTableName); + MiniHBaseCluster cluster = UTIL.getHBaseCluster(); + cluster.abortMaster(0); + cluster.startMaster(); + cluster.waitOnMaster(0); + cluster.waitForActiveAndReadyMaster(); + Thread.sleep(1000); + assertTrue("User table should be enabled.", admin.isTableEnabled(tableName)); + assertTrue("Index table should be enabled.", admin.isTableEnabled(indexTableName)); + } + + @Test(timeout = 180000) + public void testEnabledIndexTableShouldBeDisabledIfUserTableDisabledAndMasterRestarted() + throws Exception { + String tableName = "testEnabledIndexTableDisabledIfUserTableDisabledAndMasterRestarted"; + String indexTableName = IndexUtils.getIndexTableName(tableName); + HTableDescriptor iHtd = + TestUtils.createIndexedHTableDescriptor(tableName, "cf", "index_name", "cf", "cq"); + admin.createTable(iHtd); + admin.disableTable(tableName); + admin.enableTable(indexTableName); + MiniHBaseCluster cluster = UTIL.getHBaseCluster(); + cluster.abortMaster(0); + cluster.startMaster(); + cluster.waitOnMaster(0); + cluster.waitForActiveAndReadyMaster(); + Thread.sleep(1000); + assertTrue("User table should be disabled.", admin.isTableDisabled(tableName)); + assertTrue("Index table should be disabled.", admin.isTableDisabled(indexTableName)); + } + + @Test(timeout = 180000) + public void testDisabledIndexTableShouldBeEnabledIfUserTableInEnablingAndMasterRestarted() + throws Exception { + String tableName = "testDisabledIndexTableEnabledIfUserTableInEnablingAndMasterRestarted"; + String indexTableName = IndexUtils.getIndexTableName(tableName); + HTableDescriptor iHtd = + TestUtils.createIndexedHTableDescriptor(tableName, "cf", "index_name", "cf", "cq"); + admin.createTable(iHtd); + admin.disableTable(indexTableName); + MiniHBaseCluster cluster = UTIL.getHBaseCluster(); + HMaster master = cluster.getMaster(); + master.getAssignmentManager().getZKTable().setEnablingTable(TableName.valueOf(tableName)); + cluster.abortMaster(0); + cluster.startMaster(); + cluster.waitOnMaster(0); + cluster.waitForActiveAndReadyMaster(); + Thread.sleep(1000); + assertTrue("User table should be enabled.", admin.isTableEnabled(tableName)); + assertTrue("Index table should be enabled.", admin.isTableEnabled(indexTableName)); + } + + @Test(timeout = 180000) + public void testEnabledIndexTableShouldBeDisabledIfUserTableInDisablingAndMasterRestarted() + throws Exception { + String tableName = "testEnabledIndexTableDisabledIfUserTableInDisablingAndMasterRestarted"; + String indexTableName = IndexUtils.getIndexTableName(tableName); + HTableDescriptor iHtd = + TestUtils.createIndexedHTableDescriptor(tableName, "cf", "index_name", "cf", "cq"); + admin.createTable(iHtd); + MiniHBaseCluster cluster = UTIL.getHBaseCluster(); + HMaster master = cluster.getMaster(); + master.getAssignmentManager().getZKTable().setDisablingTable(TableName.valueOf(tableName)); + cluster.abortMaster(0); + cluster.startMaster(); + cluster.waitOnMaster(0); + cluster.waitForActiveAndReadyMaster(); + Thread.sleep(1000); + assertTrue("User table should be disabled.", admin.isTableDisabled(tableName)); + assertTrue("Index table should be disabled.", admin.isTableDisabled(indexTableName)); + } + + @Test(timeout = 180000) + public void testShouldModifyTableWithIndexDetails() throws Exception { + String tableName = "testShouldModifyTableWithIndexDetails"; + HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName)); + htd.addFamily(new HColumnDescriptor(Bytes.toBytes("f1"))); + htd.addFamily(new HColumnDescriptor(Bytes.toBytes("f2"))); + admin.createTable(htd); + TableName indexTableName = TableName.valueOf(IndexUtils.getIndexTableName(tableName)); + admin.disableTable(tableName); + HTableDescriptor ihtd = + TestUtils.createIndexedHTableDescriptor(tableName, "f1", "idx1", "f1", "q1"); + admin.modifyTable(Bytes.toBytes(tableName), ihtd); + List regionsOfTable = + UTIL.getHBaseCluster().getMaster().getAssignmentManager().getRegionStates() + .getRegionsOfTable(indexTableName); + while (regionsOfTable.size() != 1) { + regionsOfTable = + UTIL.getHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsOfTable(indexTableName); + } + admin.enableTable(tableName); + assertTrue(admin.isTableEnabled(Bytes.toBytes(IndexUtils.getIndexTableName(tableName)))); + } + + @Test(timeout = 180000) + public void testCreateIndexTableFromExistingTable() throws Exception { + String tableName = "testCreateIndexTableFromExistingTable"; + HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName)); + htd.addFamily(new HColumnDescriptor(Bytes.toBytes("f1"))); + htd.addFamily(new HColumnDescriptor(Bytes.toBytes("f2"))); + byte[][] split = new byte[][] { Bytes.toBytes("A"), Bytes.toBytes("B") }; + admin.createTable(htd, split); + TableName indexTableName = TableName.valueOf(IndexUtils.getIndexTableName(tableName)); + HTable table = new HTable(admin.getConfiguration(), tableName); + Put p = new Put(Bytes.toBytes("row1")); + p.add(Bytes.toBytes("f1"), Bytes.toBytes("q1"), Bytes.toBytes("2")); + p.add(Bytes.toBytes("f2"), Bytes.toBytes("q2"), Bytes.toBytes("3")); + table.put(p); + table.flushCommits(); + admin.flush(tableName); + UTIL.getConfiguration().set("table.columns.index", + "IDX1=>f1:[q1->Int&10],[q2],[q3];f2:[q1->String&15],[q2->Int&15]#IDX2=>f1:[q5]"); + IndexUtils.createIndexTable(tableName, UTIL.getConfiguration(), null); + List regionsOfTable = + UTIL.getHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsOfTable(indexTableName); + while (regionsOfTable.size() != 3) { + Thread.sleep(500); + regionsOfTable = + UTIL.getHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsOfTable(indexTableName); + } + HTableDescriptor indexedTableDesc = admin.getTableDescriptor(TableName.valueOf(tableName)); + byte[] value = indexedTableDesc.getValue(Constants.INDEX_SPEC_KEY); + TableIndices indices = new TableIndices(); + indices.readFields(value); + assertEquals(indices.getIndices().size(), 2); + Scan s = new Scan(); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(10); + List cf1 = next[0].getColumnCells(Bytes.toBytes("f1"), Bytes.toBytes("q1")); + List cf2 = next[0].getColumnCells(Bytes.toBytes("f2"), Bytes.toBytes("q2")); + assertTrue(cf1.size() > 0 && cf2.size() > 0); + } + + @Test(timeout = 180000) + public void testShouldRetainTheExistingCFsInHTD() throws Exception { + String tableName = "testShouldRetainTheExistingCFsInHTD"; + HBaseTestingUtility.getZooKeeperWatcher(UTIL); + HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName)); + htd.addFamily(new HColumnDescriptor(Bytes.toBytes("f1"))); + htd.addFamily(new HColumnDescriptor(Bytes.toBytes("f2"))); + admin.createTable(htd); + TableName indexTableName = TableName.valueOf(IndexUtils.getIndexTableName(tableName)); + HTable table = new HTable(admin.getConfiguration(), tableName); + Put p = new Put(Bytes.toBytes("row1")); + p.add(Bytes.toBytes("f1"), Bytes.toBytes("q1"), Bytes.toBytes("2")); + p.add(Bytes.toBytes("f2"), Bytes.toBytes("q2"), Bytes.toBytes("3")); + table.put(p); + table.flushCommits(); + admin.flush(tableName); + UTIL.getConfiguration().set("table.columns.index", "IDX1=>f1:[q1->Int&10],[q2],[q3];"); + IndexUtils.createIndexTable(tableName, UTIL.getConfiguration(), null); + List regionsOfTable = + UTIL.getHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsOfTable(indexTableName); + while (regionsOfTable.size() != 1) { + Thread.sleep(500); + regionsOfTable = + UTIL.getHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsOfTable(indexTableName); + } + HTableDescriptor indexedTableDesc = admin.getTableDescriptor(TableName.valueOf(tableName)); + byte[] value = indexedTableDesc.getValue(Constants.INDEX_SPEC_KEY); + TableIndices indices = new TableIndices(); + indices.readFields(value); + assertEquals(indices.getIndices().size(), 1); + Scan s = new Scan(); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(10); + List cf1 = next[0].getColumnCells(Bytes.toBytes("f1"), Bytes.toBytes("q1")); + List cf2 = next[0].getColumnCells(Bytes.toBytes("f2"), Bytes.toBytes("q2")); + assertTrue(cf1.size() > 0 && cf2.size() > 0); + } + + @Test(timeout = 180000) + public void testBlockEncoding() throws Exception { + Configuration conf = admin.getConfiguration(); + String userTableName = "testBlockEncoding"; + HTableDescriptor htd = TestUtils.createIndexedHTableDescriptor(userTableName, "col1", "Index1", "col1", "ql"); + admin.createTable(htd); + HTableDescriptor tableDescriptor = + admin.getTableDescriptor(Bytes.toBytes(IndexUtils.getIndexTableName(userTableName))); + assertEquals(DataBlockEncoding.PREFIX, + tableDescriptor.getColumnFamilies()[0].getDataBlockEncoding()); + } +} Index: hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestAcidGuaranteesForIndex.java =================================================================== --- hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestAcidGuaranteesForIndex.java (revision 0) +++ hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestAcidGuaranteesForIndex.java (working copy) @@ -0,0 +1,364 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import java.io.IOException; +import java.util.List; +import java.util.Random; +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.TableExistsException; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.MultithreadedTestUtil.RepeatingTestThread; +import org.apache.hadoop.hbase.MultithreadedTestUtil.TestContext; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; +import org.apache.hadoop.hbase.filter.Filter; +import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; +import org.apache.hadoop.hbase.index.ColumnQualifier.ValueType; +import org.apache.hadoop.hbase.index.IndexSpecification; +import org.apache.hadoop.hbase.index.SecIndexLoadBalancer; +import org.apache.hadoop.hbase.index.TableIndices; +import org.apache.hadoop.hbase.index.client.IndexAdmin; +import org.apache.hadoop.hbase.index.coprocessor.master.IndexMasterObserver; +import org.apache.hadoop.hbase.index.coprocessor.wal.IndexWALObserver; +import org.apache.hadoop.hbase.master.LoadBalancer; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import com.google.common.collect.Lists; + +/** + * Test case that uses multiple threads to read and write multifamily rows into a table, verifying + * that reads never see partially-complete writes. This can run as a junit test, or with a main() + * function which runs against a real cluster (eg for testing with failures, region movement, etc) + */ +@Category(MediumTests.class) +public class TestAcidGuaranteesForIndex { + protected static final Log LOG = LogFactory.getLog(TestAcidGuaranteesForIndex.class); + public static final byte[] TABLE_NAME = Bytes.toBytes("TestAcidGuaranteesForIndex"); + public static final byte[] FAMILY_A = Bytes.toBytes("A"); + public static final byte[] FAMILY_B = Bytes.toBytes("B"); + public static final byte[] FAMILY_C = Bytes.toBytes("C"); + public static final byte[] QUALIFIER_NAME = Bytes.toBytes("data"); + + public static final byte[][] FAMILIES = new byte[][] { FAMILY_A, FAMILY_B, FAMILY_C }; + + private HBaseTestingUtility util; + + public static int NUM_COLS_TO_CHECK = 10; + + private void createTableIfMissing() throws IOException { + try { + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(TABLE_NAME)); + for (byte[] family : FAMILIES) { + ihtd.addFamily(new HColumnDescriptor(family)); + } + IndexSpecification iSpec = new IndexSpecification("ScanIndex"); + iSpec.addIndexColumn(new HColumnDescriptor(Bytes.toString(FAMILY_A)), + Bytes.toString(QUALIFIER_NAME) + "1", ValueType.String, 10); + TableIndices indices = new TableIndices(); + indices.addIndex(iSpec); + HBaseAdmin admin = new IndexAdmin(util.getConfiguration()); + admin.createTable(ihtd); + admin.close(); + } catch (TableExistsException tee) { + } + } + + public TestAcidGuaranteesForIndex() { + // Set small flush size for minicluster so we exercise reseeking scanners + Configuration conf = HBaseConfiguration.create(); + conf.set(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, String.valueOf(128 * 1024)); + conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, IndexMasterObserver.class.getName()); + conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, IndexRegionObserver.class.getName()); + conf.set(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY, IndexWALObserver.class.getName()); + conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, SecIndexLoadBalancer.class, + LoadBalancer.class); + util = new HBaseTestingUtility(conf); + } + + /** + * Thread that does random full-row writes into a table. + */ + public static class AtomicityWriter extends RepeatingTestThread { + Random rand = new Random(); + byte data[] = "value".getBytes(); + byte targetRows[][]; + byte targetFamilies[][]; + HTable table; + AtomicLong numWritten = new AtomicLong(); + + public AtomicityWriter(TestContext ctx, byte targetRows[][], byte targetFamilies[][]) + throws IOException { + super(ctx); + this.targetRows = targetRows; + this.targetFamilies = targetFamilies; + table = new HTable(ctx.getConf(), TABLE_NAME); + } + + public void doAnAction() throws Exception { + // Pick a random row to write into + byte[] targetRow = targetRows[rand.nextInt(targetRows.length)]; + Put p = new Put(targetRow); + for (byte[] family : targetFamilies) { + for (int i = 0; i < NUM_COLS_TO_CHECK; i++) { + byte qualifier[] = Bytes.toBytes("data" + i); + p.add(family, qualifier, data); + } + } + table.put(p); + numWritten.getAndIncrement(); + } + } + + /** + * Thread that does single-row reads in a table, looking for partially completed rows. + */ + public static class AtomicGetReader extends RepeatingTestThread { + byte targetRow[]; + byte targetFamilies[][]; + HTable table; + int numVerified = 0; + AtomicLong numRead = new AtomicLong(); + + public AtomicGetReader(TestContext ctx, byte targetRow[], byte targetFamilies[][]) + throws IOException { + super(ctx); + this.targetRow = targetRow; + this.targetFamilies = targetFamilies; + table = new HTable(ctx.getConf(), TABLE_NAME); + } + + public void doAnAction() throws Exception { + Get g = new Get(targetRow); + Result res = table.get(g); + byte[] gotValue = null; + if (res.getRow() == null) { + // Trying to verify but we didn't find the row - the writing + // thread probably just hasn't started writing yet, so we can + // ignore this action + return; + } + + for (byte[] family : targetFamilies) { + for (int i = 0; i < NUM_COLS_TO_CHECK; i++) { + byte qualifier[] = Bytes.toBytes("data" + i); + byte thisValue[] = res.getValue(family, qualifier); + if (gotValue != null && !Bytes.equals(gotValue, thisValue)) { + gotFailure(gotValue, res); + } + numVerified++; + gotValue = thisValue; + } + } + numRead.getAndIncrement(); + } + + private void gotFailure(byte[] expected, Result res) { + StringBuilder msg = new StringBuilder(); + msg.append("Failed after ").append(numVerified).append("!"); + msg.append("Expected=").append(Bytes.toStringBinary(expected)); + msg.append("Got:\n"); + for (Cell kv : res.list()) { + msg.append(kv.toString()); + msg.append(" val= "); + msg.append(Bytes.toStringBinary(kv.getValue())); + msg.append("\n"); + } + throw new RuntimeException(msg.toString()); + } + } + + /** + * Thread that does full scans of the table looking for any partially completed rows. + */ + public static class AtomicScanReader extends RepeatingTestThread { + byte targetFamilies[][]; + HTable table; + AtomicLong numScans = new AtomicLong(); + AtomicLong numRowsScanned = new AtomicLong(); + Random rand = new Random(); + byte data[] = "value".getBytes(); + + public AtomicScanReader(TestContext ctx, byte targetFamilies[][]) throws IOException { + super(ctx); + this.targetFamilies = targetFamilies; + table = new HTable(ctx.getConf(), TABLE_NAME); + } + + public void doAnAction() throws Exception { + Scan s = new Scan(); + for (byte[] family : targetFamilies) { + s.addFamily(family); + } + Filter filter = + new SingleColumnValueFilter(FAMILY_A, (Bytes.toString(QUALIFIER_NAME) + "1").getBytes(), + CompareOp.EQUAL, data); + s.setFilter(filter); + ResultScanner scanner = table.getScanner(s); + + for (Result res : scanner) { + byte[] gotValue = null; + + for (byte[] family : targetFamilies) { + for (int i = 0; i < NUM_COLS_TO_CHECK; i++) { + byte qualifier[] = Bytes.toBytes("data" + i); + byte thisValue[] = res.getValue(family, qualifier); + if (gotValue != null && !Bytes.equals(gotValue, thisValue)) { + gotFailure(gotValue, res); + } + gotValue = thisValue; + } + } + numRowsScanned.getAndIncrement(); + } + numScans.getAndIncrement(); + } + + private void gotFailure(byte[] expected, Result res) { + StringBuilder msg = new StringBuilder(); + msg.append("Failed after ").append(numRowsScanned).append("!"); + msg.append("Expected=").append(Bytes.toStringBinary(expected)); + msg.append("Got:\n"); + for (Cell kv : res.list()) { + msg.append(kv.toString()); + msg.append(" val= "); + msg.append(Bytes.toStringBinary(kv.getValue())); + msg.append("\n"); + } + throw new RuntimeException(msg.toString()); + } + } + + public void runTestAtomicity(long millisToRun, int numWriters, int numGetters, int numScanners, + int numUniqueRows) throws Exception { + createTableIfMissing(); + TestContext ctx = new TestContext(util.getConfiguration()); + + byte rows[][] = new byte[numUniqueRows][]; + for (int i = 0; i < numUniqueRows; i++) { + rows[i] = Bytes.toBytes("test_row_" + i); + } + + List writers = Lists.newArrayList(); + for (int i = 0; i < numWriters; i++) { + AtomicityWriter writer = new AtomicityWriter(ctx, rows, FAMILIES); + writers.add(writer); + ctx.addThread(writer); + } + // Add a flusher + ctx.addThread(new RepeatingTestThread(ctx) { + public void doAnAction() throws Exception { + util.flush(); + } + }); + + List getters = Lists.newArrayList(); + for (int i = 0; i < numGetters; i++) { + AtomicGetReader getter = new AtomicGetReader(ctx, rows[i % numUniqueRows], FAMILIES); + getters.add(getter); + ctx.addThread(getter); + } + + List scanners = Lists.newArrayList(); + for (int i = 0; i < numScanners; i++) { + AtomicScanReader scanner = new AtomicScanReader(ctx, FAMILIES); + scanners.add(scanner); + ctx.addThread(scanner); + } + + ctx.startThreads(); + ctx.waitFor(millisToRun); + ctx.stop(); + + LOG.info("Finished test. Writers:"); + for (AtomicityWriter writer : writers) { + LOG.info(" wrote " + writer.numWritten.get()); + } + LOG.info("Readers:"); + for (AtomicGetReader reader : getters) { + LOG.info(" read " + reader.numRead.get()); + } + LOG.info("Scanners:"); + for (AtomicScanReader scanner : scanners) { + LOG.info(" scanned " + scanner.numScans.get()); + LOG.info(" verified " + scanner.numRowsScanned.get() + " rows"); + } + } + + @Test + public void testGetAtomicity() throws Exception { + util.startMiniCluster(1); + try { + runTestAtomicity(20000, 5, 5, 0, 3); + } finally { + util.shutdownMiniCluster(); + } + } + + @Test + public void testScanAtomicity() throws Exception { + util.startMiniCluster(1); + try { + runTestAtomicity(20000, 5, 0, 5, 3); + } finally { + util.shutdownMiniCluster(); + } + } + + @Test + public void testMixedAtomicity() throws Exception { + util.startMiniCluster(1); + try { + runTestAtomicity(20000, 5, 2, 2, 3); + } finally { + util.shutdownMiniCluster(); + } + } + + public static void main(String args[]) throws Exception { + Configuration c = HBaseConfiguration.create(); + TestAcidGuaranteesForIndex test = new TestAcidGuaranteesForIndex(); + test.setConf(c); + test.runTestAtomicity(5000, 50, 2, 2, 3); + } + + private void setConf(Configuration c) { + util = new HBaseTestingUtility(c); + } + +} Index: hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestDelete.java =================================================================== --- hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestDelete.java (revision 0) +++ hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestDelete.java (working copy) @@ -0,0 +1,261 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import static org.apache.hadoop.hbase.util.Bytes.toBytes; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.TreeSet; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.backup.HFileArchiver; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Durability; +import org.apache.hadoop.hbase.client.Mutation; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.index.ColumnQualifier.ValueType; +import org.apache.hadoop.hbase.index.Constants; +import org.apache.hadoop.hbase.index.IndexSpecification; +import org.apache.hadoop.hbase.index.util.IndexUtils; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.FSUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(LargeTests.class) +public class TestDelete { + private static final String CF_EMP = "emp"; + private static final String CF_DEPT = "dept"; + private static final String CQ_ENAME = "ename"; + private static final String CQ_SAL = "salary"; + private static final String CQ_DNO = "dno"; + private static final String CQ_DNAME = "dname"; + private static final String START_KEY = "100"; + private static final String END_KEY = "200"; + + private HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private String DIR = TEST_UTIL.getDataTestDir("TestHRegion").toString(); + + private Path basedir; + private HRegion userRegion; + private HRegion indexRegion; + private Map indexMap; + private IndexRegionObserver indexer; + private Collection indexPuts; + private Collection indexDeletes; + + @Before + public void setup() throws Exception { + prepare(); + + indexMap = new HashMap(); + index("idx_ename", CF_EMP, CQ_ENAME, ValueType.String, 10); + index("idx_sal", CF_EMP, CQ_SAL, ValueType.String, 10); + index("idx_dname", CF_DEPT, CQ_DNAME, ValueType.String, 10); + index("idx_dno_ename", CF_DEPT, CQ_DNO, ValueType.String, 10); + index("idx_dno_ename", CF_EMP, CQ_ENAME, ValueType.String, 10); + indexer = new IndexRegionObserver(); + + indexPuts = new TreeSet(); + indexDeletes = new TreeSet(); + } + + @After + public void teardown() throws IOException { + // Pass null table directory path to delete region. + HFileArchiver.archiveRegion(basedir.getFileSystem(TEST_UTIL.getConfiguration()), basedir, null, + new Path(FSUtils.getTableDir(basedir, userRegion.getRegionInfo().getTable()), userRegion + .getRegionInfo().getEncodedName())); + } + + @Test + public void testDeleteVersion() throws IOException { + // prepare test data + put(101, 1, 1230); + put(101, 1, 1240); + + // Delete version. Boundary scenario + deleteVersion(101, CF_EMP, CQ_SAL, 1230); + + // should delete only one cell - one version + assertTrue("Should delete exactly one nearest version of index entry (salary)", + indexDeletes.size() == 1); + // verify deletes against puts + assertTrue("Index-deletes should be a subset of index puts", + indexPuts.containsAll(indexDeletes)); + } + + @Test + public void testDeleteCells() throws IOException { + // prepare test data + put(101, 0, 1230); + put(101, 0, 1240); + put(102, 2, 1240); + + // Delete cell - all versions of a qualifier + deleteColumn(101, CF_EMP, CQ_SAL); + + assertTrue("Should delete all versions of a cell index entry (salary)", + indexDeletes.size() == 2); + // verify deletes against puts + assertTrue("Index-deletes should be a subset of index puts", + indexPuts.containsAll(indexDeletes)); + } + + @Test + public void testDeleteFamily() throws IOException { + // prepare test data + put(101, 1, 1230); + put(101, 2, 1240); + put(102, 1, 1230); + put(102, 0, 1240); + + // Delete family - All cells of the family + deleteFamily(101, CF_EMP); + + // verify deletes against puts + assertTrue("Index-deletes should be a subset of index puts", + indexPuts.size() > indexDeletes.size()); + assertTrue("Index-deletes should be a subset of index puts", + indexPuts.containsAll(indexDeletes)); + } + + @Test + public void testDeleteRow() throws IOException { + // prepare test data + put(101, 1, 1230); + put(101, 2, 1240); + put(102, 1, 1250); + + // Delete row - all families & all cells + deleteRow(101); + deleteRow(102); + + // verify deletes against puts + assertEquals("Puts and deletes are not same", indexPuts, indexDeletes); + } + + private void prepare() throws IOException { + basedir = new Path(DIR + "TestIndexDelete"); + Configuration conf = TEST_UTIL.getConfiguration(); + + // Prepare the 'employee' table region + HTableDescriptor desc = new HTableDescriptor(TableName.valueOf("employee")); + desc.addFamily(new HColumnDescriptor(CF_EMP).setMaxVersions(3)); + desc.addFamily(new HColumnDescriptor(CF_DEPT).setMaxVersions(3)); + HRegionInfo info = + new HRegionInfo(desc.getTableName(), START_KEY.getBytes(), END_KEY.getBytes(), false); + userRegion = HRegion.createHRegion(info, basedir, conf, desc); + + // Prepare the 'employee_idx' index table region + HTableDescriptor idxDesc = new HTableDescriptor(TableName.valueOf("employee_idx")); + idxDesc.addFamily(new HColumnDescriptor(Constants.IDX_COL_FAMILY)); + HRegionInfo idxInfo = + new HRegionInfo(idxDesc.getTableName(), START_KEY.getBytes(), END_KEY.getBytes(), false); + indexRegion = HRegion.createHRegion(idxInfo, basedir, conf, idxDesc); + } + + private void index(String name, String cf, String cq, ValueType type, int maxSize) { + IndexSpecification index = indexMap.get(name); + if (index == null) { + index = new IndexSpecification(name); + } + index.addIndexColumn(new HColumnDescriptor(cf), cq, type, maxSize); + indexMap.put(name, index); + } + + // For simplicity try to derive all details from eno and dno + // Don't add department details when dno is 0 - Equivalent to adding EMP column family only + private void put(int eno, int dno, long ts) throws IOException { + Put put = new Put(toBytes("" + eno)); + + put.add(toBytes(CF_EMP), toBytes(CQ_ENAME), ts, toBytes("emp_" + eno)); + put.add(toBytes(CF_EMP), toBytes(CQ_SAL), ts, toBytes("" + eno * 100)); + // Don't add department details when dno is 0 + // Equivalent to adding EMP column family only + if (dno != 0) { + put.add(toBytes(CF_DEPT), toBytes(CQ_DNO), ts, toBytes("" + dno)); + put.add(toBytes(CF_DEPT), toBytes(CQ_DNAME), ts, toBytes("dept_" + dno)); + } + put.setDurability(Durability.SKIP_WAL); + userRegion.put(put); + for (IndexSpecification spec : indexMap.values()) { + Put idxPut = IndexUtils.prepareIndexPut(put, spec, indexRegion); + if (idxPut != null) { + Cell kv = idxPut.get(Constants.IDX_COL_FAMILY, new byte[0]).get(0); + indexPuts.add(Bytes.toString(idxPut.getRow()) + "_" + kv.getTimestamp()); + } + } + } + + private void deleteRow(int eno) throws IOException { + Delete delete = new Delete(toBytes("" + eno)); + + // Make the delete ready-to-eat by indexer + for (byte[] family : userRegion.getTableDesc().getFamiliesKeys()) { + delete.deleteFamily(family, delete.getTimeStamp()); + } + delete(delete); + } + + private void deleteFamily(int eno, String cf) throws IOException { + Delete delete = new Delete(toBytes("" + eno)); + delete.deleteFamily(toBytes(cf)); + delete(delete); + } + + private void deleteColumn(int eno, String cf, String cq) throws IOException { + Delete delete = new Delete(toBytes("" + eno)); + delete.deleteColumns(toBytes(cf), toBytes(cq)); + delete(delete); + } + + private void deleteVersion(int eno, String cf, String cq, long ts) throws IOException { + Delete delete = new Delete(toBytes("" + eno)); + delete.deleteColumn(toBytes(cf), toBytes(cq), ts); + delete(delete); + } + + private void delete(Delete delete) throws IOException { + Collection deletes = + indexer.prepareIndexDeletes(delete, userRegion, + new ArrayList(indexMap.values()), indexRegion); + for (Mutation idxdelete : deletes) { + Cell kv = idxdelete.getFamilyCellMap().get(Constants.IDX_COL_FAMILY).get(0); + indexDeletes.add(Bytes.toString(idxdelete.getRow()) + "_" + kv.getTimestamp()); + } + } +} Index: hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestExtendedPutOps.java =================================================================== --- hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestExtendedPutOps.java (revision 0) +++ hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestExtendedPutOps.java (working copy) @@ -0,0 +1,790 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import static org.junit.Assert.assertTrue; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.index.ColumnQualifier; +import org.apache.hadoop.hbase.index.ColumnQualifier.ValueType; +import org.apache.hadoop.hbase.index.IndexSpecification; +import org.apache.hadoop.hbase.index.SeparatorPartition; +import org.apache.hadoop.hbase.index.SpatialPartition; +import org.apache.hadoop.hbase.index.util.IndexUtils; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Assert; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(LargeTests.class) +public class TestExtendedPutOps { + private HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private final String DIR = TEST_UTIL.getDataTestDir("TestHRegion").toString(); + + @Test(timeout = 180000) + public void testPutWithOneUnitLengthSeparator() throws IOException { + Path basedir = new Path(DIR + "TestIndexPut"); + Configuration conf = TEST_UTIL.getConfiguration(); + HTableDescriptor htd = + new HTableDescriptor(TableName.valueOf("testPutWithOneUnitLengthSeparator")); + HRegionInfo info = new HRegionInfo(htd.getTableName(), "ABC".getBytes(), "BBB".getBytes(), false); + HRegion region = HRegion.createHRegion(info, basedir, conf, htd); + IndexSpecification spec = new IndexSpecification("index"); + spec.addIndexColumn(new HColumnDescriptor("col"), "ql1", new SeparatorPartition("_", 4), + ValueType.String, 10); + + byte[] value1 = "2ndFloor_solitaire_huawei_bangalore_karnataka".getBytes(); + Put p = new Put("row".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + Put indexPut = IndexUtils.prepareIndexPut(p, spec, region); + byte[] indexRowKey = indexPut.getRow(); + byte[] actualResult = new byte[10]; + System.arraycopy(indexRowKey, 22, actualResult, 0, actualResult.length); + byte[] expectedResult = new byte[10]; + System.arraycopy("bangalore".getBytes(), 0, expectedResult, 0, "bangalore".getBytes().length); + Assert.assertTrue(Bytes.equals(actualResult, expectedResult)); + + value1 = "2ndFloor_solitaire_huawei_bangal".getBytes(); + p = new Put("row1".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + indexPut = IndexUtils.prepareIndexPut(p, spec, region); + indexRowKey = indexPut.getRow(); + actualResult = new byte[10]; + System.arraycopy(indexRowKey, 22, actualResult, 0, actualResult.length); + expectedResult = new byte[10]; + System.arraycopy("bangal".getBytes(), 0, expectedResult, 0, "bangal".getBytes().length); + Assert.assertTrue(Bytes.equals(actualResult, expectedResult)); + + value1 = "2ndFloor_solitaire_huawei_".getBytes(); + p = new Put("row2".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + indexPut = IndexUtils.prepareIndexPut(p, spec, region); + indexRowKey = indexPut.getRow(); + actualResult = new byte[10]; + System.arraycopy(indexRowKey, 22, actualResult, 0, actualResult.length); + expectedResult = new byte[10]; + Assert.assertTrue(Bytes.equals(actualResult, expectedResult)); + } + + @Test(timeout = 180000) + public void testPutWithOneAsSplit() throws IOException { + Path basedir = new Path(DIR + "TestIndexPut"); + Configuration conf = TEST_UTIL.getConfiguration(); + HTableDescriptor htd = + new HTableDescriptor(TableName.valueOf("testPutWithOneUnitLengthSeparator")); + HRegionInfo info = new HRegionInfo(htd.getTableName(), "ABC".getBytes(), "BBB".getBytes(), false); + HRegion region = HRegion.createHRegion(info, basedir, conf, htd); + IndexSpecification spec = new IndexSpecification("index"); + spec.addIndexColumn(new HColumnDescriptor("col"), "ql1", new SeparatorPartition("---", 1), + ValueType.String, 10); + + byte[] value1 = "AB---CD---EF---GH---IJ---KL---MN---OP---".getBytes(); + Put p = new Put("row".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + Put indexPut = IndexUtils.prepareIndexPut(p, spec, region); + byte[] indexRowKey = indexPut.getRow(); + byte[] actualResult = new byte[10]; + System.arraycopy(indexRowKey, 22, actualResult, 0, actualResult.length); + byte[] expectedResult = new byte[10]; + System.arraycopy("AB".getBytes(), 0, expectedResult, 0, "AB".getBytes().length); + Assert.assertTrue(Bytes.equals(actualResult, expectedResult)); + + value1 = "---CD---EF---GH---IJ---KL---MN---OP---".getBytes(); + p = new Put("row1".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + indexPut = IndexUtils.prepareIndexPut(p, spec, region); + indexRowKey = indexPut.getRow(); + actualResult = new byte[10]; + System.arraycopy(indexRowKey, 22, actualResult, 0, actualResult.length); + expectedResult = new byte[10]; + Assert.assertTrue(Bytes.equals(actualResult, expectedResult)); + + value1 = "AB".getBytes(); + p = new Put("row1".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + indexPut = IndexUtils.prepareIndexPut(p, spec, region); + indexRowKey = indexPut.getRow(); + actualResult = new byte[10]; + System.arraycopy(indexRowKey, 22, actualResult, 0, actualResult.length); + expectedResult = new byte[10]; + System.arraycopy("AB".getBytes(), 0, expectedResult, 0, "AB".getBytes().length); + Assert.assertTrue(Bytes.equals(actualResult, expectedResult)); + + value1 = "".getBytes(); + p = new Put("row1".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + indexPut = IndexUtils.prepareIndexPut(p, spec, region); + indexRowKey = indexPut.getRow(); + actualResult = new byte[10]; + System.arraycopy(indexRowKey, 22, actualResult, 0, actualResult.length); + expectedResult = new byte[10]; + Assert.assertTrue(Bytes.equals(actualResult, expectedResult)); + } + + @Test(timeout = 180000) + public void testPutWithOneUnitLengthSeparatorWithoutValue() throws IOException { + Path basedir = new Path(DIR + "TestIndexPut"); + Configuration conf = TEST_UTIL.getConfiguration(); + HTableDescriptor htd = + new HTableDescriptor(TableName.valueOf("testPutWithOneUnitLengthSeparatorWithoutValue")); + HRegionInfo info = new HRegionInfo(htd.getTableName(), "ABC".getBytes(), "BBB".getBytes(), false); + HRegion region = HRegion.createHRegion(info, basedir, conf, htd); + IndexSpecification spec = new IndexSpecification("index"); + spec.addIndexColumn(new HColumnDescriptor("col"), "ql1", new SeparatorPartition("_", 4), + ValueType.String, 10); + byte[] value1 = "2ndFloor_solitaire_huawei__karnataka".getBytes(); + Put p = new Put("row".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + Put indexPut = IndexUtils.prepareIndexPut(p, spec, region); + byte[] indexRowKey = indexPut.getRow(); + byte[] actualResult = new byte[10]; + System.arraycopy(indexRowKey, 22, actualResult, 0, actualResult.length); + byte[] expectedResult = new byte[10]; + Assert.assertTrue(Bytes.equals(actualResult, expectedResult)); + } + + @Test(timeout = 180000) + public void testIndexPutWithMultipleUnitLengthSeparator() throws IOException { + Path basedir = new Path(DIR + "TestIndexPut"); + Configuration conf = TEST_UTIL.getConfiguration(); + HTableDescriptor htd = + new HTableDescriptor(TableName.valueOf("testIndexPutWithMultipleUnitLengthSeparator")); + HRegionInfo info = new HRegionInfo(htd.getTableName(), "ABC".getBytes(), "BBB".getBytes(), false); + HRegion region = HRegion.createHRegion(info, basedir, conf, htd); + IndexSpecification spec = new IndexSpecification("index"); + spec.addIndexColumn(new HColumnDescriptor("col"), "ql1", new SeparatorPartition("---", 6), + ValueType.String, 10); + + byte[] value1 = "AB---CD---EF---GH---IJ---KL---MN---OP---".getBytes(); + Put p = new Put("row".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + Put indexPut = IndexUtils.prepareIndexPut(p, spec, region); + byte[] indexRowKey = indexPut.getRow(); + byte[] actualResult = new byte[10]; + System.arraycopy(indexRowKey, 22, actualResult, 0, actualResult.length); + byte[] expectedResult = new byte[10]; + System.arraycopy("KL".getBytes(), 0, expectedResult, 0, "KL".getBytes().length); + Assert.assertTrue(Bytes.equals(actualResult, expectedResult)); + + value1 = "AB---CD---EF---GH---IJ---K".getBytes(); + p = new Put("row2".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + indexPut = IndexUtils.prepareIndexPut(p, spec, region); + indexRowKey = indexPut.getRow(); + actualResult = new byte[10]; + System.arraycopy(indexRowKey, 22, actualResult, 0, actualResult.length); + expectedResult = new byte[10]; + System.arraycopy("K".getBytes(), 0, expectedResult, 0, "K".getBytes().length); + Assert.assertTrue(Bytes.equals(actualResult, expectedResult)); + + value1 = "AB---CD---EF---GH---".getBytes(); + p = new Put("row2".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + indexPut = IndexUtils.prepareIndexPut(p, spec, region); + indexRowKey = indexPut.getRow(); + actualResult = new byte[10]; + System.arraycopy(indexRowKey, 22, actualResult, 0, actualResult.length); + expectedResult = new byte[10]; + Assert.assertTrue(Bytes.equals(actualResult, expectedResult)); + } + + @Test(timeout = 180000) + public void testIndexPutWithMultipleUnitLengthWithSimilarStringPattern() throws IOException { + Path basedir = new Path(DIR + "TestIndexPut"); + Configuration conf = TEST_UTIL.getConfiguration(); + HTableDescriptor htd = + new HTableDescriptor(TableName.valueOf("testIndexPutWithMultipleUnitLengthSeparator")); + HRegionInfo info = new HRegionInfo(htd.getTableName(), "ABC".getBytes(), "BBB".getBytes(), false); + HRegion region = HRegion.createHRegion(info, basedir, conf, htd); + IndexSpecification spec = new IndexSpecification("index"); + spec.addIndexColumn(new HColumnDescriptor("col"), "ql1", new SeparatorPartition("---", 6), + ValueType.String, 10); + + byte[] value1 = "AB---CD---EF---GH---IJ---K-L---MN---OP---".getBytes(); + Put p = new Put("row".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + Put indexPut = IndexUtils.prepareIndexPut(p, spec, region); + byte[] indexRowKey = indexPut.getRow(); + byte[] actualResult = new byte[10]; + System.arraycopy(indexRowKey, 22, actualResult, 0, actualResult.length); + byte[] expectedResult = new byte[10]; + System.arraycopy("K-L".getBytes(), 0, expectedResult, 0, "K-L".getBytes().length); + Assert.assertTrue(Bytes.equals(actualResult, expectedResult)); + + value1 = "AB---CD---EF---GH---IJ---K--L".getBytes(); + p = new Put("row2".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + indexPut = IndexUtils.prepareIndexPut(p, spec, region); + indexRowKey = indexPut.getRow(); + actualResult = new byte[10]; + System.arraycopy(indexRowKey, 22, actualResult, 0, actualResult.length); + expectedResult = new byte[10]; + System.arraycopy("K--L".getBytes(), 0, expectedResult, 0, "K--L".getBytes().length); + Assert.assertTrue(Bytes.equals(actualResult, expectedResult)); + + value1 = "AB---CD---EF---GH---IJ----".getBytes(); + p = new Put("row2".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + indexPut = IndexUtils.prepareIndexPut(p, spec, region); + indexRowKey = indexPut.getRow(); + actualResult = new byte[10]; + System.arraycopy(indexRowKey, 22, actualResult, 0, actualResult.length); + expectedResult = new byte[10]; + expectedResult[0] = '-'; + Assert.assertTrue(Bytes.equals(actualResult, expectedResult)); + } + + @Test(timeout = 180000) + public void testIndexPutWithOffsetAndLength() throws IOException { + Path basedir = new Path(DIR + "TestIndexPut"); + Configuration conf = TEST_UTIL.getConfiguration(); + HTableDescriptor htd = + new HTableDescriptor(TableName.valueOf("testIndexPutWithOffsetAndLength")); + HRegionInfo info = new HRegionInfo(htd.getTableName(), "ABC".getBytes(), "BBB".getBytes(), false); + HRegion region = HRegion.createHRegion(info, basedir, conf, htd); + IndexSpecification spec = new IndexSpecification("index"); + spec.addIndexColumn(new HColumnDescriptor("col"), "ql1", new SpatialPartition(20, 2), + ValueType.String, 18); + + byte[] value1 = "AB---CD---EF---GH---IJ---KL---MN---OP---".getBytes(); + Put p = new Put("row".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + Put indexPut = IndexUtils.prepareIndexPut(p, spec, region); + byte[] indexRowKey = indexPut.getRow(); + byte[] actualResult = new byte[2]; + System.arraycopy(indexRowKey, 22, actualResult, 0, actualResult.length); + byte[] expectedResult = new byte[2]; + System.arraycopy("IJ".getBytes(), 0, expectedResult, 0, "IJ".getBytes().length); + Assert.assertTrue(Bytes.equals(actualResult, expectedResult)); + } + + @Test(timeout = 180000) + public void testIndexPutWithOffsetAndLengthWhenPutIsSmallerThanOffset() throws IOException { + Path basedir = new Path(DIR + "TestIndexPut"); + Configuration conf = TEST_UTIL.getConfiguration(); + HTableDescriptor htd = + new HTableDescriptor( + TableName.valueOf("testIndexPutWithOffsetAndLengthWhenPutIsSmallerThanOffset")); + HRegionInfo info = new HRegionInfo(htd.getTableName(), "ABC".getBytes(), "BBB".getBytes(), false); + HRegion region = HRegion.createHRegion(info, basedir, conf, htd); + IndexSpecification spec = new IndexSpecification("index"); + spec.addIndexColumn(new HColumnDescriptor("col"), "ql1", new SpatialPartition(20, 2), + ValueType.String, 18); + + byte[] value1 = "AB---CD---EF---GH".getBytes(); + Put p = new Put("row".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + Put indexPut = IndexUtils.prepareIndexPut(p, spec, region); + byte[] indexRowKey = indexPut.getRow(); + byte[] actualResult = new byte[2]; + System.arraycopy(indexRowKey, 22, actualResult, 0, actualResult.length); + byte[] expectedResult = new byte[2]; + Assert.assertTrue(Bytes.equals(actualResult, expectedResult)); + + value1 = "AB---CD---EF---GH---I".getBytes(); + p = new Put("row".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + indexPut = IndexUtils.prepareIndexPut(p, spec, region); + indexRowKey = indexPut.getRow(); + actualResult = new byte[2]; + System.arraycopy(indexRowKey, 22, actualResult, 0, actualResult.length); + expectedResult = new byte[2]; + expectedResult[0] = 'I'; + Assert.assertTrue(Bytes.equals(actualResult, expectedResult)); + } + + @Test(timeout = 180000) + public void testExtentedParametersValidityFailScenarios() throws IOException { + IndexSpecification spec = new IndexSpecification("index"); + + // When separator length is zero + try { + spec.addIndexColumn(new HColumnDescriptor("col"), "ql1", new SeparatorPartition("", 4), + ValueType.String, 10); + Assert.fail("Testcase should fail if separator length is zero."); + } catch (IllegalArgumentException e) { + } + + // when the valuePosition is zero with separator + try { + spec.addIndexColumn(new HColumnDescriptor("col"), "ql1", new SeparatorPartition("--", 0), + ValueType.String, 10); + Assert + .fail("the testcase should fail if the valuePosition with the separator is passed as zero."); + } catch (IllegalArgumentException e) { + } + + } + + public void testExtentedParametersValidityPassScenarios() throws IOException { + IndexSpecification spec = new IndexSpecification("index"); + + // When the provided arguments are correct + try { + spec.addIndexColumn(new HColumnDescriptor("col"), "ql1", new SpatialPartition(4, 10), + ValueType.String, 10); + } catch (IllegalArgumentException e) { + Assert.fail("the testcase should not throw exception as the arguments passed are correct."); + } + + // When the provided arguments are correct + try { + spec.addIndexColumn(new HColumnDescriptor("col"), "ql1", new SeparatorPartition("--", 1), + ValueType.String, 10); + } catch (IllegalArgumentException e) { + Assert.fail("the testcase should not throw exception as the arguments passed are correct."); + } + + try { + spec.addIndexColumn(new HColumnDescriptor("col"), "ql2", ValueType.String, 10); + } catch (IllegalArgumentException e) { + Assert.fail("the testcase should not throw exception as the arguments passed are correct."); + } + } + + @Test(timeout = 180000) + public void testColumnQualifierSerialization() throws Exception { + ByteArrayOutputStream bos = null; + DataOutputStream dos = null; + ByteArrayInputStream bis = null; + DataInputStream dis = null; + try { + bos = new ByteArrayOutputStream(); + dos = new DataOutputStream(bos); + ColumnQualifier cq = + new ColumnQualifier("cf", "cq", ValueType.String, 10, new SpatialPartition(0, 5)); + cq.write(dos); + dos.flush(); + byte[] byteArray = bos.toByteArray(); + bis = new ByteArrayInputStream(byteArray); + dis = new DataInputStream(bis); + ColumnQualifier c = new ColumnQualifier(); + c.readFields(dis); + assertTrue("ColumnQualifier state mismatch.", c.equals(cq)); + } finally { + if (null != bos) { + bos.close(); + } + if (null != dos) { + dos.close(); + } + if (null != bis) { + bis.close(); + } + if (null != dis) { + dis.close(); + } + } + + } + + @Test(timeout = 180000) + public void testIndexPutwithPositiveIntDataTypes() throws IOException { + Path basedir = new Path(DIR + "TestIndexPut"); + Configuration conf = TEST_UTIL.getConfiguration(); + HTableDescriptor htd = + new HTableDescriptor(TableName.valueOf("testIndexPutwithPositiveIntDataTypes")); + HRegionInfo info = new HRegionInfo(htd.getTableName(), "ABC".getBytes(), "BBB".getBytes(), false); + // HLog hlog = UTIL.getMiniHBaseCluster().getRegionServer(0).getWAL(); + HRegion region = HRegion.createHRegion(info, basedir, conf, htd); + IndexSpecification spec = new IndexSpecification("index"); + spec.addIndexColumn(new HColumnDescriptor("col"), "ql1", ValueType.Int, 4); + spec.addIndexColumn(new HColumnDescriptor("col"), "ql2", ValueType.Float, 4); + + byte[] value1 = Bytes.toBytes(1000); + Put p = new Put("row".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + Put indexPut1 = IndexUtils.prepareIndexPut(p, spec, region); + int a = 1000; + byte[] expectedResult = Bytes.toBytes(a ^ (1 << 31)); + byte[] actualResult = new byte[4]; + byte[] indexRowKey = indexPut1.getRow(); + System.arraycopy(indexRowKey, 22, actualResult, 0, actualResult.length); + Assert.assertTrue(Bytes.equals(expectedResult, actualResult)); + } + + @Test(timeout = 180000) + public void testIndexPutWithNegativeIntDataTypes() throws IOException { + Path basedir = new Path(DIR + "TestIndexPut"); + Configuration conf = TEST_UTIL.getConfiguration(); + HTableDescriptor htd = + new HTableDescriptor(TableName.valueOf("testIndexPutWithNegativeIntDataTypes")); + HRegionInfo info = new HRegionInfo(htd.getTableName(), "ABC".getBytes(), "BBB".getBytes(), false); + // HLog hlog = UTIL.getMiniHBaseCluster().getRegionServer(0).getWAL(); + HRegion region = HRegion.createHRegion(info, basedir, conf, htd); + IndexSpecification spec = new IndexSpecification("index"); + spec.addIndexColumn(new HColumnDescriptor("col"), "ql1", ValueType.Int, 4); + spec.addIndexColumn(new HColumnDescriptor("col"), "ql2", ValueType.Float, 4); + + byte[] value1 = Bytes.toBytes(-2562351); + Put p = new Put("row".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + Put indexPut = IndexUtils.prepareIndexPut(p, spec, region); + int a = -2562351; + byte[] expectedResult = Bytes.toBytes(a ^ (1 << 31)); + byte[] actualResult = new byte[4]; + byte[] indexRowKey = indexPut.getRow(); + System.arraycopy(indexRowKey, 22, actualResult, 0, actualResult.length); + Assert.assertTrue(Bytes.equals(expectedResult, actualResult)); + } + + @Test(timeout = 180000) + public void testIndexPutWithLongDataTypes() throws IOException { + Path basedir = new Path(DIR + "TestIndexPut"); + Configuration conf = TEST_UTIL.getConfiguration(); + HTableDescriptor htd = new HTableDescriptor("testIndexPutWithNegativeIntDataTypes"); + HRegionInfo info = new HRegionInfo(htd.getTableName(), "ABC".getBytes(), "BBB".getBytes(), false); + // HLog hlog = UTIL.getMiniHBaseCluster().getRegionServer(0).getWAL(); + HRegion region = HRegion.createHRegion(info, basedir, conf, htd); + IndexSpecification spec = new IndexSpecification("index"); + spec.addIndexColumn(new HColumnDescriptor("col"), "ql1", ValueType.Long, 4); + + byte[] value1 = Bytes.toBytes(-2562351L); + Put p = new Put("row".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + Put indexPut = IndexUtils.prepareIndexPut(p, spec, region); + long a = -2562351L; + byte[] expectedResult = Bytes.toBytes(a ^ (1L << 63)); + byte[] actualResult = new byte[8]; + byte[] indexRowKey = indexPut.getRow(); + System.arraycopy(indexRowKey, 22, actualResult, 0, actualResult.length); + Assert.assertTrue(Bytes.equals(expectedResult, actualResult)); + } + + @Test(timeout = 180000) + public void testIndexPutWithShortDataTypes() throws IOException { + Path basedir = new Path(DIR + "TestIndexPut"); + Configuration conf = TEST_UTIL.getConfiguration(); + HTableDescriptor htd = new HTableDescriptor("testIndexPutWithNegativeIntDataTypes"); + HRegionInfo info = new HRegionInfo(htd.getTableName(), "ABC".getBytes(), "BBB".getBytes(), false); + // HLog hlog = UTIL.getMiniHBaseCluster().getRegionServer(0).getWAL(); + HRegion region = HRegion.createHRegion(info, basedir, conf, htd); + IndexSpecification spec = new IndexSpecification("index"); + spec.addIndexColumn(new HColumnDescriptor("col"), "ql1", ValueType.Short, 4); + + short s = 1000; + byte[] value1 = Bytes.toBytes(s); + Put p = new Put("row".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + Put indexPut = IndexUtils.prepareIndexPut(p, spec, region); + byte[] expectedResult = Bytes.toBytes(s); + expectedResult[0] ^= 1 << 7; + byte[] actualResult = new byte[2]; + byte[] indexRowKey = indexPut.getRow(); + System.arraycopy(indexRowKey, 22, actualResult, 0, actualResult.length); + Assert.assertTrue(Bytes.equals(expectedResult, actualResult)); + } + + @Test(timeout = 180000) + public void testIndexPutWithByteDataTypes() throws IOException { + Path basedir = new Path(DIR + "TestIndexPut"); + Configuration conf = TEST_UTIL.getConfiguration(); + HTableDescriptor htd = new HTableDescriptor("testIndexPutWithNegativeIntDataTypes"); + HRegionInfo info = new HRegionInfo(htd.getTableName(), "ABC".getBytes(), "BBB".getBytes(), false); + // HLog hlog = UTIL.getMiniHBaseCluster().getRegionServer(0).getWAL(); + HRegion region = HRegion.createHRegion(info, basedir, conf, htd); + IndexSpecification spec = new IndexSpecification("index"); + spec.addIndexColumn(new HColumnDescriptor("col"), "ql1", ValueType.Short, 4); + + byte b = 100; + byte[] value1 = Bytes.toBytes(b); + Put p = new Put("row".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + Put indexPut = IndexUtils.prepareIndexPut(p, spec, region); + byte[] expectedResult = Bytes.toBytes(b); + expectedResult[0] ^= 1 << 7; + byte[] actualResult = new byte[2]; + byte[] indexRowKey = indexPut.getRow(); + System.arraycopy(indexRowKey, 22, actualResult, 0, actualResult.length); + Assert.assertTrue(Bytes.equals(expectedResult, actualResult)); + } + + @Test(timeout = 180000) + public void testIndexPutWithCharDataTypes() throws IOException { + Path basedir = new Path(DIR + "TestIndexPut"); + Configuration conf = TEST_UTIL.getConfiguration(); + HTableDescriptor htd = new HTableDescriptor("testIndexPutWithNegativeIntDataTypes"); + HRegionInfo info = new HRegionInfo(htd.getTableName(), "ABC".getBytes(), "BBB".getBytes(), false); + // HLog hlog = UTIL.getMiniHBaseCluster().getRegionServer(0).getWAL(); + HRegion region = HRegion.createHRegion(info, basedir, conf, htd); + IndexSpecification spec = new IndexSpecification("index"); + spec.addIndexColumn(new HColumnDescriptor("col"), "ql1", ValueType.Char, 4); + + char c = 'A'; + byte[] value1 = new byte[2]; + value1[1] = (byte) c; + c >>= 8; + value1[0] = (byte) c; + Put p = new Put("row".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + Put indexPut = IndexUtils.prepareIndexPut(p, spec, region); + byte[] actualResult = new byte[2]; + byte[] indexRowKey = indexPut.getRow(); + System.arraycopy(indexRowKey, 22, actualResult, 0, actualResult.length); + Assert.assertTrue(Bytes.equals(value1, actualResult)); + } + + @Test(timeout = 180000) + public void testIndexPutWithDoubleDataTypes() throws IOException { + Path basedir = new Path(DIR + "TestIndexPut"); + Configuration conf = TEST_UTIL.getConfiguration(); + HTableDescriptor htd = + new HTableDescriptor(TableName.valueOf("testIndexPutWithNegativeIntDataTypes")); + HRegionInfo info = new HRegionInfo(htd.getTableName(), "ABC".getBytes(), "BBB".getBytes(), false); + // HLog hlog = UTIL.getMiniHBaseCluster().getRegionServer(0).getWAL(); + HRegion region = HRegion.createHRegion(info, basedir, conf, htd); + IndexSpecification spec = new IndexSpecification("index"); + spec.addIndexColumn(new HColumnDescriptor("col"), "ql1", ValueType.Double, 8); + + byte[] value1 = Bytes.toBytes(109.4548957D); + Put p = new Put("row".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + Put indexPut = IndexUtils.prepareIndexPut(p, spec, region); + double d = 109.4548957D; + byte[] expectedResult = Bytes.toBytes(d); + expectedResult[0] ^= 1 << 7; + byte[] actualResult = new byte[8]; + byte[] indexRowKey = indexPut.getRow(); + System.arraycopy(indexRowKey, 22, actualResult, 0, actualResult.length); + Assert.assertTrue(Bytes.equals(expectedResult, actualResult)); + + value1 = Bytes.toBytes(-109.4548957D); + p = new Put("row".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + indexPut = IndexUtils.prepareIndexPut(p, spec, region); + d = -109.4548957D; + expectedResult = Bytes.toBytes(d); + for (int i = 0; i < 8; i++) { + expectedResult[i] ^= 0xff; + } + actualResult = new byte[8]; + indexRowKey = indexPut.getRow(); + System.arraycopy(indexRowKey, 22, actualResult, 0, actualResult.length); + Assert.assertTrue(Bytes.equals(expectedResult, actualResult)); + } + + @Test(timeout = 180000) + public void testSequenceOfIndexPutsWithNegativeInteger() throws IOException { + Path basedir = new Path(DIR + "TestIndexPut"); + Configuration conf = TEST_UTIL.getConfiguration(); + HTableDescriptor htd = + new HTableDescriptor(TableName.valueOf("testSequenceOfIndexPutsWithDataTypes")); + HRegionInfo info = new HRegionInfo(htd.getTableName(), "ABC".getBytes(), "BBB".getBytes(), false); + HRegion region = HRegion.createHRegion(info, basedir, conf, htd); + IndexSpecification spec = new IndexSpecification("index"); + spec.addIndexColumn(new HColumnDescriptor("col"), "ql1", ValueType.Int, 4); + + byte[] value1 = Bytes.toBytes(-1000); + Put p = new Put("row".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + Put indexPut = IndexUtils.prepareIndexPut(p, spec, region); + int a = -1000; + byte[] expectedResult = Bytes.toBytes(a ^ (1 << 31)); + byte[] actualResult = new byte[4]; + byte[] indexRowKey = indexPut.getRow(); + System.arraycopy(indexRowKey, 22, actualResult, 0, actualResult.length); + Assert.assertTrue(Bytes.equals(expectedResult, actualResult)); + + value1 = Bytes.toBytes(-1500); + p = new Put("row".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + Put indexPut1 = IndexUtils.prepareIndexPut(p, spec, region); + a = -1500; + byte[] expectedResult1 = Bytes.toBytes(a ^ (1 << 31)); + byte[] actualResult1 = new byte[4]; + byte[] indexRowKey1 = indexPut1.getRow(); + System.arraycopy(indexRowKey1, 22, actualResult1, 0, actualResult1.length); + Assert.assertTrue(Bytes.equals(expectedResult1, actualResult1)); + + Assert.assertTrue(Bytes.compareTo(indexPut.getRow(), indexPut1.getRow()) > 0); + + value1 = Bytes.toBytes(1500); + p = new Put("row".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + Put indexPut2 = IndexUtils.prepareIndexPut(p, spec, region); + a = 1500; + byte[] expectedResult2 = Bytes.toBytes(a ^ (1 << 31)); + byte[] actualResult2 = new byte[4]; + byte[] indexRowKey2 = indexPut2.getRow(); + System.arraycopy(indexRowKey2, 22, actualResult2, 0, actualResult2.length); + Assert.assertTrue(Bytes.equals(expectedResult2, actualResult2)); + + Assert.assertTrue(Bytes.compareTo(indexPut2.getRow(), indexPut.getRow()) > 0); + + value1 = Bytes.toBytes(2000); + p = new Put("row".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + Put indexPut3 = IndexUtils.prepareIndexPut(p, spec, region); + a = 2000; + byte[] expectedResult3 = Bytes.toBytes(a ^ (1 << 31)); + byte[] actualResult3 = new byte[4]; + byte[] indexRowKey3 = indexPut3.getRow(); + System.arraycopy(indexRowKey3, 22, actualResult3, 0, actualResult3.length); + Assert.assertTrue(Bytes.equals(expectedResult3, actualResult3)); + + Assert.assertTrue(Bytes.compareTo(indexPut3.getRow(), indexPut2.getRow()) > 0); + } + + @Test(timeout = 180000) + public void testSequenceOfIndexPutsWithNegativeFloat() throws IOException { + Path basedir = new Path(DIR + "TestIndexPut"); + Configuration conf = TEST_UTIL.getConfiguration(); + HTableDescriptor htd = + new HTableDescriptor(TableName.valueOf("testSequenceOfIndexPutsWithDataTypes")); + HRegionInfo info = new HRegionInfo(htd.getTableName(), "ABC".getBytes(), "BBB".getBytes(), false); + HRegion region = HRegion.createHRegion(info, basedir, conf, htd); + IndexSpecification spec = new IndexSpecification("index"); + spec.addIndexColumn(new HColumnDescriptor("col"), "ql1", ValueType.Float, 4); + + byte[] value1 = Bytes.toBytes(-10.40f); + Put p = new Put("row".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + Put indexPut = IndexUtils.prepareIndexPut(p, spec, region); + byte[] expectedResult = Bytes.toBytes(-10.40f); + expectedResult[0] ^= 0xff; + expectedResult[1] ^= 0xff; + expectedResult[2] ^= 0xff; + expectedResult[3] ^= 0xff; + byte[] actualResult = new byte[4]; + byte[] indexRowKey = indexPut.getRow(); + System.arraycopy(indexRowKey, 22, actualResult, 0, actualResult.length); + Assert.assertTrue(Bytes.equals(expectedResult, actualResult)); + + value1 = Bytes.toBytes(-15.20f); + p = new Put("row".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + Put indexPut1 = IndexUtils.prepareIndexPut(p, spec, region); + byte[] expectedResult1 = Bytes.toBytes(-15.20f); + expectedResult1[0] ^= 0xff; + expectedResult1[1] ^= 0xff; + expectedResult1[2] ^= 0xff; + expectedResult1[3] ^= 0xff; + byte[] actualResult1 = new byte[4]; + byte[] indexRowKey1 = indexPut1.getRow(); + System.arraycopy(indexRowKey1, 22, actualResult1, 0, actualResult1.length); + Assert.assertTrue(Bytes.equals(expectedResult1, actualResult1)); + + Assert.assertTrue(Bytes.compareTo(indexPut.getRow(), indexPut1.getRow()) > 0); + + value1 = Bytes.toBytes(30.50f); + p = new Put("row".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + Put indexPut2 = IndexUtils.prepareIndexPut(p, spec, region); + byte[] expectedResult2 = Bytes.toBytes(30.50f); + expectedResult2[0] ^= 1 << 7; + byte[] actualResult2 = new byte[4]; + byte[] indexRowKey2 = indexPut2.getRow(); + System.arraycopy(indexRowKey2, 22, actualResult2, 0, actualResult2.length); + Assert.assertTrue(Bytes.equals(expectedResult2, actualResult2)); + + Assert.assertTrue(Bytes.compareTo(indexPut2.getRow(), indexPut.getRow()) > 0); + + value1 = Bytes.toBytes(40.54f); + p = new Put("row".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + Put indexPut3 = IndexUtils.prepareIndexPut(p, spec, region); + byte[] expectedResult3 = Bytes.toBytes(40.54f); + expectedResult3[0] ^= 1 << 7; + byte[] actualResult3 = new byte[4]; + byte[] indexRowKey3 = indexPut3.getRow(); + System.arraycopy(indexRowKey3, 22, actualResult3, 0, actualResult3.length); + Assert.assertTrue(Bytes.equals(expectedResult3, actualResult3)); + + Assert.assertTrue(Bytes.compareTo(indexPut3.getRow(), indexPut2.getRow()) > 0); + } + + @Test(timeout = 180000) + public void testSequenceOfIndexPutsWithDataTypes() throws IOException { + Path basedir = new Path(DIR + "TestIndexPut"); + Configuration conf = TEST_UTIL.getConfiguration(); + HTableDescriptor htd = + new HTableDescriptor(TableName.valueOf("testSequenceOfIndexPutsWithDataTypes")); + HRegionInfo info = new HRegionInfo(htd.getTableName(), "ABC".getBytes(), "BBB".getBytes(), false); + HRegion region = HRegion.createHRegion(info, basedir, conf, htd); + IndexSpecification spec = new IndexSpecification("index"); + spec.addIndexColumn(new HColumnDescriptor("col"), "ql1", ValueType.Int, 4); + + byte[] value1 = Bytes.toBytes(1000); + Put p = new Put("row".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + Put indexPut = IndexUtils.prepareIndexPut(p, spec, region); + int a = 1000; + byte[] expectedResult = Bytes.toBytes(a ^ (1 << 31)); + byte[] actualResult = new byte[4]; + byte[] indexRowKey = indexPut.getRow(); + System.arraycopy(indexRowKey, 22, actualResult, 0, actualResult.length); + Assert.assertTrue(Bytes.equals(expectedResult, actualResult)); + + value1 = Bytes.toBytes(-2562351); + p = new Put("row".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + Put indexPut1 = IndexUtils.prepareIndexPut(p, spec, region); + a = -2562351; + byte[] expectedResult1 = Bytes.toBytes(a ^ (1 << 31)); + byte[] actualResult1 = new byte[4]; + byte[] indexRowKey1 = indexPut1.getRow(); + System.arraycopy(indexRowKey1, 22, actualResult1, 0, actualResult1.length); + Assert.assertTrue(Bytes.equals(expectedResult1, actualResult1)); + + Assert.assertTrue(Bytes.compareTo(indexPut.getRow(), indexPut1.getRow()) > 0); + } + + @Test(timeout = 180000) + public void testIndexPutWithSeparatorAndDataType() throws IOException { + Path basedir = new Path(DIR + "TestIndexPut"); + Configuration conf = TEST_UTIL.getConfiguration(); + HTableDescriptor htd = + new HTableDescriptor(TableName.valueOf("testIndexPutWithSeparatorAndDataType")); + HRegionInfo info = new HRegionInfo(htd.getTableName(), "ABC".getBytes(), "BBB".getBytes(), false); + HRegion region = HRegion.createHRegion(info, basedir, conf, htd); + IndexSpecification spec = new IndexSpecification("index"); + spec.addIndexColumn(new HColumnDescriptor("col"), "ql1", new SeparatorPartition("---", 4), + ValueType.Int, 4); + + byte[] putValue = new byte[19]; + byte[] value1 = "AB---CD---EF---".getBytes(); + byte[] value2 = Bytes.toBytes(100000); + System.arraycopy(value1, 0, putValue, 0, value1.length); + System.arraycopy(value2, 0, putValue, value1.length, value2.length); + Put p = new Put("row".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), putValue); + Put indexPut = IndexUtils.prepareIndexPut(p, spec, region); + int a = 100000; + byte[] expectedResult = Bytes.toBytes(a ^ (1 << 31)); + byte[] actualResult = new byte[4]; + byte[] indexRowKey = indexPut.getRow(); + System.arraycopy(indexRowKey, 22, actualResult, 0, actualResult.length); + Assert.assertTrue(Bytes.equals(expectedResult, actualResult)); + } + +} Index: hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestForComplexIssues.java =================================================================== --- hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestForComplexIssues.java (revision 0) +++ hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestForComplexIssues.java (working copy) @@ -0,0 +1,426 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import java.util.List; +import java.util.concurrent.CountDownLatch; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseIOException; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.MasterNotRunningException; +import org.apache.hadoop.hbase.MiniHBaseCluster; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.UnknownRegionException; +import org.apache.hadoop.hbase.ZooKeeperConnectionException; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Mutation; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.apache.hadoop.hbase.coprocessor.ObserverContext; +import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; +import org.apache.hadoop.hbase.index.ColumnQualifier.ValueType; +import org.apache.hadoop.hbase.index.IndexSpecification; +import org.apache.hadoop.hbase.index.SecIndexLoadBalancer; +import org.apache.hadoop.hbase.index.TableIndices; +import org.apache.hadoop.hbase.index.TestUtils; +import org.apache.hadoop.hbase.index.client.IndexAdmin; +import org.apache.hadoop.hbase.index.coprocessor.master.IndexMasterObserver; +import org.apache.hadoop.hbase.index.coprocessor.wal.IndexWALObserver; +import org.apache.hadoop.hbase.index.mapreduce.TableIndexer; +import org.apache.hadoop.hbase.index.util.IndexUtils; +import org.apache.hadoop.hbase.index.Constants; +import org.apache.hadoop.hbase.master.LoadBalancer; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.regionserver.InternalScanner; +import org.apache.hadoop.hbase.regionserver.KeyValueScanner; +import org.apache.hadoop.hbase.regionserver.MiniBatchOperationInProgress; +import org.apache.hadoop.hbase.regionserver.ScanType; +import org.apache.hadoop.hbase.regionserver.Store; +import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.After; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(LargeTests.class) +public class TestForComplexIssues { + private static HBaseTestingUtility UTIL = new HBaseTestingUtility(); + private static HBaseAdmin admin = null; + public static CountDownLatch latch = new CountDownLatch(1); + public static CountDownLatch latchForCompact = new CountDownLatch(1); + private static boolean compactionCalled = false; + private static volatile boolean closeCalled = false; + private static volatile boolean preCloseCalled = false; + private static volatile boolean delayPostBatchMutate = false; + private static int openCount = 0; + private static int closeCount = 0; + + @BeforeClass + public static void setupBeforeClass() throws Exception { + Configuration conf = UTIL.getConfiguration(); + conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, IndexMasterObserver.class.getName()); + conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, LocalIndexRegionObserver.class.getName()); + conf.set(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY, IndexWALObserver.class.getName()); + conf.setBoolean("hbase.use.secondary.index", true); + conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, SecIndexLoadBalancer.class, + LoadBalancer.class); + } + + @Before + public void setUp() throws Exception { + UTIL.startMiniCluster(2); + admin = new IndexAdmin(UTIL.getConfiguration()); + } + + @After + public void tearDown() throws Exception { + compactionCalled = false; + closeCalled = false; + preCloseCalled = false; + delayPostBatchMutate = false; + if(admin != null) admin.close(); + UTIL.shutdownMiniCluster(); + } + + @Test(timeout = 180000) + public void testHDP2989() throws Exception { + Configuration conf = UTIL.getConfiguration(); + conf.setInt("hbase.client.retries.number", 1); + String userTableName = "testHDP2989"; + HTableDescriptor ihtd = + TestUtils.createIndexedHTableDescriptor(userTableName, "col", "index_name", "cf", "ql"); + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + // test put with the indexed column + for (int i = 0; i < 10; i++) { + String row = "row" + i; + Put p = new Put(row.getBytes()); + String val = "Val" + i; + p.add("col".getBytes(), "ql".getBytes(), val.getBytes()); + table.put(p); + admin.flush(userTableName); + } + + MiniHBaseCluster hBaseCluster = UTIL.getHBaseCluster(); + List regions = hBaseCluster.getRegions(Bytes.toBytes(userTableName)); + HRegion hRegion = regions.get(0); + byte[] encodedNameAsBytes = hRegion.getRegionInfo().getEncodedNameAsBytes(); + int serverWith = hBaseCluster.getServerWith(hRegion.getRegionName()); + int destServerNo = (serverWith == 1 ? 0 : 1); + HRegionServer destServer = hBaseCluster.getRegionServer(destServerNo); + + admin.compact(userTableName); + + while (!compactionCalled) { + Thread.sleep(1000); + } + + byte[] dstRSName = Bytes.toBytes(destServer.getServerName().getServerName()); + // Move the main region + MoveThread move = new MoveThread(admin, encodedNameAsBytes, dstRSName); + move.start(); + + while (!preCloseCalled) { + Thread.sleep(200); + } + regions = hBaseCluster.getRegions(Bytes.toBytes(userTableName + "_idx")); + HRegion hRegion1 = regions.get(0); + encodedNameAsBytes = hRegion1.getRegionInfo().getEncodedNameAsBytes(); + move = new MoveThread(admin, encodedNameAsBytes, dstRSName); + move.start(); + while (!closeCalled) { + Thread.sleep(200); + } + + String row = "row" + 46; + Put p = new Put(row.getBytes()); + String val = "Val" + 46; + p.add("col".getBytes(), "ql".getBytes(), val.getBytes()); + try { + table.put(p); + } catch (Exception e) { + e.printStackTrace(); + } + latchForCompact.countDown(); + closeCount++; + latch.countDown(); + List onlineRegions = destServer.getOnlineRegions(TableName.valueOf(userTableName)); + List onlineIdxRegions = + destServer.getOnlineRegions(TableName.valueOf(IndexUtils.getIndexTableName(userTableName))); + while (onlineRegions.size() == 0 || onlineIdxRegions.size() == 0) { + Thread.sleep(1000); + onlineRegions = destServer.getOnlineRegions(TableName.valueOf(userTableName)); + onlineIdxRegions = + destServer + .getOnlineRegions(TableName.valueOf(IndexUtils.getIndexTableName(userTableName))); + } + + /* + * closeCount++; latch.countDown(); + */ + + Scan s = new Scan(); + conf.setInt("hbase.client.retries.number", 10); + table = new HTable(conf, userTableName); + ResultScanner scanner = table.getScanner(s); + int i = 0; + for (Result result : scanner) { + i++; + } + HTable indextable = new HTable(conf, userTableName + "_idx"); + s = new Scan(); + scanner = indextable.getScanner(s); + int j = 0; + for (Result result : scanner) { + j++; + } + assertEquals("", i, j); + + } + + @Test(timeout = 180000) + public void testHDP3015() throws Exception { + Configuration conf = UTIL.getConfiguration(); + final String userTableName = "testHDP3015"; + HTableDescriptor ihtd = + TestUtils.createIndexedHTableDescriptor(userTableName, "col", "index_name", "col", "ql"); + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + // test put with the indexed column + for (int i = 0; i < 4; i++) { + String row = "row" + i; + Put p = new Put(row.getBytes()); + String val = "Val" + i; + p.add("col".getBytes(), "ql".getBytes(), val.getBytes()); + if (i == 3) { + delayPostBatchMutate = true; + } + table.put(p); + if (i == 2) { + new Thread() { + public void run() { + try { + admin.flush(userTableName + "_idx"); + } catch (Exception e) { + + } + } + }.start(); + } + } + MiniHBaseCluster hBaseCluster = UTIL.getHBaseCluster(); + List userRegions = hBaseCluster.getRegions(Bytes.toBytes(userTableName)); + + List indexRegions = hBaseCluster.getRegions(Bytes.toBytes(userTableName + "_idx")); + HRegion indexHRegion = indexRegions.get(0); + + int serverWith = hBaseCluster.getServerWith((userRegions.get(0).getRegionName())); + HRegionServer regionServer = hBaseCluster.getRegionServer(serverWith); + byte[][] indexColumns = new byte[1][]; + indexColumns[0] = Constants.IDX_COL_FAMILY; + List storeFileList = indexHRegion.getStoreFileList(indexColumns); + while (storeFileList.size() == 0) { + storeFileList = indexHRegion.getStoreFileList(indexColumns); + } + + table = new HTable(conf, userTableName); + ResultScanner scanner = table.getScanner(new Scan()); + int i = 0; + for (Result result : scanner) { + i++; + } + table = new HTable(conf, userTableName + "_idx"); + scanner = table.getScanner(new Scan()); + int j = 0; + for (Result result : scanner) { + j++; + } + assertEquals("", i, j); + + hBaseCluster.abortRegionServer(serverWith); + hBaseCluster.waitOnRegionServer(serverWith); + + boolean regionOnline = + hBaseCluster.getMaster().getAssignmentManager() + .getRegionStates().isRegionOnline(indexHRegion.getRegionInfo()); + while (!regionOnline) { + regionOnline = + hBaseCluster.getMaster().getAssignmentManager() + .getRegionStates().isRegionOnline(indexHRegion.getRegionInfo()); + } + + table = new HTable(conf, userTableName); + scanner = table.getScanner(new Scan()); + i = 0; + for (Result result : scanner) { + i++; + } + table = new HTable(conf, userTableName + "_idx"); + scanner = table.getScanner(new Scan()); + j = 0; + for (Result result : scanner) { + j++; + } + assertEquals("", i, j); + + } + + public static class LocalIndexRegionObserver extends IndexRegionObserver { + + @Override + public InternalScanner preCompactScannerOpen(ObserverContext c, + Store store, List scanners, ScanType scanType, + long earliestPutTs, InternalScanner s, CompactionRequest request) throws IOException { + if (store.getTableName().getNameAsString().equals("testHDP2989")) { + try { + compactionCalled = true; + latchForCompact.await(); + } catch (InterruptedException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + return super.preCompactScannerOpen(c, store, scanners, scanType, earliestPutTs, s, request); + } + + @Override + public void postOpen(ObserverContext e) { + if (e.getEnvironment().getRegion().getRegionInfo().getTable().getNameAsString() + .equals("testCompactionOfIndexRegionBeforeMainRegionOpens") + && openCount > 0) { + try { + latch.await(); + } catch (InterruptedException e1) { + // TODO Auto-generated catch block + e1.printStackTrace(); + } + } + + super.postOpen(e); + } + + @Override + public void preClose(ObserverContext c, boolean abortRequested) + throws IOException { + if (c.getEnvironment().getRegion().getTableDesc().getNameAsString().equals("testHDP2989") + && closeCount == 0) { + preCloseCalled = true; + } + super.preClose(c, abortRequested); + } + + @Override + public void postClose(ObserverContext e, boolean abortRequested) { + if (e.getEnvironment().getRegion().getTableDesc().getNameAsString().equals("testHDP2989_idx") + && closeCount == 0) { + try { + closeCalled = true; + latch.await(); + } catch (InterruptedException e1) { + // TODO Auto-generated catch block + e1.printStackTrace(); + } + } + super.postClose(e, abortRequested); + } + + @Override + public void postBatchMutate(ObserverContext ctx, + MiniBatchOperationInProgress miniBatchOp) { + if (ctx.getEnvironment().getRegion().getTableDesc().getNameAsString().contains("testHDP3015")) { + if (delayPostBatchMutate) { + try { + latch.await(); + } catch (InterruptedException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + super.postBatchMutate(ctx, miniBatchOp); + } else { + super.postBatchMutate(ctx, miniBatchOp); + } + } + + @Override + public void preFlush(ObserverContext e) throws IOException { + if (e.getEnvironment().getRegion().getTableDesc().getNameAsString().contains("testHDP3015")) { + while (!delayPostBatchMutate) { + try { + Thread.sleep(200); + } catch (InterruptedException e1) { + // TODO Auto-generated catch block + e1.printStackTrace(); + } + } + + super.preFlush(e); + latch.countDown(); + } else { + super.preFlush(e); + } + } + } + + public static class MoveThread extends Thread { + private HBaseAdmin admin; + byte[] regionName; + byte[] dstRSName; + + public MoveThread(HBaseAdmin admin, byte[] regionName, byte[] dstRSName) { + this.admin = admin; + this.regionName = regionName; + this.dstRSName = dstRSName; + } + + @Override + public void run() { + try { + this.admin.move(regionName, dstRSName); + } catch (UnknownRegionException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch (MasterNotRunningException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch (ZooKeeperConnectionException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch (HBaseIOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + } + +} Index: hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestIndexPutsWithRegionServerRestart.java =================================================================== --- hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestIndexPutsWithRegionServerRestart.java (revision 0) +++ hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestIndexPutsWithRegionServerRestart.java (working copy) @@ -0,0 +1,139 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.apache.hadoop.hbase.index.Constants; +import org.apache.hadoop.hbase.index.SecIndexLoadBalancer; +import org.apache.hadoop.hbase.index.TestUtils; +import org.apache.hadoop.hbase.index.client.IndexAdmin; +import org.apache.hadoop.hbase.index.coprocessor.master.IndexMasterObserver; +import org.apache.hadoop.hbase.index.coprocessor.regionserver.TestIndexRegionObserver.MockIndexRegionObserver; +import org.apache.hadoop.hbase.index.coprocessor.wal.IndexWALObserver; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.master.LoadBalancer; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.zookeeper.KeeperException; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(LargeTests.class) +public class TestIndexPutsWithRegionServerRestart { + private static HBaseTestingUtility UTIL = new HBaseTestingUtility(); + private static HBaseAdmin admin = null; + + @BeforeClass + public static void setupBeforeClass() throws Exception { + Configuration conf = UTIL.getConfiguration(); + conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, IndexMasterObserver.class.getName()); + conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, MockIndexRegionObserver.class.getName()); + conf.set(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY, IndexWALObserver.class.getName()); + conf.setBoolean("hbase.use.secondary.index", true); + conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, SecIndexLoadBalancer.class, + LoadBalancer.class); + UTIL.startMiniCluster(1); + admin = new IndexAdmin(conf); + + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + if (admin != null) admin.close(); + UTIL.shutdownMiniCluster(); + } + + @Test(timeout = 180000) + public void testShouldRetrieveIndexPutsOnRSRestart() throws IOException, KeeperException, + InterruptedException { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testPutContainingTheIndexedColumn"; + HTableDescriptor ihtd = + TestUtils.createIndexedHTableDescriptor(userTableName, "col", "Index1", "col", "ql"); + admin.createTable(ihtd); + + HTable table = new HTable(conf, userTableName); + // test put with the indexed column + Put p = new Put("row1".getBytes()); + p.add("col".getBytes(), "ql".getBytes(), "myValue".getBytes()); + table.put(p); + + // Thread.sleep(2000); + int i = countNumberOfRows(userTableName); + Assert.assertEquals(1, i); + i = countNumberOfRows(userTableName + Constants.INDEX_TABLE_SUFFIX); + Assert.assertEquals(1, i); + + HRegionServer regionServer = UTIL.getHBaseCluster().getRegionServer(0); + HMaster master = UTIL.getHBaseCluster().getMaster(); + regionServer.abort("Aborting region server"); + while (master.getServerManager().areDeadServersInProgress()) { + Thread.sleep(1000); + } + UTIL.getHBaseCluster().startRegionServer(); + i = countNumberOfRows(userTableName); + Assert.assertEquals(1, i); + i = countNumberOfRows(userTableName + Constants.INDEX_TABLE_SUFFIX); + Assert.assertEquals(1, i); + } + + public int countNumberOfRows(String tableName) throws IOException { + Configuration conf = UTIL.getConfiguration(); + HTable table = new HTable(conf, tableName); + Scan s = new Scan(); + int i = 0; + ResultScanner scanner = table.getScanner(s); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + i++; + result = scanner.next(1); + } + return i; + } + + public Result[] getTheLastRow(String tableName) throws IOException { + Configuration conf = UTIL.getConfiguration(); + HTable table = new HTable(conf, tableName); + Scan s = new Scan(); + ResultScanner scanner = table.getScanner(s); + Result[] result = scanner.next(1); + Result[] result1 = result; + while (result1 != null && result1.length > 0) { + result1 = scanner.next(1); + if (null == result1 || result1.length <= 0) break; + else result = result1; + } + return result; + } + +} Index: hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestIndexRegionObserver.java =================================================================== --- hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestIndexRegionObserver.java (revision 0) +++ hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestIndexRegionObserver.java (working copy) @@ -0,0 +1,2046 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Mutation; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException; +import org.apache.hadoop.hbase.client.RowMutations; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.apache.hadoop.hbase.coprocessor.ObserverContext; +import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; +import org.apache.hadoop.hbase.index.ColumnQualifier.ValueType; +import org.apache.hadoop.hbase.index.Constants; +import org.apache.hadoop.hbase.index.IndexSpecification; +import org.apache.hadoop.hbase.index.SecIndexLoadBalancer; +import org.apache.hadoop.hbase.index.TableIndices; +import org.apache.hadoop.hbase.index.TestUtils; +import org.apache.hadoop.hbase.index.client.IndexAdmin; +import org.apache.hadoop.hbase.index.coprocessor.master.IndexMasterObserver; +import org.apache.hadoop.hbase.index.coprocessor.wal.IndexWALObserver; +import org.apache.hadoop.hbase.index.manager.IndexManager; +import org.apache.hadoop.hbase.index.util.IndexUtils; +import org.apache.hadoop.hbase.master.LoadBalancer; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.regionserver.SplitTransaction; +import org.apache.hadoop.hbase.regionserver.StoreFile; +import org.apache.hadoop.hbase.regionserver.wal.HLog; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Threads; +import org.apache.hadoop.hbase.zookeeper.ZKAssign; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.zookeeper.KeeperException; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(LargeTests.class) +public class TestIndexRegionObserver { + + private static HBaseTestingUtility UTIL = new HBaseTestingUtility(); + private static final int TTL_SECONDS = 2; + private static final int TTL_MS = TTL_SECONDS * 1000; + private static HBaseAdmin admin = null; + + @BeforeClass + public static void setupBeforeClass() throws Exception { + Configuration conf = UTIL.getConfiguration(); + conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, IndexMasterObserver.class.getName()); + conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, MockIndexRegionObserver.class.getName()); + conf.set(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY, IndexWALObserver.class.getName()); + conf.setBoolean("hbase.use.secondary.index", true); + conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, SecIndexLoadBalancer.class, + LoadBalancer.class); + conf.setInt("hbase.hstore.compactionThreshold",5); + UTIL.startMiniCluster(1); + admin = new IndexAdmin(conf); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + if (admin != null) admin.close(); + UTIL.shutdownMiniCluster(); + } + + @Before + public void setUp() { + MockIndexRegionObserver.count = 0; + MockIndexRegionObserver.isnullIndexRegion = false; + } + + @Test(timeout = 180000) + public void testPutWithAndWithoutTheIndexedColumn() throws IOException, KeeperException, + InterruptedException { + Configuration conf = UTIL.getConfiguration(); + + String userTableName = "testPutContainingTheIndexedColumn"; + HTableDescriptor ihtd = + TestUtils.createIndexedHTableDescriptor(userTableName, "col", "Index1", "col", "ql"); + admin.createTable(ihtd); + + HTable table = new HTable(conf, userTableName); + // test put with the indexed column + Put p = new Put("row1".getBytes()); + p.add("col".getBytes(), "ql".getBytes(), "myValue".getBytes()); + table.put(p); + + int i = countNumberOfRows(userTableName); + Assert.assertEquals(1, i); + i = countNumberOfRows(userTableName + Constants.INDEX_TABLE_SUFFIX); + Assert.assertEquals(1, i); + + // Test put without the indexed column + Put p1 = new Put("row2".getBytes()); + p1.add("col".getBytes(), "ql1".getBytes(), "myValue".getBytes()); + table.put(p1); + + i = countNumberOfRows(userTableName); + Assert.assertEquals(2, i); + i = countNumberOfRows(userTableName + Constants.INDEX_TABLE_SUFFIX); + Assert.assertEquals(1, i); + } + + public int countNumberOfRows(String tableName) throws IOException { + Configuration conf = UTIL.getConfiguration(); + HTable table = new HTable(conf, tableName); + Scan s = new Scan(); + int i = 0; + ResultScanner scanner = table.getScanner(s); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + i++; + result = scanner.next(1); + } + return i; + } + + public Result[] getTheLastRow(String tableName) throws IOException { + Configuration conf = UTIL.getConfiguration(); + HTable table = new HTable(conf, tableName); + Scan s = new Scan(); + ResultScanner scanner = table.getScanner(s); + Result[] result = scanner.next(1); + Result[] result1 = result; + while (result1 != null && result1.length > 0) { + result1 = scanner.next(1); + if (null == result1 || result1.length <= 0) break; + else result = result1; + } + return result; + } + + @Test(timeout = 180000) + public void testPostOpenCoprocessor() throws IOException, KeeperException, InterruptedException { + String userTableName = "testPostOpenCoprocessor"; + + HTableDescriptor ihtd = + TestUtils.createIndexedHTableDescriptor(userTableName, "col", "Index1", "col", "ql"); + admin.createTable(ihtd); + + // Check the number of indices + List list = IndexManager.getInstance().getIndicesForTable(userTableName); + Assert.assertEquals(1, list.size()); + + // Check the index name + boolean bool = false; + for (IndexSpecification e : list) { + if (e.getName().equals("Index1")) bool = true; + } + Assert.assertTrue(bool); + } + + @Test(timeout = 180000) + public void testMultipleIndicesOnUniqueColumns() throws IOException, KeeperException, + InterruptedException { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testMultipleIndicesOnUniqueColumns"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + IndexSpecification iSpec1 = new IndexSpecification("Index1"); + IndexSpecification iSpec2 = new IndexSpecification("Index2"); + iSpec1.addIndexColumn(hcd, "ql1", ValueType.String, 10); + iSpec2.addIndexColumn(hcd, "ql2", ValueType.String, 10); + ihtd.addFamily(hcd); + TableIndices indices = new TableIndices(); + indices.addIndex(iSpec1); + indices.addIndex(iSpec2); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + + HTable table = new HTable(conf, userTableName); + Put p = new Put("row1".getBytes()); + p.add("col".getBytes(), "ql3".getBytes(), "myValue".getBytes()); + p.add("col".getBytes(), "ql4".getBytes(), "myValue".getBytes()); + table.put(p); + int i = countNumberOfRows(userTableName); + Assert.assertEquals(1, i); + i = countNumberOfRows(userTableName + Constants.INDEX_TABLE_SUFFIX); + Assert.assertEquals(0, i); + + p = new Put("row2".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), "myValue".getBytes()); + p.add("col".getBytes(), "ql2".getBytes(), "myValue".getBytes()); + table.put(p); + i = countNumberOfRows(userTableName); + Assert.assertEquals(2, i); + i = countNumberOfRows(userTableName + Constants.INDEX_TABLE_SUFFIX); + Assert.assertEquals(2, i); + } + + @Test(timeout = 180000) + public void testIndexOnMultipleCols() throws IOException, KeeperException, InterruptedException { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testSingleIndexOnMultipleCols"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + + // Creating and adding the column families + HColumnDescriptor hcd1 = new HColumnDescriptor("col1"); + HColumnDescriptor hcd2 = new HColumnDescriptor("col2"); + HColumnDescriptor hcd3 = new HColumnDescriptor("col3"); + ihtd.addFamily(hcd1); + ihtd.addFamily(hcd2); + ihtd.addFamily(hcd3); + + // Create and add indices + IndexSpecification iSpec1 = new IndexSpecification("Index1"); + IndexSpecification iSpec2 = new IndexSpecification("Index2"); + iSpec1.addIndexColumn(hcd1, "ql1", ValueType.String, 10); + iSpec2.addIndexColumn(hcd2, "ql1", ValueType.String, 10); + iSpec2.addIndexColumn(hcd3, "ql1", ValueType.String, 10); + TableIndices indices = new TableIndices(); + indices.addIndex(iSpec1); + indices.addIndex(iSpec2); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + + HTable table = new HTable(conf, userTableName); + Put p = new Put("row1".getBytes()); + p.add("col1".getBytes(), "ql1".getBytes(), "myValue".getBytes()); + p.add("col2".getBytes(), "ql1".getBytes(), "myValue".getBytes()); + p.add("col3".getBytes(), "ql1".getBytes(), "myValue".getBytes()); + table.put(p); + int i = countNumberOfRows(userTableName); + Assert.assertEquals(1, i); + i = countNumberOfRows(userTableName + Constants.INDEX_TABLE_SUFFIX); + Assert.assertEquals(2, i); + + p = new Put("row2".getBytes()); + p.add("col1".getBytes(), "ql1".getBytes(), "myValue".getBytes()); + p.add("col2".getBytes(), "ql1".getBytes(), "myValue".getBytes()); + table.put(p); + i = countNumberOfRows(userTableName); + Assert.assertEquals(2, i); + i = countNumberOfRows(userTableName + Constants.INDEX_TABLE_SUFFIX); + Assert.assertEquals(4, i); + + p = new Put("row3".getBytes()); + p.add("col1".getBytes(), "ql1".getBytes(), "myValue".getBytes()); + p.add("col3".getBytes(), "ql1".getBytes(), "myValue".getBytes()); + table.put(p); + i = countNumberOfRows(userTableName); + Assert.assertEquals(3, i); + i = countNumberOfRows(userTableName + Constants.INDEX_TABLE_SUFFIX); + Assert.assertEquals(6, i); + + p = new Put("row4".getBytes()); + p.add("col2".getBytes(), "ql1".getBytes(), "myValue".getBytes()); + p.add("col3".getBytes(), "ql1".getBytes(), "myValue".getBytes()); + table.put(p); + i = countNumberOfRows(userTableName); + Assert.assertEquals(4, i); + i = countNumberOfRows(userTableName + Constants.INDEX_TABLE_SUFFIX); + Assert.assertEquals(7, i); + } + + @Test(timeout = 180000) + public void testPutsWithPadding() throws IOException, KeeperException, InterruptedException { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testPutsWithPadding"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + + // Creating and adding the column families + HColumnDescriptor hcd2 = new HColumnDescriptor("col2"); + HColumnDescriptor hcd3 = new HColumnDescriptor("col3"); + HColumnDescriptor hcd4 = new HColumnDescriptor("col4"); + + ihtd.addFamily(hcd2); + ihtd.addFamily(hcd3); + ihtd.addFamily(hcd4); + + // Create and add indices + IndexSpecification iSpec2 = new IndexSpecification("Index2"); + iSpec2.addIndexColumn(hcd2, "ql1", ValueType.String, 10); + iSpec2.addIndexColumn(hcd3, "ql1", ValueType.String, 10); + iSpec2.addIndexColumn(hcd4, "ql1", ValueType.String, 10); + TableIndices indices = new TableIndices(); + indices.addIndex(iSpec2); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + + HTable table = new HTable(conf, userTableName); + Put p = new Put("row1".getBytes()); + p.add("col2".getBytes(), "ql1".getBytes(), "myValue".getBytes()); + table.put(p); + int i = countNumberOfRows(userTableName); + Assert.assertEquals(1, i); + i = countNumberOfRows(userTableName + Constants.INDEX_TABLE_SUFFIX); + Assert.assertEquals(1, i); + + Result[] result = getTheLastRow(userTableName + Constants.INDEX_TABLE_SUFFIX); + byte[] rowKey1 = result[0].getRow(); + + p = new Put("row2".getBytes()); + p.add("col3".getBytes(), "ql1".getBytes(), "myValue".getBytes()); + table.put(p); + i = countNumberOfRows(userTableName); + Assert.assertEquals(2, i); + i = countNumberOfRows(userTableName + Constants.INDEX_TABLE_SUFFIX); + Assert.assertEquals(2, i); + + result = getTheLastRow(userTableName + Constants.INDEX_TABLE_SUFFIX); + byte[] rowKey2 = result[0].getRow(); + + Assert.assertEquals(rowKey1.length, rowKey2.length); + + p = new Put("row3".getBytes()); + p.add("col3".getBytes(), "ql1".getBytes(), "myValue".getBytes()); + table.put(p); + i = countNumberOfRows(userTableName); + Assert.assertEquals(3, i); + i = countNumberOfRows(userTableName + Constants.INDEX_TABLE_SUFFIX); + Assert.assertEquals(3, i); + + result = getTheLastRow(userTableName + Constants.INDEX_TABLE_SUFFIX); + byte[] rowKey3 = result[0].getRow(); + Assert.assertEquals(rowKey2.length, rowKey3.length); + + /* + * p = new Put("row4".getBytes()); p.add("col3".getBytes(), "ql1".getBytes(), + * "myValuefgacfgn".getBytes()); p.add("col2".getBytes(), "ql1".getBytes(), + * "myValue".getBytes()); p.add("col4".getBytes(), "ql1".getBytes(), "myValue".getBytes()); + * table.put(p); i = countNumberOfRows(userTableName); Assert.assertEquals(4, i); i = + * countNumberOfRows(userTableName + Constants.INDEX_TABLE_SUFFIX); Assert.assertEquals(4, i); + * result = getTheLastRow(userTableName + Constants.INDEX_TABLE_SUFFIX); byte[] rowKey4 = + * result[0].getRow(); Assert.assertEquals(rowKey3.length, rowKey4.length); + */ + } + + @Test(timeout = 180000) + public void testBulkPut() throws IOException, KeeperException, InterruptedException { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testBulkPut"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + + // Creating and adding the column families + HColumnDescriptor hcd1 = new HColumnDescriptor("col1"); + HColumnDescriptor hcd2 = new HColumnDescriptor("col2"); + HColumnDescriptor hcd3 = new HColumnDescriptor("col3"); + ihtd.addFamily(hcd1); + ihtd.addFamily(hcd2); + ihtd.addFamily(hcd3); + + // Create and add indices + IndexSpecification iSpec1 = new IndexSpecification("Index1"); + IndexSpecification iSpec2 = new IndexSpecification("Index2"); + iSpec1.addIndexColumn(hcd1, "ql1", ValueType.String, 10); + iSpec2.addIndexColumn(hcd3, "ql1", ValueType.String, 10); + TableIndices indices = new TableIndices(); + indices.addIndex(iSpec1); + indices.addIndex(iSpec2); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + + Thread[] t = new Thread[10]; + for (int i = 0; i < 10; i++) { + t[i] = new Testthread(conf, userTableName); + } + for (int i = 0; i < 10; i++) { + t[i].start(); + } + + for (int i = 0; i < 10; i++) { + t[i].join(); + } + + // System.out.println("Woke up"); + int i = countNumberOfRows(userTableName); + Assert.assertEquals(5000, i); + i = countNumberOfRows(userTableName + Constants.INDEX_TABLE_SUFFIX); + Assert.assertEquals(10000, i); + /* + * HLog log = UTIL.getHBaseCluster().getRegionServer(0).getWAL(); log.getHLogDirectoryName + * (UTIL.getHBaseCluster().getRegionServer(0).toString()); log.getReader( + * UTIL.getMiniHBaseCluster().getRegionServer(0).getFileSystem(), path, + * UTIL.getMiniHBaseCluster().getConfiguration()); + */ + } + + class Testthread extends Thread { + Configuration conf; + String userTableName; + HTable table; + + public Testthread(Configuration conf, String userTableName) throws IOException { + this.conf = conf; + this.userTableName = userTableName; + this.table = new HTable(conf, userTableName); + } + + public void run() { + for (int j = 0; j < 500; j++) { + double d = Math.random(); + byte[] rowKey = Bytes.toBytes(d); + Put p = new Put(rowKey); + p.add(Bytes.toBytes("col1"), Bytes.toBytes("ql1"), Bytes.toBytes("myValue")); + p.add(Bytes.toBytes("col2"), Bytes.toBytes("ql1"), Bytes.toBytes("myValue")); + p.add(Bytes.toBytes("col3"), Bytes.toBytes("ql1"), Bytes.toBytes("myValue")); + try { + table.put(p); + } catch (IOException e) { + } + } + } + } + + @Test(timeout = 180000) + public void testBulkPutWithRepeatedRows() throws IOException, KeeperException, + InterruptedException { + final Configuration conf = UTIL.getConfiguration(); + final String userTableName = "TestBulkPutWithRepeatedRows"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + + // Creating and adding the column families + HColumnDescriptor hcd1 = new HColumnDescriptor("col1"); + + ihtd.addFamily(hcd1); + + // Create and add indices + IndexSpecification iSpec1 = new IndexSpecification("Index1"); + + iSpec1.addIndexColumn(hcd1, "ql1", ValueType.String, 10); + TableIndices indices = new TableIndices(); + indices.addIndex(iSpec1); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + + new Thread() { + @Override + public void run() { + try { + HTable table = new HTable(conf, userTableName); + List puts = new ArrayList(5); + Put p = new Put("row1".getBytes()); + p.add(Bytes.toBytes("col1"), Bytes.toBytes("ql1"), Bytes.toBytes("myValue")); + puts.add(p); + p = new Put("row2".getBytes()); + p.add(Bytes.toBytes("col1"), Bytes.toBytes("ql1"), Bytes.toBytes("myValue")); + puts.add(p); + p = new Put("row3".getBytes()); + p.add(Bytes.toBytes("col1"), Bytes.toBytes("ql1"), Bytes.toBytes("myValue")); + puts.add(p); + p = new Put("row4".getBytes()); + p.add(Bytes.toBytes("col1"), Bytes.toBytes("ql1"), Bytes.toBytes("myValue")); + puts.add(p); + p = new Put("row5".getBytes()); + p.add(Bytes.toBytes("col1"), Bytes.toBytes("ql1"), Bytes.toBytes("myValue")); + puts.add(p); + table.put(puts); + } catch (IOException e) { + } + } + }.start(); + + new Thread() { + @Override + public void run() { + try { + HTable table = new HTable(conf, userTableName); + List puts = new ArrayList(5); + Put p = new Put("row6".getBytes()); + p.add(Bytes.toBytes("col1"), Bytes.toBytes("ql1"), Bytes.toBytes("myValue")); + puts.add(p); + p = new Put("row7".getBytes()); + p.add(Bytes.toBytes("col1"), Bytes.toBytes("ql1"), Bytes.toBytes("myValue")); + puts.add(p); + p = new Put("row3".getBytes()); + p.add(Bytes.toBytes("col1"), Bytes.toBytes("ql1"), Bytes.toBytes("myValue")); + puts.add(p); + p = new Put("row4".getBytes()); + p.add(Bytes.toBytes("col1"), Bytes.toBytes("ql1"), Bytes.toBytes("myValue")); + puts.add(p); + p = new Put("row10".getBytes()); + p.add(Bytes.toBytes("col1"), Bytes.toBytes("ql1"), Bytes.toBytes("myValue")); + puts.add(p); + table.put(puts); + } catch (IOException e) { + } + } + }.start(); + Thread.sleep(2000); + int i = countNumberOfRows(userTableName); + Assert.assertEquals(8, i); + i = countNumberOfRows(userTableName + Constants.INDEX_TABLE_SUFFIX); + Assert.assertEquals(8, i); + + } + + @Test(timeout = 180000) + public void testIndexPutRowkeyWithAllTheValues() throws IOException { + String DIR = UTIL.getDataTestDir("TestStore").toString(); + Path basedir = new Path(DIR + "TestIndexPut"); + // Path logdir = new Path(DIR+"TestIndexPut"+"/logs"); + FileSystem fs = UTIL.getTestFileSystem(); + Configuration conf = UTIL.getConfiguration(); + HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("TestIndexPut")); + HRegionInfo info = new HRegionInfo(htd.getTableName(), "A".getBytes(), "B".getBytes(), false); + HLog hlog = UTIL.getMiniHBaseCluster().getRegionServer(0).getWAL(); + HRegion region = new HRegion(basedir, hlog, fs, conf, info, htd, null); + IndexSpecification spec = new IndexSpecification("testSpec"); + spec.addIndexColumn(new HColumnDescriptor("cf1"), "ql1", ValueType.String, 10); + spec.addIndexColumn(new HColumnDescriptor("cf2"), "ql1", ValueType.String, 10); + + // Scenario where both the indexed cols are there in the put + byte[] rowKey = "Arow1".getBytes(); + Put p = new Put(rowKey); + long time = 1234567; + p.add("cf1".getBytes(), "ql1".getBytes(), time, "testvalue1".getBytes()); + p.add("cf2".getBytes(), "ql1".getBytes(), time + 10, "testvalue1".getBytes()); + Put indexPut = IndexUtils.prepareIndexPut(p, spec, region); + Assert.assertEquals(region.getStartKey().length + 1 + Constants.DEF_MAX_INDEX_NAME_LENGTH + 2 + * 10 + rowKey.length, indexPut.getRow().length); + Assert.assertEquals(time + 10, indexPut.get(Constants.IDX_COL_FAMILY, "".getBytes()).get(0) + .getTimestamp()); + + } + + @Test(timeout = 180000) + public void testIndexPutWithOnlyOneValue() throws IOException { + String DIR = UTIL.getDataTestDir("TestStore").toString(); + Path basedir = new Path(DIR + "TestIndexPut"); + // Path logdir = new Path(DIR+"TestIndexPut"+"/logs"); + FileSystem fs = UTIL.getTestFileSystem(); + Configuration conf = UTIL.getConfiguration(); + HTableDescriptor htd = new HTableDescriptor("TestIndexPut"); + HRegionInfo info = new HRegionInfo(htd.getTableName(), "A".getBytes(), "B".getBytes(), false); + HLog hlog = UTIL.getMiniHBaseCluster().getRegionServer(0).getWAL(); + HRegion region = new HRegion(basedir, hlog, fs, conf, info, htd, null); + IndexSpecification spec = new IndexSpecification("testSpec"); + spec.addIndexColumn(new HColumnDescriptor("cf1"), "ql1", ValueType.String, 10); + spec.addIndexColumn(new HColumnDescriptor("cf2"), "ql1", ValueType.String, 10); + + byte[] rowKey = "Arow1".getBytes(); + Put p = new Put(rowKey); + long time = 1234567; + p.add("cf1".getBytes(), "ql1".getBytes(), time, "testvalue1".getBytes()); + Put indexPut = IndexUtils.prepareIndexPut(p, spec, region); + Assert.assertEquals(region.getStartKey().length + 1 + Constants.DEF_MAX_INDEX_NAME_LENGTH + 2 + * 10 + rowKey.length, indexPut.getRow().length); + Assert.assertEquals(time, indexPut.get(Constants.IDX_COL_FAMILY, "".getBytes()).get(0) + .getTimestamp()); + // asserting pad........this has to be hardcoded. + byte[] pad = new byte[10]; + System.arraycopy(indexPut.getRow(), region.getStartKey().length + 1 + + Constants.DEF_MAX_INDEX_NAME_LENGTH + 10, pad, 0, 10); + Assert.assertTrue(Bytes.equals(pad, new byte[10])); + } + + @Test(timeout = 180000) + public void testIndexPutWithValueGreaterThanLength() throws IOException { + String DIR = UTIL.getDataTestDir("TestStore").toString(); + Path basedir = new Path(DIR + "TestIndexPut"); + // Path logdir = new Path(DIR+"TestIndexPut"+"/logs"); + FileSystem fs = UTIL.getTestFileSystem(); + Configuration conf = UTIL.getConfiguration(); + HTableDescriptor htd = new HTableDescriptor("TestIndexPut"); + HRegionInfo info = new HRegionInfo(htd.getTableName(), "A".getBytes(), "B".getBytes(), false); + HLog hlog = UTIL.getMiniHBaseCluster().getRegionServer(0).getWAL(); + HRegion region = new HRegion(basedir, hlog, fs, conf, info, htd, null); + IndexSpecification spec = new IndexSpecification("testSpec"); + spec.addIndexColumn(new HColumnDescriptor("cf1"), "ql1", ValueType.String, 10); + spec.addIndexColumn(new HColumnDescriptor("cf2"), "ql1", ValueType.String, 10); + + // assert IOException when value length goes beyond the limit. + byte[] rowKey = "Arow1".getBytes(); + Put p = new Put(rowKey); + long time = 1234567; + boolean returnVal = false; + try { + p.add("cf1".getBytes(), "ql1".getBytes(), time, "testvalue11".getBytes()); + IndexUtils.prepareIndexPut(p, spec, region); + } catch (IOException e) { + returnVal = true; + } + Assert.assertTrue(returnVal); + } + + @Test(timeout = 180000) + public void testIndexPutSequence() throws IOException { + String DIR = UTIL.getDataTestDir("TestStore").toString(); + Path basedir = new Path(DIR + "TestIndexPut"); + // Path logdir = new Path(DIR+"TestIndexPut"+"/logs"); + FileSystem fs = UTIL.getTestFileSystem(); + Configuration conf = UTIL.getConfiguration(); + HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("TestIndexPut")); + HRegionInfo info = new HRegionInfo(htd.getTableName(), "A".getBytes(), "B".getBytes(), false); + HLog hlog = UTIL.getMiniHBaseCluster().getRegionServer(0).getWAL(); + HRegion region = new HRegion(basedir, hlog, fs, conf, info, htd, null); + IndexSpecification spec = new IndexSpecification("index"); + spec.addIndexColumn(new HColumnDescriptor("cf1"), "ql1", ValueType.String, 10); + spec.addIndexColumn(new HColumnDescriptor("cf2"), "ql1", ValueType.String, 10); + + // scenario where same indexName but diff colvalues can disturb the order if + // used without pad + byte[] rowKey = "Arow1".getBytes(); + Put p1 = new Put(rowKey); + long time = 1234567; + p1.add("cf1".getBytes(), "ql1".getBytes(), time, "testcase".getBytes()); + p1.add("cf2".getBytes(), "ql1".getBytes(), time, "value".getBytes()); + Put indexPut1 = IndexUtils.prepareIndexPut(p1, spec, region); + Put p2 = new Put(rowKey); + p2.add("cf1".getBytes(), "ql1".getBytes(), time, "test".getBytes()); + p2.add("cf2".getBytes(), "ql1".getBytes(), time, "value".getBytes()); + Put indexPut2 = IndexUtils.prepareIndexPut(p2, spec, region); + // (spaces just for easier reading...not present in actual) + // Index Row key For p1 = "A index testcase value Arow1" + // Index Row key For p2 = "A index test value Arow1" + // NOW acc. to the lexographical ordering p2 should come second but we need + // it to come 1st datz where pad is needed. + byte[] rowKey1 = indexPut1.getRow(); + byte[] rowKey2 = indexPut2.getRow(); + + int result = Bytes.compareTo(rowKey1, rowKey2); + Assert.assertTrue(result > 0); + + // scenario where the index names are diff and padding is needed. + IndexSpecification spec1 = new IndexSpecification("ind"); + spec1.addIndexColumn(new HColumnDescriptor("cf3"), "ql1", ValueType.String, 10); + p1 = new Put(rowKey); + p1.add("cf3".getBytes(), "ql1".getBytes(), time, "testcase".getBytes()); + indexPut1 = IndexUtils.prepareIndexPut(p1, spec1, region); + // (spaces just for easier reading...not present in actual) + // Index Row key For p1 = "A ind testcase value Arow1" + // Index Row key For p2 = "A index test value Arow1" + // NOW acc. to the lexographical ordering p1 should come second but we need + // it to come 1st datz where pad is needed. + rowKey1 = indexPut1.getRow(); + result = Bytes.compareTo(rowKey1, rowKey2); + Assert.assertTrue(result < 0); + + } + + @Test(timeout = 180000) + public void testIndexTableValue() throws IOException { + String DIR = UTIL.getDataTestDir("TestStore").toString(); + Path basedir = new Path(DIR + "TestIndexPut"); + // Path logdir = new Path(DIR+"TestIndexPut"+"/logs"); + FileSystem fs = UTIL.getTestFileSystem(); + Configuration conf = UTIL.getConfiguration(); + HTableDescriptor htd = new HTableDescriptor("TestIndexPut"); + HRegionInfo info = new HRegionInfo(htd.getTableName(), "ABC".getBytes(), "BBB".getBytes(), false); + HLog hlog = UTIL.getMiniHBaseCluster().getRegionServer(0).getWAL(); + HRegion region = new HRegion(basedir, hlog, fs, conf, info, htd, null); + IndexSpecification spec = new IndexSpecification("index"); + spec.addIndexColumn(new HColumnDescriptor("cf1"), "ql1", ValueType.String, 10); + spec.addIndexColumn(new HColumnDescriptor("cf2"), "ql1", ValueType.String, 10); + + byte[] rowKey = "Arow1".getBytes(); + Put p1 = new Put(rowKey); + long time = 1234567; + p1.add("cf1".getBytes(), "ql1".getBytes(), time, "testcase".getBytes()); + p1.add("cf2".getBytes(), "ql1".getBytes(), time, "value".getBytes()); + Put indexPut1 = IndexUtils.prepareIndexPut(p1, spec, region); + + List kvs = indexPut1.get(Constants.IDX_COL_FAMILY, "".getBytes()); + Cell kv = null; + if (null != kvs) { + kv = kvs.get(0); + } + byte[] val = kv.getValue(); + byte[] startKeyLengthInBytes = new byte[2]; + System.arraycopy(val, 0, startKeyLengthInBytes, 0, startKeyLengthInBytes.length); + int startkeylen = (int) (Bytes.toShort(startKeyLengthInBytes)); + Assert.assertEquals(3, startkeylen); + + byte[] rowKeyOffset = new byte[2]; + System.arraycopy(val, startKeyLengthInBytes.length, rowKeyOffset, 0, rowKeyOffset.length); + int rowKeyOffsetInt = Bytes.toShort(rowKeyOffset); + Assert.assertEquals(42, rowKeyOffsetInt); + } + + @Test(timeout = 180000) + public void testIndexedRegionAfterSplitShouldReplaceStartKeyAndValue() throws IOException, + KeeperException, InterruptedException { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testIndexRegionSplit"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + IndexSpecification iSpec = new IndexSpecification("ScanIndexf"); + iSpec.addIndexColumn(new HColumnDescriptor("col"), "ql", ValueType.String, 10); + ihtd.addFamily(hcd); + TableIndices indices = new TableIndices(); + indices.addIndex(iSpec); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + TableName userTable = TableName.valueOf(userTableName); + HTable table = new HTable(conf, userTableName); + // test put with the indexed column + for (int i = 0; i < 10; i++) { + String row = "row" + i; + Put p = new Put(row.getBytes()); + String val = "Val" + i; + p.add("col".getBytes(), "ql".getBytes(), val.getBytes()); + table.put(p); + } + + List regionsOfTable = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsOfTable(userTable); + HRegionInfo hRegionInfo = regionsOfTable.get(0); + hRegionInfo.getRegionName(); + Scan s = new Scan(); + int i = 0; + ResultScanner scanner = table.getScanner(s); + for (Result rr = scanner.next(); rr != null; rr = scanner.next()) { + i++; + } + admin.split(hRegionInfo.getRegionName(), "row5".getBytes()); + Scan s1 = new Scan(); + TableName indexTable = TableName.valueOf(IndexUtils.getIndexTableName(userTableName)); + List mainTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsOfTable(userTable); + while (mainTableRegions.size() != 2) { + Thread.sleep(2000); + mainTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsOfTable(userTable); + } + + List indexTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsOfTable(indexTable); + while (indexTableRegions.size() != 2) { + Thread.sleep(2000); + indexTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsOfTable(indexTable); + } + Assert.assertEquals(2, mainTableRegions.size()); + Assert.assertEquals(2, indexTableRegions.size()); + int i1 = 0; + HTable tableidx = new HTable(conf, userTableName + "_idx"); + ResultScanner scanner1 = tableidx.getScanner(s1); + for (Result rr = scanner1.next(); rr != null; rr = scanner1.next()) { + i1++; + } + Assert.assertEquals("count should be equal", i, i1); + } + + @Test(timeout = 180000) + public void + testIndexedRegionAfterSplitShouldNotThrowExceptionIfThereAreNoSplitFilesForIndexedTable() + throws IOException, KeeperException, InterruptedException { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testIndexRegionSplit1"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + HColumnDescriptor hcd2 = new HColumnDescriptor("col2"); + IndexSpecification iSpec = new IndexSpecification("ScanIndexf"); + iSpec.addIndexColumn(hcd, "ql", ValueType.String, 10); + ihtd.addFamily(hcd); + ihtd.addFamily(hcd2); + TableIndices indices = new TableIndices(); + indices.addIndex(iSpec); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + TableName userTable = TableName.valueOf(userTableName); + HTable table = new HTable(conf, userTableName); + // test put with the indexed column + for (int i = 0; i < 10; i++) { + String row = "row" + i; + Put p = new Put(row.getBytes()); + String val = "Val" + i; + p.add("col2".getBytes(), "ql".getBytes(), val.getBytes()); + table.put(p); + } + + List regionsOfTable = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsOfTable(userTable); + HRegionInfo hRegionInfo = regionsOfTable.get(0); + HTable tableidx = new HTable(conf, userTableName + "_idx"); + Scan s = new Scan(); + int i = 0; + ResultScanner scanner = table.getScanner(s); + for (Result rr = scanner.next(); rr != null; rr = scanner.next()) { + i++; + } + admin.split(hRegionInfo.getRegionName(), "row5".getBytes()); + Scan s1 = new Scan(); + int i1 = 0; + TableName indexTable = TableName.valueOf(IndexUtils.getIndexTableName(userTableName)); + List mainTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsOfTable(userTable); + + List indexTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsOfTable(indexTable); + while (mainTableRegions.size() != 2) { + Thread.sleep(2000); + mainTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsOfTable(userTable); + } + while (indexTableRegions.size() != 2) { + Thread.sleep(2000); + indexTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsOfTable(indexTable); + } + Assert.assertEquals(2, mainTableRegions.size()); + Assert.assertEquals(2, indexTableRegions.size()); + Assert.assertEquals("Main table count shud be 10", 10, i); + ResultScanner scanner1 = tableidx.getScanner(s1); + for (Result rr = scanner1.next(); rr != null; rr = scanner1.next()) { + i1++; + } + Assert.assertEquals("Index table count shud be 0", 0, i1); + + // Trying to put data for indexed col + for (int k = 10; k < 20; k++) { + String row = "row" + k; + Put p = new Put(row.getBytes()); + String val = "Val" + k; + p.add("col".getBytes(), "ql".getBytes(), val.getBytes()); + table.put(p); + } + int z = 0; + Scan s2 = new Scan(); + ResultScanner scanner2 = tableidx.getScanner(s2); + + admin.flush(userTableName + "_idx"); + + for (Result rr = scanner2.next(); rr != null; rr = scanner2.next()) { + z++; + } + Assert.assertEquals("Index table count shud be now 10", 10, z); + + List regionsOfTable1 = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsOfTable(userTable); + HRegionInfo hRegionInfo1 = regionsOfTable1.get(0); + + admin.split(hRegionInfo1.getRegionName(), "row3".getBytes()); + List mainTableRegions1 = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsOfTable(userTable); + + List indexTableRegions1 = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsOfTable(indexTable); + while (mainTableRegions1.size() != 3) { + Thread.sleep(2000); + mainTableRegions1 = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsOfTable(userTable); + } + while (indexTableRegions1.size() != 3) { + Thread.sleep(2000); + indexTableRegions1 = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsOfTable(indexTable); + } + // It shud be 3 after one more split + Assert.assertEquals(3, mainTableRegions1.size()); + Assert.assertEquals(3, indexTableRegions1.size()); + Scan s3 = new Scan(); + int a = 0; + ResultScanner scanner3 = tableidx.getScanner(s3); + for (Result rr = scanner3.next(); rr != null; rr = scanner3.next()) { + a++; + } + int b = 0; + Scan s4 = new Scan(); + ResultScanner scanner4 = table.getScanner(s4); + for (Result rr = scanner4.next(); rr != null; rr = scanner4.next()) { + b++; + } + // subracting 10 coz addtional 10 is for other col family which is not indexed + Assert.assertEquals("count should be equal", a, b - 10); + + } + + @Test(timeout = 180000) + public void testSeekToInIndexHalfStoreFileReaderShouldRetreiveClosestRowCorrectly() + throws IOException, KeeperException, InterruptedException { + Configuration conf = UTIL.getConfiguration(); + conf.setInt("hbase.regionserver.lease.period", 900000000); + conf.setBoolean("hbase.use.secondary.index", true); + String userTableName = "testIndexSplitWithClosestRow"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + IndexSpecification iSpec = new IndexSpecification("ScanIndexf"); + iSpec.addIndexColumn(new HColumnDescriptor("col"), "ql", ValueType.String, 10); + ihtd.addFamily(hcd); + TableIndices indices = new TableIndices(); + indices.addIndex(iSpec); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + TableName userTable = TableName.valueOf(userTableName); + HTable table = new HTable(conf, userTableName); + // test put with the indexed column + for (int i = 0; i <= 10; i++) { + // Don't put row4 alone so that when getRowOrBefore for row4 is specifiesd row3 shud be + // returned + if (i != 4) { + String row = "row" + i; + Put p = new Put(row.getBytes()); + String val = "Val" + i; + p.add("col".getBytes(), "ql".getBytes(), val.getBytes()); + table.put(p); + } + } + TableName indexTable = TableName.valueOf(IndexUtils.getIndexTableName(userTableName)); + List regionsOfTable = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsOfTable(userTable); + HRegionInfo hRegionInfo = regionsOfTable.get(0); + hRegionInfo.getRegionName(); + HTable tableidx = new HTable(conf, userTableName + "_idx"); + admin.split(hRegionInfo.getRegionName(), "row6".getBytes()); + List mainTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsOfTable(userTable); + + List indexTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsOfTable(indexTable); + while (mainTableRegions.size() != 2) { + Thread.sleep(2000); + mainTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsOfTable(userTable); + } + while (indexTableRegions.size() != 2) { + Thread.sleep(2000); + indexTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsOfTable(indexTable); + } + Assert.assertEquals(2, mainTableRegions.size()); + Assert.assertEquals(2, indexTableRegions.size()); + Scan s1 = new Scan(); + ResultScanner scanner1 = tableidx.getScanner(s1); + Result next = null; + for (int i = 0; i < 6; i++) { + next = scanner1.next(); + } + String row1 = "row4"; + Put row4Put = new Put(row1.getBytes()); + String val = "Val4"; + row4Put.add("col".getBytes(), "ql".getBytes(), val.getBytes()); + + byte[] row4 = row4Put.getRow(); + byte[] nextRow = next.getRow(); + byte[] replacedRow = new byte[nextRow.length]; + System.arraycopy(nextRow, 0, replacedRow, 0, nextRow.length); + System.arraycopy(row4, 0, replacedRow, replacedRow.length - row4.length, row4.length); + Result rowOrBefore = tableidx.getRowOrBefore(replacedRow, "d".getBytes()); + + String expectedRow = "row3"; + Put p1 = new Put(expectedRow.getBytes()); + String actualStr = Bytes.toString(rowOrBefore.getRow()); + int lastIndexOf = actualStr.lastIndexOf("row3"); + Assert.assertEquals("SeekTo should return row3 as closestRowBefore", + Bytes.toString(p1.getRow()), actualStr.substring(lastIndexOf, actualStr.length())); + } + + @Test(timeout = 180000) + public + void + testSeekToInIndexHalfStoreFileReaderShouldRetreiveClosestRowCorrectlyWhenRowIsNotFoundInMainTable() + throws IOException, KeeperException, InterruptedException { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testIndexSplitWithClosestRowNotInMainTable"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + IndexSpecification iSpec = new IndexSpecification("ScanIndexf"); + iSpec.addIndexColumn(new HColumnDescriptor("col"), "ql", ValueType.String, 10); + ihtd.addFamily(hcd); + TableIndices indices = new TableIndices(); + indices.addIndex(iSpec); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + TableName userTable = TableName.valueOf(userTableName); + HTable table = new HTable(conf, userTableName); + // test put with the indexed column + for (int i = 0; i <= 10; i++) { + // Don't put row8 alone so that when getRowOrBefore for row8 is specifiesd row7 shud be + // returned + if (i != 8) { + String row = "row" + i; + Put p = new Put(row.getBytes()); + String val = "Val" + i; + p.add("col".getBytes(), "ql".getBytes(), val.getBytes()); + table.put(p); + } + } + List regionsOfTable = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsOfTable(userTable); + HRegionInfo hRegionInfo = regionsOfTable.get(0); + hRegionInfo.getRegionName(); + TableName indexTable = TableName.valueOf(IndexUtils.getIndexTableName(userTableName)); + HTable tableidx = new HTable(conf, indexTable); + admin.split(hRegionInfo.getRegionName(), "row4".getBytes()); + + List mainTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsOfTable(userTable); + + List indexTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsOfTable(indexTable); + while (mainTableRegions.size() != 2) { + Thread.sleep(2000); + mainTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsOfTable(userTable); + } + while (indexTableRegions.size() != 2) { + Thread.sleep(2000); + indexTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsOfTable(indexTable); + } + Assert.assertEquals(2, mainTableRegions.size()); + Assert.assertEquals(2, indexTableRegions.size()); + + Scan s1 = new Scan(); + ResultScanner scanner1 = tableidx.getScanner(s1); + Result next1 = null; + Thread.sleep(3000); + for (int i = 0; i < 10; i++) { + next1 = scanner1.next(); + } + String row1 = "row8"; + Put row4Put = new Put(row1.getBytes()); + String val = "Val8"; + row4Put.add("col".getBytes(), "ql".getBytes(), val.getBytes()); + byte[] row4 = row4Put.getRow(); + byte[] nextRow = next1.getRow(); + byte[] replacedRow = new byte[nextRow.length]; + System.arraycopy(nextRow, 0, replacedRow, 0, nextRow.length); + System.arraycopy(row4, 0, replacedRow, replacedRow.length - row4.length, row4.length); + Result rowOrBefore = tableidx.getRowOrBefore(replacedRow, "d".getBytes()); + + String expectedRow = "row7"; + Put p1 = new Put(expectedRow.getBytes()); + String actualStr = Bytes.toString(rowOrBefore.getRow()); + int lastIndexOf = actualStr.lastIndexOf("row7"); + Assert.assertTrue("Expected row should have the start key replaced to split key ", Bytes + .toString(rowOrBefore.getRow()).startsWith("row4")); + Assert.assertEquals("SeekTo should return row7 as closestRowBefore and split is completed ", + Bytes.toString(p1.getRow()), actualStr.substring(lastIndexOf, actualStr.length())); + } + + @Test(timeout = 180000) + public void testPutWithValueLengthMoreThanMaxValueLength() throws IOException, KeeperException, + InterruptedException { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testPutWithValueLengthMoreThanMaxValueLength"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + IndexSpecification iSpec1 = new IndexSpecification("Index1"); + iSpec1.addIndexColumn(hcd, "ql1", ValueType.String, 10); + ihtd.addFamily(hcd); + TableIndices indices = new TableIndices(); + indices.addIndex(iSpec1); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + + HTable table = new HTable(conf, userTableName); + table.setAutoFlush(false, false); + List putList = new ArrayList(3); + putList.add(new Put("row1".getBytes()).add("col".getBytes(), "ql1".getBytes(), + "valueLengthMoreThanMaxValueLength".getBytes())); + putList.add(new Put("row2".getBytes()).add("col".getBytes(), "ql1".getBytes(), + "myValue".getBytes())); + putList.add(new Put("row3".getBytes()).add("col".getBytes(), "ql1".getBytes(), + "myValue".getBytes())); + table.put(putList); + try { + table.flushCommits(); + } catch (RetriesExhaustedWithDetailsException e) { + // nothing to do. + } + Assert.assertEquals(1, table.getWriteBuffer().size()); + } + + @Test(timeout = 180000) + public void testIfPrepareFailsFor2ndSplitShouldFailTheMainTableSplitAlso() throws Exception { + Configuration conf = UTIL.getConfiguration(); + conf.setInt("hbase.regionserver.lease.period", 90000000); + String userTableName = "testPrepareFailForIdx"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + IndexSpecification iSpec = new IndexSpecification("ScanIndexf"); + iSpec.addIndexColumn(hcd, "ql", ValueType.String, 10); + ihtd.addFamily(hcd); + TableIndices indices = new TableIndices(); + indices.addIndex(iSpec); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + TableName userTable = TableName.valueOf(userTableName); + TableName indexTable = TableName.valueOf(IndexUtils.getIndexTableName(userTableName)); + HTable table = new HTable(conf, userTableName); + // test put with the indexed column + for (int i = 0; i < 10; i++) { + String row = "row" + i; + Put p = new Put(row.getBytes()); + String val = "Val" + i; + p.add("col".getBytes(), "ql".getBytes(), val.getBytes()); + table.put(p); + } + List regionsOfTable = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsOfTable(userTable); + HRegionInfo hRegionInfo = regionsOfTable.get(0); + MockIndexRegionObserver.count++; + admin.split(hRegionInfo.getRegionName(), "row5".getBytes()); + List mainTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsOfTable(userTable); + List indexTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsOfTable(indexTable); + while (mainTableRegions.size() != 2) { + Thread.sleep(2000); + mainTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsOfTable(userTable); + } + while (indexTableRegions.size() != 2) { + Thread.sleep(2000); + indexTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsOfTable(indexTable); + } + Assert.assertEquals(2, mainTableRegions.size()); + Assert.assertEquals(2, indexTableRegions.size()); + + MockIndexRegionObserver.count++; + List regionsOfTable2 = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsOfTable(userTable); + HRegionInfo hRegionInfo1 = regionsOfTable2.get(0); + admin.split(hRegionInfo1.getRegionName(), "row3".getBytes()); + List mainTableRegions1 = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsOfTable(userTable); + List indexTableRegions1 = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsOfTable(indexTable); + Assert.assertEquals(2, mainTableRegions1.size()); + Assert.assertEquals(2, indexTableRegions1.size()); + } + + public static class MockIndexRegionObserver extends IndexRegionObserver { + static int count = 0; + static boolean isnullIndexRegion = false; + + public void preSplitBeforePONR(ObserverContext e, + byte[] splitKey, List metaEntries) throws IOException { + if (e.getEnvironment().getRegion().getRegionInfo().getTable().getNameAsString() + .equals("testIndexManagerWithFailedSplitOfIndexRegion")) { + throw new IOException(); + } + if (isnullIndexRegion) { + e.bypass(); + return; + } + if (count == 2) { + splitThreadLocal.remove(); + e.bypass(); + return; + } else { + super.preSplitBeforePONR(e, splitKey,metaEntries); + } + } + + @Override + public void preSplit(ObserverContext e) throws IOException { + super.preSplit(e); + } + } + + @Test(timeout = 180000) + public void testShouldNotSplitIfIndexRegionIsNullForIndexTable() throws Exception { + Configuration conf = UTIL.getConfiguration(); + conf.setBoolean("hbase.use.secondary.index", true); + conf.setInt("hbase.regionserver.lease.period", 90000000); + String userTableName = "testIndexRegionSplit12"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + IndexSpecification iSpec = new IndexSpecification("ScanIndexf"); + iSpec.addIndexColumn(hcd, "ql", ValueType.String, 10); + ihtd.addFamily(hcd); + TableIndices indices = new TableIndices(); + indices.addIndex(iSpec); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + TableName userTable = TableName.valueOf(userTableName); + TableName indexTable = TableName.valueOf(IndexUtils.getIndexTableName(userTableName)); + HTable table = new HTable(conf, userTableName); + // test put with the indexed column + for (int i = 0; i < 10; i++) { + String row = "row" + i; + Put p = new Put(row.getBytes()); + String val = "Val" + i; + p.add("col".getBytes(), "ql".getBytes(), val.getBytes()); + table.put(p); + } + List regionsOfTable = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsOfTable(userTable); + HRegionInfo hRegionInfo = regionsOfTable.get(0); + MockIndexRegionObserver.isnullIndexRegion = true; + admin.split(hRegionInfo.getRegionName(), "row5".getBytes()); + List mainTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsOfTable(userTable); + List indexTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsOfTable(indexTable); + Assert.assertEquals(1, mainTableRegions.size()); + Assert.assertEquals(1, indexTableRegions.size()); + } + + @Test(timeout = 180000) + public void testCheckAndPutFor1PutShouldHav2PutsInIndexTableAndShouldReplaceWithNewValue() + throws Exception { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testCheckAndPutContainingTheIndexedColumn"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + IndexSpecification iSpec = new IndexSpecification("Index1"); + iSpec.addIndexColumn(hcd, "q1", ValueType.String, 10); + ihtd.addFamily(hcd); + TableIndices indices = new TableIndices(); + indices.addIndex(iSpec); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + String idxTableName = userTableName + Constants.INDEX_TABLE_SUFFIX; + HTable table = new HTable(conf, userTableName); + // test put with the indexed column + Put p = new Put("row1".getBytes()); + p.add("col".getBytes(), "q1".getBytes(), "myValue".getBytes()); + table.put(p); + + int usertableCount = countNumberOfRows(userTableName); + Assert.assertEquals(1, usertableCount); + int idxtableCount = countNumberOfRows(idxTableName); + Assert.assertEquals(1, idxtableCount); + + // Test check and put + Put p1 = new Put("row1".getBytes()); + p1.add("col".getBytes(), "q1".getBytes(), "myNewValue".getBytes()); + Assert.assertTrue(table.checkAndPut("row1".getBytes(), "col".getBytes(), "q1".getBytes(), + "myValue".getBytes(), p1)); + usertableCount = countNumberOfRows(userTableName); + Assert.assertEquals(1, usertableCount); + idxtableCount = countNumberOfRows(idxTableName); + Assert.assertEquals(2, idxtableCount); + + Get get = new Get("row1".getBytes()); + get.addColumn(Bytes.toBytes("col"), Bytes.toBytes("q1")); + Result result = table.get(get); + byte[] val = result.getValue(Bytes.toBytes("col"), Bytes.toBytes("q1")); + Assert.assertEquals("myNewValue", Bytes.toString(val)); + } + + @Test(timeout = 180000) + public void testCheckAndPutAndNormalPutInParallel() throws Exception { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testCheckAndPutAndNormalPutInParallel"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + IndexSpecification iSpec = new IndexSpecification("Index1"); + iSpec.addIndexColumn(hcd, "q1", ValueType.String, 10); + ihtd.addFamily(hcd); + TableIndices indices = new TableIndices(); + indices.addIndex(iSpec); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + String idxTableName = userTableName + Constants.INDEX_TABLE_SUFFIX; + final HTable table = new HTable(conf, userTableName); + // test put with the indexed column + Put p = new Put("row1".getBytes()); + p.add("col".getBytes(), "q1".getBytes(), "myValue".getBytes()); + table.put(p); + + int usertableCount = countNumberOfRows(userTableName); + Assert.assertEquals(1, usertableCount); + int idxtableCount = countNumberOfRows(idxTableName); + Assert.assertEquals(1, idxtableCount); + + // Test check and put + Put p1 = new Put("row1".getBytes()); + p1.add("col".getBytes(), "q1".getBytes(), "myNewValue".getBytes()); + Assert.assertTrue(table.checkAndPut("row1".getBytes(), "col".getBytes(), "q1".getBytes(), + "myValue".getBytes(), p1)); + new Thread() { + public void run() { + Put p = new Put("row1".getBytes()); + p.add("col".getBytes(), "q1".getBytes(), "myValue1".getBytes()); + try { + table.put(p); + } catch (IOException e) { + e.printStackTrace(); + } + } + }.start(); + Thread.sleep(3000); + usertableCount = countNumberOfRows(userTableName); + Assert.assertEquals(1, usertableCount); + idxtableCount = countNumberOfRows(idxTableName); + Assert.assertEquals(3, idxtableCount); + + } + + @Test(timeout = 180000) + public void testCheckAndDeleteShudDeleteTheRowSuccessfullyInBothIndexAndMainTable() + throws Exception { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testCheckAndDeleteContainingTheIndexedColumn"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + IndexSpecification iSpec = new IndexSpecification("Index1"); + iSpec.addIndexColumn(hcd, "q1", ValueType.String, 10); + ihtd.addFamily(hcd); + TableIndices indices = new TableIndices(); + indices.addIndex(iSpec); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + String idxTableName = userTableName + Constants.INDEX_TABLE_SUFFIX; + HTable table = new HTable(conf, userTableName); + // test put with the indexed column + Put p = new Put("row1".getBytes()); + p.add("col".getBytes(), "q1".getBytes(), "myValue".getBytes()); + table.put(p); + + int usertableCount = countNumberOfRows(userTableName); + Assert.assertEquals(1, usertableCount); + int idxtableCount = countNumberOfRows(idxTableName); + Assert.assertEquals(1, idxtableCount); + + Delete delete = new Delete("row1".getBytes()); + delete.deleteFamily("col".getBytes()); + // CheckandDelete + Assert.assertTrue(table.checkAndDelete("row1".getBytes(), "col".getBytes(), "q1".getBytes(), + "myValue".getBytes(), delete)); + + Get get = new Get("row1".getBytes()); + get.addFamily("col".getBytes()); + Result r = table.get(get); + Assert.assertEquals(0, r.size()); + + usertableCount = countNumberOfRows(userTableName); + Assert.assertEquals(0, usertableCount); + idxtableCount = countNumberOfRows(idxTableName); + Assert.assertEquals(0, idxtableCount); + } + +/* @Test(timeout = 180000) + public void testRowMutations() throws Exception { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testMutateRows"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + IndexSpecification iSpec = new IndexSpecification("Index1"); + iSpec.addIndexColumn(hcd, "q1", ValueType.String, 10); + iSpec.addIndexColumn(hcd, "q2", ValueType.String, 10); + ihtd.addFamily(hcd); + TableIndices indices = new TableIndices(); + indices.addIndex(iSpec); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + String idxTableName = userTableName + Constants.INDEX_TABLE_SUFFIX; + HTable table = new HTable(conf, userTableName); + HTable idxtable = new HTable(conf, idxTableName); + byte[] row = Bytes.toBytes("rowA"); + + Put put = new Put(row); + put.add("col".getBytes(), "q2".getBytes(), "delValue".getBytes()); + table.put(put); + + int usertableCount = countNumberOfRows(userTableName); + Assert.assertEquals(1, usertableCount); + int idxtableCount = countNumberOfRows(idxTableName); + Assert.assertEquals(1, idxtableCount); + + RowMutations rm = new RowMutations(row); + Put p = new Put(row); + p.add("col".getBytes(), "q1".getBytes(), "q1value".getBytes()); + rm.add(p); + Delete d = new Delete(row); + d.deleteColumns("col".getBytes(), "q2".getBytes()); + rm.add(d); + + // Mutate rows + table.mutateRow(rm); + + admin.flush(userTableName); + admin.majorCompact(userTableName); + // Now after one put and one delete it shud be 1 + usertableCount = countNumberOfRows(userTableName); + Assert.assertEquals(1, usertableCount); + + int idxTableCount = countNumberOfRows(idxTableName); + Assert.assertEquals(1, idxTableCount); + + Get get = new Get(row); + get.addFamily("col".getBytes()); + Result r = table.get(get); + Assert.assertEquals(1, r.size()); + + String del = Bytes.toString(r.getValue("col".getBytes(), "q2".getBytes())); + Assert.assertNull(del); + String putval = Bytes.toString(r.getValue("col".getBytes(), "q1".getBytes())); + Assert.assertEquals("q1value", putval); + + // Result of index table shud contain one keyvalue for "q1value" put only + Scan s = new Scan(); + ResultScanner scanner = idxtable.getScanner(s); + Result result = scanner.next(); + Assert.assertEquals(1, result.size()); + + String idxRow = Bytes.toString(result.getRow()); + int len = IndexUtils.getMaxIndexNameLength() + "q1value".length(); + String value = idxRow.substring(IndexUtils.getMaxIndexNameLength() + 1, len + 1); + // asserting value in idx table is for the put which has the value "q1value" + Assert.assertEquals("q1value", value); + }*/ + + @Test(timeout = 180000) + public void testWithPutsAndDeletesAfterSplitShouldRetreiveTheRowsCorrectly() throws Exception { + Configuration conf = UTIL.getConfiguration(); + conf.setInt("hbase.regionserver.lease.period", 900000000); + conf.setBoolean("hbase.use.secondary.index", true); + String userTableName = "testPutsAndDeletes"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + IndexSpecification iSpec = new IndexSpecification("ScanIndexf"); + iSpec.addIndexColumn(new HColumnDescriptor("col"), "ql", ValueType.String, 10); + ihtd.addFamily(hcd); + TableIndices indices = new TableIndices(); + indices.addIndex(iSpec); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + TableName userTable = TableName.valueOf(userTableName); + TableName indexTable = TableName.valueOf(IndexUtils.getIndexTableName(userTableName)); + HTable table = new HTable(conf, userTableName); + // test put with the indexed column + for (int i = 0; i <= 5; i++) { + String row = "row" + i; + Put p = new Put(row.getBytes()); + String val = "Val" + i; + p.add("col".getBytes(), "ql".getBytes(), val.getBytes()); + table.put(p); + admin.flush(userTableName); + admin.flush(userTableName + "_idx"); + + if (i < 3) { + Delete d = new Delete(row.getBytes()); + // Do a normal delete + table.delete(d); + } else { + if (i > 4) { + Delete d = new Delete(row.getBytes()); + // Do column family delete + d.deleteFamily("col".getBytes()); + table.delete(d); + } else { + Delete d = new Delete(row.getBytes()); + // Do delete column + d.deleteColumn("col".getBytes(), "ql".getBytes()); + table.delete(d); + } + } + admin.flush(userTableName); + admin.flush(userTableName + "_idx"); + } + List regionsOfTable = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsOfTable(userTable); + HRegionInfo hRegionInfo = regionsOfTable.get(0); + hRegionInfo.getRegionName(); + HTable tableidx = new HTable(conf,indexTable); + admin.split(hRegionInfo.getRegionName(), "row5".getBytes()); + List mainTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsOfTable(userTable); + + List indexTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsOfTable(indexTable); + while (mainTableRegions.size() != 2) { + Thread.sleep(2000); + mainTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsOfTable(userTable); + } + while (indexTableRegions.size() != 2) { + Thread.sleep(2000); + indexTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsOfTable(indexTable); + } + Assert.assertEquals(2, mainTableRegions.size()); + Assert.assertEquals(2, indexTableRegions.size()); + Scan s = new Scan(); + ResultScanner scanner = table.getScanner(s); + int mainTableCount = 0; + for (Result rr = scanner.next(); rr != null; rr = scanner.next()) { + mainTableCount++; + } + Assert.assertEquals(0, mainTableCount); + Scan s1 = new Scan(); + ResultScanner scanner1 = tableidx.getScanner(s1); + + int indexTableCount = 0; + for (Result rr = scanner1.next(); rr != null; rr = scanner1.next()) { + indexTableCount++; + } + Assert.assertEquals(0, indexTableCount); + + Put p = new Put("row7".getBytes()); + String val = "Val" + "7"; + p.add("col".getBytes(), "ql".getBytes(), val.getBytes()); + table.put(p); + Scan s2 = new Scan(); + ResultScanner scanner2 = table.getScanner(s2); + for (Result rr = scanner2.next(); rr != null; rr = scanner2.next()) { + mainTableCount++; + } + Scan s3 = new Scan(); + ResultScanner scanner3 = tableidx.getScanner(s3); + for (Result rr = scanner3.next(); rr != null; rr = scanner3.next()) { + indexTableCount++; + } + Assert.assertEquals(1, mainTableCount); + Assert.assertEquals(1, indexTableCount); + + } + + @Test(timeout = 180000) + public void test6PutsAnd3DeletesAfterSplitShouldRetreiveRowsSuccessfully() throws Exception { + Configuration conf = UTIL.getConfiguration(); + conf.setInt("hbase.regionserver.lease.period", 900000000); + conf.setBoolean("hbase.use.secondary.index", true); + String userTableName = "test6Puts3Deletes"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + IndexSpecification iSpec = new IndexSpecification("ScanIndexf"); + iSpec.addIndexColumn(new HColumnDescriptor("col"), "ql", ValueType.String, 10); + ihtd.addFamily(hcd); + TableIndices indices = new TableIndices(); + indices.addIndex(iSpec); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + TableName userTable = TableName.valueOf(userTableName); + TableName indexTable = TableName.valueOf(IndexUtils.getIndexTableName(userTableName)); + HTable table = new HTable(conf, userTableName); + // test put with the indexed column + for (int i = 0; i <= 5; i++) { + String row = "row" + i; + Put p = new Put(row.getBytes()); + String val = "Val" + i; + p.add("col".getBytes(), "ql".getBytes(), val.getBytes()); + table.put(p); + admin.flush(userTableName); + admin.flush(userTableName + "_idx"); + if (i < 3) { + Delete d = new Delete(row.getBytes()); + // Do a normal delete + table.delete(d); + } + admin.flush(userTableName); + admin.flush(userTableName + "_idx"); + } + List regionsOfTable = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsOfTable(userTable); + HRegionInfo hRegionInfo = regionsOfTable.get(0); + hRegionInfo.getRegionName(); + + HTable tableidx = new HTable(conf, indexTable); + admin.split(hRegionInfo.getRegionName(), "row5".getBytes()); + List mainTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsOfTable(userTable); + + List indexTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsOfTable(indexTable); + while (mainTableRegions.size() != 2) { + Thread.sleep(2000); + mainTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsOfTable(userTable); + } + while (indexTableRegions.size() != 2) { + Thread.sleep(2000); + indexTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsOfTable(indexTable); + } + Assert.assertEquals(2, mainTableRegions.size()); + Assert.assertEquals(2, indexTableRegions.size()); + Scan s = new Scan(); + ResultScanner scanner = table.getScanner(s); + int mainTableCount = 0; + for (Result rr = scanner.next(); rr != null; rr = scanner.next()) { + mainTableCount++; + } + Assert.assertEquals(3, mainTableCount); + Scan s1 = new Scan(); + ResultScanner scanner1 = tableidx.getScanner(s1); + + int indexTableCount = 0; + for (Result rr = scanner1.next(); rr != null; rr = scanner1.next()) { + indexTableCount++; + } + Assert.assertEquals(3, indexTableCount); + + } + + @Test(timeout = 180000) + public void testSplitForMainTableWithoutIndexShouldBeSuccessfUl() throws Exception { + Configuration conf = UTIL.getConfiguration(); + conf.setInt("hbase.regionserver.lease.period", 900000000); + conf.setBoolean("hbase.use.secondary.index", true); + String userTableName = "test6Puts3Deletes34534"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + ihtd.addFamily(hcd); + admin.createTable(ihtd); + TableName userTable = TableName.valueOf(userTableName); + TableName indexTable = TableName.valueOf(IndexUtils.getIndexTableName(userTableName)); + HTable table = new HTable(conf, userTableName); + + for (int i = 0; i < 10; i++) { + String row = "row" + i; + Put p = new Put(row.getBytes()); + String val = "Val" + i; + p.add("col".getBytes(), "ql".getBytes(), val.getBytes()); + table.put(p); + } + List regionsOfTable = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsOfTable(userTable); + HRegionInfo hRegionInfo = regionsOfTable.get(0); + hRegionInfo.getRegionName(); + admin.split(hRegionInfo.getRegionName(), "row5".getBytes()); + + List mainTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsOfTable(userTable); + + while (mainTableRegions.size() != 2) { + Thread.sleep(2000); + mainTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsOfTable(userTable); + } + for (int i = 20; i < 30; i++) { + String row = "row" + i; + Put p = new Put(row.getBytes()); + String val = "Val" + i; + p.add("col".getBytes(), "ql".getBytes(), val.getBytes()); + table.put(p); + } + Scan s = new Scan(); + ResultScanner scanner = table.getScanner(s); + int mainTableCount = 0; + for (Result rr = scanner.next(); rr != null; rr = scanner.next()) { + mainTableCount++; + } + Assert.assertEquals(20, mainTableCount); + + } + + @Test(timeout = 180000) + public void testSplittingIndexRegionExplicitly() throws Exception { + Configuration conf = UTIL.getConfiguration(); + conf.setInt("hbase.regionserver.lease.period", 900000000); + conf.setBoolean("hbase.use.secondary.index", true); + String userTableName = "testSplitTransaction"; + String indexTableName = "testSplitTransaction_idx"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + IndexSpecification iSpec = new IndexSpecification("ScanIndexf"); + iSpec.addIndexColumn(new HColumnDescriptor("col"), "ql", ValueType.String, 10); + ihtd.addFamily(hcd); + TableIndices indices = new TableIndices(); + indices.addIndex(iSpec); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + TableName userTable = TableName.valueOf(userTableName); + TableName indexTable = TableName.valueOf(IndexUtils.getIndexTableName(userTableName)); + HTable table = new HTable(conf, userTableName); + + for (int i = 0; i < 10; i++) { + String row = "row" + i; + Put p = new Put(row.getBytes()); + String val = "Val" + i; + p.add("col".getBytes(), "ql".getBytes(), val.getBytes()); + table.put(p); + } + + List regionsOfUserTable = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsOfTable(userTable); + + List regionsOfIndexTable = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsOfTable(indexTable); + + // try splitting index. + admin.split(indexTableName.getBytes()); + Thread.sleep(2000); + regionsOfIndexTable = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsOfTable(indexTable); + Assert.assertEquals("Index table should not get splited", 1, regionsOfIndexTable.size()); + + // try splitting the user region. + admin.split(userTableName.getBytes(), "row5".getBytes()); + while (regionsOfUserTable.size() != 2) { + Thread.sleep(2000); + regionsOfUserTable = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsOfTable(userTable); + } + while (regionsOfIndexTable.size() != 2) { + Thread.sleep(2000); + regionsOfIndexTable = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsOfTable(indexTable); + } + Assert.assertEquals(2, regionsOfUserTable.size()); + Assert.assertEquals(2, regionsOfIndexTable.size()); + } + + @Test(timeout = 180000) + public void testIndexManagerCleanUp() throws Exception { + Configuration conf = UTIL.getConfiguration(); + conf.setBoolean("hbase.use.secondary.index", true); + String userTableName = "testIndexManagerCleanUp"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd = new HColumnDescriptor("col1"); + ihtd.addFamily(hcd); + IndexSpecification iSpec = new IndexSpecification("Index1"); + iSpec.addIndexColumn(hcd, "ql", ValueType.String, 10); + TableIndices indices = new TableIndices(); + indices.addIndex(iSpec); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + + byte[][] splits = new byte[10][]; + char c = 'A'; + for (int i = 0; i < 10; i++) { + byte[] b = { (byte) c }; + splits[i] = b; + c++; + } + admin.createTable(ihtd, splits); + IndexManager instance = IndexManager.getInstance(); + int regionCount = instance.getTableRegionCount(userTableName); + Assert.assertEquals(11, regionCount); + + admin.disableTable(Bytes.toBytes(userTableName)); + regionCount = instance.getTableRegionCount(userTableName); + Assert.assertEquals(0, regionCount); + + admin.enableTable(userTableName); + regionCount = instance.getTableRegionCount(userTableName); + Assert.assertEquals(11, regionCount); + } + + @Test(timeout = 180000) + public void testHDP2938() throws Exception { + String userTableName = "testHDP2938"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + + HColumnDescriptor hcd1 = new HColumnDescriptor("col1").setMaxVersions(Integer.MAX_VALUE); + IndexSpecification iSpec = new IndexSpecification("ScanIndexf"); + iSpec.addIndexColumn(hcd1, "q2", ValueType.String, 10); + ihtd.addFamily(hcd1); + TableIndices indices = new TableIndices(); + indices.addIndex(iSpec); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + + admin.disableTable(userTableName); + admin.deleteTable(userTableName); + while(admin.isTableAvailable(IndexUtils.getIndexTableName(userTableName))){ + Threads.sleep(100); + } + ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + hcd1 = + new HColumnDescriptor("col1").setMaxVersions(Integer.MAX_VALUE).setTimeToLive( + TTL_SECONDS - 1); + iSpec = new IndexSpecification("ScanIndexf"); + iSpec.addIndexColumn(hcd1, "q2", ValueType.String, 10); + ihtd.addFamily(hcd1); + indices = new TableIndices(); + indices.addIndex(iSpec); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + + Configuration conf = UTIL.getConfiguration(); + HTable table = new HTable(conf, userTableName); + + // test put with the indexed column + + Put p = new Put("row1".getBytes()); + p.add("col1".getBytes(), "q2".getBytes(), Bytes.toBytes("ValForCF2")); + table.put(p); + admin.flush(userTableName + "_idx"); + + Put p1 = new Put("row01".getBytes()); + p1.add("col1".getBytes(), "q2".getBytes(), Bytes.toBytes("ValForCF2")); + table.put(p1); + admin.flush(userTableName + "_idx"); + + Put p2 = new Put("row010".getBytes()); + p2.add("col1".getBytes(), "q2".getBytes(), Bytes.toBytes("ValForCF2")); + table.put(p2); + admin.flush(userTableName + "_idx"); + + Put p3 = new Put("row001".getBytes()); + p3.add("col1".getBytes(), "q2".getBytes(), Bytes.toBytes("ValForCF2")); + table.put(p3); + + admin.flush(userTableName); + admin.flush(userTableName + "_idx"); + + HRegionServer regionServer = UTIL.getHBaseCluster().getRegionServer(0); + List onlineRegions = regionServer.getOnlineRegions(TableName.valueOf(userTableName)); + byte[][] columns = new byte[1][]; + columns[0] = hcd1.getName(); + byte[][] indexColumns = new byte[1][]; + indexColumns[0] = Constants.IDX_COL_FAMILY; + + List storeFileList = + onlineRegions.get(0).getStoreFileList(columns); + onlineRegions = + regionServer + .getOnlineRegions(TableName.valueOf(IndexUtils.getIndexTableName(userTableName))); + storeFileList = onlineRegions.get(0).getStoreFileList(indexColumns); + while (storeFileList.size() < 4) { + Thread.sleep(1000); + storeFileList = onlineRegions.get(0).getStoreFileList(indexColumns); + } + int prevSize = storeFileList.size(); + Assert.assertEquals("The total store files for the index table should be 4", 4, prevSize); + Scan s = new Scan(); + HTable indexTable = new HTable(conf, userTableName + "_idx"); + ResultScanner scanner = indexTable.getScanner(s); + // Result res = scanner.next(); + for (Result result : scanner) { + System.out.println(result); + } + for (String store : storeFileList) { + Threads.sleepWithoutInterrupt(TTL_MS); + } + admin.majorCompact(userTableName + "_idx"); + + onlineRegions = + regionServer + .getOnlineRegions(TableName.valueOf(IndexUtils.getIndexTableName(userTableName))); + storeFileList = onlineRegions.get(0).getStoreFileList(indexColumns);; + while (storeFileList.size() != 1) { + Thread.sleep(1000); + storeFileList = onlineRegions.get(0).getStoreFileList(indexColumns); + } + Assert.assertEquals("The total store files for the index table should be 1", 1, + storeFileList.size()); + s = new Scan(); + indexTable = new HTable(conf, userTableName + "_idx"); + scanner = indexTable.getScanner(s); + // Result res = scanner.next(); + boolean dataAvailable = false; + for (Result result : scanner) { + dataAvailable = true; + System.out.println(result); + } + Assert.assertFalse("dataShould not be retrieved", dataAvailable); + } + + @Test(timeout = 180000) + public void testIndexManagerWithSplitTransactions() throws Exception { + Configuration conf = UTIL.getConfiguration(); + ZooKeeperWatcher zkw = UTIL.getZooKeeperWatcher(UTIL); + conf.setBoolean("hbase.use.secondary.index", true); + String userTableName = "testIndexManagerWithSplitTransactions"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd = new HColumnDescriptor("col1"); + ihtd.addFamily(hcd); + IndexSpecification iSpec = new IndexSpecification("Index1"); + iSpec.addIndexColumn(hcd, "ql", ValueType.String, 10); + TableIndices indices = new TableIndices(); + indices.addIndex(iSpec); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + + IndexManager manager = IndexManager.getInstance(); + int count = manager.getTableRegionCount(userTableName); + Assert.assertEquals(1, count); + + HTable table = new HTable(conf, userTableName); + Put p = null; + for (int i = 0; i < 10; i++) { + p = new Put(Bytes.toBytes("row" + i)); + p.add(Bytes.toBytes("col1"), Bytes.toBytes("ql"), Bytes.toBytes("test_val")); + table.put(p); + } + + admin.split(userTableName, "row5"); + Threads.sleep(10000); + ZKAssign.blockUntilNoRIT(zkw); + UTIL.waitUntilAllRegionsAssigned(TableName.valueOf(userTableName)); + count = manager.getTableRegionCount(userTableName); + Assert.assertEquals(2, count); + } + + @Test(timeout = 180000) + public void testIndexManagerWithFailedSplitOfIndexRegion() throws Exception { + Configuration conf = UTIL.getConfiguration(); + conf.setBoolean("hbase.use.secondary.index", true); + String userTableName = "testIndexManagerWithFailedSplitOfIndexRegion"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd = new HColumnDescriptor("col1"); + ihtd.addFamily(hcd); + IndexSpecification iSpec = new IndexSpecification("Index1"); + iSpec.addIndexColumn(hcd, "ql", ValueType.String, 10); + TableIndices indices = new TableIndices(); + indices.addIndex(iSpec); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + + IndexManager manager = IndexManager.getInstance(); + int count = manager.getTableRegionCount(userTableName); + Assert.assertEquals(1, count); + + HTable table = new HTable(conf, userTableName); + Put p = null; + for (int i = 0; i < 10; i++) { + p = new Put(Bytes.toBytes("row" + i)); + p.add(Bytes.toBytes("col1"), Bytes.toBytes("ql"), Bytes.toBytes("test_val")); + table.put(p); + } + + admin.split(userTableName); + + count = manager.getTableRegionCount(userTableName); + Assert.assertEquals(1, count); + } + + @Test(timeout = 180000) + public void testIndexManagerWithFailedSplitTransaction() throws Exception { + Configuration conf = UTIL.getConfiguration(); + conf.setBoolean("hbase.use.secondary.index", true); + String userTableName = "testIndexManagerWithFailedSplitTransaction"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd = new HColumnDescriptor("col1"); + ihtd.addFamily(hcd); + IndexSpecification iSpec = new IndexSpecification("Index1"); + iSpec.addIndexColumn(hcd, "ql", ValueType.String, 10); + TableIndices indices = new TableIndices(); + indices.addIndex(iSpec); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + + IndexManager manager = IndexManager.getInstance(); + int count = manager.getTableRegionCount(userTableName); + Assert.assertEquals(1, count); + + HTable table = new HTable(conf, userTableName); + Put p = null; + for (int i = 0; i < 10; i++) { + p = new Put(Bytes.toBytes("row" + i)); + p.add(Bytes.toBytes("col1"), Bytes.toBytes("ql"), Bytes.toBytes("test_val")); + table.put(p); + } + List regions = UTIL.getMiniHBaseCluster().getRegions(Bytes.toBytes(userTableName)); + HRegionServer rs = UTIL.getMiniHBaseCluster().getRegionServer(0); + SplitTransaction st = null; + + st = new MockedSplitTransaction(regions.get(0), "row5".getBytes()); + try { + st.prepare(); + st.execute(rs, rs); + } catch (IOException e) { + st.rollback(rs, rs); + } + + count = manager.getTableRegionCount(userTableName); + Assert.assertEquals(1, count); + } + + public static class MockedSplitTransaction extends SplitTransaction { + + public MockedSplitTransaction(HRegion r, byte[] splitrow) { + super(r, splitrow); + } + + public void splitStoreFiles(Map> hstoreFilesToSplit) throws IOException { + throw new IOException(); + } + + } +} \ No newline at end of file Index: hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestIndexRegionObserverForScan.java =================================================================== --- hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestIndexRegionObserverForScan.java (revision 0) +++ hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestIndexRegionObserverForScan.java (working copy) @@ -0,0 +1,1450 @@ +/** +f * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.MasterNotRunningException; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.UnknownRegionException; +import org.apache.hadoop.hbase.ZooKeeperConnectionException; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.apache.hadoop.hbase.coprocessor.ObserverContext; +import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; +import org.apache.hadoop.hbase.filter.BinaryComparator; +import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; +import org.apache.hadoop.hbase.filter.Filter; +import org.apache.hadoop.hbase.filter.FilterList; +import org.apache.hadoop.hbase.filter.FilterList.Operator; +import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter; +import org.apache.hadoop.hbase.filter.RowFilter; +import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; +import org.apache.hadoop.hbase.index.ColumnQualifier.ValueType; +import org.apache.hadoop.hbase.index.Constants; +import org.apache.hadoop.hbase.index.IndexSpecification; +import org.apache.hadoop.hbase.index.SecIndexLoadBalancer; +import org.apache.hadoop.hbase.index.TableIndices; +import org.apache.hadoop.hbase.index.TestUtils; +import org.apache.hadoop.hbase.index.client.IndexAdmin; +import org.apache.hadoop.hbase.index.coprocessor.master.IndexMasterObserver; +import org.apache.hadoop.hbase.index.coprocessor.wal.IndexWALObserver; +import org.apache.hadoop.hbase.index.util.IndexUtils; +import org.apache.hadoop.hbase.master.LoadBalancer; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.regionserver.RegionScanner; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Threads; +import org.apache.zookeeper.KeeperException; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(LargeTests.class) +public class TestIndexRegionObserverForScan { + + private static HBaseTestingUtility UTIL = new HBaseTestingUtility(); + private static HBaseAdmin admin = null; + + @BeforeClass + public static void setupBeforeClass() throws Exception { + Configuration conf = UTIL.getConfiguration(); + conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, IndexMasterObserver.class.getName()); + conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, IndexRegionObserver.class.getName()); + conf.set(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY, IndexWALObserver.class.getName()); + conf.setBoolean("hbase.use.secondary.index", true); + conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, SecIndexLoadBalancer.class, + LoadBalancer.class); + UTIL.startMiniCluster(1); + admin = new IndexAdmin(conf); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + if (admin != null) admin.close(); + UTIL.shutdownMiniCluster(); + } + + @Before + public void setUp() throws Exception { + IndexRegionObserver.setIndexedFlowUsed(false); + IndexRegionObserver.setSeekpointAdded(false); + IndexRegionObserver.setSeekPoints(null); + IndexRegionObserver.setIsTestingEnabled(true); + } + + @After + public void tearDown() throws Exception { + IndexRegionObserver.setIsTestingEnabled(false); + } + + @Test(timeout = 180000) + public void testScanIndexedColumnWithOnePutShouldRetreiveOneRowSuccessfully() throws IOException, + KeeperException, InterruptedException { + String userTableName = "testScanIndexedColumnWithOnePutShouldRetreiveOneRowSuccessfully"; + HTableDescriptor ihtd = + TestUtils.createIndexedHTableDescriptor(userTableName, "col", "ScanIndexf", "col", "ql"); + admin.createTable(ihtd); + Configuration conf = UTIL.getConfiguration(); + HTable table = new HTable(conf, userTableName); + + // test put with the indexed column + Put p = new Put("row1".getBytes()); + p.add("col".getBytes(), "ql".getBytes(), "Val".getBytes()); + table.put(p); + int i = countNumberOfRowsWithFilter(userTableName, "Val", true, false, 0); + Assert.assertEquals("Should match for 1 row successfully ", 1, i); + Assert.assertTrue("Indexed table should be used ", IndexRegionObserver.getIndexedFlowUsed()); + Assert.assertTrue("Seek points should be added ", IndexRegionObserver.getSeekpointAdded()); + + } + + @Test(timeout = 180000) + public void testScanIndexedColumnWithOnePutAndSplitKeyatBorderShouldRetreiveOneRowSuccessfully() + throws IOException, KeeperException, InterruptedException { + String userTableName = + "testScanIndexedColumnWithOnePutAndSplitKeyatBorderShouldRetreiveOneRowSuccessfully"; + HTableDescriptor ihtd = + TestUtils.createIndexedHTableDescriptor(userTableName, "col", "ScanIndexf", "col", "ql"); + byte[][] split = + new byte[][] { "row1".getBytes(), "row21".getBytes(), "row41".getBytes(), + "row61".getBytes(), "row81".getBytes(), "row101".getBytes(), "row121".getBytes(), + "row141".getBytes(), }; + + // create table with splits this will create 9 regions + admin.createTable(ihtd, split); + Configuration conf = UTIL.getConfiguration(); + HTable table = new HTable(conf, userTableName); + + // test put with the indexed column + Put p = new Put("row1".getBytes()); + p.add("col".getBytes(), "ql".getBytes(), "Val".getBytes()); + table.put(p); + + Put p1 = new Put("row01".getBytes()); + p1.add("col".getBytes(), "ql".getBytes(), "Val".getBytes()); + table.put(p1); + + Put p2 = new Put("row010".getBytes()); + p2.add("col".getBytes(), "ql".getBytes(), "Val".getBytes()); + table.put(p2); + + Put p3 = new Put("row001".getBytes()); + p3.add("col".getBytes(), "ql".getBytes(), "Val".getBytes()); + table.put(p3); + + validateCountOfMainTableIndIndexedTable(conf, userTableName, table); + + int i = countNumberOfRowsWithFilter(userTableName, "Val", true, false, 0); + Assert.assertEquals("Should match for 1 row successfully ", 4, i); + Assert.assertTrue("Indexed table should be used ", IndexRegionObserver.getIndexedFlowUsed()); + Assert.assertTrue("Seek points should be added ", IndexRegionObserver.getSeekpointAdded()); + + } + + @Test(timeout = 180000) + public + void + testScanIndexedColumnWithOnePutAndSplitKeyAndStartKeyRegionEmptyShouldRetreiveOneRowSuccessfully() + throws IOException, KeeperException, InterruptedException { + Configuration conf = UTIL.getConfiguration(); + String userTableName = + "testScanIndexedColumnWithOnePutAndSplitKeyAndStartKeyRegionEmptyShouldRetreiveOneRowSuccessfully"; + HTableDescriptor ihtd = + TestUtils.createIndexedHTableDescriptor(userTableName, "col", "ScanIndexf", "col", "ql"); + + byte[][] split = new byte[][] { "A".getBytes(), "B".getBytes(), "C".getBytes() }; + + // create table with splits this will create 9 regions + admin.createTable(ihtd, split); + + HTable table = new HTable(conf, userTableName); + Put p = new Put("00row1".getBytes()); + p.add("col".getBytes(), "ql".getBytes(), "Val".getBytes()); + table.put(p); + + Put p1 = new Put("0row1".getBytes()); + p1.add("col".getBytes(), "ql".getBytes(), "Val".getBytes()); + table.put(p1); + + Put p2 = new Put("000row1".getBytes()); + p2.add("col".getBytes(), "ql".getBytes(), "Val".getBytes()); + table.put(p2); + + Put p3 = new Put("0000row1".getBytes()); + p3.add("col".getBytes(), "ql".getBytes(), "Val".getBytes()); + table.put(p3); + + // Check for verification of number of rows in main table and user table + validateCountOfMainTableIndIndexedTable(conf, userTableName, table); + + // test put with the indexed column + + int i = countNumberOfRowsWithFilter(userTableName, "Val", true, false, 0); + Assert.assertEquals("Should match for 1 row successfully ", 4, i); + Assert.assertTrue("Indexed table should be used ", IndexRegionObserver.getIndexedFlowUsed()); + Assert.assertTrue("Seek points should be added ", IndexRegionObserver.getSeekpointAdded()); + + } + + @Test(timeout = 180000) + public void + testScanIndexedColumnWithOnePutAndSplitKeyHavingSpaceShouldRetreiveOneRowSuccessfully() + throws IOException, KeeperException, InterruptedException { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testScanIndexedColumnWithOnePutAndSplitKeyHavingSpaceShouldRetreiveOneRowSuccessfully"; + HTableDescriptor ihtd = + TestUtils.createIndexedHTableDescriptor(userTableName, "col", "ScanIndexf", "col", "ql"); + + byte[][] split = + new byte[][] { " row1".getBytes(), "row21".getBytes(), "row41".getBytes(), + "row61".getBytes(), "row81".getBytes(), "row101 ".getBytes(), "row121".getBytes(), + "row141".getBytes(), }; + + // create table with splits this will create 9 regions + admin.createTable(ihtd, split); + + HTable table = new HTable(conf, userTableName); + + // test put with the indexed column + Put p = new Put("row1".getBytes()); + p.add("col".getBytes(), "ql".getBytes(), "Val".getBytes()); + table.put(p); + validateCountOfMainTableIndIndexedTable(conf, userTableName, table); + int i = countNumberOfRowsWithFilter(userTableName, "Val", true, false, 0); + Assert.assertEquals("Should match for 1 row successfully ", 1, i); + Assert.assertTrue("Indexed table should be used ", IndexRegionObserver.getIndexedFlowUsed()); + Assert.assertTrue("Seek points should be added ", IndexRegionObserver.getSeekpointAdded()); + + } + + @Test(timeout = 180000) + public void testScanIndexedColumnShouldNotRetreiveRowIfThereIsNoMatch() throws IOException, + KeeperException, InterruptedException { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testScanIndexedColumnShouldNotRetreiveRowIfThereIsNoMatch"; + HTableDescriptor ihtd = + TestUtils.createIndexedHTableDescriptor(userTableName, "col", "ScanIndexf", "col", "ql"); + admin.createTable(ihtd); + + HTable table = new HTable(conf, userTableName); + + // test put with the indexed column + Put p = new Put("row1".getBytes()); + p.add("col".getBytes(), "ql".getBytes(), "Val".getBytes()); + table.put(p); + + Put p1 = new Put("row2".getBytes()); + p1.add("col".getBytes(), "ql".getBytes(), "Val1".getBytes()); + table.put(p1); + int i = countNumberOfRowsWithFilter(userTableName, "unmatch", true, false, 0); + Assert.assertEquals("Should not match any rows ", 0, i); + Assert.assertFalse("Seek points should not be added ", IndexRegionObserver.getSeekpointAdded()); + Assert.assertTrue("Indexed table should be used ", IndexRegionObserver.getIndexedFlowUsed()); + } + + @Test(timeout = 180000) + public void testScanIndexedColumnWith5PutsAnd3EqualPutValuesShouldRetreive3RowsSuccessfully() + throws IOException, KeeperException, InterruptedException { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testScanIndexedColumnWith5PutsAnd3EqualPutValuesShouldRetreive3RowsSuccessfully"; + HTableDescriptor ihtd = + TestUtils.createIndexedHTableDescriptor(userTableName, "col", "ScanIndexf", "col", "ql"); + admin.createTable(ihtd); + + HTable table = new HTable(conf, userTableName); + Put p1 = new Put("row1".getBytes()); + p1.add("col".getBytes(), "ql".getBytes(), "cat".getBytes()); + table.put(p1); + + Put p4 = new Put("row3".getBytes()); + p4.add("col".getBytes(), "ql".getBytes(), "cat".getBytes()); + table.put(p4); + + Put p5 = new Put("row2".getBytes()); + p5.add("col".getBytes(), "ql".getBytes(), "dog".getBytes()); + table.put(p5); + + Put p2 = new Put("row4".getBytes()); + p2.add("col".getBytes(), "ql".getBytes(), "cat".getBytes()); + table.put(p2); + + Put p3 = new Put("row5".getBytes()); + p3.add("col".getBytes(), "ql".getBytes(), "dogs".getBytes()); + table.put(p3); + + int i = countNumberOfRowsWithFilter(userTableName, "cat", true, false, 0); + + Assert.assertEquals("Should match for exactly 3 rows ", 3, i); + Assert.assertTrue("Seek points should be added ", IndexRegionObserver.getSeekpointAdded()); + Assert.assertTrue("Indexed table should be used ", IndexRegionObserver.getIndexedFlowUsed()); + } + + @Test(timeout = 180000) + public void testPerformanceOfScanOnIndexedColumnShouldBeMoreWith1LakhRowsIfFlushIsNotMade() + throws Exception { + + long withoutIndex = doPerformanceTest(false, "tableWithoutIndexAndWithoutFlush", false); + long withIndex = doPerformanceTest(true, "tableWithIndexAndWithoutFlush", false); + Assert.assertTrue("Seek points should be added ", IndexRegionObserver.getSeekpointAdded()); + Assert.assertTrue("Indexed table should be used ", IndexRegionObserver.getIndexedFlowUsed()); + IndexRegionObserver.setSeekpointAdded(false); + IndexRegionObserver.setIndexedFlowUsed(false); + Assert.assertTrue( + "Without flush time taken for Indexed scan should be less than without index ", + withIndex < withoutIndex); + + Assert.assertFalse("Seek points should not be added ", IndexRegionObserver.getSeekpointAdded()); + Assert.assertFalse("Indexed table should not be used ", + IndexRegionObserver.getIndexedFlowUsed()); + } + + @Test(timeout = 180000) + public void testShouldSuccessfullyReturn3RowsIfOnly3RowsMatchesIn1LakhRowsWithParallelPuts() + throws Exception { + String userTableName = "tableToCheck3Rows"; + doBulkParallelPuts(true, userTableName, false); + int i = countNumberOfRowsWithFilter(userTableName, "cat", true, false, 0); + Assert.assertEquals("Should match for exactly 3 rows in 1 lakh rows ", 3, i); + Assert.assertTrue("Indexed table should be used ", IndexRegionObserver.getSeekpointAdded()); + } + + @Test(timeout = 180000) + public void testPerformanceOfScanOnIndexedColumnShouldBeMoreWith1LakhRowsIfFlushIsMade() + throws Exception { + long withIndex = doPerformanceTest(true, "tableWithIndexAndWithFlush", true); + + Assert.assertTrue("Seek points should be added ", IndexRegionObserver.getSeekpointAdded()); + Assert.assertTrue("Indexed table should be used ", IndexRegionObserver.getIndexedFlowUsed()); + IndexRegionObserver.setSeekpointAdded(false); + IndexRegionObserver.setIndexedFlowUsed(false); + long withoutIndex = doPerformanceTest(false, "tableWithoutIndexAndWithFlush", true); + Assert.assertTrue("With flush time taken for Indexed scan should be less than without index ", + withIndex < withoutIndex); + Assert.assertFalse("Seek points should not be added ", IndexRegionObserver.getSeekpointAdded()); + Assert.assertFalse("Indexed table should not be used ", + IndexRegionObserver.getIndexedFlowUsed()); + } + + @Test(timeout = 180000) + public void testParallelScansShouldRetreiveRowsCorrectlyForIndexedColumn() throws Exception { + String userTableName = "testParallelScansOnIndexedColumn"; + doParallelScanPuts(userTableName); + ParallelScanThread p1 = new ParallelScanThread(userTableName, "cat", false, 0); + p1.start(); + + ParallelScanThread p2 = new ParallelScanThread(userTableName, "dog", false, 0); + p2.start(); + + ParallelScanThread p3 = new ParallelScanThread(userTableName, "pup", false, 0); + p3.start(); + + // wait for scan to complete + p1.join(); + p2.join(); + p3.join(); + Assert.assertEquals("Should match for exactly 700 cats ", 700, p1.count); + Assert.assertEquals("Should match for exactly 500 dogs ", 500, p2.count); + Assert.assertEquals("Should match for exactly 300 pups ", 300, p3.count); + Assert.assertTrue("Seek points should be added ", IndexRegionObserver.getSeekpointAdded()); + Assert.assertTrue("Indexed table should be used ", IndexRegionObserver.getIndexedFlowUsed()); + } + + @Test(timeout = 180000) + public void testParallelScansWithCacheShouldRetreiveRowsCorrectlyForIndexedColumn() + throws Exception { + String userTableName = "testParallelScansWithCacheOnIndexedColumn"; + doParallelScanPuts(userTableName); + // In parallel scan setting the cache + ParallelScanThread p1 = new ParallelScanThread(userTableName, "cat", true, 200); + p1.start(); + + ParallelScanThread p2 = new ParallelScanThread(userTableName, "dog", true, 200); + p2.start(); + + ParallelScanThread p3 = new ParallelScanThread(userTableName, "pup", true, 200); + p3.start(); + + // wait for scan to complete + p1.join(); + p2.join(); + p3.join(); + Assert.assertEquals("Should match for exactly 700 cats ", 700, p1.count); + Assert.assertEquals("Should match for exactly 500 dogs ", 500, p2.count); + Assert.assertEquals("Should match for exactly 300 pups ", 300, p3.count); + Assert.assertTrue("Seek points should be added ", IndexRegionObserver.getSeekpointAdded()); + Assert.assertTrue("Indexed table should be used ", IndexRegionObserver.getIndexedFlowUsed()); + } + + @Test(timeout = 180000) + public void testScanShouldBeSuccessfulEvenIfExceptionIsThrownFromPostScannerOpen() + throws Exception { + + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testScanShouldBeSuccessfulEvenIfExceptionIsThrownFromPostScannerOpen"; + HTableDescriptor ihtd = + TestUtils.createIndexedHTableDescriptor(userTableName, "col", "ScanIndexf", "col", "ql"); + admin.createTable(ihtd); + + HTable table = new HTable(conf, userTableName); + + // test put with the indexed column + Put p = new Put("row1".getBytes()); + p.add("col".getBytes(), "ql".getBytes(), "Val".getBytes()); + table.put(p); + Scan s = new Scan(); + s.setLoadColumnFamiliesOnDemand(false); + Filter filter = new FilterList(); + s.setFilter(filter); + int i = 0; + ResultScanner scanner = table.getScanner(s); + for (Result result : scanner) { + i++; + } + + Assert.assertEquals("Should match for 1 row successfully ", 1, i); + Assert.assertFalse("Seek points should not be added ", IndexRegionObserver.getSeekpointAdded()); + Assert.assertFalse("Indexed table should not be used ", + IndexRegionObserver.getIndexedFlowUsed()); + + } + + private String doParallelScanPuts(String userTableName) throws IOException, + ZooKeeperConnectionException, KeeperException, InterruptedException { + Configuration conf = UTIL.getConfiguration(); + HTableDescriptor ihtd = + TestUtils.createIndexedHTableDescriptor(userTableName, "col", "ScanIndexf", "col", "ql"); + admin.createTable(ihtd); + + HTable table = new HTable(conf, userTableName); + List puts = new ArrayList(); + + for (int i = 1; i <= 500; i++) { + Put p1 = new Put(("row" + i).getBytes()); + p1.add("col".getBytes(), "ql".getBytes(), "cat".getBytes()); + puts.add(p1); + } + table.put(puts); + + for (int i = 501; i <= 1000; i++) { + Put p1 = new Put(("row" + i).getBytes()); + p1.add("col".getBytes(), "ql".getBytes(), "dog".getBytes()); + puts.add(p1); + } + table.put(puts); + + for (int i = 1001; i <= 1300; i++) { + Put p1 = new Put(("row" + i).getBytes()); + p1.add("col".getBytes(), "ql".getBytes(), "pup".getBytes()); + puts.add(p1); + } + table.put(puts); + + for (int i = 1301; i <= 1500; i++) { + Put p1 = new Put(("row" + i).getBytes()); + p1.add("col".getBytes(), "ql".getBytes(), "cat".getBytes()); + puts.add(p1); + } + table.put(puts); + return userTableName; + } + + @Test(timeout = 180000) + public void testScanShouldNotRetreiveRowsIfRowsArePresentOnlyInIndexedTableAndNotInMainTable() + throws Exception { + + Configuration conf = UTIL.getConfiguration(); + final String userTableName = "testScanOnIndexedColumnForFalsePositve"; + HTableDescriptor ihtd = + TestUtils.createIndexedHTableDescriptor(userTableName, "col", "ScanIndex", "col", "ql"); + admin.createTable(ihtd); + + HTable table = new HTable(conf, userTableName); + + List puts = new ArrayList(); + + for (int i = 1; i <= 100; i++) { + Put p1 = new Put(("row" + i).getBytes()); + p1.add("col".getBytes(), "ql".getBytes(), "cat".getBytes()); + puts.add(p1); + } + table.put(puts); + + puts = new ArrayList(); + for (int i = 101; i <= 200; i++) { + Put p1 = new Put(("row" + i).getBytes()); + p1.add("col".getBytes(), "ql".getBytes(), "dog".getBytes()); + puts.add(p1); + } + table.put(puts); + + puts = new ArrayList(); + for (int i = 201; i <= 300; i++) { + Put p1 = new Put(("row" + i).getBytes()); + p1.add("col".getBytes(), "ql".getBytes(), "pup".getBytes()); + puts.add(p1); + } + table.put(puts); + + // Doing one extra put explicitly into indexed table + HTable indexTable = new HTable(conf, userTableName + Constants.INDEX_TABLE_SUFFIX); + + List regionsOfTable = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates() + .getRegionsOfTable(TableName.valueOf(IndexUtils.getIndexTableName(userTableName))); + + byte[] startRow = generateStartKey(regionsOfTable); + + Put p1 = new Put(startRow); + p1.add("d".getBytes(), "ql".getBytes(), "idxCat".getBytes()); + indexTable.put(p1); + + int i = countNumberOfRowsWithFilter(userTableName, "idxCat", true, false, 0); + Assert.assertEquals("Should not match any rows in main table ", 0, i); + Assert.assertTrue("Indexed table should be used ", IndexRegionObserver.getIndexedFlowUsed()); + Assert.assertFalse("Seek points should not be added ", IndexRegionObserver.getSeekpointAdded()); + + } + + @Test(timeout = 180000) + public void testScanWithPutsAndCacheSetShouldRetreiveMatchingRows() throws Exception { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testcachedColumn"; + HTableDescriptor ihtd = + TestUtils.createIndexedHTableDescriptor(userTableName, "col", "ScanIndexf", "col", "ql"); + admin.createTable(ihtd); + int i = singleIndexPutAndCache(conf, userTableName); + Assert.assertEquals("Should match for exactly 5 rows ", 5, i); + Assert.assertTrue("Seek points should be added ", IndexRegionObserver.getSeekpointAdded()); + Assert.assertTrue("Indexed table should be used ", IndexRegionObserver.getIndexedFlowUsed()); + Assert.assertEquals("Remaining rows in cache should be 2 ", 2, IndexRegionObserver + .getSeekpoints().size()); + + } + + @Test(timeout = 180000) + public void testScanMultipleIdxWithSameColFamilyAndDifferentQualifierShouldBeSuccessful() + throws Exception { + + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testScanWithMultIndexedSameColFamilyColumn"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd1 = new HColumnDescriptor("col1"); + ihtd.addFamily(hcd1); + IndexSpecification idx1 = new IndexSpecification("ScanMulIndex"); + idx1.addIndexColumn(hcd1, "ql", ValueType.String, 10); + idx1.addIndexColumn(hcd1, "q2", ValueType.String, 10); + TableIndices indices = new TableIndices(); + indices.addIndex(idx1); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + + // test put with the indexed column + Put p1 = new Put("row1".getBytes()); + p1.add("col1".getBytes(), "ql".getBytes(), "cat".getBytes()); + p1.add("col1".getBytes(), "q2".getBytes(), "dog".getBytes()); + table.put(p1); + + Put p2 = new Put("row2".getBytes()); + p2.add("col1".getBytes(), "ql".getBytes(), "dog".getBytes()); + p2.add("col1".getBytes(), "q2".getBytes(), "cat".getBytes()); + table.put(p2); + + Put p3 = new Put("row3".getBytes()); + p3.add("col1".getBytes(), "ql".getBytes(), "cat".getBytes()); + p3.add("col1".getBytes(), "q2".getBytes(), "dog".getBytes()); + table.put(p3); + + int i = 0; + Scan s = new Scan(); + FilterList filterList = new FilterList(); + // check for combination of cat in q1 and dog in q2 + SingleColumnValueFilter filter1 = + new SingleColumnValueFilter("col1".getBytes(), "ql".getBytes(), CompareOp.EQUAL, + "cat".getBytes()); + filter1.setFilterIfMissing(true); + SingleColumnValueFilter filter2 = + new SingleColumnValueFilter("col1".getBytes(), "q2".getBytes(), CompareOp.EQUAL, + "dog".getBytes()); + filter2.setFilterIfMissing(true); + filterList.addFilter(filter1); + + filterList.addFilter(filter2); + s.setFilter(filterList); + + ResultScanner scanner = table.getScanner(s); + for (Result result : scanner) { + i++; + } + + Assert.assertEquals("Should match for 2 rows in multiple index successfully ", 2, i); + Assert.assertTrue("Indexed table should be used ", IndexRegionObserver.getIndexedFlowUsed()); + Assert.assertTrue("Seek points should be added ", IndexRegionObserver.getSeekpointAdded()); + + } + + @Test(timeout = 180000) + public void testScanMultipleIdxWithDifferentColFamilyShouldBeSuccessful() throws Exception { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testScanWithMultIndexedDiffColFamilyColumn"; + putMulIndex(userTableName); + int i = 0; + Scan s = new Scan(); + FilterList filterList = new FilterList(); + // check for combination of cat in q1 and dog in q1 + SingleColumnValueFilter filter1 = + new SingleColumnValueFilter("col1".getBytes(), "ql".getBytes(), CompareOp.EQUAL, + "cat".getBytes()); + filter1.setFilterIfMissing(true); + SingleColumnValueFilter filter2 = + new SingleColumnValueFilter("col2".getBytes(), "ql".getBytes(), CompareOp.EQUAL, + "dog".getBytes()); + filter2.setFilterIfMissing(true); + filterList.addFilter(filter1); + filterList.addFilter(filter2); + s.setFilter(filterList); + HTable table = new HTable(conf, userTableName); + ResultScanner scanner = table.getScanner(s); + for (Result result : scanner) { + i++; + } + Assert.assertEquals( + "Should match for 5 rows in multiple index with diff column family successfully ", 5, i); + Assert.assertTrue("Seek points should be added ", IndexRegionObserver.getSeekpointAdded()); + Assert.assertTrue("Indexed table should be used ", IndexRegionObserver.getIndexedFlowUsed()); + } + + @Test(timeout = 180000) + public void testScanMultipleIdxWithDifferentColFamilyAndCacheShouldBeSuccessful() + throws Exception { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testScanWithMultIndexedCacheDiffColFamilyColumn"; + putMulIndex(userTableName); + int i = 0; + Scan s = new Scan(); + FilterList filterList = new FilterList(); + // check for combination of cat in q1 and dog in q1 + SingleColumnValueFilter filter1 = + new SingleColumnValueFilter("col1".getBytes(), "ql".getBytes(), CompareOp.EQUAL, + "cat".getBytes()); + filter1.setFilterIfMissing(true); + SingleColumnValueFilter filter2 = + new SingleColumnValueFilter("col2".getBytes(), "ql".getBytes(), CompareOp.EQUAL, + "dog".getBytes()); + filter2.setFilterIfMissing(true); + filterList.addFilter(filter1); + filterList.addFilter(filter2); + s.setCaching(4); + s.setFilter(filterList); + HTable table = new HTable(conf, userTableName); + ResultScanner scanner = table.getScanner(s); + for (Result result : scanner) { + i++; + } + Assert.assertEquals( + "Should match for 5 rows in multiple index with diff column family successfully ", 5, i); + Assert.assertTrue("Seek points should be added ", IndexRegionObserver.getSeekpointAdded()); + Assert.assertTrue("Indexed table should be used ", IndexRegionObserver.getIndexedFlowUsed()); + Assert.assertEquals("Remaining rows in cache should be 1 ", 1, IndexRegionObserver + .getSeekpoints().size()); + } + + @Test(timeout = 180000) + public void + testScanMultipleIdxWithDifferentFiltersShouldBeSuccessfulAndShouldNotGoWithIndexedFlow() + throws Exception { + + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testScanWithMultIndexedDiffFilters"; + putMulIndex(userTableName); + HTable table = new HTable(conf, userTableName); + int i = 0; + Scan s = new Scan(); + FilterList filterList = new FilterList(); + // check for combination of cat in q1 and dog in q1 + Filter filter1 = + new RowFilter(CompareOp.LESS_OR_EQUAL, new BinaryComparator("row5".getBytes())); + Filter filter2 = new FirstKeyOnlyFilter(); + filterList.addFilter(filter1); + filterList.addFilter(filter2); + s.setFilter(filterList); + ResultScanner scanner = table.getScanner(s); + for (Result result : scanner) { + i++; + } + Assert.assertEquals( + "Should match for 5 rows in multiple index with diff column family successfully ", 5, i); + Assert.assertFalse("Seek points should not be added ", IndexRegionObserver.getSeekpointAdded()); + Assert.assertFalse("Indexed table should not be used ", + IndexRegionObserver.getIndexedFlowUsed()); + } + + @Test(timeout = 180000) + public void testScanWithIndexOn2ColumnsAndFiltersOn2ColumnsInReverseWayShouldBeSuccessful() + throws Exception { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testScan2Indexed2ReversedFilters"; + putMulIndex(userTableName); + HTable table = new HTable(conf, userTableName); + int i = 0; + Scan s = new Scan(); + FilterList filterList = new FilterList(); + // check for combination of cat in q1 and dog in q1 + SingleColumnValueFilter filter1 = + new SingleColumnValueFilter("col2".getBytes(), "ql".getBytes(), CompareOp.EQUAL, + "dog".getBytes()); + filter1.setFilterIfMissing(true); + SingleColumnValueFilter filter2 = + new SingleColumnValueFilter("col1".getBytes(), "ql".getBytes(), CompareOp.EQUAL, + "cat".getBytes()); + filter2.setFilterIfMissing(true); + filterList.addFilter(filter1); + filterList.addFilter(filter2); + s.setFilter(filterList); + + ResultScanner scanner = table.getScanner(s); + for (Result result : scanner) { + i++; + } + Assert.assertEquals( + "Should match for 5 rows in multiple index with diff column family successfully ", 5, i); + Assert.assertTrue("Seek points should be added ", IndexRegionObserver.getSeekpointAdded()); + Assert.assertTrue("Indexed table should be used ", IndexRegionObserver.getIndexedFlowUsed()); + } + + @Test(timeout = 180000) + public + void + testScanMultipleIdxWithDifferentColumnsInFiltersShouldBeSuccessfulAndShouldNotGoWithIndexedFlow() + throws Exception { + + Configuration conf = UTIL.getConfiguration(); + String userTableName = "test11ScanWithMultIndexedDiff11Filters"; + + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + + HColumnDescriptor hcd1 = new HColumnDescriptor("col1"); + HColumnDescriptor hcd2 = new HColumnDescriptor("col2"); + HColumnDescriptor hcd3 = new HColumnDescriptor("col3"); + ihtd.addFamily(hcd1); + ihtd.addFamily(hcd2); + ihtd.addFamily(hcd3); + IndexSpecification idx1 = new IndexSpecification("ScanMulIndex"); + idx1.addIndexColumn(hcd1, "ql", ValueType.String, 10); + idx1.addIndexColumn(hcd2, "ql", ValueType.String, 10); + TableIndices indices = new TableIndices(); + indices.addIndex(idx1); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + + // test put with the multiple indexed column in diffrent column families + Put p1 = new Put("row1".getBytes()); + p1.add("col1".getBytes(), "ql".getBytes(), "cat".getBytes()); + p1.add("col2".getBytes(), "ql".getBytes(), "dog".getBytes()); + table.put(p1); + + Put p2 = new Put("row2".getBytes()); + p2.add("col1".getBytes(), "ql".getBytes(), "dog".getBytes()); + p2.add("col2".getBytes(), "ql".getBytes(), "cat".getBytes()); + table.put(p2); + + Put p3 = new Put("row3".getBytes()); + p3.add("col1".getBytes(), "ql".getBytes(), "cat".getBytes()); + p3.add("col2".getBytes(), "ql".getBytes(), "dog".getBytes()); + table.put(p3); + + Put p4 = new Put("row4".getBytes()); + p4.add("col1".getBytes(), "ql".getBytes(), "dog".getBytes()); + p4.add("col2".getBytes(), "ql".getBytes(), "dog".getBytes()); + table.put(p4); + + Put p5 = new Put("row5".getBytes()); + p5.add("col1".getBytes(), "ql".getBytes(), "cat".getBytes()); + p5.add("col2".getBytes(), "ql".getBytes(), "dog".getBytes()); + table.put(p5); + + Put p6 = new Put("row6".getBytes()); + p6.add("col1".getBytes(), "ql".getBytes(), "cat".getBytes()); + p6.add("col2".getBytes(), "ql".getBytes(), "cat".getBytes()); + table.put(p6); + + Put p7 = new Put("row7".getBytes()); + p7.add("col1".getBytes(), "ql".getBytes(), "cat".getBytes()); + p7.add("col2".getBytes(), "ql".getBytes(), "dog".getBytes()); + table.put(p7); + + Put p9 = new Put("row8".getBytes()); + p9.add("col1".getBytes(), "ql".getBytes(), "cat".getBytes()); + p9.add("col3".getBytes(), "ql".getBytes(), "dog".getBytes()); + table.put(p9); + + Put p8 = new Put("row9".getBytes()); + p8.add("col1".getBytes(), "ql".getBytes(), "cat".getBytes()); + p8.add("col2".getBytes(), "ql".getBytes(), "dog".getBytes()); + table.put(p8); + int i = 0; + Scan s = new Scan(); + s.setLoadColumnFamiliesOnDemand(false); + FilterList filterList = new FilterList(Operator.MUST_PASS_ALL); + // check for combination of cat in q1 and dog in q1 + SingleColumnValueFilter filter1 = + new SingleColumnValueFilter("col1".getBytes(), "ql".getBytes(), CompareOp.EQUAL, + "cat".getBytes()); + filter1.setFilterIfMissing(true); + SingleColumnValueFilter filter2 = + new SingleColumnValueFilter("col3".getBytes(), "ql".getBytes(), CompareOp.EQUAL, + "dog".getBytes()); + filter2.setFilterIfMissing(true); + filterList.addFilter(filter1); + filterList.addFilter(filter2); + s.setFilter(filterList); + ResultScanner scanner = table.getScanner(s); + for (Result result : scanner) { + i++; + } + Assert.assertEquals( + "Should match for 1 rows in multiple index with diff column family successfully ", 1, i); + Assert.assertTrue("Seek points should be added ", IndexRegionObserver.getSeekpointAdded()); + Assert.assertTrue("Indexed table should be used ", IndexRegionObserver.getIndexedFlowUsed()); + + // Different values in column family should not retreive the rows.. Below + // ensures the same + Scan s1 = new Scan(); + s1.setLoadColumnFamiliesOnDemand(false); + FilterList filterList1 = new FilterList(); + // check for combination of cat in q1 and dog in q1 + SingleColumnValueFilter filter11 = + new SingleColumnValueFilter("col1".getBytes(), "ql".getBytes(), CompareOp.EQUAL, + "cat".getBytes()); + filter11.setFilterIfMissing(true); + SingleColumnValueFilter filter12 = + new SingleColumnValueFilter("col3".getBytes(), "ql".getBytes(), CompareOp.EQUAL, + "dog1".getBytes()); + filter12.setFilterIfMissing(true); + filterList1.addFilter(filter11); + filterList1.addFilter(filter12); + s1.setFilter(filterList1); + i = 0; + ResultScanner scanner1 = table.getScanner(s1); + for (Result result : scanner1) { + i++; + } + Assert.assertEquals( + "Should match for 0 rows in multiple index with diff column family successfully ", 0, i); + Assert.assertTrue("Seek points should be added ", IndexRegionObserver.getSeekpointAdded()); + Assert.assertTrue("Indexed table should be used ", IndexRegionObserver.getIndexedFlowUsed()); + + } + + @Test(timeout = 180000) + public void testScanWith4IdxAnd2ColumnsInFiltersShouldBeSuccessful() throws Exception { + HTable table = put4ColumnIndex(); + int i = 0; + Scan s = new Scan(); + FilterList filterList = new FilterList(); + // check for combination of cat in q1 and dog in q1 + Filter filter1 = + new SingleColumnValueFilter("col1".getBytes(), "ql".getBytes(), CompareOp.EQUAL, + "cat".getBytes()); + Filter filter2 = + new SingleColumnValueFilter("col2".getBytes(), "ql".getBytes(), CompareOp.EQUAL, + "dog".getBytes()); + filterList.addFilter(filter1); + filterList.addFilter(filter2); + s.setFilter(filterList); + + ResultScanner scanner = table.getScanner(s); + for (Result result : scanner) { + i++; + } + Assert.assertEquals( + "Should match for 5 rows in multiple index with diff column family successfully ", 5, i); + Assert.assertTrue("Seek points should be added ", IndexRegionObserver.getSeekpointAdded()); + Assert.assertTrue("Indexed table should be used ", IndexRegionObserver.getIndexedFlowUsed()); + } + + @Test(timeout = 180000) + public void testScanShouldBeSuccessfulEvenIfUserRegionAndIndexRegionAreNotCollocated() + throws Exception { + + // starting 3 RS + UTIL.getMiniHBaseCluster().startRegionServer(); + UTIL.getMiniHBaseCluster().startRegionServer(); + Configuration conf = UTIL.getConfiguration(); + final String userTableName = "testCollocatedScansOnIndexedColumn"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + IndexSpecification iSpec = new IndexSpecification("ScanIndex"); + iSpec.addIndexColumn(hcd, "ql", ValueType.String, 10); + ihtd.addFamily(hcd); + TableIndices indices = new TableIndices(); + indices.addIndex(iSpec); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + + byte[][] split = + new byte[][] { "row1".getBytes(), "row21".getBytes(), "row41".getBytes(), + "row61".getBytes(), "row81".getBytes(), "row101".getBytes(), "row121".getBytes(), + "row141".getBytes(), }; + + // create table with splits this will create 9 regions + admin.createTable(ihtd, split); + doPuts(conf, userTableName); + + HTable table = new HTable(conf, userTableName); + // Now collocation between the RS's is done so put the rows + + UTIL.getMiniHBaseCluster().getMaster().balanceSwitch(false); + // Now move the 3 rows that has the string "cat" to different RS + + collocateRowToDifferentRS(admin, table, "row1001"); + collocateRowToDifferentRS(admin, table, "row11004"); + collocateRowToDifferentRS(admin, table, "row11007"); + + // Scan should be successful + int i = countNumberOfRowsWithFilter(userTableName, "cat", true, false, 0); + Assert.assertEquals("Should match for exactly 6000 rows ", 6000, i); + + UTIL.getMiniHBaseCluster().abortRegionServer(1); + UTIL.getMiniHBaseCluster().abortRegionServer(2); + } + + private void putMulIndex(String userTableName) throws IOException, KeeperException, + InterruptedException { + Configuration conf = UTIL.getConfiguration(); + + + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + + HColumnDescriptor hcd1 = new HColumnDescriptor("col1"); + HColumnDescriptor hcd2 = new HColumnDescriptor("col2"); + ihtd.addFamily(hcd1); + ihtd.addFamily(hcd2); + IndexSpecification idx1 = new IndexSpecification("ScanMulIndex"); + idx1.addIndexColumn(hcd1, "ql", ValueType.String, 10); + idx1.addIndexColumn(hcd2, "ql", ValueType.String, 10); + TableIndices indices = new TableIndices(); + indices.addIndex(idx1); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + + // test put with the multiple indexed column in diffrent column families + Put p1 = new Put("row1".getBytes()); + p1.add("col1".getBytes(), "ql".getBytes(), "cat".getBytes()); + p1.add("col2".getBytes(), "ql".getBytes(), "dog".getBytes()); + table.put(p1); + + Put p2 = new Put("row2".getBytes()); + p2.add("col1".getBytes(), "ql".getBytes(), "dog".getBytes()); + p2.add("col2".getBytes(), "ql".getBytes(), "cat".getBytes()); + table.put(p2); + + Put p3 = new Put("row3".getBytes()); + p3.add("col1".getBytes(), "ql".getBytes(), "cat".getBytes()); + p3.add("col2".getBytes(), "ql".getBytes(), "dog".getBytes()); + table.put(p3); + + Put p4 = new Put("row4".getBytes()); + p4.add("col1".getBytes(), "ql".getBytes(), "dog".getBytes()); + p4.add("col2".getBytes(), "ql".getBytes(), "dog".getBytes()); + table.put(p4); + + Put p5 = new Put("row5".getBytes()); + p5.add("col1".getBytes(), "ql".getBytes(), "cat".getBytes()); + p5.add("col2".getBytes(), "ql".getBytes(), "dog".getBytes()); + table.put(p5); + + Put p6 = new Put("row6".getBytes()); + p6.add("col1".getBytes(), "ql".getBytes(), "cat".getBytes()); + p6.add("col2".getBytes(), "ql".getBytes(), "cat".getBytes()); + table.put(p6); + + Put p7 = new Put("row7".getBytes()); + p7.add("col1".getBytes(), "ql".getBytes(), "cat".getBytes()); + p7.add("col2".getBytes(), "ql".getBytes(), "dog".getBytes()); + table.put(p7); + + Put p8 = new Put("row8".getBytes()); + p8.add("col1".getBytes(), "ql".getBytes(), "cat".getBytes()); + p8.add("col2".getBytes(), "ql".getBytes(), "dog".getBytes()); + table.put(p8); + + } + + private void collocateRowToDifferentRS(HBaseAdmin admin, HTable table, String rowKey) + throws IOException, UnknownRegionException, MasterNotRunningException, + ZooKeeperConnectionException { + + HRegionInfo regionInfo = table.getRegionLocation(rowKey).getRegionInfo(); + int originServerNum = UTIL.getMiniHBaseCluster().getServerWith(regionInfo.getRegionName()); + int targetServerNum = 3 - 1 - originServerNum; + HRegionServer targetServer = UTIL.getMiniHBaseCluster().getRegionServer(targetServerNum); + admin.move(regionInfo.getEncodedNameAsBytes(), + Bytes.toBytes(targetServer.getServerName().getServerName())); + Threads.sleep(10000); + } + + private void doPuts(Configuration conf, String userTableName) throws IOException { + + HTable table = new HTable(conf, userTableName); + List puts = new ArrayList(); + + for (int i = 1; i <= 2000; i++) { + Put p1 = new Put(("row" + i).getBytes()); + p1.add("col".getBytes(), "ql".getBytes(), "cat".getBytes()); + puts.add(p1); + } + table.put(puts); + puts = new ArrayList(); + + for (int i = 2001; i <= 4000; i++) { + Put p1 = new Put(("row" + i).getBytes()); + p1.add("col".getBytes(), "ql".getBytes(), "dog".getBytes()); + puts.add(p1); + } + table.put(puts); + + puts = new ArrayList(); + for (int i = 4001; i <= 6000; i++) { + Put p1 = new Put(("row" + i).getBytes()); + p1.add("col".getBytes(), "ql".getBytes(), "pup".getBytes()); + puts.add(p1); + } + table.put(puts); + + puts = new ArrayList(); + for (int i = 6001; i <= 8000; i++) { + Put p1 = new Put(("row" + i).getBytes()); + p1.add("col".getBytes(), "ql".getBytes(), "cats".getBytes()); + puts.add(p1); + } + table.put(puts); + + puts = new ArrayList(); + for (int i = 8001; i <= 10000; i++) { + Put p1 = new Put(("row" + i).getBytes()); + p1.add("col".getBytes(), "ql".getBytes(), "dogs".getBytes()); + puts.add(p1); + } + table.put(puts); + + puts = new ArrayList(); + for (int i = 10001; i <= 12000; i++) { + Put p1 = new Put(("row" + i).getBytes()); + p1.add("col".getBytes(), "ql".getBytes(), "cat".getBytes()); + puts.add(p1); + } + table.put(puts); + + puts = new ArrayList(); + for (int i = 12001; i <= 14000; i++) { + Put p1 = new Put(("row" + i).getBytes()); + p1.add("col".getBytes(), "ql".getBytes(), "pup".getBytes()); + puts.add(p1); + } + table.put(puts); + + puts = new ArrayList(); + for (int i = 14001; i <= 16000; i++) { + Put p1 = new Put(("row" + i).getBytes()); + p1.add("col".getBytes(), "ql".getBytes(), "cat".getBytes()); + puts.add(p1); + } + table.put(puts); + + puts = new ArrayList(); + for (int i = 16001; i <= 18000; i++) { + Put p1 = new Put(("row" + i).getBytes()); + p1.add("col".getBytes(), "ql".getBytes(), "dog".getBytes()); + puts.add(p1); + } + table.put(puts); + puts = new ArrayList(); + + } + + class ParallelScanThread extends Thread { + + String filterString; + String userTableName; + int count; + int cacheNumber; + boolean iscached; + + ParallelScanThread(String userTableName, String filterString, boolean iscached, int cacheNumber) + throws IOException { + this.filterString = filterString; + this.userTableName = userTableName; + this.iscached = iscached; + this.cacheNumber = cacheNumber; + } + + public void run() { + try { + count = + countNumberOfRowsWithFilter(userTableName, filterString, true, iscached, cacheNumber); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + + class ParallelPutsValueThread extends Thread { + + int start; + int end; + String userTableName; + Configuration conf; + HTable table; + String value; + List puts; + + ParallelPutsValueThread(int start, Configuration conf, String userTableName, int end, + String value) throws IOException { + this.start = start; + this.end = end; + this.conf = conf; + this.userTableName = userTableName; + this.table = new HTable(this.conf, this.userTableName); + this.value = value; + puts = new ArrayList(); + } + + public void run() { + + for (int i = this.start; i <= this.end; i++) { + Put p1 = new Put(("row" + i).getBytes()); + p1.add("col".getBytes(), "ql".getBytes(), value.getBytes()); + puts.add(p1); + } + try { + this.table.put(puts); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + + private void doBulkParallelPuts(boolean toIndex, final String userTableName, boolean toFlush) + throws IOException, KeeperException, InterruptedException { + final Configuration conf = UTIL.getConfiguration(); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + HTableDescriptor htd = null; + if (toIndex) { + htd = + TestUtils.createIndexedHTableDescriptor(userTableName, "col", "ScanIndexf", "col", "ql"); + } else { + htd = new HTableDescriptor(TableName.valueOf(userTableName)); + htd.addFamily(hcd); + } + admin.createTable(htd); + + ParallelPutsValueThread p1 = new ParallelPutsValueThread(1, conf, userTableName, 10000, "dogs"); + p1.start(); + + Put a1 = new Put(("row" + 10001).getBytes()); + a1.add("col".getBytes(), "ql".getBytes(), "cat".getBytes()); + HTable table2 = new HTable(conf, userTableName); + table2.put(a1); + ParallelPutsValueThread p2 = + new ParallelPutsValueThread(10002, conf, userTableName, 30000, "dogs"); + p2.start(); + Put a2 = new Put(("row" + 30001).getBytes()); + a2.add("col".getBytes(), "ql".getBytes(), "cat".getBytes()); + HTable table4 = new HTable(conf, userTableName); + table4.put(a2); + + ParallelPutsValueThread p3 = + new ParallelPutsValueThread(30002, conf, userTableName, 60000, "dogs"); + p3.start(); + + Put a3 = new Put(("row" + 60001).getBytes()); + a3.add("col".getBytes(), "ql".getBytes(), "cat".getBytes()); + HTable table6 = new HTable(conf, userTableName); + table6.put(a3); + + ParallelPutsValueThread p4 = + new ParallelPutsValueThread(60002, conf, userTableName, 100000, "dogs"); + p4.start(); + + p1.join(); + p2.join(); + p3.join(); + p4.join(); + + if (toFlush) { + if (toIndex) { + admin.flush(userTableName + Constants.INDEX_TABLE_SUFFIX); + admin.flush(userTableName); + } else { + admin.flush(userTableName); + } + } + } + + private long doPerformanceTest(boolean toIndex, final String userTableName, boolean toFlush) + throws IOException, KeeperException, InterruptedException { + doBulkParallelPuts(toIndex, userTableName, toFlush); + long before = System.currentTimeMillis(); + int i = countNumberOfRowsWithFilter(userTableName, "cat", toIndex, false, 0); + long after = System.currentTimeMillis(); + return (after - before); + + } + + private int countNumberOfRowsWithFilter(String tableName, String filterVal, boolean isIndexed, + boolean isCached, int cacheNumber) throws IOException { + Configuration conf = UTIL.getConfiguration(); + HTable table = new HTable(conf, tableName); + Scan s = new Scan(); + Filter filter = null; + if (isIndexed) { + filter = + new SingleColumnValueFilter("col".getBytes(), "ql".getBytes(), CompareOp.EQUAL, + filterVal.getBytes()); + } else { + filter = + new SingleColumnValueFilter("col".getBytes(), "ql".getBytes(), CompareOp.EQUAL, + "cat".getBytes()); + } + s.setFilter(filter); + if (isCached) { + s.setCaching(cacheNumber); + } + int i = 0; + ResultScanner scanner = table.getScanner(s); + for (Result result : scanner) { + i++; + } + return i; + } + + private HTable put4ColumnIndex() throws IOException, ZooKeeperConnectionException, + KeeperException, InterruptedException { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testcan4hMultIndexed2DiffFilters"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd1 = new HColumnDescriptor("col1"); + HColumnDescriptor hcd2 = new HColumnDescriptor("col2"); + HColumnDescriptor hcd3 = new HColumnDescriptor("col3"); + HColumnDescriptor hcd4 = new HColumnDescriptor("col4"); + ihtd.addFamily(hcd1); + ihtd.addFamily(hcd2); + ihtd.addFamily(hcd3); + ihtd.addFamily(hcd4); + IndexSpecification idx1 = new IndexSpecification("ScanMulIndex"); + idx1.addIndexColumn(hcd1, "ql", ValueType.String, 10); + idx1.addIndexColumn(hcd2, "ql", ValueType.String, 10); + idx1.addIndexColumn(hcd3, "ql", ValueType.String, 10); + idx1.addIndexColumn(hcd4, "ql", ValueType.String, 10); + TableIndices indices = new TableIndices(); + indices.addIndex(idx1); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + + // test put with the multiple indexed column in diffrent column families + Put p1 = new Put("row1".getBytes()); + p1.add("col1".getBytes(), "ql".getBytes(), "cat".getBytes()); + p1.add("col2".getBytes(), "ql".getBytes(), "dog".getBytes()); + p1.add("col3".getBytes(), "ql".getBytes(), "dog".getBytes()); + p1.add("col4".getBytes(), "ql".getBytes(), "dog".getBytes()); + table.put(p1); + + Put p2 = new Put("row2".getBytes()); + p2.add("col1".getBytes(), "ql".getBytes(), "dog".getBytes()); + p2.add("col2".getBytes(), "ql".getBytes(), "cat".getBytes()); + p2.add("col3".getBytes(), "ql".getBytes(), "dog".getBytes()); + p2.add("col4".getBytes(), "ql".getBytes(), "dog".getBytes()); + table.put(p2); + + Put p3 = new Put("row3".getBytes()); + p3.add("col1".getBytes(), "ql".getBytes(), "cat".getBytes()); + p3.add("col2".getBytes(), "ql".getBytes(), "dog".getBytes()); + p3.add("col3".getBytes(), "ql".getBytes(), "dog".getBytes()); + p3.add("col4".getBytes(), "ql".getBytes(), "dog".getBytes()); + table.put(p3); + + Put p4 = new Put("row4".getBytes()); + p4.add("col1".getBytes(), "ql".getBytes(), "dog".getBytes()); + p4.add("col2".getBytes(), "ql".getBytes(), "dog".getBytes()); + p4.add("col3".getBytes(), "ql".getBytes(), "dog".getBytes()); + p4.add("col4".getBytes(), "ql".getBytes(), "dog".getBytes()); + table.put(p4); + + Put p5 = new Put("row5".getBytes()); + p5.add("col1".getBytes(), "ql".getBytes(), "cat".getBytes()); + p5.add("col2".getBytes(), "ql".getBytes(), "dog".getBytes()); + p5.add("col3".getBytes(), "ql".getBytes(), "dog".getBytes()); + p5.add("col4".getBytes(), "ql".getBytes(), "dog".getBytes()); + table.put(p5); + + Put p6 = new Put("row8".getBytes()); + p6.add("col1".getBytes(), "ql".getBytes(), "cat".getBytes()); + p6.add("col2".getBytes(), "ql".getBytes(), "cat".getBytes()); + p6.add("col3".getBytes(), "ql".getBytes(), "dog".getBytes()); + p6.add("col4".getBytes(), "ql".getBytes(), "dog".getBytes()); + table.put(p6); + + Put p8 = new Put("row6".getBytes()); + p8.add("col1".getBytes(), "ql".getBytes(), "cat".getBytes()); + p8.add("col2".getBytes(), "ql".getBytes(), "dog".getBytes()); + p8.add("col3".getBytes(), "ql".getBytes(), "dog".getBytes()); + p8.add("col4".getBytes(), "ql".getBytes(), "dog".getBytes()); + table.put(p8); + + Put p7 = new Put("row7".getBytes()); + p7.add("col1".getBytes(), "ql".getBytes(), "cat".getBytes()); + p7.add("col2".getBytes(), "ql".getBytes(), "dog".getBytes()); + p7.add("col3".getBytes(), "ql".getBytes(), "dog".getBytes()); + p7.add("col4".getBytes(), "ql".getBytes(), "dog".getBytes()); + table.put(p7); + return table; + } + + private byte[] generateStartKey(List regionsOfTable) { + byte[] startKey = regionsOfTable.get(0).getStartKey(); + + byte[] startRow = + new byte[startKey.length + Constants.DEF_MAX_INDEX_NAME_LENGTH + 10 + + "row999".getBytes().length]; + + System.arraycopy(startKey, 0, startRow, 0, startKey.length); + System.arraycopy("ScanIndex".getBytes(), 0, startRow, startKey.length, "ScanIndex".length()); + + byte[] arr = new byte[18 - "ScanIndex".length()]; + byte e[] = new byte[10]; + + System.arraycopy(arr, 0, startRow, startKey.length + "ScanIndex".length(), arr.length); + System.arraycopy(e, 0, startRow, startKey.length + Constants.DEF_MAX_INDEX_NAME_LENGTH, 10); + + System.arraycopy("idxCat".getBytes(), 0, startRow, startKey.length + + Constants.DEF_MAX_INDEX_NAME_LENGTH, "idxCat".getBytes().length); + + System.arraycopy("row99".getBytes(), 0, startRow, startKey.length + + Constants.DEF_MAX_INDEX_NAME_LENGTH + 10, "row99".getBytes().length); + + System.out.println("constructed rowkey for indexed table " + Bytes.toString(startRow)); + return startRow; + } + + private int singleIndexPutAndCache(Configuration conf, String userTableName) throws IOException { + HTable table = new HTable(conf, userTableName); + Put p1 = new Put("row1".getBytes()); + p1.add("col".getBytes(), "ql".getBytes(), "cat".getBytes()); + table.put(p1); + + Put p4 = new Put("row3".getBytes()); + p4.add("col".getBytes(), "ql".getBytes(), "cat".getBytes()); + table.put(p4); + + Put p5 = new Put("row2".getBytes()); + p5.add("col".getBytes(), "ql".getBytes(), "dog".getBytes()); + table.put(p5); + + Put p2 = new Put("row4".getBytes()); + p2.add("col".getBytes(), "ql".getBytes(), "cat".getBytes()); + table.put(p2); + + Put p3 = new Put("row5".getBytes()); + p3.add("col".getBytes(), "ql".getBytes(), "dogs".getBytes()); + table.put(p3); + + Put p6 = new Put("row6".getBytes()); + p6.add("col".getBytes(), "ql".getBytes(), "dog".getBytes()); + table.put(p6); + + Put p7 = new Put("row7".getBytes()); + p7.add("col".getBytes(), "ql".getBytes(), "cat".getBytes()); + table.put(p7); + + Put p8 = new Put("row8".getBytes()); + p8.add("col".getBytes(), "ql".getBytes(), "cat".getBytes()); + table.put(p8); + + int i = 0; + Scan s = new Scan(); + Filter filter = null; + filter = + new SingleColumnValueFilter("col".getBytes(), "ql".getBytes(), CompareOp.EQUAL, + "cat".getBytes()); + + s.setFilter(filter); + s.setCaching(3); + ResultScanner scanner = table.getScanner(s); + for (Result result : scanner) { + i++; + } + return i; + } + + private void validateCountOfMainTableIndIndexedTable(Configuration conf, String userTableName, + HTable table) throws IOException { + Scan s = new Scan(); + int j = 0; + ResultScanner scanner = table.getScanner(s); + for (Result result : scanner) { + j++; + } + + HTable table1 = new HTable(conf, userTableName + Constants.INDEX_TABLE_SUFFIX); + Scan s1 = new Scan(); + int k = 0; + ResultScanner scanner1 = table1.getScanner(s1); + for (Result result : scanner1) { + k++; + } + + Assert.assertEquals("COunt of rows in main tbale and indexed table shoudl be same ", j, k); + } + + public static class MockedRegionObserver extends IndexRegionObserver { + @Override + public RegionScanner postScannerOpen(ObserverContext e, + Scan scan, RegionScanner s) { + if (e.getEnvironment().getRegion().getTableDesc().getTableName().getNameAsString() + .equals("testScanShouldBeSuccessfulEvenIfExceptionIsThrownFromPostScannerOpen")){ + throw new RuntimeException("Exception thrwn from postScannerOpen "); + } + return super.postScannerOpen(e, scan, s); + } + } +} Index: hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestMultipleIndicesInScan.java =================================================================== --- hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestMultipleIndicesInScan.java (revision 0) +++ hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestMultipleIndicesInScan.java (working copy) @@ -0,0 +1,3416 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.apache.hadoop.hbase.filter.BinaryComparator; +import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; +import org.apache.hadoop.hbase.filter.Filter; +import org.apache.hadoop.hbase.filter.FilterList; +import org.apache.hadoop.hbase.filter.FilterList.Operator; +import org.apache.hadoop.hbase.filter.RowFilter; +import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; +import org.apache.hadoop.hbase.index.ColumnQualifier.ValueType; +import org.apache.hadoop.hbase.index.Constants; +import org.apache.hadoop.hbase.index.IndexSpecification; +import org.apache.hadoop.hbase.index.SecIndexLoadBalancer; +import org.apache.hadoop.hbase.index.TableIndices; +import org.apache.hadoop.hbase.index.client.IndexAdmin; +import org.apache.hadoop.hbase.index.coprocessor.master.IndexMasterObserver; +import org.apache.hadoop.hbase.index.coprocessor.wal.IndexWALObserver; +import org.apache.hadoop.hbase.index.util.ByteArrayBuilder; +import org.apache.hadoop.hbase.index.util.DoubleComparator; +import org.apache.hadoop.hbase.index.util.FloatComparator; +import org.apache.hadoop.hbase.index.util.IndexUtils; +import org.apache.hadoop.hbase.index.util.IntComparator; +import org.apache.hadoop.hbase.master.LoadBalancer; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.zookeeper.KeeperException; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(LargeTests.class) +public class TestMultipleIndicesInScan { + + private static HBaseTestingUtility UTIL = new HBaseTestingUtility(); + + private static HBaseAdmin admin; + + @BeforeClass + public static void setupBeforeClass() throws Exception { + Configuration conf = UTIL.getConfiguration(); + conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, IndexMasterObserver.class.getName()); + conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, IndexRegionObserver.class.getName()); + conf.set(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY, IndexWALObserver.class.getName()); + UTIL.startMiniCluster(1); + conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, SecIndexLoadBalancer.class, + LoadBalancer.class); + admin = new IndexAdmin(conf); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + if (admin != null) admin.close(); + UTIL.shutdownMiniCluster(); + } + + @After + public void tearDown() throws Exception { + IndexRegionObserver.setIsTestingEnabled(false); + } + + @Before + public void setUp() throws Exception { + IndexRegionObserver.setIndexedFlowUsed(false); + IndexRegionObserver.setSeekpointAdded(false); + IndexRegionObserver.setSeekPoints(null); + IndexRegionObserver.setIsTestingEnabled(true); + IndexRegionObserver.addSeekPoints(null); + } + + @Test(timeout = 180000) + public void testAndOrCombinationWithMultipleIndices() throws IOException, KeeperException, + InterruptedException { + + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testSimpleScenarioForMultipleIndices"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, + new String[] { "c3", "c4", "c5", "c6" }, "idx1"); + TableIndices indices = new TableIndices(); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c1" }, "idx2"); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2" }, "idx3"); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2", "c1" }, "idx4"); + indices.addIndex(indexSpecification); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + HTable table = new HTable(conf, "testSimpleScenarioForMultipleIndices"); + + putforIDX1(Bytes.toBytes("row0"), table); + putforIDX1(Bytes.toBytes("row1"), table); + putforIDX2(Bytes.toBytes("row2"), table); + putforIDX3(Bytes.toBytes("row3"), table); + + putforIDX1(Bytes.toBytes("row4"), table); + putforIDX2(Bytes.toBytes("row4"), table); + putforIDX3(Bytes.toBytes("row4"), table); + + putforIDX1(Bytes.toBytes("row5"), table); + putforIDX1(Bytes.toBytes("row6"), table); + putforIDX2(Bytes.toBytes("row7"), table); + putforIDX3(Bytes.toBytes("row8"), table); + + putforIDX1(Bytes.toBytes("row9"), table); + putforIDX2(Bytes.toBytes("row9"), table); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + // create the filter + FilterList filter = new FilterList(Operator.MUST_PASS_ONE); + SingleColumnValueFilter iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "apple".getBytes()); + SingleColumnValueFilter iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.EQUAL, + "bat".getBytes()); + filter.addFilter(iscvf1); + filter.addFilter(iscvf2); + + FilterList filter1 = new FilterList(Operator.MUST_PASS_ALL); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + "cat".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c4".getBytes(), CompareOp.EQUAL, + "dog".getBytes()); + filter1.addFilter(iscvf1); + filter1.addFilter(iscvf2); + + FilterList filter2 = new FilterList(Operator.MUST_PASS_ALL); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c5".getBytes(), CompareOp.EQUAL, + "ele".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c6".getBytes(), CompareOp.EQUAL, + "fan".getBytes()); + filter2.addFilter(iscvf1); + filter2.addFilter(iscvf2); + + masterFilter.addFilter(filter); + masterFilter.addFilter(filter1); + masterFilter.addFilter(filter2); + + Scan scan = new Scan(); + scan.setFilter(masterFilter); + int i = 0; + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + + assertTrue("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); + assertTrue("Seekpoints should get added by index scanner", + IndexRegionObserver.getSeekpointAdded()); + assertEquals("It should get two seek points from index scanner.", 2, IndexRegionObserver + .getMultipleSeekPoints().size()); + assertTrue("Overall result should have only 2 rows", testRes.size() == 2); + } + + @Test(timeout = 180000) + public void testReseekWhenSomeScannerAlreadyScannedGreaterValueThanSeekPoint() + throws IOException, KeeperException, InterruptedException { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testReseekWhenSomeScannerAlreadyScannedGreaterValueThanSeekPoint"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + TableIndices indices = new TableIndices(); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, + new String[] { "c3", "c4", "c5", "c6" }, "idx1"); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c1" }, "idx2"); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2" }, "idx3"); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2", "c1" }, "idx4"); + indices.addIndex(indexSpecification); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + HTable table = + new HTable(conf, "testReseekWhenSomeScannerAlreadyScannedGreaterValueThanSeekPoint"); + + putforIDX1(Bytes.toBytes("row0"), table); + putforIDX1(Bytes.toBytes("row3"), table); + putforIDX1(Bytes.toBytes("row5"), table); + putforIDX2(Bytes.toBytes("row5"), table); + putforIDX2(Bytes.toBytes("row6"), table); + putforIDX3(Bytes.toBytes("row1"), table); + putforIDX3(Bytes.toBytes("row3"), table); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + // create the filter + FilterList filter = new FilterList(Operator.MUST_PASS_ONE); + SingleColumnValueFilter iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "apple".getBytes()); + SingleColumnValueFilter iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.EQUAL, + "bat".getBytes()); + filter.addFilter(iscvf1); + filter.addFilter(iscvf2); + + FilterList filter1 = new FilterList(Operator.MUST_PASS_ALL); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + "cat".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c4".getBytes(), CompareOp.EQUAL, + "dog".getBytes()); + filter1.addFilter(iscvf1); + filter1.addFilter(iscvf2); + + FilterList filter2 = new FilterList(Operator.MUST_PASS_ALL); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c5".getBytes(), CompareOp.EQUAL, + "ele".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c6".getBytes(), CompareOp.EQUAL, + "fan".getBytes()); + filter2.addFilter(iscvf1); + filter2.addFilter(iscvf2); + + masterFilter.addFilter(filter); + masterFilter.addFilter(filter1); + masterFilter.addFilter(filter2); + + Scan scan = new Scan(); + scan.setFilter(masterFilter); + int i = 0; + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + + assertTrue("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); + assertTrue("Seekpoints should get added by index scanner", + IndexRegionObserver.getSeekpointAdded()); + assertEquals("It should get two seek points from index scanner.", 2, IndexRegionObserver + .getMultipleSeekPoints().size()); + assertTrue("Overall result should have only 2 rows", testRes.size() == 2); + } + + private void putforIDX3(byte[] row, HTable htable) throws IOException { + Put p = new Put(row); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c2"), Bytes.toBytes("bat")); + htable.put(p); + } + + private void putforIDX2(byte[] row, HTable htable) throws IOException { + Put p = new Put(row); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes("apple")); + htable.put(p); + } + + private void putforIDX1(byte[] row, HTable htable) throws IOException { + Put p = new Put(row); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c3"), Bytes.toBytes("cat")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c4"), Bytes.toBytes("dog")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c5"), Bytes.toBytes("ele")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c6"), Bytes.toBytes("fan")); + htable.put(p); + } + + private IndexSpecification createIndexSpecification(HColumnDescriptor hcd, ValueType type, + int maxValueLength, String[] qualifiers, String name) { + IndexSpecification index = new IndexSpecification(name.getBytes()); + for (String qualifier : qualifiers) { + index.addIndexColumn(hcd, qualifier, type, maxValueLength); + } + return index; + } + + @Test(timeout = 180000) + public void testWhenAppliedFilterGetsNoScanScheme() throws Exception { + + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testWhenAppliedFilterGetsNoScanScheme"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + TableIndices indices = new TableIndices(); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, + new String[] { "c3", "c4", "c5", "c6" }, "idx1"); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c1" }, "idx2"); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2" }, "idx3"); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2", "c1" }, "idx4"); + indices.addIndex(indexSpecification); + + SingleColumnValueFilter filter = + new SingleColumnValueFilter("cf1".getBytes(), "c5".getBytes(), CompareOp.EQUAL, + "ele".getBytes()); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + HTable table = new HTable(conf, "testWhenAppliedFilterGetsNoScanScheme"); + + putforIDX1(Bytes.toBytes("row1"), table); + putforIDX1(Bytes.toBytes("row2"), table); + putforIDX1(Bytes.toBytes("row3"), table); + putforIDX1(Bytes.toBytes("row4"), table); + + Scan scan = new Scan(); + scan.setFilter(filter); + int i = 0; + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + + assertFalse("Index table should not be used", IndexRegionObserver.getIndexedFlowUsed()); + assertFalse("No seek points should get added from index flow", + IndexRegionObserver.getSeekpointAdded()); + assertTrue(testRes.size() == 4); + + } + + @Test(timeout = 180000) + public void testTheOROperation() throws Exception { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testTheOROperation"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + TableIndices indices = new TableIndices(); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, + new String[] { "c3", "c4", "c5", "c6" }, "idx1"); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c1" }, "idx2"); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2" }, "idx3"); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2", "c1" }, "idx4"); + indices.addIndex(indexSpecification); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + SingleColumnValueFilter filter = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "apple".getBytes()); + + SingleColumnValueFilter filter1 = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.EQUAL, + "bat".getBytes()); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ONE); + masterFilter.addFilter(filter1); + masterFilter.addFilter(filter); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + HTable table = new HTable(conf, "testTheOROperation"); + + putforIDX2(Bytes.toBytes("row1"), table); + putforIDX3(Bytes.toBytes("row1"), table); + putforIDX2(Bytes.toBytes("row2"), table); + putforIDX2(Bytes.toBytes("row3"), table); + putforIDX3(Bytes.toBytes("row4"), table); + putforIDX3(Bytes.toBytes("row5"), table); + putforIDX2(Bytes.toBytes("row5"), table); + putforIDX2(Bytes.toBytes("row6"), table); + putforIDX3(Bytes.toBytes("row7"), table); + + Scan scan = new Scan(); + scan.setFilter(masterFilter); + int i = 0; + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + + assertTrue("Index flow should be used.", IndexRegionObserver.getIndexedFlowUsed()); + assertTrue("Index should fetch some seek points.", + IndexRegionObserver.getSeekpointAdded()); + assertEquals("Index should fetch 7 seek points", 7, IndexRegionObserver + .getMultipleSeekPoints().size()); + assertEquals("Final result should have 7 rows.", 7, testRes.size()); + } + + @Test(timeout = 180000) + public void testTheANDOpeartion() throws Exception { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testTheANDOpeartion"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + TableIndices indices = new TableIndices(); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, + new String[] { "c3", "c4", "c5", "c6" }, "idx1"); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c1" }, "idx2"); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2" }, "idx3"); + indices.addIndex(indexSpecification); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + SingleColumnValueFilter filter = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "apple".getBytes()); + + SingleColumnValueFilter filter1 = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.EQUAL, + "bat".getBytes()); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + masterFilter.addFilter(filter1); + masterFilter.addFilter(filter); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + HTable table = new HTable(conf, "testTheANDOpeartion"); + + putforIDX2(Bytes.toBytes("row1"), table); + putforIDX3(Bytes.toBytes("row1"), table); + putforIDX2(Bytes.toBytes("row2"), table); + putforIDX2(Bytes.toBytes("row3"), table); + putforIDX3(Bytes.toBytes("row4"), table); + putforIDX3(Bytes.toBytes("row5"), table); + putforIDX2(Bytes.toBytes("row5"), table); + putforIDX2(Bytes.toBytes("row6"), table); + putforIDX3(Bytes.toBytes("row7"), table); + + Scan scan = new Scan(); + scan.setFilter(masterFilter); + int i = 0; + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + + assertTrue("Index flow should be used.", IndexRegionObserver.getIndexedFlowUsed()); + assertTrue("Index should fetch some seek points.", + IndexRegionObserver.getSeekpointAdded()); + assertEquals("Index should fetch 2 seek points", 2, IndexRegionObserver + .getMultipleSeekPoints().size()); + assertEquals("Final result should have 2 rows.", 2, testRes.size()); + } + + @Test(timeout = 180000) + public void testAndOperationWithProperStartAndStopRow() throws Exception { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testAndOperationWithProperStartAndStopRow"; + HTableDescriptor ihtd = new HTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + TableIndices indices = new TableIndices(); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c1" }, "idx1"); + indices.addIndex(indexSpecification); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + rangePutForIdx2(table); + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + Bytes.toBytes("5")); + masterFilter.addFilter(scvf); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + result = scanner.next(1); + } + assertTrue(testRes.size() == 1); + + } + + @Test(timeout = 180000) + public void testAndOperationWithSimilarValuePattern() throws Exception { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testAndOperationWithSimilarValuePattern"; + HTableDescriptor ihtd = new HTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + TableIndices indices = new TableIndices(); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c1" }, "idx1"); + indices.addIndex(indexSpecification); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + Put p = new Put(Bytes.toBytes("row0")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes("aaa")); + table.put(p); + p = new Put(Bytes.toBytes("row9")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes("aaa1")); + table.put(p); + p = new Put(Bytes.toBytes("row1")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes("aaa3")); + table.put(p); + p = new Put(Bytes.toBytes("row3")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes("aaa4")); + table.put(p); + p = new Put(Bytes.toBytes("row7")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes("aaa5")); + table.put(p); + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + Bytes.toBytes("aaa")); + masterFilter.addFilter(scvf); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + result = scanner.next(1); + } + assertTrue(testRes.size() == 1); + assertTrue("Index should fetch some seek points.", + IndexRegionObserver.getSeekpointAdded()); + assertEquals("Index should fetch 2 seek points", 1, IndexRegionObserver + .getMultipleSeekPoints().size()); + } + + @Test(timeout = 180000) + public void testScanWithMutlipleIndicesOnTheSameColAndSimilarPattern() throws Exception { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testScanWithMutlipleIndicesOnTheSameColAndSimilarPattern"; + HTableDescriptor ihtd = new HTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + TableIndices indices = new TableIndices(); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, + new String[] { "c3", "c4", "c5", "c6" }, "idx1"); + indices.addIndex(indexSpecification); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + byte[][] val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("ele"), + Bytes.toBytes("goat") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row1"), val, table); + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("ele"), + Bytes.toBytes("goat1") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row2"), val, table); + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("elef"), + Bytes.toBytes("goat1") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row3"), val, table); + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog1"), Bytes.toBytes("elef"), + Bytes.toBytes("goat1") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row4"), val, table); + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("elef"), + Bytes.toBytes("goat1"), Bytes.toBytes("ant") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row5"), val, table); + + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("elefe"), + Bytes.toBytes("goat1") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row6"), val, table); + + table.flushCommits(); + admin.flush(userTableName); + admin.flush(userTableName + "_idx"); + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + Bytes.toBytes("cat")); + SingleColumnValueFilter scvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c4".getBytes(), CompareOp.EQUAL, + Bytes.toBytes("dog")); + SingleColumnValueFilter scvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c5".getBytes(), CompareOp.EQUAL, + Bytes.toBytes("elef")); + masterFilter.addFilter(scvf); + masterFilter.addFilter(scvf1); + masterFilter.addFilter(scvf2); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + result = scanner.next(1); + } + assertTrue(testRes.size() == 2); + } + + @Test(timeout = 180000) + public void testScanWithMutlipleIndicesWithGreaterthanEqualCondOnTheSameColAndSimilarPattern() + throws Exception { + Configuration conf = UTIL.getConfiguration(); + String userTableName = + "testScanWithMutlipleIndicesWithGreaterthanEqualCondOnTheSameColAndSimilarPattern"; + HTableDescriptor ihtd = new HTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + TableIndices indices = new TableIndices(); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, + new String[] { "c3", "c4", "c5", "c6" }, "idx1"); + indices.addIndex(indexSpecification); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + byte[][] val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("ele"), + Bytes.toBytes("goat") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row1"), val, table); + + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("ele"), + Bytes.toBytes("goat1") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row2"), val, table); + + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("elef"), + Bytes.toBytes("goat1") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row3"), val, table); + + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog1"), Bytes.toBytes("elef"), + Bytes.toBytes("goat1") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row4"), val, table); + + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("elef"), + Bytes.toBytes("goat1"), Bytes.toBytes("ant") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row5"), val, table); + + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("elefe"), + Bytes.toBytes("goat1") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row6"), val, table); + + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("elef"), + Bytes.toBytes("goat") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row7"), val, table); + + table.flushCommits(); + admin.flush(userTableName); + admin.flush(userTableName + "_idx"); + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + Bytes.toBytes("cat")); + SingleColumnValueFilter scvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c4".getBytes(), CompareOp.EQUAL, + Bytes.toBytes("dog")); + SingleColumnValueFilter scvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c5".getBytes(), CompareOp.EQUAL, + Bytes.toBytes("elef")); + SingleColumnValueFilter scvf3 = + new SingleColumnValueFilter("cf1".getBytes(), "c6".getBytes(), CompareOp.GREATER_OR_EQUAL, + Bytes.toBytes("goat")); + masterFilter.addFilter(scvf); + masterFilter.addFilter(scvf1); + masterFilter.addFilter(scvf2); + masterFilter.addFilter(scvf3); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + result = scanner.next(1); + } + assertEquals(testRes.size(), 3); + } + + @Test(timeout = 180000) + public void testScanWithMutlipleIndicesWithGreaterCondOnTheSameColAndSimilarPattern() + throws Exception { + Configuration conf = UTIL.getConfiguration(); + String userTableName = + "testScanWithMutlipleIndicesWithGreaterCondOnTheSameColAndSimilarPattern"; + HTableDescriptor ihtd = new HTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + TableIndices indices = new TableIndices(); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, + new String[] { "c3", "c4", "c5", "c6" }, "idx1"); + indices.addIndex(indexSpecification); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + byte[][] val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("ele"), + Bytes.toBytes("goat") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row1"), val, table); + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("ele"), + Bytes.toBytes("goat1") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row2"), val, table); + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("elef"), + Bytes.toBytes("goat1") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row3"), val, table); + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog1"), Bytes.toBytes("elef"), + Bytes.toBytes("goat1") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row4"), val, table); + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("elef"), + Bytes.toBytes("goat1"), Bytes.toBytes("ant") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row5"), val, table); + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("elefe"), + Bytes.toBytes("goat1") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row6"), val, table); + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("elef"), + Bytes.toBytes("goat") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row7"), val, table); + + table.flushCommits(); + admin.flush(userTableName); + admin.flush(userTableName + "_idx"); + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + Bytes.toBytes("cat")); + SingleColumnValueFilter scvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c4".getBytes(), CompareOp.EQUAL, + Bytes.toBytes("dog")); + SingleColumnValueFilter scvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c5".getBytes(), CompareOp.EQUAL, + Bytes.toBytes("elef")); + SingleColumnValueFilter scvf3 = + new SingleColumnValueFilter("cf1".getBytes(), "c6".getBytes(), CompareOp.GREATER, + Bytes.toBytes("goat")); + + masterFilter.addFilter(scvf); + masterFilter.addFilter(scvf1); + masterFilter.addFilter(scvf2); + masterFilter.addFilter(scvf3); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + result = scanner.next(1); + } + assertEquals(testRes.size(), 2); + } + + @Test(timeout = 180000) + public void testScanWithMutlipleIndicesWithLesserCondOnTheSameColAndSimilarPattern() + throws Exception { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testScanWithMutlipleIndicesWithLesserCondOnTheSameColAndSimilarPattern"; + HTableDescriptor ihtd = new HTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + TableIndices indices = new TableIndices(); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, + new String[] { "c3", "c4", "c5", "c6" }, "idx1"); + indices.addIndex(indexSpecification); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + byte[][] val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("ele"), + Bytes.toBytes("goat") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row1"), val, table); + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("ele"), + Bytes.toBytes("goat1") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row2"), val, table); + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("elef"), + Bytes.toBytes("goat1") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row3"), val, table); + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog1"), Bytes.toBytes("elef"), + Bytes.toBytes("goat1") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row4"), val, table); + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("elef"), + Bytes.toBytes("goat1"), Bytes.toBytes("ant") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row5"), val, table); + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("elefe"), + Bytes.toBytes("goat1") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row6"), val, table); + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("elef"), + Bytes.toBytes("goat") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row7"), val, table); + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("elef"), + Bytes.toBytes("gda") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row8"), val, table); + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("elef"), + Bytes.toBytes("goa") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row9"), val, table); + + table.flushCommits(); + admin.flush(userTableName); + admin.flush(userTableName + "_idx"); + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + Bytes.toBytes("cat")); + SingleColumnValueFilter scvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c4".getBytes(), CompareOp.EQUAL, + Bytes.toBytes("dog")); + SingleColumnValueFilter scvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c5".getBytes(), CompareOp.EQUAL, + Bytes.toBytes("elef")); + SingleColumnValueFilter scvf3 = + new SingleColumnValueFilter("cf1".getBytes(), "c6".getBytes(), CompareOp.LESS, + Bytes.toBytes("goat")); + + masterFilter.addFilter(scvf); + masterFilter.addFilter(scvf1); + masterFilter.addFilter(scvf2); + masterFilter.addFilter(scvf3); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + result = scanner.next(1); + } + assertEquals(testRes.size(), 2); + } + + @Test(timeout = 180000) + public void testScanWithMutlipleIndicesWithLesserEqualCondOnTheSameColAndSimilarPattern() + throws Exception { + Configuration conf = UTIL.getConfiguration(); + String userTableName = + "testScanWithMutlipleIndicesWithLesserEqualCondOnTheSameColAndSimilarPattern"; + HTableDescriptor ihtd = new HTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + TableIndices indices = new TableIndices(); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, + new String[] { "c3", "c4", "c5", "c6" }, "idx1"); + indices.addIndex(indexSpecification); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + byte[][] val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("ele"), + Bytes.toBytes("goat") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row1"), val, table); + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("ele"), + Bytes.toBytes("goat1") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row2"), val, table); + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("elef"), + Bytes.toBytes("goat1") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row3"), val, table); + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog1"), Bytes.toBytes("elef"), + Bytes.toBytes("goat1") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row4"), val, table); + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("elef"), + Bytes.toBytes("goat1"), Bytes.toBytes("ant") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row5"), val, table); + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("elefe"), + Bytes.toBytes("goat1") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row6"), val, table); + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("elef"), + Bytes.toBytes("goat") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row7"), val, table); + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("elef"), + Bytes.toBytes("gda") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row8"), val, table); + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("elef"), + Bytes.toBytes("goa") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row9"), val, table); + + table.flushCommits(); + admin.flush(userTableName); + admin.flush(userTableName + "_idx"); + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + Bytes.toBytes("cat")); + SingleColumnValueFilter scvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c4".getBytes(), CompareOp.EQUAL, + Bytes.toBytes("dog")); + SingleColumnValueFilter scvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c5".getBytes(), CompareOp.EQUAL, + Bytes.toBytes("elef")); + SingleColumnValueFilter scvf3 = + new SingleColumnValueFilter("cf1".getBytes(), "c6".getBytes(), CompareOp.LESS_OR_EQUAL, + Bytes.toBytes("goat")); + + masterFilter.addFilter(scvf); + masterFilter.addFilter(scvf1); + masterFilter.addFilter(scvf2); + masterFilter.addFilter(scvf3); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + result = scanner.next(1); + } + assertEquals(testRes.size(), 3); + } + + private void putsForIdx1WithDiffValues(byte[] row, byte[][] valList, HTable table) + throws IOException { + Put p = new Put(row); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c3"), (valList[0])); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c4"), valList[1]); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c5"), valList[2]); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c6"), valList[3]); + table.put(p); + } + + @Test(timeout = 180000) + public void testWhenSomePointsAreFetchedFromIndexButMainScanStillHasSomeFiltersToApply() + throws Exception { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "MainScanStillHasSomeFiltersToApply"; + HTableDescriptor ihtd = new HTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + TableIndices indices = new TableIndices(); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c1" }, "idx2"); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2" }, "idx3"); + indices.addIndex(indexSpecification); + + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "apple".getBytes()); + scvf.setFilterIfMissing(true); + SingleColumnValueFilter scvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.EQUAL, + "bat".getBytes()); + scvf1.setFilterIfMissing(true); + FilterList orFilter = new FilterList(Operator.MUST_PASS_ONE); + orFilter.addFilter(scvf); + orFilter.addFilter(scvf1); + + FilterList andFilter = new FilterList(Operator.MUST_PASS_ALL); + SingleColumnValueFilter scvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + "cat".getBytes()); + scvf2.setFilterIfMissing(true); + SingleColumnValueFilter scvf3 = + new SingleColumnValueFilter("cf1".getBytes(), "c4".getBytes(), CompareOp.EQUAL, + "dog".getBytes()); + scvf3.setFilterIfMissing(true); + SingleColumnValueFilter scvf4 = + new SingleColumnValueFilter("cf1".getBytes(), "c5".getBytes(), CompareOp.EQUAL, + "ele".getBytes()); + scvf4.setFilterIfMissing(true); + SingleColumnValueFilter scvf5 = + new SingleColumnValueFilter("cf1".getBytes(), "c6".getBytes(), CompareOp.EQUAL, + "fan".getBytes()); + scvf5.setFilterIfMissing(true); + andFilter.addFilter(scvf5); + andFilter.addFilter(scvf4); + andFilter.addFilter(scvf3); + andFilter.addFilter(scvf2); + + FilterList master = new FilterList(Operator.MUST_PASS_ALL); + master.addFilter(andFilter); + master.addFilter(orFilter); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + HTable table = new HTable(conf, "MainScanStillHasSomeFiltersToApply"); + + putforIDX1(Bytes.toBytes("row0"), table); + putforIDX1(Bytes.toBytes("row1"), table); + putforIDX2(Bytes.toBytes("row2"), table); + putforIDX3(Bytes.toBytes("row3"), table); + + putforIDX1(Bytes.toBytes("row4"), table); + putforIDX2(Bytes.toBytes("row4"), table); + putforIDX3(Bytes.toBytes("row4"), table); + + putforIDX1(Bytes.toBytes("row5"), table); + putforIDX1(Bytes.toBytes("row6"), table); + putforIDX2(Bytes.toBytes("row7"), table); + putforIDX3(Bytes.toBytes("row8"), table); + + putforIDX1(Bytes.toBytes("row9"), table); + putforIDX2(Bytes.toBytes("row9"), table); + + Scan scan = new Scan(); + scan.setFilter(master); + // scan.setCaching(10); + int i = 0; + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + + assertTrue("Index flow should be used.", IndexRegionObserver.getIndexedFlowUsed()); + assertTrue("Index should fetch some seek points.", + IndexRegionObserver.getSeekpointAdded()); + assertEquals("Index should fetch 6 seek points", 6, IndexRegionObserver + .getMultipleSeekPoints().size()); + assertEquals("Final result should have 2 rows.", 2, testRes.size()); + } + + @Test(timeout = 180000) + public void testWhenThereIsNoDataInIndexRegion() throws Exception { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testWhenThereIsNoDataInIndexRegion"; + HTableDescriptor ihtd = new HTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + TableIndices indices = new TableIndices(); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, + new String[] { "c3", "c4", "c5", "c6" }, "idx1"); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c1" }, "idx2"); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2" }, "idx3"); + indices.addIndex(indexSpecification); + + SingleColumnValueFilter filter = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "apple".getBytes()); + + SingleColumnValueFilter filter1 = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.EQUAL, + "bat".getBytes()); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + masterFilter.addFilter(filter1); + masterFilter.addFilter(filter); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + HTable table = new HTable(conf, "testWhenThereIsNoDataInIndexRegion"); + + Scan scan = new Scan(); + scan.setFilter(masterFilter); + int i = 0; + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + + assertTrue("Index flow should be used.", IndexRegionObserver.getIndexedFlowUsed()); + assertFalse("Index should fetch some seek points.", + IndexRegionObserver.getSeekpointAdded()); + } + + @Test(timeout = 180000) + public void testMultipleScansOnTheIndexRegion() throws Exception { + final Configuration conf = UTIL.getConfiguration(); + String userTableName = "testMultipleScansOnTheIndexRegion"; + HTableDescriptor ihtd = new HTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + TableIndices indices = new TableIndices(); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, + new String[] { "c3", "c4", "c5", "c6" }, "idx1"); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c1" }, "idx2"); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2" }, "idx3"); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2", "c1" }, "idx4"); + indices.addIndex(indexSpecification); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + HTable table = new HTable(conf, "testMultipleScansOnTheIndexRegion"); + + putforIDX1(Bytes.toBytes("row0"), table); + putforIDX1(Bytes.toBytes("row1"), table); + putforIDX2(Bytes.toBytes("row2"), table); + putforIDX3(Bytes.toBytes("row3"), table); + + putforIDX1(Bytes.toBytes("row4"), table); + putforIDX2(Bytes.toBytes("row4"), table); + putforIDX3(Bytes.toBytes("row4"), table); + + putforIDX1(Bytes.toBytes("row5"), table); + putforIDX1(Bytes.toBytes("row6"), table); + putforIDX2(Bytes.toBytes("row7"), table); + putforIDX3(Bytes.toBytes("row8"), table); + + putforIDX1(Bytes.toBytes("row9"), table); + putforIDX2(Bytes.toBytes("row9"), table); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + // create the filter + FilterList filter = new FilterList(Operator.MUST_PASS_ONE); + SingleColumnValueFilter iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "apple".getBytes()); + SingleColumnValueFilter iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.EQUAL, + "bat".getBytes()); + filter.addFilter(iscvf1); + filter.addFilter(iscvf2); + + FilterList filter1 = new FilterList(Operator.MUST_PASS_ALL); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + "cat".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c4".getBytes(), CompareOp.EQUAL, + "dog".getBytes()); + filter1.addFilter(iscvf1); + filter1.addFilter(iscvf2); + + FilterList filter2 = new FilterList(Operator.MUST_PASS_ALL); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c5".getBytes(), CompareOp.EQUAL, + "ele".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c6".getBytes(), CompareOp.EQUAL, + "fan".getBytes()); + filter2.addFilter(iscvf1); + filter2.addFilter(iscvf2); + + masterFilter.addFilter(filter); + masterFilter.addFilter(filter1); + masterFilter.addFilter(filter2); + + Scan scan = new Scan(); + scan.setFilter(masterFilter); + + int i = 0; + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + Thread test = new testThread(masterFilter, "testMultipleScansOnTheIndexRegion"); + test.start(); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + + test.join(); + assertTrue("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); + assertTrue("Seekpoints should get added by index scanner", + IndexRegionObserver.getSeekpointAdded()); + assertEquals("It should get two seek points from index scanner.", 4, IndexRegionObserver + .getMultipleSeekPoints().size()); + assertTrue("Overall result should have only 2 rows", testRes.size() == 2); + } + + private class testThread extends Thread { + Filter filter = null; + String tableName = null; + + public testThread(Filter filter, String tableName) { + this.filter = filter; + this.tableName = tableName; + } + + @Override + public synchronized void start() { + try { + HTable table = new HTable(UTIL.getConfiguration(), tableName); + Scan scan = new Scan(); + scan.setFilter(filter); + int i = 0; + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + assertTrue("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); + assertTrue("Seekpoints should get added by index scanner", + IndexRegionObserver.getSeekpointAdded()); + assertTrue("Overall result should have only 2 rows", testRes.size() == 2); + } catch (IOException e) { + } + } + } + + @Test(timeout = 180000) + public void testFalsePositiveCases() throws Exception { + final Configuration conf = UTIL.getConfiguration(); + String userTableName = "testFalsePositiveCases"; + HTableDescriptor ihtd = new HTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + TableIndices indices = new TableIndices(); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c1" }, "idx2"); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2" }, "idx3"); + indices.addIndex(indexSpecification); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + HTable table = new HTable(conf, "testFalsePositiveCases"); + + HTable idx_table = new HTable(conf, "testFalsePositiveCases_idx"); + + ByteArrayBuilder byteArray = new ByteArrayBuilder(33); + byteArray.put(new byte[1]); + byteArray.put(Bytes.toBytes("idx2")); + byteArray.put(new byte[14]); + byteArray.put(Bytes.toBytes("apple")); + byteArray.put(new byte[5]); + int offset = byteArray.position(); + byteArray.put(Bytes.toBytes("row1")); + + ByteArrayBuilder value = new ByteArrayBuilder(4); + value.put(Bytes.toBytes((short) byteArray.array().length)); + value.put(Bytes.toBytes((short) offset)); + Put p = new Put(byteArray.array()); + p.add(Constants.IDX_COL_FAMILY, Constants.IDX_COL_QUAL, value.array()); + idx_table.put(p); + SingleColumnValueFilter filter = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "apple".getBytes()); + + SingleColumnValueFilter filter1 = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.EQUAL, + "bat".getBytes()); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ONE); + masterFilter.addFilter(filter1); + masterFilter.addFilter(filter); + + Scan scan = new Scan(); + scan.setFilter(masterFilter); + + int i = 0; + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + + assertTrue("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); + assertTrue("Seekpoints should get added by index scanner", + IndexRegionObserver.getSeekpointAdded()); + assertEquals("It should get two seek points from index scanner.", 1, IndexRegionObserver + .getMultipleSeekPoints().size()); + assertEquals("Overall result should have only 2 rows", 0, testRes.size()); + } + + @Test(timeout = 180000) + public void testSingleLevelRangeScanForAND() throws IOException, KeeperException, + InterruptedException { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testSingleLevelRangeScanForAND"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + TableIndices indices = new TableIndices(); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, + new String[] { "c3", "c4", "c5", "c6" }, "idx1"); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c1" }, "idx2"); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2" }, "idx3"); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c3" }, "idx4"); + indices.addIndex(indexSpecification); + + // indexSpecification = createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2", + // "c1" }, "idx4"); + // ihtd.addIndex(indexSpecification); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + rangePutForIdx2(table); + rangePutForIdx3(table); + rangePutForIdx4(table); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + + FilterList filterList1 = new FilterList(Operator.MUST_PASS_ONE); + + // SingleColumnValueFilter scvfsub = new SingleColumnValueFilter("cf1".getBytes(), + // "c1".getBytes(), + // CompareOp.EQUAL, "5".getBytes()); + SingleColumnValueFilter scvf2sub = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS_OR_EQUAL, + "6".getBytes()); + SingleColumnValueFilter scvf3sub = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.GREATER_OR_EQUAL, + "2".getBytes()); + // filterList1.addFilter(scvfsub); + // filterList1.addFilter(scvf2sub); + // filterList1.addFilter(scvf3sub); + + // SingleColumnValueFilter scvf = new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), + // CompareOp.GREATER_OR_EQUAL, "5".getBytes()); + // SingleColumnValueFilter scvf2 = new SingleColumnValueFilter("cf1".getBytes(), + // "c2".getBytes(), + // CompareOp.LESS_OR_EQUAL, "5".getBytes()); + SingleColumnValueFilter scvf3 = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.GREATER_OR_EQUAL, + "5".getBytes()); + + // masterFilter.addFilter(scvf); + // masterFilter.addFilter(scvf2); + masterFilter.addFilter(scvf2sub); + masterFilter.addFilter(scvf3sub); + masterFilter.addFilter(scvf3); + // masterFilter.addFilter(filterList1); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + int i = 0; + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + System.out.println("************* result count......*********** " + testRes.size() + + " ###### " + testRes); + assertTrue("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); + assertTrue("Seekpoints should get added by index scanner", + IndexRegionObserver.getSeekpointAdded()); + assertEquals("It should get two seek points from index scanner.", 2, IndexRegionObserver + .getMultipleSeekPoints().size()); + assertTrue("Overall result should have only 2 rows", testRes.size() == 2); + } + + @Test(timeout = 180000) + public void testSingleLevelRangeScanForOR() throws IOException, KeeperException, + InterruptedException { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testSingleLevelRangeScanForOR"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + TableIndices indices = new TableIndices(); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, + new String[] { "c3", "c4", "c5", "c6" }, "idx1"); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c1" }, "idx2"); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2" }, "idx3"); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c3" }, "idx4"); + indices.addIndex(indexSpecification); + + // indexSpecification = createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2", + // "c1" }, "idx4"); + // ihtd.addIndex(indexSpecification); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + rangePutForIdx2(table); + rangePutForIdx3(table); + rangePutForIdx4(table); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ONE); + + FilterList filterList1 = new FilterList(Operator.MUST_PASS_ONE); + + // SingleColumnValueFilter scvfsub = new SingleColumnValueFilter("cf1".getBytes(), + // "c1".getBytes(), + // CompareOp.EQUAL, "5".getBytes()); + SingleColumnValueFilter scvf2sub = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS_OR_EQUAL, + "6".getBytes()); + SingleColumnValueFilter scvf3sub = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.GREATER_OR_EQUAL, + "2".getBytes()); + // filterList1.addFilter(scvfsub); + // filterList1.addFilter(scvf2sub); + // filterList1.addFilter(scvf3sub); + + // SingleColumnValueFilter scvf = new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), + // CompareOp.GREATER_OR_EQUAL, "5".getBytes()); + // SingleColumnValueFilter scvf2 = new SingleColumnValueFilter("cf1".getBytes(), + // "c2".getBytes(), + // CompareOp.LESS_OR_EQUAL, "5".getBytes()); + SingleColumnValueFilter scvf3 = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.GREATER_OR_EQUAL, + "5".getBytes()); + + // masterFilter.addFilter(scvf); + // masterFilter.addFilter(scvf2); + masterFilter.addFilter(scvf2sub); + masterFilter.addFilter(scvf3sub); + masterFilter.addFilter(scvf3); + // masterFilter.addFilter(filterList1); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + int i = 0; + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + System.out.println("************* result count......*********** " + testRes.size() + + " ###### " + testRes); + assertTrue("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); + assertTrue("Seekpoints should get added by index scanner", + IndexRegionObserver.getSeekpointAdded()); + assertEquals("It should get two seek points from index scanner.", 6, IndexRegionObserver + .getMultipleSeekPoints().size()); + assertTrue("Overall result should have only 6 rows", testRes.size() == 6); + } + + @Test(timeout = 180000) + public void testEqualAndRangeCombinationWithMultipleIndices() throws IOException, + KeeperException, InterruptedException { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testEqualAndRangeCombinationWithMultipleIndices"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + TableIndices indices = new TableIndices(); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, + new String[] { "c3", "c4", "c5", "c6" }, "idx1"); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c1" }, "idx2"); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2" }, "idx3"); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c3" }, "idx4"); + indices.addIndex(indexSpecification); + + // indexSpecification = createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2", + // "c1" }, "idx4"); + // ihtd.addIndex(indexSpecification); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + rangePutForIdx2(table); + rangePutForIdx3(table); + rangePutForIdx4(table); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + + FilterList filterList1 = new FilterList(Operator.MUST_PASS_ONE); + + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + "2".getBytes()); + SingleColumnValueFilter scvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.LESS_OR_EQUAL, + "6".getBytes()); + SingleColumnValueFilter scvf3 = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.GREATER_OR_EQUAL, + "4".getBytes()); + filterList1.addFilter(scvf); + filterList1.addFilter(scvf2); + filterList1.addFilter(scvf3); + + SingleColumnValueFilter scvfsub = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "2".getBytes()); + SingleColumnValueFilter scvf2sub = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.EQUAL, + "2".getBytes()); + SingleColumnValueFilter scvf3sub = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + "2".getBytes()); + + masterFilter.addFilter(scvfsub); + masterFilter.addFilter(scvf2sub); + masterFilter.addFilter(scvf3sub); + masterFilter.addFilter(filterList1); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + int i = 0; + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + System.out.println("************* result count......*********** " + testRes.size() + + " ###### " + testRes); + assertTrue("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); + assertTrue("Seekpoints should get added by index scanner", + IndexRegionObserver.getSeekpointAdded()); + assertEquals("It should get two seek points from index scanner.", 1, IndexRegionObserver + .getMultipleSeekPoints().size()); + assertTrue("Overall result should have only 2 rows", testRes.size() == 1); + } + + @Test(timeout = 180000) + public void testEqualAndRangeCombinationWithMultipleIndicesPart2() throws IOException, + KeeperException, InterruptedException { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testEqualAndRangeCombinationWithMultipleIndicesPart2"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + TableIndices indices = new TableIndices(); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, + new String[] { "c3", "c4", "c5", "c6" }, "idx1"); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c1" }, "idx2"); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2" }, "idx3"); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c3" }, "idx4"); + indices.addIndex(indexSpecification); + + // indexSpecification = createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2", + // "c1" }, "idx4"); + // ihtd.addIndex(indexSpecification); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + rangePutForIdx2(table); + rangePutForIdx3(table); + rangePutForIdx4(table); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ONE); + + FilterList filterList1 = new FilterList(Operator.MUST_PASS_ALL); + + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + "2".getBytes()); + SingleColumnValueFilter scvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.LESS_OR_EQUAL, + "4".getBytes()); + SingleColumnValueFilter scvf3 = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.GREATER_OR_EQUAL, + "1".getBytes()); + filterList1.addFilter(scvf); + filterList1.addFilter(scvf2); + filterList1.addFilter(scvf3); + + SingleColumnValueFilter scvfsub = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "6".getBytes()); + SingleColumnValueFilter scvf2sub = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.EQUAL, + "6".getBytes()); + SingleColumnValueFilter scvf3sub = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + "6".getBytes()); + + masterFilter.addFilter(scvfsub); + masterFilter.addFilter(scvf2sub); + masterFilter.addFilter(scvf3sub); + masterFilter.addFilter(filterList1); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + int i = 0; + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + System.out.println("************* result count......*********** " + testRes.size() + + " ###### " + testRes); + assertTrue("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); + assertTrue("Seekpoints should get added by index scanner", + IndexRegionObserver.getSeekpointAdded()); + assertEquals("It should get two seek points from index scanner.", 4, IndexRegionObserver + .getMultipleSeekPoints().size()); + assertTrue("Overall result should have only 2 rows", testRes.size() == 4); + } + + @Test(timeout = 180000) + public void testOREvaluatorWithMultipleOperatorsInEachLevel() throws IOException, + KeeperException, InterruptedException { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testOREvaluatorWithMultipleOperatorsInEachLevel"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + TableIndices indices = new TableIndices(); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, + new String[] { "c3", "c4", "c5", "c6" }, "idx1"); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c1" }, "idx2"); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2" }, "idx3"); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c3" }, "idx4"); + indices.addIndex(indexSpecification); + + // indexSpecification = createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2", + // "c1" }, "idx4"); + // ihtd.addIndex(indexSpecification); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + rangePutForIdx2(table); + rangePutForIdx3(table); + rangePutForIdx4(table); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ONE); + + FilterList filterList1 = new FilterList(Operator.MUST_PASS_ONE); + + SingleColumnValueFilter scvfsub = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "5".getBytes()); + SingleColumnValueFilter scvf2sub = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS_OR_EQUAL, + "5".getBytes()); + SingleColumnValueFilter scvf3sub = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + "2".getBytes()); + filterList1.addFilter(scvfsub); + filterList1.addFilter(scvf2sub); + filterList1.addFilter(scvf3sub); + + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + "4".getBytes()); + SingleColumnValueFilter scvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS_OR_EQUAL, + "4".getBytes()); + SingleColumnValueFilter scvf3 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "4".getBytes()); + + masterFilter.addFilter(scvf); + masterFilter.addFilter(scvf2); + masterFilter.addFilter(scvf3); + masterFilter.addFilter(filterList1); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + int i = 0; + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + System.out.println("************* result count......*********** " + testRes.size() + + " ###### " + testRes); + assertTrue("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); + assertTrue("Seekpoints should get added by index scanner", + IndexRegionObserver.getSeekpointAdded()); + assertEquals("It should get two seek points from index scanner.", 6, IndexRegionObserver + .getMultipleSeekPoints().size()); + assertTrue("Overall result should have only 6 rows", testRes.size() == 6); + } + + @Test(timeout = 180000) + public void testIfAllScannersAreRangeInAllLevels() throws IOException, KeeperException, + InterruptedException { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testIfAllScannersAreRangeInAllLevels"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + TableIndices indices = new TableIndices(); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, + new String[] { "c3", "c4", "c5", "c6" }, "idx1"); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c1" }, "idx2"); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2" }, "idx3"); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c3" }, "idx4"); + indices.addIndex(indexSpecification); + + // indexSpecification = createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2", + // "c1" }, "idx4"); + // ihtd.addIndex(indexSpecification); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + rangePutForIdx2(table); + rangePutForIdx3(table); + rangePutForIdx4(table); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ONE); + + FilterList filterList1 = new FilterList(Operator.MUST_PASS_ALL); + + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + "2".getBytes()); + SingleColumnValueFilter scvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.LESS_OR_EQUAL, + "4".getBytes()); + SingleColumnValueFilter scvf3 = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.GREATER_OR_EQUAL, + "1".getBytes()); + filterList1.addFilter(scvf); + filterList1.addFilter(scvf2); + filterList1.addFilter(scvf3); + + SingleColumnValueFilter scvfsub = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "2".getBytes()); + SingleColumnValueFilter scvf2sub = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.LESS_OR_EQUAL, + "6".getBytes()); + SingleColumnValueFilter scvf3sub = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + "2".getBytes()); + + masterFilter.addFilter(scvfsub); + masterFilter.addFilter(scvf2sub); + masterFilter.addFilter(scvf3sub); + masterFilter.addFilter(filterList1); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + int i = 0; + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + System.out.println("************* result count......*********** " + testRes.size() + + " ###### " + testRes); + assertTrue("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); + assertTrue("Seekpoints should get added by index scanner", + IndexRegionObserver.getSeekpointAdded()); + assertEquals("It should get two seek points from index scanner.", 6, IndexRegionObserver + .getMultipleSeekPoints().size()); + assertTrue("Overall result should have only 2 rows", testRes.size() == 6); + } + + @Test(timeout = 180000) + public void testANDWithORbranchesWhereEachBranchHavingAtleastOneFilterOtherThanSCVF() + throws IOException, KeeperException, InterruptedException { + Configuration conf = UTIL.getConfiguration(); + String userTableName = + "testANDWithORbranchesWhereEachBranchHavingAtleastOneFilterOtherThanSCVF"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + TableIndices indices = new TableIndices(); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, + new String[] { "c3", "c4", "c5", "c6" }, "idx1"); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c1" }, "idx2"); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2" }, "idx3"); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c3" }, "idx4"); + indices.addIndex(indexSpecification); + + // indexSpecification = createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2", + // "c1" }, "idx4"); + // ihtd.addIndex(indexSpecification); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + rangePutForIdx2(table); + rangePutForIdx3(table); + rangePutForIdx4(table); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + + FilterList filterList1 = new FilterList(Operator.MUST_PASS_ONE); + + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + "2".getBytes()); + SingleColumnValueFilter scvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS_OR_EQUAL, + "4".getBytes()); + SingleColumnValueFilter scvf3 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + "1".getBytes()); + filterList1.addFilter(scvf); + filterList1.addFilter(scvf2); + filterList1.addFilter(scvf3); + + SingleColumnValueFilter scvfsub = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "2".getBytes()); + SingleColumnValueFilter scvf2sub = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.LESS_OR_EQUAL, + "6".getBytes()); + SingleColumnValueFilter scvf3sub = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + "2".getBytes()); + RowFilter rowFilter = + new RowFilter(CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("row1"))); + FilterList filterList2 = new FilterList(Operator.MUST_PASS_ONE); + filterList2.addFilter(scvfsub); + filterList2.addFilter(scvf2sub); + filterList2.addFilter(scvf3sub); + filterList2.addFilter(rowFilter); + + SingleColumnValueFilter scvfsub1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "2".getBytes()); + SingleColumnValueFilter scvf2sub2 = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.LESS_OR_EQUAL, + "6".getBytes()); + SingleColumnValueFilter scvf3sub3 = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + "2".getBytes()); + RowFilter rowFilter2 = + new RowFilter(CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("row1"))); + FilterList subFilterList = new FilterList(Operator.MUST_PASS_ONE); + subFilterList.addFilter(scvfsub1); + subFilterList.addFilter(scvf2sub2); + subFilterList.addFilter(scvf3sub3); + subFilterList.addFilter(rowFilter2); + + filterList1.addFilter(subFilterList); + masterFilter.addFilter(filterList1); + masterFilter.addFilter(filterList2); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + int i = 0; + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + System.out.println("************* result count......*********** " + testRes.size() + + " ###### " + testRes); + assertFalse("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); + } + + @Test(timeout = 180000) + public void testORIfEachBranchHavingAtleastOneOtherFilterThanSCVF() throws IOException, + KeeperException, InterruptedException { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testORIfEachBranchHavingAtleastOneOtherFilterThanSCVF"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + TableIndices indices = new TableIndices(); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, + new String[] { "c3", "c4", "c5", "c6" }, "idx1"); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c1" }, "idx2"); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2" }, "idx3"); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c3" }, "idx4"); + indices.addIndex(indexSpecification); + + // indexSpecification = createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2", + // "c1" }, "idx4"); + // ihtd.addIndex(indexSpecification); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + rangePutForIdx2(table); + rangePutForIdx3(table); + rangePutForIdx4(table); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ONE); + + FilterList filterList1 = new FilterList(Operator.MUST_PASS_ONE); + + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + "2".getBytes()); + SingleColumnValueFilter scvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS_OR_EQUAL, + "4".getBytes()); + SingleColumnValueFilter scvf3 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + "1".getBytes()); + filterList1.addFilter(scvf); + filterList1.addFilter(scvf2); + filterList1.addFilter(scvf3); + + SingleColumnValueFilter scvfsub = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "2".getBytes()); + SingleColumnValueFilter scvf2sub = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.LESS_OR_EQUAL, + "6".getBytes()); + SingleColumnValueFilter scvf3sub = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + "2".getBytes()); + RowFilter rowFilter = + new RowFilter(CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("row1"))); + FilterList filterList2 = new FilterList(Operator.MUST_PASS_ONE); + filterList2.addFilter(scvfsub); + filterList2.addFilter(scvf2sub); + filterList2.addFilter(scvf3sub); + filterList2.addFilter(rowFilter); + + SingleColumnValueFilter scvfsub1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "2".getBytes()); + SingleColumnValueFilter scvf2sub2 = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.LESS_OR_EQUAL, + "6".getBytes()); + SingleColumnValueFilter scvf3sub3 = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + "2".getBytes()); + RowFilter rowFilter2 = + new RowFilter(CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("row1"))); + FilterList subFilterList = new FilterList(Operator.MUST_PASS_ONE); + subFilterList.addFilter(scvfsub1); + subFilterList.addFilter(scvf2sub2); + subFilterList.addFilter(scvf3sub3); + subFilterList.addFilter(rowFilter2); + + filterList1.addFilter(subFilterList); + masterFilter.addFilter(filterList1); + masterFilter.addFilter(filterList2); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + int i = 0; + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + System.out.println("************* result count......*********** " + testRes.size() + + " ###### " + testRes); + assertFalse("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); + } + + @Test(timeout = 180000) + public void testORBranchesInWhichOneBranchHavingOtherFiltersThanSCVF() throws IOException, + KeeperException, InterruptedException { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testORBranchesInWhichOneBranchHavingOtherFiltersThanSCVF"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + TableIndices indices = new TableIndices(); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, + new String[] { "c3", "c4", "c5", "c6" }, "idx1"); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c1" }, "idx2"); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2" }, "idx3"); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c3" }, "idx4"); + indices.addIndex(indexSpecification); + + // indexSpecification = createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2", + // "c1" }, "idx4"); + // ihtd.addIndex(indexSpecification); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + rangePutForIdx2(table); + rangePutForIdx3(table); + rangePutForIdx4(table); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ONE); + + FilterList filterList1 = new FilterList(Operator.MUST_PASS_ONE); + + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + "2".getBytes()); + SingleColumnValueFilter scvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS_OR_EQUAL, + "4".getBytes()); + SingleColumnValueFilter scvf3 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + "1".getBytes()); + filterList1.addFilter(scvf); + filterList1.addFilter(scvf2); + filterList1.addFilter(scvf3); + + SingleColumnValueFilter scvfsub = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "2".getBytes()); + SingleColumnValueFilter scvf2sub = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.LESS_OR_EQUAL, + "6".getBytes()); + SingleColumnValueFilter scvf3sub = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + "2".getBytes()); + RowFilter rowFilter = + new RowFilter(CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("row1"))); + FilterList filterList2 = new FilterList(Operator.MUST_PASS_ONE); + filterList2.addFilter(scvfsub); + filterList2.addFilter(scvf2sub); + filterList2.addFilter(scvf3sub); + filterList2.addFilter(rowFilter); + + masterFilter.addFilter(filterList1); + masterFilter.addFilter(filterList2); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + int i = 0; + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + System.out.println("************* result count......*********** " + testRes.size() + + " ###### " + testRes); + assertFalse("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); + } + + @Test(timeout = 180000) + public void testANDhavingORbranchWithOtherFilterThanSCVF() throws IOException, KeeperException, + InterruptedException { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testANDhavingORbranchWithOtherFilterThanSCVF"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + TableIndices indices = new TableIndices(); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, + new String[] { "c3", "c4", "c5", "c6" }, "idx1"); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c1" }, "idx2"); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2" }, "idx3"); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c3" }, "idx4"); + indices.addIndex(indexSpecification); + + // indexSpecification = createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2", + // "c1" }, "idx4"); + // ihtd.addIndex(indexSpecification); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + rangePutForIdx2(table); + rangePutForIdx3(table); + rangePutForIdx4(table); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + + FilterList filterList1 = new FilterList(Operator.MUST_PASS_ONE); + + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + "2".getBytes()); + SingleColumnValueFilter scvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS_OR_EQUAL, + "4".getBytes()); + SingleColumnValueFilter scvf3 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + "1".getBytes()); + filterList1.addFilter(scvf); + filterList1.addFilter(scvf2); + filterList1.addFilter(scvf3); + + SingleColumnValueFilter scvfsub = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "2".getBytes()); + SingleColumnValueFilter scvf2sub = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.LESS_OR_EQUAL, + "6".getBytes()); + SingleColumnValueFilter scvf3sub = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + "2".getBytes()); + RowFilter rowFilter = + new RowFilter(CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("row1"))); + FilterList filterList2 = new FilterList(Operator.MUST_PASS_ONE); + filterList2.addFilter(scvfsub); + filterList2.addFilter(scvf2sub); + filterList2.addFilter(scvf3sub); + filterList2.addFilter(rowFilter); + + masterFilter.addFilter(filterList1); + masterFilter.addFilter(filterList2); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + int i = 0; + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + System.out.println("************* result count......*********** " + testRes.size() + + " ###### " + testRes); + assertTrue("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); + assertTrue("Seekpoints should get added by index scanner", + IndexRegionObserver.getSeekpointAdded()); + assertEquals("It should get two seek points from index scanner.", 6, IndexRegionObserver + .getMultipleSeekPoints().size()); + assertTrue("Overall result should have only 2 rows", testRes.size() == 6); + } + + @Test(timeout = 180000) + public void testIfAllScannersAreRangeInAllLevelsPart2() throws IOException, KeeperException, + InterruptedException { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testIfAllScannersAreRangeInAllLevelsPart2"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + TableIndices indices = new TableIndices(); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, + new String[] { "c3", "c4", "c5", "c6" }, "idx1"); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c1" }, "idx2"); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2" }, "idx3"); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c3" }, "idx4"); + indices.addIndex(indexSpecification); + + // indexSpecification = createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2", + // "c1" }, "idx4"); + // ihtd.addIndex(indexSpecification); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + rangePutForIdx2(table); + rangePutForIdx3(table); + rangePutForIdx4(table); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + + FilterList filterList1 = new FilterList(Operator.MUST_PASS_ONE); + + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + "2".getBytes()); + SingleColumnValueFilter scvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.LESS_OR_EQUAL, + "4".getBytes()); + SingleColumnValueFilter scvf3 = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + "2".getBytes()); + filterList1.addFilter(scvf); + filterList1.addFilter(scvf2); + filterList1.addFilter(scvf3); + + SingleColumnValueFilter scvfsub = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "1".getBytes()); + SingleColumnValueFilter scvf2sub = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.LESS_OR_EQUAL, + "6".getBytes()); + SingleColumnValueFilter scvf3sub = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + "4".getBytes()); + + masterFilter.addFilter(scvfsub); + masterFilter.addFilter(scvf2sub); + masterFilter.addFilter(scvf3sub); + masterFilter.addFilter(filterList1); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + int i = 0; + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + System.out.println("************* result count......*********** " + testRes.size() + + " ###### " + testRes); + assertTrue("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); + assertTrue("Seekpoints should get added by index scanner", + IndexRegionObserver.getSeekpointAdded()); + assertEquals("It should get two seek points from index scanner.", 1, IndexRegionObserver + .getMultipleSeekPoints().size()); + assertTrue("Overall result should have only 2 rows", testRes.size() == 1); + } + + @Test(timeout = 180000) + public void testIfAllScannersAreRangeInAllLevelsPart3() throws IOException, KeeperException, + InterruptedException { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testIfAllScannersAreRangeInAllLevelsPart3"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + TableIndices indices = new TableIndices(); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, + new String[] { "c3", "c4", "c5", "c6" }, "idx1"); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c1" }, "idx2"); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2" }, "idx3"); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c3" }, "idx4"); + indices.addIndex(indexSpecification); + + // indexSpecification = createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2", + // "c1" }, "idx4"); + // ihtd.addIndex(indexSpecification); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + rangePutForIdx2(table); + rangePutForIdx3(table); + rangePutForIdx4(table); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + + FilterList filterList1 = new FilterList(Operator.MUST_PASS_ALL); + + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + "2".getBytes()); + SingleColumnValueFilter scvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.LESS_OR_EQUAL, + "4".getBytes()); + SingleColumnValueFilter scvf3 = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + "2".getBytes()); + filterList1.addFilter(scvf); + filterList1.addFilter(scvf2); + filterList1.addFilter(scvf3); + + SingleColumnValueFilter scvfsub = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "1".getBytes()); + SingleColumnValueFilter scvf2sub = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.LESS_OR_EQUAL, + "6".getBytes()); + SingleColumnValueFilter scvf3sub = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + "2".getBytes()); + + masterFilter.addFilter(scvfsub); + masterFilter.addFilter(scvf2sub); + masterFilter.addFilter(scvf3sub); + masterFilter.addFilter(filterList1); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + int i = 0; + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + System.out.println("************* result count......*********** " + testRes.size() + + " ###### " + testRes); + assertTrue("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); + assertTrue("Seekpoints should get added by index scanner", + IndexRegionObserver.getSeekpointAdded()); + assertEquals("It should get two seek points from index scanner.", 1, IndexRegionObserver + .getMultipleSeekPoints().size()); + assertTrue("Overall result should have only 2 rows", testRes.size() == 1); + } + + @Test(timeout = 180000) + public void testOREvaluationFromMultipleLevels() throws IOException, KeeperException, + InterruptedException { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testOREvaluationFromMultipleLevels"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + TableIndices indices = new TableIndices(); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, + new String[] { "c3", "c4", "c5", "c6" }, "idx1"); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c1" }, "idx2"); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2" }, "idx3"); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c3" }, "idx4"); + indices.addIndex(indexSpecification); + + // indexSpecification = createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2", + // "c1" }, "idx4"); + // ihtd.addIndex(indexSpecification); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + rangePutForIdx2(table); + rangePutForIdx3(table); + rangePutForIdx4(table); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ONE); + + FilterList filterList1 = new FilterList(Operator.MUST_PASS_ONE); + + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS_OR_EQUAL, + "1".getBytes()); + SingleColumnValueFilter scvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.GREATER_OR_EQUAL, + "5".getBytes()); + SingleColumnValueFilter scvf3 = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + "4".getBytes()); + filterList1.addFilter(scvf); + filterList1.addFilter(scvf2); + filterList1.addFilter(scvf3); + + SingleColumnValueFilter scvfsub = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS_OR_EQUAL, + "2".getBytes()); + SingleColumnValueFilter scvf2sub = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.GREATER_OR_EQUAL, + "6".getBytes()); + SingleColumnValueFilter scvf3sub = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + "3".getBytes()); + + masterFilter.addFilter(scvfsub); + masterFilter.addFilter(scvf2sub); + masterFilter.addFilter(scvf3sub); + masterFilter.addFilter(filterList1); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + int i = 0; + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + System.out.println("************* result count......*********** " + testRes.size() + + " ###### " + testRes); + assertTrue("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); + assertTrue("Seekpoints should get added by index scanner", + IndexRegionObserver.getSeekpointAdded()); + assertEquals("It should get two seek points from index scanner.", 6, IndexRegionObserver + .getMultipleSeekPoints().size()); + assertTrue("Overall result should have only 2 rows", testRes.size() == 6); + } + + private void rangePutForIdx2(HTable table) throws IOException { + Put p = new Put(Bytes.toBytes("row0")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes("1")); + table.put(p); + p = new Put(Bytes.toBytes("row9")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes("2")); + table.put(p); + p = new Put(Bytes.toBytes("row1")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes("3")); + table.put(p); + p = new Put(Bytes.toBytes("row3")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes("4")); + table.put(p); + p = new Put(Bytes.toBytes("row7")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes("5")); + table.put(p); + p = new Put(Bytes.toBytes("row15")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes("6")); + table.put(p); + } + + private void rangePutForIdx3(HTable table) throws IOException { + Put p = new Put(Bytes.toBytes("row0")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c2"), Bytes.toBytes("1")); + table.put(p); + p = new Put(Bytes.toBytes("row9")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c2"), Bytes.toBytes("2")); + table.put(p); + p = new Put(Bytes.toBytes("row1")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c2"), Bytes.toBytes("3")); + table.put(p); + p = new Put(Bytes.toBytes("row3")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c2"), Bytes.toBytes("4")); + table.put(p); + p = new Put(Bytes.toBytes("row7")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c2"), Bytes.toBytes("5")); + table.put(p); + p = new Put(Bytes.toBytes("row15")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c2"), Bytes.toBytes("6")); + table.put(p); + } + + private void rangePutForIdx4(HTable table) throws IOException { + Put p = new Put(Bytes.toBytes("row0")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c3"), Bytes.toBytes("1")); + table.put(p); + p = new Put(Bytes.toBytes("row9")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c3"), Bytes.toBytes("2")); + table.put(p); + p = new Put(Bytes.toBytes("row1")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c3"), Bytes.toBytes("3")); + table.put(p); + p = new Put(Bytes.toBytes("row3")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c3"), Bytes.toBytes("4")); + table.put(p); + p = new Put(Bytes.toBytes("row7")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c3"), Bytes.toBytes("5")); + table.put(p); + p = new Put(Bytes.toBytes("row15")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c3"), Bytes.toBytes("6")); + table.put(p); + } + + @Test(timeout = 180000) + public void testCombinationOfLESSorGREATERwithEQUAL() throws Exception { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testCombinationOfLESSorGREATERwithEQUAL"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + TableIndices indices = new TableIndices(); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, + new String[] { "c3", "c4", "c5", "c6" }, "idx1"); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c1" }, "idx2"); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2" }, "idx3"); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c3" }, "idx4"); + indices.addIndex(indexSpecification); + + // indexSpecification = createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2", + // "c1" }, "idx4"); + // ihtd.addIndex(indexSpecification); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + rangePutForIdx2(table); + rangePutForIdx3(table); + rangePutForIdx4(table); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ONE); + + FilterList filterList1 = new FilterList(Operator.MUST_PASS_ONE); + + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS, + "2".getBytes()); + SingleColumnValueFilter scvf3 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "2".getBytes()); + SingleColumnValueFilter scvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "5".getBytes()); + SingleColumnValueFilter scvf4 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "5".getBytes()); + + filterList1.addFilter(scvf); + filterList1.addFilter(scvf2); + filterList1.addFilter(scvf3); + filterList1.addFilter(scvf4); + masterFilter.addFilter(filterList1); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + int i = 0; + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + System.out.println("************* result count......*********** " + testRes.size() + + " ###### " + testRes); + assertTrue("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); + assertTrue("Seekpoints should get added by index scanner", + IndexRegionObserver.getSeekpointAdded()); + assertEquals("It should get two seek points from index scanner.", 4, IndexRegionObserver + .getMultipleSeekPoints().size()); + assertTrue("Overall result should have only 2 rows", testRes.size() == 4); + } + + @Test(timeout = 180000) + public void testIndexScanWithCaching() throws Exception { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testIndexScanWithCaching"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + TableIndices indices = new TableIndices(); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c1" }, "idx2"); + indices.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2" }, "idx3"); + indices.addIndex(indexSpecification); + + SingleColumnValueFilter filter = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "apple".getBytes()); + + SingleColumnValueFilter filter1 = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.EQUAL, + "bat".getBytes()); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ONE); + masterFilter.addFilter(filter1); + masterFilter.addFilter(filter); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + HTable table = new HTable(conf, "testIndexScanWithCaching"); + + putforIDX2(Bytes.toBytes("row1"), table); + putforIDX3(Bytes.toBytes("row1"), table); + putforIDX2(Bytes.toBytes("row2"), table); + putforIDX2(Bytes.toBytes("row3"), table); + putforIDX3(Bytes.toBytes("row4"), table); + putforIDX3(Bytes.toBytes("row5"), table); + putforIDX2(Bytes.toBytes("row5"), table); + putforIDX2(Bytes.toBytes("row6"), table); + putforIDX3(Bytes.toBytes("row7"), table); + + Scan scan = new Scan(); + scan.setFilter(masterFilter); + scan.setCaching(10); + int i = 0; + ResultScanner scanner = table.getScanner(scan); + int nextCalled = 0; + List testRes = new ArrayList(); + Result[] result = scanner.next(10); + nextCalled++; + while (result != null && result.length > 1) { + for (int j = 0; j < result.length; j++) { + testRes.add(result[j]); + } + i++; + result = scanner.next(10); + nextCalled++; + } + + assertTrue("Index flow should be used.", IndexRegionObserver.getIndexedFlowUsed()); + assertTrue("Index should fetch some seek points.", + IndexRegionObserver.getSeekpointAdded()); + assertEquals("Index should fetch 7 seek points", 7, IndexRegionObserver + .getMultipleSeekPoints().size()); + assertEquals("Final result should have 7 rows.", 7, testRes.size()); + assertEquals("All rows should be fetched in single next call.", 2, nextCalled); + } + + @Test(timeout = 180000) + public void testShouldRetrieveNegtiveIntValueWithEqualCondition() throws Exception { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testOtherDataTypes"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + TableIndices indices = new TableIndices(); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.Int, 10, new String[] { "c1" }, "idx1"); + indices.addIndex(indexSpecification); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + rangePutForIdx2WithInteger(table); + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + Bytes.toBytes(-4)); + masterFilter.addFilter(scvf); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + result = scanner.next(1); + } + assertTrue(testRes.size() == 1); + assertTrue(testRes.toString().contains("row3")); + } + + @Test(timeout = 180000) + public void testShouldRetrieveNegativeIntValueWithLessCondition() throws Exception { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testShouldRetrieveNegativeIntValueWithLessCondition"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + TableIndices indices = new TableIndices(); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.Int, 10, new String[] { "c1" }, "idx1"); + indices.addIndex(indexSpecification); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + rangePutForIdx2WithInteger(table); + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS, + (Bytes.toBytes(-4))); + + masterFilter.addFilter(scvf); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + result = scanner.next(1); + } + assertTrue(testRes.size() == 2); + } + + @Test(timeout = 180000) + public void testShouldRetriveNegativeIntValueWithGreaterCondition() throws Exception { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testShouldRetriveNegativeIntValueWithGreaterCondition"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + TableIndices indices = new TableIndices(); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.Int, 10, new String[] { "c1" }, "idx1"); + indices.addIndex(indexSpecification); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + rangePutForIdx2WithInteger(table); + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + new IntComparator(Bytes.toBytes(-6))); + masterFilter.addFilter(scvf); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + result = scanner.next(1); + } + assertTrue(testRes.size() == 5); + } + + @Test(timeout = 180000) + public void testShouldRetrieveNegativeIntValue() throws Exception { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testShouldRetrieveNegativeIntValue"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + rangePutForIdx2WithInteger(table); + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + new IntComparator(Bytes.toBytes(-6))); + masterFilter.addFilter(scvf); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + result = scanner.next(1); + } + assertTrue(testRes.size() == 5); + } + + @Test(timeout = 180000) + public void testShouldRetrieveNegativeFloatValueWithGreaterCondition() throws Exception { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testShouldRetrieveNegativeFloatValueWithGreaterCondition"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + TableIndices indices = new TableIndices(); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.Float, 10, new String[] { "c1" }, "idx1"); + indices.addIndex(indexSpecification); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + rangePutForIdx2WithFloat(table); + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + new FloatComparator(Bytes.toBytes(-5f))); + masterFilter.addFilter(scvf); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + result = scanner.next(1); + } + assertTrue(testRes.size() == 4); + } + + @Test(timeout = 180000) + public void testShouldRetrieveNegativeFloatValueWithLessCondition() throws Exception { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testShouldRetrieveNegativeFloatValueWithLessCondition"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + TableIndices indices = new TableIndices(); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.Float, 10, new String[] { "c1" }, "idx1"); + indices.addIndex(indexSpecification); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + rangePutForIdx2WithFloat(table); + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS, + new FloatComparator(Bytes.toBytes(-5f))); + + masterFilter.addFilter(scvf); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + result = scanner.next(1); + } + assertTrue(testRes.size() == 2); + } + + @Test(timeout = 180000) + public void testShouldRetrieveNegativeFloatValueWithEqualsCondition() throws Exception { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testShouldRetrieveNegativeFloatValueWithEqualsCondition"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + TableIndices indices = new TableIndices(); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.Float, 10, new String[] { "c1" }, "idx1"); + indices.addIndex(indexSpecification); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + rangePutForIdx2WithFloat(table); + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + new FloatComparator(Bytes.toBytes(-5.3f))); + masterFilter.addFilter(scvf); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + result = scanner.next(1); + } + assertTrue(testRes.size() == 1); + } + + @Test(timeout = 180000) + public void testShouldRetrieveNegativeDoubleValueWithLesserThanEqualsCondition() throws Exception { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testShouldRetrieveNegativeDoubleValueWithLesserThanEqualsCondition"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + TableIndices indices = new TableIndices(); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.Double, 10, new String[] { "c1" }, "idx1"); + indices.addIndex(indexSpecification); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + rangePutForIdx2WithDouble(table); + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS_OR_EQUAL, + new DoubleComparator(Bytes.toBytes(-5.3d))); + masterFilter.addFilter(scvf); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + result = scanner.next(1); + } + assertTrue(testRes.size() == 2); + } + + @Test//(timeout = 180000) + public void testShouldRetrieveNegativeDoubleValueWithGreaterThanEqualsCondition() + throws Exception { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testShouldRetrieveNegativeDoubleValueWithGreaterThanEqualsCondition"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + TableIndices indices = new TableIndices(); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.Double, 10, new String[] { "c1" }, "idx1"); + indices.addIndex(indexSpecification); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + rangePutForIdx2WithDouble(table); + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + new DoubleComparator(Bytes.toBytes(-5.3d))); + masterFilter.addFilter(scvf); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + result = scanner.next(1); + } + assertTrue(testRes.size() == 5); + } + + @Test(timeout = 180000) + public void testCachingWithValuesDistributedAmongMulitpleRegions() throws Exception { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testCachingWithValuesDistributedAmongMulitpleRegions"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + TableIndices indices = new TableIndices(); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c1" }, "idx1"); + indices.addIndex(indexSpecification); + byte[][] split = + new byte[][] { Bytes.toBytes("row1"), Bytes.toBytes("row2"), Bytes.toBytes("row3"), + Bytes.toBytes("row4") }; + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd, split); + HTable table = new HTable(conf, userTableName); + insert100Rows(table); + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + Bytes.toBytes("5")); + masterFilter.addFilter(scvf); + Scan scan = new Scan(); + scan.setCaching(5); + scan.setFilter(masterFilter); + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + int i = 0; + while (result != null && result.length > 0) { + System.out.println(Bytes.toString(result[0].getRow())); + testRes.add(result[0]); + result = scanner.next(1); + i++; + } + assertEquals(8, i); + } + + @Test(timeout = 180000) + public void testCachingWithValuesWhereSomeRegionsDontHaveAnyData() throws Exception { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testCachingWithValuesWhereSomeRegionsDontHaveAnyData"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + TableIndices indices = new TableIndices(); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c1" }, "idx1"); + indices.addIndex(indexSpecification); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + byte[][] split = + new byte[][] { Bytes.toBytes("row1"), Bytes.toBytes("row3"), Bytes.toBytes("row5"), + Bytes.toBytes("row7") }; + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd, split); + HTable table = new HTable(conf, userTableName); + for (int i = 0; i < 10; i++) { + if (i > 4 && i < 8) { + continue; + } + Put p = new Put(Bytes.toBytes("row" + i)); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes("5")); + table.put(p); + } + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + Bytes.toBytes("5")); + masterFilter.addFilter(scvf); + Scan scan = new Scan(); + scan.setCaching(5); + scan.setFilter(masterFilter); + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + int i = 0; + while (result != null && result.length > 0) { + System.out.println(Bytes.toString(result[0].getRow())); + testRes.add(result[0]); + result = scanner.next(1); + i++; + } + assertEquals(7, i); + } + + @Test(timeout = 180000) + public void testCachingWithLessNumberOfRowsThanCaching() throws Exception { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testCachingWithLessNumberOfRowsThanCaching"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + TableIndices indices = new TableIndices(); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c1" }, "idx1"); + indices.addIndex(indexSpecification); + byte[][] split = + new byte[][] { Bytes.toBytes("row1"), Bytes.toBytes("row3"), Bytes.toBytes("row5"), + Bytes.toBytes("row7") }; + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd, split); + HTable table = new HTable(conf, userTableName); + for (int i = 0; i < 10; i++) { + if (i > 4 && i < 8) { + continue; + } + Put p = new Put(Bytes.toBytes("row" + i)); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes("5")); + table.put(p); + } + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + Bytes.toBytes("5")); + masterFilter.addFilter(scvf); + Scan scan = new Scan(); + scan.setCaching(10); + scan.setFilter(masterFilter); + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + int i = 0; + while (result != null && result.length > 0) { + System.out.println(Bytes.toString(result[0].getRow())); + testRes.add(result[0]); + result = scanner.next(1); + i++; + } + assertEquals(7, i); + } + + private void insert100Rows(HTable table) throws IOException { + for (int i = 0; i < 8; i++) { + Put p = new Put(Bytes.toBytes("row" + i)); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes("5")); + + table.put(p); + } + } + + private void rangePutForIdx2WithInteger(HTable table) throws IOException { + Put p = new Put(Bytes.toBytes("row0")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes(1)); + table.put(p); + p = new Put(Bytes.toBytes("row1")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes(2)); + table.put(p); + p = new Put(Bytes.toBytes("row2")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes(3)); + table.put(p); + p = new Put(Bytes.toBytes("row3")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes(-4)); + table.put(p); + p = new Put(Bytes.toBytes("row4")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes(-5)); + table.put(p); + p = new Put(Bytes.toBytes("row5")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes(-6)); + table.put(p); + } + + private void rangePutForIdx2WithFloat(HTable table) throws IOException { + Put p = new Put(Bytes.toBytes("row0")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes(1.5f)); + table.put(p); + p = new Put(Bytes.toBytes("row1")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes(2.89f)); + table.put(p); + p = new Put(Bytes.toBytes("row2")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes(3.9f)); + table.put(p); + p = new Put(Bytes.toBytes("row3")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes(-4.7f)); + table.put(p); + p = new Put(Bytes.toBytes("row4")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes(-5.3f)); + table.put(p); + p = new Put(Bytes.toBytes("row5")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes(-6.456f)); + table.put(p); + } + + private void rangePutForIdx2WithDouble(HTable table) throws IOException { + Put p = new Put(Bytes.toBytes("row0")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes(1.5d)); + table.put(p); + p = new Put(Bytes.toBytes("row1")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes(2.89d)); + table.put(p); + p = new Put(Bytes.toBytes("row2")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes(3.9d)); + table.put(p); + p = new Put(Bytes.toBytes("row3")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes(-4.7d)); + table.put(p); + p = new Put(Bytes.toBytes("row4")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes(-5.3d)); + table.put(p); + p = new Put(Bytes.toBytes("row5")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes(-6.456d)); + table.put(p); + } + + @Test(timeout = 180000) + public void testComplexRangeScan() throws Exception { + Configuration conf = UTIL.getConfiguration(); + String tableName = "testComplexRangeScan"; + IndexSpecification spec1 = new IndexSpecification("idx1"); + IndexSpecification spec2 = new IndexSpecification("idx2"); + IndexSpecification spec3 = new IndexSpecification("idx3"); + HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName)); + // HTableDescriptor htd = new HTableDescriptor(tableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf"); + spec1.addIndexColumn(hcd, "detail", ValueType.String, 10); + spec2.addIndexColumn(hcd, "info", ValueType.String, 10); + spec3.addIndexColumn(hcd, "value", ValueType.String, 10); + htd.addFamily(hcd); + TableIndices indices = new TableIndices(); + indices.addIndex(spec1); + indices.addIndex(spec2); + indices.addIndex(spec3); + String[] splitkeys = new String[9]; + + for (int i = 100, j = 0; i <= 900; i += 100, j++) { + splitkeys[j] = new Integer(i).toString(); + } + htd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(htd, Bytes.toByteArrays(splitkeys)); + String rowname = "row"; + String startrow = ""; + int keys = 0; + List put = new ArrayList(); + for (int i = 1, j = 999; i < 1000; i++, j--) { + if (i % 100 == 0) { + startrow = splitkeys[keys++]; + } + Put p = new Put(Bytes.toBytes(startrow + rowname + i)); + p.add(Bytes.toBytes("cf"), Bytes.toBytes("detail"), Bytes.toBytes(new Integer(i).toString())); + p.add(Bytes.toBytes("cf"), Bytes.toBytes("info"), Bytes.toBytes(new Integer(j).toString())); + p.add(Bytes.toBytes("cf"), Bytes.toBytes("value"), + Bytes.toBytes(new Integer(i % 100).toString())); + System.out.println(p); + put.add(p); + } + HTable table = new HTable(conf, tableName); + table.put(put); + + Scan s = new Scan(); + s.setCacheBlocks(true); + FilterList master = new FilterList(Operator.MUST_PASS_ONE); + SingleColumnValueFilter filter1 = + new SingleColumnValueFilter("cf".getBytes(), "detail".getBytes(), CompareOp.LESS_OR_EQUAL, + "6".getBytes()); + filter1.setFilterIfMissing(true); + SingleColumnValueFilter filter2 = + new SingleColumnValueFilter("cf".getBytes(), "info".getBytes(), CompareOp.GREATER_OR_EQUAL, + "992".getBytes()); + filter2.setFilterIfMissing(true); + SingleColumnValueFilter filter3 = + new SingleColumnValueFilter("cf".getBytes(), "value".getBytes(), CompareOp.EQUAL, + "9".getBytes()); + filter3.setFilterIfMissing(true); + master.addFilter(filter1); + master.addFilter(filter2); + master.addFilter(filter3); + s.setFilter(master); + // scanOperation(s, conf, tableName); + assertEquals("data consistency is missed ", 563, scanOperation(s, conf, tableName)); + System.out.println("Done ************"); + s = new Scan(); + s.setFilter(master); + s.setCaching(5); + // scanOperation(s, conf, tableName); + assertEquals("data consistency is missed ", 563, scanOperation(s, conf, tableName)); + } + + @Test(timeout = 180000) + public void testComplexRangeScanWithAnd() throws Exception { + Configuration conf = UTIL.getConfiguration(); + String tableName = "RangeScanMetrix_2_new_id"; + IndexSpecification spec1 = new IndexSpecification("idx1"); + IndexSpecification spec2 = new IndexSpecification("idx2"); + IndexSpecification spec3 = new IndexSpecification("idx3"); + HTableDescriptor htd = new HTableDescriptor(tableName); + // HTableDescriptor htd = new HTableDescriptor(tableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf"); + spec1.addIndexColumn(hcd, "detail", ValueType.String, 10); + spec2.addIndexColumn(hcd, "info", ValueType.String, 10); + spec3.addIndexColumn(hcd, "value", ValueType.String, 10); + htd.addFamily(hcd); + TableIndices indices = new TableIndices(); + indices.addIndex(spec1); + indices.addIndex(spec2); + indices.addIndex(spec3); + String[] splitkeys = new String[9]; + + for (int i = 100, j = 0; i <= 900; i += 100, j++) { + splitkeys[j] = new Integer(i).toString(); + } + htd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(htd, Bytes.toByteArrays(splitkeys)); + String rowname = "row"; + String startrow = ""; + int keys = 0; + List put = new ArrayList(); + for (int i = 1, j = 999; i < 1000; i++, j--) { + if (i % 100 == 0) { + startrow = splitkeys[keys++]; + } + Put p = new Put(Bytes.toBytes(startrow + rowname + i)); + p.add(Bytes.toBytes("cf"), Bytes.toBytes("detail"), Bytes.toBytes(new Integer(i).toString())); + p.add(Bytes.toBytes("cf"), Bytes.toBytes("info"), Bytes.toBytes(new Integer(j).toString())); + p.add(Bytes.toBytes("cf"), Bytes.toBytes("value"), + Bytes.toBytes(new Integer(i % 100).toString())); + System.out.println(p); + put.add(p); + } + HTable table = new HTable(conf, tableName); + table.put(put); + + Scan s = new Scan(); + s.setCacheBlocks(true); + s.setCaching(1); + FilterList master = new FilterList(Operator.MUST_PASS_ALL); + SingleColumnValueFilter filter1 = + new SingleColumnValueFilter("cf".getBytes(), "detail".getBytes(), CompareOp.LESS_OR_EQUAL, + "65".getBytes()); + filter1.setFilterIfMissing(true); + SingleColumnValueFilter filter2 = + new SingleColumnValueFilter("cf".getBytes(), "info".getBytes(), CompareOp.GREATER, + "900".getBytes()); + filter2.setFilterIfMissing(true); + SingleColumnValueFilter filter3 = + new SingleColumnValueFilter("cf".getBytes(), "value".getBytes(), + CompareOp.GREATER_OR_EQUAL, "5".getBytes()); + filter3.setFilterIfMissing(true); + master.addFilter(filter1); + master.addFilter(filter2); + master.addFilter(filter3); + s.setFilter(master); + // scanOperation(s, conf, tableName); + assertEquals("data consistency is missed ", 18, scanOperation(s, conf, tableName)); + System.out.println("Done ************"); + s = new Scan(); + s.setFilter(master); + s.setCaching(5); + // scanOperation(s, conf, tableName); + assertEquals("data consistency is missed ", 18, scanOperation(s, conf, tableName)); + } + + @Test(timeout = 180000) + public void testVerifyTheStopRowIsCorrectInCaseOfGreaterOperatorsInSCVF() throws Exception { + Configuration conf = UTIL.getConfiguration(); + String tableName = "testDeleteIncosistent"; + IndexSpecification spec1 = new IndexSpecification("idx1"); + HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName)); + // HTableDescriptor htd = new HTableDescriptor(tableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf"); + spec1.addIndexColumn(hcd, "detail", ValueType.String, 10); + htd.addFamily(hcd); + TableIndices indices = new TableIndices(); + indices.addIndex(spec1); + htd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(htd); + HTable table = new HTable(conf, "testDeleteIncosistent"); + HTable table2 = new HTable(conf, "testDeleteIncosistent_idx"); + Put p = new Put(Bytes.toBytes("row5")); + p.add("cf".getBytes(), "detail".getBytes(), "5".getBytes()); + table2.put(IndexUtils.prepareIndexPut(p, spec1, HConstants.EMPTY_START_ROW)); + p = new Put(Bytes.toBytes("row6")); + p.add("cf".getBytes(), "detail".getBytes(), "6".getBytes()); + table.put(p); + Scan s = new Scan(); + SingleColumnValueFilter filter1 = + new SingleColumnValueFilter("cf".getBytes(), "detail".getBytes(), CompareOp.GREATER, + "5".getBytes()); + s.setFilter(filter1); + ResultScanner scanner = table.getScanner(s); + int i = 0; + for (Result result : scanner) { + i++; + } + assertEquals(1, i); + } + + private int scanOperation(Scan s, Configuration conf, String tableName) { + ResultScanner scanner = null; + int i = 0; + try { + HTable table = new HTable(conf, tableName); + scanner = table.getScanner(s); + + for (Result result : scanner) { + System.out.println(Bytes.toString(result.getRow())); + i++; + } + } catch (IOException e) { + e.printStackTrace(); + } finally { + if (scanner != null) { + scanner.close(); + } + } + System.out.println("******* Return value " + i); + return i; + } + +} Index: hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestScanFilterEvaluator.java =================================================================== --- hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestScanFilterEvaluator.java (revision 0) +++ hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestScanFilterEvaluator.java (working copy) @@ -0,0 +1,1594 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import junit.framework.TestCase; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; +import org.apache.hadoop.hbase.filter.Filter; +import org.apache.hadoop.hbase.filter.FilterList; +import org.apache.hadoop.hbase.filter.FilterList.Operator; +import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; +import org.apache.hadoop.hbase.index.ColumnQualifier.ValueType; +import org.apache.hadoop.hbase.index.IndexSpecification; +import org.apache.hadoop.hbase.index.filter.SingleColumnRangeFilter; +import org.apache.hadoop.hbase.index.manager.IndexManager; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.experimental.categories.Category; + +@Category(MediumTests.class) +public class TestScanFilterEvaluator extends TestCase { + +/* public void testCircularList() throws Exception{ + ScanFilterEvaluator.CirularDoublyLinkedList cir = new ScanFilterEvaluator.CirularDoublyLinkedList(); + cir.add(10); + cir.add(23); + cir.add(45); + Node head = cir.getHead(); + //System.out.println(head); + Node next = head; + for(int i = 0; i< 100; i++){ + next = cir.next(next); + System.out.println(next); + } + + }*/ + +/* public void testName1() throws Exception { + ScanFilterEvaluator mapper = new ScanFilterEvaluator(); + List indices = new ArrayList(); + // create the indices. + indices.add(createIndexSpecification("cf1", ValueType.String, 10, new String[] { "c1", "c2" }, "idx1")); + indices.add(createIndexSpecification("cf1", ValueType.String, 10, new String[] { "c1" }, "idx2")); + indices.add(createIndexSpecification("cf1", ValueType.String, 10, new String[] { "c2" }, "idx3")); + indices.add(createIndexSpecification("cf1", ValueType.String, 10, new String[] { "c2", "c1" }, "idx4")); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + // create the filter + FilterList filter = new FilterList(Operator.MUST_PASS_ALL); + IndexedSingleColumnValueFilter iscvf1 = new IndexedSingleColumnValueFilter("cf1".getBytes(), + "c1".getBytes(), CompareOp.EQUAL, "a".getBytes()); + IndexedSingleColumnValueFilter iscvf2 = new IndexedSingleColumnValueFilter("cf1".getBytes(), + "c2".getBytes(), CompareOp.EQUAL, "K".getBytes()); + filter.addFilter(iscvf1); + filter.addFilter(iscvf2); + + FilterList filter1 = new FilterList(Operator.MUST_PASS_ALL); + iscvf1 = new IndexedSingleColumnValueFilter("cf1".getBytes(), + "c3".getBytes(), CompareOp.EQUAL, "a".getBytes()); + iscvf2 = new IndexedSingleColumnValueFilter("cf1".getBytes(), + "c4".getBytes(), CompareOp.EQUAL, "K".getBytes()); + filter1.addFilter(iscvf1); + filter1.addFilter(iscvf2); + + FilterList filter2 = new FilterList(Operator.MUST_PASS_ALL); + iscvf1 = new IndexedSingleColumnValueFilter("cf1".getBytes(), + "c5".getBytes(), CompareOp.EQUAL, "a".getBytes()); + iscvf2 = new IndexedSingleColumnValueFilter("cf1".getBytes(), + "c6".getBytes(), CompareOp.GREATER, "K".getBytes()); + filter2.addFilter(iscvf1); + filter2.addFilter(iscvf2); + + masterFilter.addFilter(filter); + masterFilter.addFilter(filter1); + masterFilter.addFilter(filter2); + // mapper.evaluate(masterFilter, indices); + }*/ + private static final String COLUMN_FAMILY = "MyCF"; + + static HRegion region = null; + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static final String DIR = TEST_UTIL.getDataTestDir("TestScanFilterEvaluator").toString(); + + static String method = "TestScanFilterEvaluator"; + static byte[] tableName = Bytes.toBytes(method); + static byte[] family = Bytes.toBytes("family"); + static byte[] qual = Bytes.toBytes("q"); + private final int MAX_VERSIONS = 2; + + private static HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf, + byte[]... families) throws IOException { + return initHRegion(tableName, null, null, callingMethod, conf, families); + } + + /** + * @param tableName + * @param startKey + * @param stopKey + * @param callingMethod + * @param conf + * @param families + * @throws IOException + * @return A region on which you must call {@link HRegion#closeHRegion(HRegion)} when done. + */ + private static HRegion initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey, + String callingMethod, Configuration conf, byte[]... families) throws IOException { + HTableDescriptor htd = new HTableDescriptor(tableName); + for (byte[] family : families) { + htd.addFamily(new HColumnDescriptor(family)); + } + HRegionInfo info = new HRegionInfo(htd.getTableName(), startKey, stopKey, false); + Path path = new Path(DIR + callingMethod); + FileSystem fs = FileSystem.get(conf); + if (fs.exists(path)) { + if (!fs.delete(path, true)) { + throw new IOException("Failed delete of " + path); + } + } + return HRegion.createHRegion(info, path, conf, htd); + } + + public void testName2() throws Exception { + Configuration conf = HBaseConfiguration.create(); + region = initHRegion(tableName, method, conf, family); + ArrayList arrayList = new ArrayList(); + List indices = arrayList; + ScanFilterEvaluator mapper = new ScanFilterEvaluator(); + // create the indices. + indices.add(createIndexSpecification("cf1", ValueType.String, 10, new String[] { "c1", "c2", + "c3", "c4", "c5", "c6", "c7" }, "idx1")); + indices + .add(createIndexSpecification("cf1", ValueType.String, 10, new String[] { "c1" }, "idx2")); + indices + .add(createIndexSpecification("cf1", ValueType.String, 10, new String[] { "c2" }, "idx3")); + indices.add(createIndexSpecification("cf1", ValueType.String, 10, new String[] { "c2", "c1" }, + "idx4")); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + // create the filter + FilterList filter = new FilterList(Operator.MUST_PASS_ALL); + SingleColumnValueFilter iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "a".getBytes()); + SingleColumnValueFilter iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.EQUAL, + "K".getBytes()); + filter.addFilter(iscvf1); + filter.addFilter(iscvf2); + + FilterList filter1 = new FilterList(Operator.MUST_PASS_ALL); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + "a".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c4".getBytes(), CompareOp.EQUAL, + "K".getBytes()); + filter1.addFilter(iscvf1); + filter1.addFilter(iscvf2); + + FilterList filter2 = new FilterList(Operator.MUST_PASS_ALL); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c5".getBytes(), CompareOp.EQUAL, + "a".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c6".getBytes(), CompareOp.EQUAL, + "K".getBytes()); + SingleColumnValueFilter iscvf3 = + new SingleColumnValueFilter("cf1".getBytes(), "c7".getBytes(), CompareOp.EQUAL, + "K".getBytes()); + filter2.addFilter(iscvf1); + filter2.addFilter(iscvf2); + // filter2.addFilter(iscvf3); + + masterFilter.addFilter(filter); + masterFilter.addFilter(filter1); + masterFilter.addFilter(filter2); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + // Will throw null pointer here. + IndexSpecification indexSpec = new IndexSpecification("a"); + indexSpec + .addIndexColumn(new HColumnDescriptor(family), Bytes.toString(qual), ValueType.Int, 10); + boolean add = arrayList.add(indexSpec); + IndexManager.getInstance().addIndexForTable(this.region.getTableDesc().getNameAsString(), + arrayList); + mapper.evaluate(scan, indices, new byte[0], this.region, this.region.getTableDesc() + .getNameAsString()); + } + + public void testDiffCombinations() throws Exception { + Configuration conf = HBaseConfiguration.create(); + region = initHRegion(tableName, "testDiffCombinations", conf, family); + ScanFilterEvaluator mapper = new ScanFilterEvaluator(); + List indices = new ArrayList(); + + // create the indices. + indices.add(createIndexSpecification("cf1", ValueType.String, 10, new String[] { "c2", "c3", + "c4", "c5", "c6" }, "idx1")); + + indices.add(createIndexSpecification("cf1", ValueType.String, 10, new String[] { "c2", "c1", + "c3", "c4" }, "idx4")); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + // create the filter + FilterList filter = new FilterList(Operator.MUST_PASS_ALL); + SingleColumnValueFilter iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "a".getBytes()); + SingleColumnValueFilter iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.EQUAL, + "K".getBytes()); + filter.addFilter(iscvf1); + filter.addFilter(iscvf2); + + FilterList filter1 = new FilterList(Operator.MUST_PASS_ALL); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + "a".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c4".getBytes(), CompareOp.EQUAL, + "K".getBytes()); + filter1.addFilter(iscvf1); + filter1.addFilter(iscvf2); + + FilterList filter2 = new FilterList(Operator.MUST_PASS_ALL); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c5".getBytes(), CompareOp.EQUAL, + "a".getBytes()); + filter2.addFilter(iscvf1); + + // filter2.addFilter(iscvf3); + + masterFilter.addFilter(filter); + masterFilter.addFilter(filter1); + masterFilter.addFilter(filter2); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + + IndexManager.getInstance().addIndexForTable(this.region.getTableDesc().getNameAsString(), + indices); + mapper.evaluate(scan, indices, new byte[0], this.region, this.region.getTableDesc() + .getNameAsString()); + } + + public void testDiffCombinations1() throws Exception { + Configuration conf = HBaseConfiguration.create(); + region = initHRegion(tableName, "testDiffCombinations1", conf, family); + ScanFilterEvaluator mapper = new ScanFilterEvaluator(); + List indices = new ArrayList(); + // create the indices. + indices.add(createIndexSpecification("cf1", ValueType.String, 10, new String[] { "c2", "c3", + "c4", }, "idx1")); + + indices.add(createIndexSpecification("cf1", ValueType.String, 10, new String[] { "c2", "c1" }, + "idx2")); + + indices + .add(createIndexSpecification("cf1", ValueType.String, 10, new String[] { "c5" }, "idx3")); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + // create the filter + FilterList filter = new FilterList(Operator.MUST_PASS_ALL); + SingleColumnValueFilter iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "a".getBytes()); + SingleColumnValueFilter iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.EQUAL, + "K".getBytes()); + filter.addFilter(iscvf1); + filter.addFilter(iscvf2); + + FilterList filter1 = new FilterList(Operator.MUST_PASS_ALL); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + "a".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c4".getBytes(), CompareOp.EQUAL, + "K".getBytes()); + filter1.addFilter(iscvf1); + filter1.addFilter(iscvf2); + + FilterList filter2 = new FilterList(Operator.MUST_PASS_ALL); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c5".getBytes(), CompareOp.EQUAL, + "a".getBytes()); + filter2.addFilter(iscvf1); + + // filter2.addFilter(iscvf3); + + masterFilter.addFilter(filter); + masterFilter.addFilter(filter1); + masterFilter.addFilter(filter2); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + IndexManager.getInstance().addIndexForTable(this.region.getTableDesc().getNameAsString(), + indices); + mapper.evaluate(scan, indices, new byte[0], this.region, this.region.getTableDesc() + .getNameAsString()); + } + + public void testDiffCombinations2() throws Exception { + Configuration conf = HBaseConfiguration.create(); + region = initHRegion(tableName, "testDiffCombinations2", conf, family); + ScanFilterEvaluator mapper = new ScanFilterEvaluator(); + List indices = new ArrayList(); + // create the indices. + indices.add(createIndexSpecification("cf1", ValueType.String, 10, new String[] { "c2", "c3" }, + "idx1")); + + indices.add(createIndexSpecification("cf1", ValueType.String, 10, new String[] { "c2", "c1" }, + "idx2")); + + indices + .add(createIndexSpecification("cf1", ValueType.String, 10, new String[] { "c5" }, "idx3")); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + // create the filter + FilterList filter = new FilterList(Operator.MUST_PASS_ALL); + SingleColumnValueFilter iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "a".getBytes()); + SingleColumnValueFilter iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.EQUAL, + "K".getBytes()); + filter.addFilter(iscvf1); + filter.addFilter(iscvf2); + + FilterList filter1 = new FilterList(Operator.MUST_PASS_ALL); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + "a".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c4".getBytes(), CompareOp.EQUAL, + "K".getBytes()); + filter1.addFilter(iscvf1); + filter1.addFilter(iscvf2); + + FilterList filter2 = new FilterList(Operator.MUST_PASS_ALL); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c5".getBytes(), CompareOp.EQUAL, + "a".getBytes()); + filter2.addFilter(iscvf1); + + // filter2.addFilter(iscvf3); + + masterFilter.addFilter(filter); + masterFilter.addFilter(filter1); + masterFilter.addFilter(filter2); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + + IndexManager.getInstance().addIndexForTable(this.region.getTableDesc().getNameAsString(), + indices); + mapper.evaluate(scan, indices, new byte[0], this.region, this.region.getTableDesc() + .getNameAsString()); + } + + public void testDiffCombinations3() throws Exception { + Configuration conf = HBaseConfiguration.create(); + region = initHRegion(tableName, "testDiffCombinations3", conf, family); + ScanFilterEvaluator mapper = new ScanFilterEvaluator(); + List indices = new ArrayList(); + // create the indices. + indices.add(createIndexSpecification("cf1", ValueType.String, 10, new String[] { "c2", "c3" }, + "idx1")); + + indices + .add(createIndexSpecification("cf1", ValueType.String, 10, new String[] { "c1" }, "idx2")); + + indices + .add(createIndexSpecification("cf1", ValueType.String, 10, new String[] { "c5" }, "idx3")); + + indices + .add(createIndexSpecification("cf1", ValueType.String, 10, new String[] { "c4" }, "idx4")); + + indices + .add(createIndexSpecification("cf1", ValueType.String, 10, new String[] { "c6" }, "idx5")); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + // create the filter + FilterList filter = new FilterList(Operator.MUST_PASS_ALL); + SingleColumnValueFilter iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "a".getBytes()); + SingleColumnValueFilter iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.EQUAL, + "K".getBytes()); + filter.addFilter(iscvf1); + filter.addFilter(iscvf2); + + FilterList filter1 = new FilterList(Operator.MUST_PASS_ALL); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + "a".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c4".getBytes(), CompareOp.EQUAL, + "K".getBytes()); + filter1.addFilter(iscvf1); + filter1.addFilter(iscvf2); + + FilterList filter2 = new FilterList(Operator.MUST_PASS_ALL); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c5".getBytes(), CompareOp.EQUAL, + "a".getBytes()); + filter2.addFilter(iscvf1); + + // filter2.addFilter(iscvf3); + + masterFilter.addFilter(filter); + masterFilter.addFilter(filter1); + masterFilter.addFilter(filter2); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + + IndexManager.getInstance().addIndexForTable(this.region.getTableDesc().getNameAsString(), + indices); + mapper.evaluate(scan, indices, new byte[0], this.region, this.region.getTableDesc() + .getNameAsString()); + } + + public void testWhenORWithSameColumnAppearsinDiffChild() throws Exception { + Configuration conf = HBaseConfiguration.create(); + region = initHRegion(tableName, "tesWhenORWithSameColumnAppearsinDiffChild", conf, family); + ScanFilterEvaluator mapper = new ScanFilterEvaluator(); + List indices = new ArrayList(); + // create the indices. + indices.add(createIndexSpecification("cf1", ValueType.String, 10, new String[] { "c2", "c3" }, + "idx1")); + + indices + .add(createIndexSpecification("cf1", ValueType.String, 10, new String[] { "c1" }, "idx2")); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ONE); + // create the filter + FilterList filter = new FilterList(Operator.MUST_PASS_ALL); + SingleColumnValueFilter iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "a".getBytes()); + SingleColumnValueFilter iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.EQUAL, + "K".getBytes()); + filter.addFilter(iscvf1); + filter.addFilter(iscvf2); + + SingleColumnValueFilter iscvf3 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "d".getBytes()); + // filter2.addFilter(iscvf3); + + masterFilter.addFilter(filter); + masterFilter.addFilter(iscvf3); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + + IndexManager.getInstance().addIndexForTable(this.region.getTableDesc().getNameAsString(), + indices); + mapper.evaluate(scan, indices, new byte[0], this.region, this.region.getTableDesc() + .getNameAsString()); + } + + public void testWhenORConditionAppears() throws Exception { + Configuration conf = HBaseConfiguration.create(); + region = initHRegion(tableName, "testWhenORConditionAppears", conf, family); + ScanFilterEvaluator mapper = new ScanFilterEvaluator(); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ONE); + // create the filter + FilterList filter = new FilterList(Operator.MUST_PASS_ALL); + SingleColumnValueFilter iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "a".getBytes()); + SingleColumnValueFilter iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.EQUAL, + "K".getBytes()); + filter.addFilter(iscvf1); + filter.addFilter(iscvf2); + + FilterList filter1 = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + "a".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c4".getBytes(), CompareOp.EQUAL, + "K".getBytes()); + filter1.addFilter(iscvf1); + filter1.addFilter(iscvf2); + + FilterList filter2 = new FilterList(Operator.MUST_PASS_ALL); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c5".getBytes(), CompareOp.GREATER, + Bytes.toBytes(10)); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c6".getBytes(), CompareOp.EQUAL, + Bytes.toBytes(100)); + filter2.addFilter(iscvf1); + filter2.addFilter(iscvf2); + + masterFilter.addFilter(filter); + masterFilter.addFilter(filter1); + masterFilter.addFilter(filter2); + Filter doFiltersRestruct = mapper.doFiltersRestruct(masterFilter); + // System.out.println(doFiltersRestruct); + /* + * if (doFiltersRestruct instanceof FilterList) { FilterList list = ((FilterList) + * doFiltersRestruct); assertEquals(3, list.getFilters().size()); } + */ + } + + public void testORFiltersGrouping() throws Exception { + ScanFilterEvaluator mapper = new ScanFilterEvaluator(); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ONE); + SingleColumnValueFilter iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "10".getBytes()); + SingleColumnValueFilter iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS, + "10".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + Filter resultFilter = mapper.doFiltersRestruct(masterFilter); + List filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getUpperBoundOp().equals( + CompareOp.LESS_OR_EQUAL)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS, + "10".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "10".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getUpperBoundOp().equals( + CompareOp.LESS_OR_EQUAL)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS_OR_EQUAL, + "10".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "10".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getUpperBoundOp().equals( + CompareOp.LESS_OR_EQUAL)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "10".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS_OR_EQUAL, + "10".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getUpperBoundOp().equals( + CompareOp.LESS_OR_EQUAL)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "10".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "10".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getLowerBoundOp().equals( + CompareOp.GREATER_OR_EQUAL)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + "10".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "10".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getLowerBoundOp().equals( + CompareOp.GREATER_OR_EQUAL)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "10".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "10".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getLowerBoundOp().equals( + CompareOp.GREATER_OR_EQUAL)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "10".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + "10".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getLowerBoundOp().equals( + CompareOp.GREATER_OR_EQUAL)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "10".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "10".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnValueFilter); + assertTrue(((SingleColumnValueFilter) filterList.get(0)).getOperator().equals(CompareOp.EQUAL)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS, + "10".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "10".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 2); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getUpperBoundOp().equals( + CompareOp.LESS)); + assertTrue(((SingleColumnRangeFilter) filterList.get(1)).getLowerBoundOp().equals( + CompareOp.GREATER)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "10".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS, + "10".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 2); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getLowerBoundOp().equals( + CompareOp.GREATER)); + assertTrue(((SingleColumnRangeFilter) filterList.get(1)).getUpperBoundOp().equals( + CompareOp.LESS)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + "10".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS, + "10".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 2); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getLowerBoundOp().equals( + CompareOp.GREATER_OR_EQUAL)); + assertTrue(((SingleColumnRangeFilter) filterList.get(1)).getUpperBoundOp().equals( + CompareOp.LESS)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "10".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS_OR_EQUAL, + "10".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 2); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getLowerBoundOp().equals( + CompareOp.GREATER)); + assertTrue(((SingleColumnRangeFilter) filterList.get(1)).getUpperBoundOp().equals( + CompareOp.LESS_OR_EQUAL)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "10".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "10".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getLowerBoundOp().equals( + CompareOp.GREATER)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "10".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + "10".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getLowerBoundOp().equals( + CompareOp.GREATER_OR_EQUAL)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + "10".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "10".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getLowerBoundOp().equals( + CompareOp.GREATER_OR_EQUAL)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS, + "10".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS, + "10".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getUpperBoundOp().equals( + CompareOp.LESS)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS, + "10".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS_OR_EQUAL, + "10".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getUpperBoundOp().equals( + CompareOp.LESS_OR_EQUAL)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS_OR_EQUAL, + "10".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS, + "10".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getUpperBoundOp().equals( + CompareOp.LESS_OR_EQUAL)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS_OR_EQUAL, + "9".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS, + "5".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getUpperBoundOp().equals( + CompareOp.LESS_OR_EQUAL)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS, + "9".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS_OR_EQUAL, + "5".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getUpperBoundOp().equals( + CompareOp.LESS)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS_OR_EQUAL, + "9".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS, + "5".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getUpperBoundOp().equals( + CompareOp.LESS_OR_EQUAL)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS, + "9".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS_OR_EQUAL, + "5".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getUpperBoundOp().equals( + CompareOp.LESS)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "9".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + "5".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getLowerBoundOp().equals( + CompareOp.GREATER_OR_EQUAL)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + "9".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "5".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getLowerBoundOp().equals( + CompareOp.GREATER)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "9".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + "5".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getLowerBoundOp().equals( + CompareOp.GREATER_OR_EQUAL)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + "9".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "5".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getLowerBoundOp().equals( + CompareOp.GREATER)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "9".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "5".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getLowerBoundOp().equals( + CompareOp.GREATER)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "9".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + "5".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getLowerBoundOp().equals( + CompareOp.GREATER_OR_EQUAL)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "9".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "5".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getLowerBoundOp().equals( + CompareOp.GREATER)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "9".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "5".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getLowerBoundOp().equals( + CompareOp.GREATER)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + "9".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "5".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 2); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(filterList.get(1) instanceof SingleColumnValueFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getLowerBoundOp().equals( + CompareOp.GREATER_OR_EQUAL)); + assertTrue(((SingleColumnValueFilter) filterList.get(1)).getOperator().equals(CompareOp.EQUAL)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "9".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "5".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 2); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(filterList.get(1) instanceof SingleColumnValueFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getLowerBoundOp().equals( + CompareOp.GREATER)); + assertTrue(((SingleColumnValueFilter) filterList.get(1)).getOperator().equals(CompareOp.EQUAL)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS, + "5".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "9".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 2); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(filterList.get(1) instanceof SingleColumnValueFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getUpperBoundOp().equals( + CompareOp.LESS)); + assertTrue(((SingleColumnValueFilter) filterList.get(1)).getOperator().equals(CompareOp.EQUAL)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS, + "5".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "2".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getUpperBoundOp().equals( + CompareOp.LESS)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS_OR_EQUAL, + "5".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "2".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getUpperBoundOp().equals( + CompareOp.LESS_OR_EQUAL)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "5".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "7".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getLowerBoundOp().equals( + CompareOp.GREATER)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + "5".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "7".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getLowerBoundOp().equals( + CompareOp.GREATER_OR_EQUAL)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "2".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS, + "5".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getUpperBoundOp().equals( + CompareOp.LESS)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "2".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS_OR_EQUAL, + "5".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getUpperBoundOp().equals( + CompareOp.LESS_OR_EQUAL)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "7".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "5".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getLowerBoundOp().equals( + CompareOp.GREATER)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "7".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + "5".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getLowerBoundOp().equals( + CompareOp.GREATER_OR_EQUAL)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS, + "3".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS, + "5".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getUpperBoundOp().equals( + CompareOp.LESS)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS_OR_EQUAL, + "3".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS, + "5".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getUpperBoundOp().equals( + CompareOp.LESS)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS_OR_EQUAL, + "3".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS_OR_EQUAL, + "5".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getUpperBoundOp().equals( + CompareOp.LESS_OR_EQUAL)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "3".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "5".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getLowerBoundOp().equals( + CompareOp.GREATER)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "3".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + "5".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getLowerBoundOp().equals( + CompareOp.GREATER)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + "3".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + "5".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getLowerBoundOp().equals( + CompareOp.GREATER_OR_EQUAL)); + + } + + /*public void testWhenColNamesAreRepeated() throws Exception { + ScanFilterEvaluator mapper = new ScanFilterEvaluator(); + + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + // create the filter + FilterList filter = new FilterList(Operator.MUST_PASS_ALL); + IndexedSingleColumnValueFilter iscvf1 = new IndexedSingleColumnValueFilter("cf1".getBytes(), + "c1".getBytes(), CompareOp.EQUAL, "a".getBytes()); + IndexedSingleColumnValueFilter iscvf2 = new IndexedSingleColumnValueFilter("cf1".getBytes(), + "c1".getBytes(), CompareOp.EQUAL, "K".getBytes()); + filter.addFilter(iscvf1); + filter.addFilter(iscvf2); + + FilterList filter1 = new FilterList(Operator.MUST_PASS_ALL); + iscvf1 = new IndexedSingleColumnValueFilter("cf1".getBytes(), + "c3".getBytes(), CompareOp.EQUAL, "a".getBytes()); + iscvf2 = new IndexedSingleColumnValueFilter("cf1".getBytes(), + "c4".getBytes(), CompareOp.EQUAL, "K".getBytes()); + filter1.addFilter(iscvf1); + filter1.addFilter(iscvf2); + + masterFilter.addFilter(filter); + masterFilter.addFilter(filter1); + + Filter doFiltersRestruct = mapper.doFiltersRestruct(masterFilter); + if (doFiltersRestruct instanceof FilterList) { + FilterList list = ((FilterList) doFiltersRestruct); + assertEquals(3, list.getFilters().size()); + + List filters = list.getFilters(); + for (Filter filter2 : filters) { + if(filter2 instanceof SingleColumnValueFilter){ + SingleColumnValueFilter scvf = (SingleColumnValueFilter)filter2; + if(Bytes.equals(scvf.getQualifier(), "c1".getBytes())){ + assertTrue(Bytes.equals("K".getBytes(), scvf.getComparator().getValue())); + } + } + } + } + } + + public void testWhenSameColsConditionsComeMoreThanOnce() throws Exception{ + + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + // create the filter + FilterList filter = new FilterList(Operator.MUST_PASS_ALL); + IndexedSingleColumnValueFilter iscvf1 = new IndexedSingleColumnValueFilter("cf1".getBytes(), + "c1".getBytes(), CompareOp.GREATER, Bytes.toBytes(20)); + IndexedSingleColumnValueFilter iscvf2 = new IndexedSingleColumnValueFilter("cf1".getBytes(), + "c1".getBytes(), CompareOp.LESS, Bytes.toBytes(20)); + filter.addFilter(iscvf1); + filter.addFilter(iscvf2); + + FilterList filter1 = new FilterList(Operator.MUST_PASS_ALL); + iscvf1 = new IndexedSingleColumnValueFilter("cf1".getBytes(), + "c3".getBytes(), CompareOp.EQUAL, Bytes.toBytes(10)); + iscvf2 = new IndexedSingleColumnValueFilter("cf1".getBytes(), + "c4".getBytes(), CompareOp.EQUAL, Bytes.toBytes(100)); + filter1.addFilter(iscvf1); + filter1.addFilter(iscvf2); + + FilterList filter2 = new FilterList(Operator.MUST_PASS_ALL); + iscvf1 = new IndexedSingleColumnValueFilter("cf1".getBytes(), + "c1".getBytes(), CompareOp.GREATER, Bytes.toBytes(10)); + iscvf2 = new IndexedSingleColumnValueFilter("cf1".getBytes(), + "c4".getBytes(), CompareOp.EQUAL, Bytes.toBytes(100)); + filter2.addFilter(iscvf1); + filter2.addFilter(iscvf2); + + masterFilter.addFilter(filter); + masterFilter.addFilter(filter1); + masterFilter.addFilter(filter2); + ScanFilterEvaluator mapper = new ScanFilterEvaluator(); + Filter doFiltersRestruct = mapper.doFiltersRestruct(masterFilter); + if (doFiltersRestruct instanceof FilterList) { + FilterList list = ((FilterList) doFiltersRestruct); + assertEquals(2, list.getFilters().size()); + } + + } + + public void testShouldTakeOnlyTheEqualConditionWhenGreaterAlsoComes() throws Exception { + ScanFilterEvaluator mapper = new ScanFilterEvaluator(); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + // create the filter + FilterList filter = new FilterList(Operator.MUST_PASS_ALL); + IndexedSingleColumnValueFilter iscvf1 = new IndexedSingleColumnValueFilter("cf1".getBytes(), + "c1".getBytes(), CompareOp.GREATER, Bytes.toBytes(20)); + IndexedSingleColumnValueFilter iscvf2 = new IndexedSingleColumnValueFilter("cf1".getBytes(), + "c2".getBytes(), CompareOp.EQUAL, Bytes.toBytes(20)); + filter.addFilter(iscvf1); + filter.addFilter(iscvf2); + + FilterList filter1 = new FilterList(Operator.MUST_PASS_ALL); + iscvf1 = new IndexedSingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + Bytes.toBytes(10)); + iscvf2 = new IndexedSingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + Bytes.toBytes(100)); + filter1.addFilter(iscvf1); + filter1.addFilter(iscvf2); + + masterFilter.addFilter(filter); + masterFilter.addFilter(filter1); + + Filter doFiltersRestruct = mapper.doFiltersRestruct(masterFilter); + if (doFiltersRestruct instanceof FilterList) { + FilterList list = ((FilterList) doFiltersRestruct); + assertEquals(3, list.getFilters().size()); + } + } + + public void testShouldTakeOnlyTheEqualConditionWhenLesserAlsoComes() throws Exception{ + ScanFilterEvaluator mapper = new ScanFilterEvaluator(); + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + // create the filter + FilterList filter = new FilterList(Operator.MUST_PASS_ALL); + IndexedSingleColumnValueFilter iscvf1 = new IndexedSingleColumnValueFilter("cf1".getBytes(), + "c1".getBytes(), CompareOp.EQUAL, Bytes.toBytes(20)); + IndexedSingleColumnValueFilter iscvf2 = new IndexedSingleColumnValueFilter("cf1".getBytes(), + "c2".getBytes(), CompareOp.EQUAL, Bytes.toBytes(20)); + filter.addFilter(iscvf1); + filter.addFilter(iscvf2); + + FilterList filter1 = new FilterList(Operator.MUST_PASS_ALL); + iscvf1 = new IndexedSingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + Bytes.toBytes(10)); + iscvf2 = new IndexedSingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS, + Bytes.toBytes(100)); + filter1.addFilter(iscvf1); + filter1.addFilter(iscvf2); + + masterFilter.addFilter(filter); + masterFilter.addFilter(filter1); + + Filter doFiltersRestruct = mapper.doFiltersRestruct(masterFilter); + if (doFiltersRestruct instanceof FilterList) { + FilterList list = ((FilterList) doFiltersRestruct); + assertEquals(3, list.getFilters().size()); + List filters = list.getFilters(); + for (Filter filter2 : filters) { + if (filter2 instanceof SingleColumnValueFilter) { + SingleColumnValueFilter scvf = (SingleColumnValueFilter) filter2; + if (Bytes.equals(scvf.getQualifier(), "c1".getBytes())) { + assertEquals(CompareOp.EQUAL, scvf.getOperator()); + } + } + } + } + } + + public void testShouldNotIncludeFilterIfTheRangeConditionIsWrong() throws Exception{ + ScanFilterEvaluator mapper = new ScanFilterEvaluator(); + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + // create the filter + FilterList filter = new FilterList(Operator.MUST_PASS_ALL); + IndexedSingleColumnValueFilter iscvf1 = new IndexedSingleColumnValueFilter("cf1".getBytes(), + "c1".getBytes(), CompareOp.LESS, Bytes.toBytes(10)); + IndexedSingleColumnValueFilter iscvf2 = new IndexedSingleColumnValueFilter("cf1".getBytes(), + "c2".getBytes(), CompareOp.EQUAL, Bytes.toBytes(20)); + filter.addFilter(iscvf1); + filter.addFilter(iscvf2); + + FilterList filter1 = new FilterList(Operator.MUST_PASS_ALL); + iscvf1 = new IndexedSingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + Bytes.toBytes(10)); + iscvf2 = new IndexedSingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + Bytes.toBytes(100)); + filter1.addFilter(iscvf1); + filter1.addFilter(iscvf2); + + masterFilter.addFilter(filter); + masterFilter.addFilter(filter1); + + Filter doFiltersRestruct = mapper.doFiltersRestruct(masterFilter); + if (doFiltersRestruct instanceof FilterList) { + FilterList list = ((FilterList) doFiltersRestruct); + assertEquals(2, list.getFilters().size()); + } + } + + public void testShouldTakeOnlyTheHighestFilterWhenTwoGreaterConditonsAreFound() throws Exception{ + ScanFilterEvaluator mapper = new ScanFilterEvaluator(); + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + // create the filter + FilterList filter = new FilterList(Operator.MUST_PASS_ALL); + IndexedSingleColumnValueFilter iscvf1 = new IndexedSingleColumnValueFilter("cf1".getBytes(), + "c1".getBytes(), CompareOp.GREATER, Bytes.toBytes(10)); + IndexedSingleColumnValueFilter iscvf2 = new IndexedSingleColumnValueFilter("cf1".getBytes(), + "c2".getBytes(), CompareOp.EQUAL, Bytes.toBytes(20)); + filter.addFilter(iscvf1); + filter.addFilter(iscvf2); + + FilterList filter1 = new FilterList(Operator.MUST_PASS_ALL); + iscvf1 = new IndexedSingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + Bytes.toBytes(10)); + iscvf2 = new IndexedSingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + Bytes.toBytes(100)); + filter1.addFilter(iscvf1); + filter1.addFilter(iscvf2); + + masterFilter.addFilter(filter); + masterFilter.addFilter(filter1); + + Filter doFiltersRestruct = mapper.doFiltersRestruct(masterFilter); + if (doFiltersRestruct instanceof FilterList) { + FilterList list = ((FilterList) doFiltersRestruct); + assertEquals(3, list.getFilters().size()); + List filters = list.getFilters(); + for (Filter filter2 : filters) { + if (filter2 instanceof SingleColumnValueFilter) { + SingleColumnValueFilter scvf = (SingleColumnValueFilter) filter2; + if (Bytes.equals(scvf.getQualifier(), "c1".getBytes())) { + assertTrue(Bytes.equals(Bytes.toBytes(100), scvf.getComparator().getValue())); + } + } + } + } + } + + public void testShouldTakeOnlyTheLowestFilterWhenTwoLesserConditonsAreFound() throws Exception{ + ScanFilterEvaluator mapper = new ScanFilterEvaluator(); + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + // create the filter + FilterList filter = new FilterList(Operator.MUST_PASS_ALL); + IndexedSingleColumnValueFilter iscvf1 = new IndexedSingleColumnValueFilter("cf1".getBytes(), + "c1".getBytes(), CompareOp.LESS, Bytes.toBytes(10)); + IndexedSingleColumnValueFilter iscvf2 = new IndexedSingleColumnValueFilter("cf1".getBytes(), + "c2".getBytes(), CompareOp.EQUAL, Bytes.toBytes(20)); + filter.addFilter(iscvf1); + filter.addFilter(iscvf2); + + FilterList filter1 = new FilterList(Operator.MUST_PASS_ALL); + iscvf1 = new IndexedSingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + Bytes.toBytes(10)); + iscvf2 = new IndexedSingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS, + Bytes.toBytes(1)); + filter1.addFilter(iscvf1); + filter1.addFilter(iscvf2); + + masterFilter.addFilter(filter); + masterFilter.addFilter(filter1); + + Filter doFiltersRestruct = mapper.doFiltersRestruct(masterFilter); + if (doFiltersRestruct instanceof FilterList) { + FilterList list = ((FilterList) doFiltersRestruct); + assertEquals(3, list.getFilters().size()); + List filters = list.getFilters(); + for (Filter filter2 : filters) { + if (filter2 instanceof SingleColumnValueFilter) { + SingleColumnValueFilter scvf = (SingleColumnValueFilter) filter2; + if (Bytes.equals(scvf.getQualifier(), "c1".getBytes())) { + assertTrue(Bytes.equals(Bytes.toBytes(1), scvf.getComparator().getValue())); + } + } + } + } + } +*/ + private IndexSpecification createIndexSpecification(String cf, ValueType type, + int maxValueLength, String[] qualifiers, String name) { + IndexSpecification index = new IndexSpecification(name.getBytes()); + for (String qualifier : qualifiers) { + index.addIndexColumn(new HColumnDescriptor(cf), qualifier, type, maxValueLength); + } + return index; + } + +} Index: hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestScanFilterEvaluatorForIndexInScan.java =================================================================== --- hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestScanFilterEvaluatorForIndexInScan.java (revision 0) +++ hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestScanFilterEvaluatorForIndexInScan.java (working copy) @@ -0,0 +1,257 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import static org.junit.Assert.assertNull; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; +import org.apache.hadoop.hbase.filter.Filter; +import org.apache.hadoop.hbase.filter.FilterList; +import org.apache.hadoop.hbase.filter.FilterList.Operator; +import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; +import org.apache.hadoop.hbase.index.Column; +import org.apache.hadoop.hbase.index.ColumnQualifier.ValueType; +import org.apache.hadoop.hbase.index.Constants; +import org.apache.hadoop.hbase.index.GroupingCondition; +import org.apache.hadoop.hbase.index.IndexSpecification; +import org.apache.hadoop.hbase.index.client.EqualsExpression; +import org.apache.hadoop.hbase.index.client.IndexExpression; +import org.apache.hadoop.hbase.index.client.IndexUtils; +import org.apache.hadoop.hbase.index.client.MultiIndexExpression; +import org.apache.hadoop.hbase.index.client.NoIndexExpression; +import org.apache.hadoop.hbase.index.client.RangeExpression; +import org.apache.hadoop.hbase.index.client.SingleIndexExpression; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(MediumTests.class) +public class TestScanFilterEvaluatorForIndexInScan { + + private static final byte[] FAMILY1 = Bytes.toBytes("cf1"); + private static final byte[] FAMILY2 = Bytes.toBytes("cf2"); + private static final String COL1 = "c1"; + private static final String COL2 = "c2"; + private static final String COL3 = "c3"; + private static final byte[] QUALIFIER1 = Bytes.toBytes(COL1); + private static final byte[] QUALIFIER2 = Bytes.toBytes(COL2); + private static final byte[] QUALIFIER3 = Bytes.toBytes(COL3); + private static final String tableName = "tab1"; + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static final String DIR = TEST_UTIL.getDataTestDir( + "TestScanFilterEvaluatorForIndexInScan").toString(); + + @Test + public void testSingleIndexExpressionWithOneEqualsExpression() throws Exception { + String indexName = "idx1"; + SingleIndexExpression singleIndexExpression = new SingleIndexExpression(indexName); + byte[] value = "1".getBytes(); + Column column = new Column(FAMILY1, QUALIFIER1); + EqualsExpression equalsExpression = new EqualsExpression(column, value); + singleIndexExpression.addEqualsExpression(equalsExpression); + + Scan scan = new Scan(); + scan.setAttribute(Constants.INDEX_EXPRESSION, IndexUtils.toBytes(singleIndexExpression)); + Filter filter = new SingleColumnValueFilter(FAMILY1, QUALIFIER1, CompareOp.EQUAL, value); + scan.setFilter(filter); + ScanFilterEvaluator evaluator = new ScanFilterEvaluator(); + List indices = new ArrayList(); + IndexSpecification index = new IndexSpecification(indexName); + HColumnDescriptor colDesc = new HColumnDescriptor(FAMILY1); + index.addIndexColumn(colDesc, COL1, ValueType.String, 10); + indices.add(index); + HRegion region = + initHRegion(tableName.getBytes(), null, null, + "testSingleIndexExpressionWithOneEqualsExpression", TEST_UTIL.getConfiguration(), FAMILY1); + IndexRegionScanner scanner = evaluator.evaluate(scan, indices, new byte[0], region, tableName); + // TODO add assertions + } + + @Test + public void testSingleIndexExpressionWithMoreEqualsExpsAndOneRangeExp() throws Exception { + String indexName = "idx1"; + SingleIndexExpression singleIndexExpression = new SingleIndexExpression(indexName); + byte[] value1 = "1".getBytes(); + byte[] value2 = Bytes.toBytes(1234); + Column column = new Column(FAMILY1, QUALIFIER1); + EqualsExpression equalsExpression = new EqualsExpression(column, value1); + singleIndexExpression.addEqualsExpression(equalsExpression); + column = new Column(FAMILY1, QUALIFIER2); + equalsExpression = new EqualsExpression(column, value2); + singleIndexExpression.addEqualsExpression(equalsExpression); + column = new Column(FAMILY1, QUALIFIER3); + byte[] value3_1 = Bytes.toBytes(10.4F); + byte[] value3_2 = Bytes.toBytes(16.91F); + RangeExpression re = new RangeExpression(column, value3_1, value3_2, true, false); + singleIndexExpression.setRangeExpression(re); + + Scan scan = new Scan(); + scan.setAttribute(Constants.INDEX_EXPRESSION, IndexUtils.toBytes(singleIndexExpression)); + FilterList fl = new FilterList(Operator.MUST_PASS_ALL); + Filter filter = new SingleColumnValueFilter(FAMILY1, QUALIFIER1, CompareOp.EQUAL, value1); + fl.addFilter(filter); + filter = new SingleColumnValueFilter(FAMILY1, QUALIFIER2, CompareOp.EQUAL, value2); + fl.addFilter(filter); + filter = new SingleColumnValueFilter(FAMILY1, QUALIFIER3, CompareOp.GREATER_OR_EQUAL, value3_1); + fl.addFilter(filter); + filter = new SingleColumnValueFilter(FAMILY1, QUALIFIER3, CompareOp.LESS, value3_2); + fl.addFilter(filter); + scan.setFilter(fl); + + ScanFilterEvaluator evaluator = new ScanFilterEvaluator(); + List indices = new ArrayList(); + IndexSpecification index = new IndexSpecification(indexName); + HColumnDescriptor colDesc = new HColumnDescriptor(FAMILY1); + index.addIndexColumn(colDesc, COL1, ValueType.String, 10); + index.addIndexColumn(colDesc, COL2, ValueType.Int, 4); + index.addIndexColumn(colDesc, COL3, ValueType.Float, 4); + indices.add(index); + + HRegion region = + initHRegion(tableName.getBytes(), null, null, + "testSingleIndexExpressionWithMoreEqualsExpsAndOneRangeExp", + TEST_UTIL.getConfiguration(), FAMILY1); + IndexRegionScanner scanner = evaluator.evaluate(scan, indices, new byte[0], region, tableName); + // TODO add assertions + } + + @Test + public void testMultiIndexExpression() throws Exception { + MultiIndexExpression multiIndexExpression = new MultiIndexExpression(GroupingCondition.AND); + String index1 = "idx1"; + SingleIndexExpression singleIndexExpression = new SingleIndexExpression(index1); + byte[] value2 = Bytes.toBytes(1234); + Column column = new Column(FAMILY1, QUALIFIER2); + EqualsExpression equalsExpression = new EqualsExpression(column, value2); + singleIndexExpression.addEqualsExpression(equalsExpression); + column = new Column(FAMILY1, QUALIFIER3); + byte[] value3_1 = Bytes.toBytes(10.4F); + byte[] value3_2 = Bytes.toBytes(16.91F); + RangeExpression re = new RangeExpression(column, value3_1, value3_2, true, false); + singleIndexExpression.setRangeExpression(re); + multiIndexExpression.addIndexExpression(singleIndexExpression); + + MultiIndexExpression multiIndexExpression2 = new MultiIndexExpression(GroupingCondition.OR); + String index2 = "idx2"; + singleIndexExpression = new SingleIndexExpression(index2); + byte[] value1 = Bytes.toBytes("asdf"); + column = new Column(FAMILY1, QUALIFIER1); + equalsExpression = new EqualsExpression(column, value1); + singleIndexExpression.addEqualsExpression(equalsExpression); + multiIndexExpression2.addIndexExpression(singleIndexExpression); + + String index3 = "idx3"; + singleIndexExpression = new SingleIndexExpression(index3); + byte[] value4 = Bytes.toBytes(567.009D); + column = new Column(FAMILY2, QUALIFIER1); + equalsExpression = new EqualsExpression(column, value4); + singleIndexExpression.addEqualsExpression(equalsExpression); + multiIndexExpression2.addIndexExpression(singleIndexExpression); + + multiIndexExpression.addIndexExpression(multiIndexExpression2); + + Scan scan = new Scan(); + scan.setAttribute(Constants.INDEX_EXPRESSION, IndexUtils.toBytes(multiIndexExpression)); + FilterList outerFL = new FilterList(Operator.MUST_PASS_ALL); + FilterList fl = new FilterList(Operator.MUST_PASS_ALL); + Filter filter = new SingleColumnValueFilter(FAMILY1, QUALIFIER2, CompareOp.EQUAL, value2); + fl.addFilter(filter); + filter = new SingleColumnValueFilter(FAMILY1, QUALIFIER3, CompareOp.GREATER_OR_EQUAL, value3_1); + fl.addFilter(filter); + filter = new SingleColumnValueFilter(FAMILY1, QUALIFIER3, CompareOp.LESS, value3_2); + fl.addFilter(filter); + outerFL.addFilter(fl); + FilterList innerFL = new FilterList(Operator.MUST_PASS_ONE); + innerFL.addFilter(new SingleColumnValueFilter(FAMILY1, QUALIFIER1, CompareOp.EQUAL, value1)); + innerFL.addFilter(new SingleColumnValueFilter(FAMILY2, QUALIFIER1, CompareOp.EQUAL, value4)); + outerFL.addFilter(innerFL); + scan.setFilter(outerFL); + + ScanFilterEvaluator evaluator = new ScanFilterEvaluator(); + List indices = new ArrayList(); + IndexSpecification is1 = new IndexSpecification(index1); + HColumnDescriptor colDesc = new HColumnDescriptor(FAMILY1); + is1.addIndexColumn(colDesc, COL2, ValueType.Int, 4); + is1.addIndexColumn(colDesc, COL3, ValueType.Float, 4); + indices.add(is1); + IndexSpecification is2 = new IndexSpecification(index2); + is2.addIndexColumn(colDesc, COL1, ValueType.String, 15); + indices.add(is2); + IndexSpecification is3 = new IndexSpecification(index3); + colDesc = new HColumnDescriptor(FAMILY2); + is3.addIndexColumn(colDesc, COL1, ValueType.Double, 8); + indices.add(is3); + + HRegion region = + initHRegion(tableName.getBytes(), null, null, "testMultiIndexExpression", + TEST_UTIL.getConfiguration(), FAMILY1); + IndexRegionScanner scanner = evaluator.evaluate(scan, indices, new byte[0], region, tableName); + // TODO add assertions + } + + @Test + public void testNoIndexExpression() throws Exception { + IndexExpression exp = new NoIndexExpression(); + Scan scan = new Scan(); + scan.setAttribute(Constants.INDEX_EXPRESSION, IndexUtils.toBytes(exp)); + byte[] value1 = Bytes.toBytes("asdf"); + scan.setFilter(new SingleColumnValueFilter(FAMILY1, QUALIFIER1, CompareOp.EQUAL, value1)); + List indices = new ArrayList(); + IndexSpecification is1 = new IndexSpecification("idx1"); + HColumnDescriptor colDesc = new HColumnDescriptor(FAMILY1); + is1.addIndexColumn(colDesc, COL1, ValueType.String, 15); + indices.add(is1); + ScanFilterEvaluator evaluator = new ScanFilterEvaluator(); + HRegion region = + initHRegion(tableName.getBytes(), null, null, "testNoIndexExpression", + TEST_UTIL.getConfiguration(), FAMILY1); + IndexRegionScanner scanner = evaluator.evaluate(scan, indices, new byte[0], region, tableName); + assertNull(scanner); + } + + private static HRegion initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey, + String callingMethod, Configuration conf, byte[]... families) throws IOException { + HTableDescriptor htd = new HTableDescriptor(tableName); + for (byte[] family : families) { + htd.addFamily(new HColumnDescriptor(family)); + } + HRegionInfo info = new HRegionInfo(htd.getTableName(), startKey, stopKey, false); + Path path = new Path(DIR + callingMethod); + FileSystem fs = FileSystem.get(conf); + if (fs.exists(path)) { + if (!fs.delete(path, true)) { + throw new IOException("Failed delete of " + path); + } + } + return HRegion.createHRegion(info, path, conf, htd); + } +} Index: hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestScanWhenTTLExpired.java =================================================================== --- hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestScanWhenTTLExpired.java (revision 0) +++ hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestScanWhenTTLExpired.java (working copy) @@ -0,0 +1,353 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +import java.io.IOException; +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.MiniHBaseCluster; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; +import org.apache.hadoop.hbase.filter.Filter; +import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; +import org.apache.hadoop.hbase.index.ColumnQualifier.ValueType; +import org.apache.hadoop.hbase.index.Constants; +import org.apache.hadoop.hbase.index.IndexSpecification; +import org.apache.hadoop.hbase.index.SecIndexLoadBalancer; +import org.apache.hadoop.hbase.index.TableIndices; +import org.apache.hadoop.hbase.index.client.IndexAdmin; +import org.apache.hadoop.hbase.index.coprocessor.master.IndexMasterObserver; +import org.apache.hadoop.hbase.index.coprocessor.wal.IndexWALObserver; +import org.apache.hadoop.hbase.index.util.IndexUtils; +import org.apache.hadoop.hbase.master.LoadBalancer; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Threads; +import org.apache.zookeeper.KeeperException; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(LargeTests.class) +public class TestScanWhenTTLExpired { + + private static HBaseAdmin admin = null; + private MiniHBaseCluster cluster = null; + private static final int NB_SERVERS = 1; + private static final int TTL_SECONDS = 2; + private static final int TTL_MS = TTL_SECONDS * 1000; + + private static final HBaseTestingUtility TESTING_UTIL = new HBaseTestingUtility(); + + @BeforeClass + public static void before() throws Exception { + Configuration conf = TESTING_UTIL.getConfiguration(); + conf.setInt("hbase.balancer.period", 60000); + // Needed because some tests have splits happening on RS that are killed + // We don't want to wait 3min for the master to figure it out + conf.setInt("hbase.master.assignment.timeoutmonitor.timeout", 4000); + conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, IndexMasterObserver.class.getName()); + conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, IndexRegionObserver.class.getName()); + conf.set(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY, IndexWALObserver.class.getName()); + conf.setBoolean("hbase.use.secondary.index", true); + conf.setInt("hbase.hstore.compactionThreshold",5); + conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, SecIndexLoadBalancer.class, + LoadBalancer.class); + TESTING_UTIL.startMiniCluster(NB_SERVERS); + + } + + @Before + public void setup() throws IOException { + TESTING_UTIL.ensureSomeRegionServersAvailable(NB_SERVERS); + this.admin = new IndexAdmin(TESTING_UTIL.getConfiguration()); + this.cluster = TESTING_UTIL.getMiniHBaseCluster(); + } + + @AfterClass + public static void after() throws Exception { + if (admin != null) admin.close(); + TESTING_UTIL.shutdownMiniCluster(); + } + + + @Test(timeout = 180000) + public void testScannerSelectionWhenPutHasOneColumn() throws IOException, KeeperException, + InterruptedException { + String userTableName = "testScannerSelectionWhenPutHasOneColumn"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + + HColumnDescriptor hcd = + new HColumnDescriptor("col").setMaxVersions(Integer.MAX_VALUE).setTimeToLive(TTL_SECONDS); + IndexSpecification iSpec = new IndexSpecification("ScanIndexf"); + iSpec.addIndexColumn(hcd, "q1", ValueType.String, 10); + ihtd.addFamily(hcd); + TableIndices indices = new TableIndices(); + indices.addIndex(iSpec); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + + Configuration conf = TESTING_UTIL.getConfiguration(); + HTable table = new HTable(conf, userTableName); + // test put with the indexed column + Put p = new Put("row1".getBytes()); + p.add(Bytes.toBytes("col"), Bytes.toBytes("q1"), Bytes.toBytes("Val")); + table.put(p); + + Put p1 = new Put("row01".getBytes()); + p1.add(Bytes.toBytes("col"), Bytes.toBytes("q1"), Bytes.toBytes("Val")); + table.put(p1); + + Put p2 = new Put("row010".getBytes()); + p2.add(Bytes.toBytes("col"), Bytes.toBytes("q1"), Bytes.toBytes("Val")); + table.put(p2); + + Put p3 = new Put("row001".getBytes()); + p3.add(Bytes.toBytes("col"), Bytes.toBytes("q1"), Bytes.toBytes("Val")); + table.put(p3); + + admin.flush(userTableName); + + HRegionServer regionServer = TESTING_UTIL.getHBaseCluster().getRegionServer(0); + List onlineRegions = regionServer.getOnlineRegions(TableName.valueOf(userTableName)); + byte[][] columns = new byte[1][]; + columns[0] = hcd.getName(); + List storeFileList = + onlineRegions.get(0).getStoreFileList(columns); + + for (String store : storeFileList) { + Threads.sleepWithoutInterrupt(TTL_MS); + } + int i = countNumberOfRowsWithFilter(userTableName, "Val", true, false, 0); + assertEquals("No rows should be retrieved", 0, i); + } + + @Test(timeout = 180000) + public void testScannerSelectionWhenThereAreMutlipleCFs() throws IOException, KeeperException, + InterruptedException { + String userTableName = "testScannerSelectionWhenThereAreMutlipleCFs"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + + HColumnDescriptor hcd = + new HColumnDescriptor("col").setMaxVersions(Integer.MAX_VALUE).setTimeToLive( + Integer.MAX_VALUE); + HColumnDescriptor hcd1 = + new HColumnDescriptor("col1").setMaxVersions(Integer.MAX_VALUE).setTimeToLive( + TTL_SECONDS - 1); + IndexSpecification iSpec = new IndexSpecification("ScanIndexf"); + iSpec.addIndexColumn(hcd, "q1", ValueType.String, 10); + iSpec.addIndexColumn(hcd1, "q2", ValueType.String, 10); + ihtd.addFamily(hcd); + ihtd.addFamily(hcd1); + TableIndices indices = new TableIndices(); + indices.addIndex(iSpec); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + + Configuration conf = TESTING_UTIL.getConfiguration(); + HTable table = new HTable(conf, userTableName); + + // test put with the indexed column + + Put p = new Put("row1".getBytes()); + p.add(Bytes.toBytes("col"), Bytes.toBytes("q1"), Bytes.toBytes("Val")); + p.add("col1".getBytes(), "q2".getBytes(), Bytes.toBytes("ValForCF2")); + table.put(p); + + Put p1 = new Put("row01".getBytes()); + p1.add(Bytes.toBytes("col"), Bytes.toBytes("q1"), Bytes.toBytes("Val")); + p1.add("col1".getBytes(), "q2".getBytes(), Bytes.toBytes("ValForCF2")); + table.put(p1); + + Put p2 = new Put("row010".getBytes()); + p2.add(Bytes.toBytes("col"), Bytes.toBytes("q1"), Bytes.toBytes("Val")); + p2.add("col1".getBytes(), "q2".getBytes(), Bytes.toBytes("ValForCF2")); + table.put(p2); + + Put p3 = new Put("row001".getBytes()); + p3.add(Bytes.toBytes("col"), Bytes.toBytes("q1"), Bytes.toBytes("Val")); + p3.add("col1".getBytes(), "q2".getBytes(), Bytes.toBytes("ValForCF2")); + table.put(p3); + + admin.flush(userTableName); + + HRegionServer regionServer = TESTING_UTIL.getHBaseCluster().getRegionServer(0); + List onlineRegions = regionServer.getOnlineRegions(TableName.valueOf(userTableName)); + byte[][] columns = new byte[1][]; + columns[0] = hcd.getName(); + List storeFileList = + onlineRegions.get(0).getStoreFileList(columns); + + for (String store : storeFileList) { + Threads.sleepWithoutInterrupt(TTL_MS); + } + int i = countNumberOfRowsWithFilter(userTableName, "Val", true, false, 0); + assertEquals("No rows should be retrieved", 0, i); + + } + + @Test(timeout = 180000) + public void testCompactionOnIndexTableShouldNotRetrieveTTLExpiredData() throws Exception { + String userTableName = "testCompactionOnIndexTableShouldNotRetrieveTTLExpiredData"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + + HColumnDescriptor hcd = + new HColumnDescriptor("col").setMaxVersions(Integer.MAX_VALUE).setTimeToLive( + TTL_SECONDS - 1); + HColumnDescriptor hcd1 = + new HColumnDescriptor("col1").setMaxVersions(Integer.MAX_VALUE).setTimeToLive( + TTL_SECONDS - 1); + IndexSpecification iSpec = new IndexSpecification("ScanIndexf"); + iSpec.addIndexColumn(hcd, "q1", ValueType.String, 10); + iSpec.addIndexColumn(hcd1, "q2", ValueType.String, 10); + ihtd.addFamily(hcd); + ihtd.addFamily(hcd1); + TableIndices indices = new TableIndices(); + indices.addIndex(iSpec); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + + Configuration conf = TESTING_UTIL.getConfiguration(); + HTable table = new HTable(conf, userTableName); + + // test put with the indexed column + + Put p = new Put("row1".getBytes()); + p.add(Bytes.toBytes("col"), Bytes.toBytes("q1"), Bytes.toBytes("Val")); + p.add("col1".getBytes(), "q2".getBytes(), Bytes.toBytes("ValForCF2")); + table.put(p); + admin.flush(userTableName + "_idx"); + + Put p1 = new Put("row01".getBytes()); + p1.add(Bytes.toBytes("col"), Bytes.toBytes("q1"), Bytes.toBytes("Val")); + p1.add("col1".getBytes(), "q2".getBytes(), Bytes.toBytes("ValForCF2")); + table.put(p1); + admin.flush(userTableName + "_idx"); + + Put p2 = new Put("row010".getBytes()); + p2.add(Bytes.toBytes("col"), Bytes.toBytes("q1"), Bytes.toBytes("Val")); + p2.add("col1".getBytes(), "q2".getBytes(), Bytes.toBytes("ValForCF2")); + table.put(p2); + admin.flush(userTableName + "_idx"); + + Put p3 = new Put("row001".getBytes()); + p3.add(Bytes.toBytes("col"), Bytes.toBytes("q1"), Bytes.toBytes("Val")); + p3.add("col1".getBytes(), "q2".getBytes(), Bytes.toBytes("ValForCF2")); + table.put(p3); + + admin.flush(userTableName); + admin.flush(userTableName + "_idx"); + + HRegionServer regionServer = TESTING_UTIL.getHBaseCluster().getRegionServer(0); + List onlineRegions = regionServer.getOnlineRegions(TableName.valueOf(userTableName)); + byte[][] columns = new byte[1][]; + columns[0] = hcd.getName(); + List storeFileList = + onlineRegions.get(0).getStoreFileList(columns); + onlineRegions = + regionServer + .getOnlineRegions(TableName.valueOf(IndexUtils.getIndexTableName(userTableName))); + byte[][] indexColumns = new byte[1][]; + indexColumns[0] = Constants.IDX_COL_FAMILY; + storeFileList = onlineRegions.get(0).getStoreFileList(indexColumns); + while (storeFileList.size() < 4) { + Thread.sleep(1000); + storeFileList = onlineRegions.get(0).getStoreFileList(indexColumns); + } + int prevSize = storeFileList.size(); + assertEquals("The total store files for the index table should be 4", 4, prevSize); + Scan s = new Scan(); + HTable indexTable = new HTable(conf, userTableName + "_idx"); + ResultScanner scanner = indexTable.getScanner(s); + // Result res = scanner.next(); + for (Result result : scanner) { + System.out.println(result); + } + for (String store : storeFileList) { + Threads.sleepWithoutInterrupt(TTL_MS); + } + admin.majorCompact(userTableName + "_idx"); + + onlineRegions = + regionServer + .getOnlineRegions(TableName.valueOf(IndexUtils.getIndexTableName(userTableName))); + storeFileList = onlineRegions.get(0).getStoreFileList(indexColumns); + while (storeFileList.size() != 1) { + Thread.sleep(1000); + storeFileList = onlineRegions.get(0).getStoreFileList(indexColumns); + } + assertEquals("The total store files for the index table should be 1", 1, storeFileList.size()); + s = new Scan(); + indexTable = new HTable(conf, userTableName + "_idx"); + scanner = indexTable.getScanner(s); + // Result res = scanner.next(); + boolean dataAvailable = false; + for (Result result : scanner) { + dataAvailable = true; + System.out.println(result); + } + assertFalse("dataShould not be retrieved", dataAvailable); + + } + + private int countNumberOfRowsWithFilter(String tableName, String filterVal, boolean isIndexed, + boolean isCached, int cacheNumber) throws IOException { + Configuration conf = TESTING_UTIL.getConfiguration(); + HTable table = new HTable(conf, tableName); + Scan s = new Scan(); + Filter filter = null; + if (isIndexed) { + filter = + new SingleColumnValueFilter(Bytes.toBytes("col"), Bytes.toBytes("q1"), CompareOp.EQUAL, + filterVal.getBytes()); + } else { + filter = + new SingleColumnValueFilter(Bytes.toBytes("col"), Bytes.toBytes("q1"), CompareOp.EQUAL, + "cat".getBytes()); + } + s.setFilter(filter); + if (isCached) { + s.setCaching(cacheNumber); + } + int i = 0; + ResultScanner scanner = table.getScanner(s); + for (Result result : scanner) { + i++; + } + return i; + } +} Index: hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/io/TestIndexHalfStoreFileReader.java =================================================================== --- hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/io/TestIndexHalfStoreFileReader.java (revision 0) +++ hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/io/TestIndexHalfStoreFileReader.java (working copy) @@ -0,0 +1,240 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.io; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.apache.hadoop.hbase.index.ColumnQualifier.ValueType; +import org.apache.hadoop.hbase.index.Constants; +import org.apache.hadoop.hbase.index.IndexSpecification; +import org.apache.hadoop.hbase.index.SecIndexLoadBalancer; +import org.apache.hadoop.hbase.index.TableIndices; +import org.apache.hadoop.hbase.index.client.IndexAdmin; +import org.apache.hadoop.hbase.index.coprocessor.master.IndexMasterObserver; +import org.apache.hadoop.hbase.index.coprocessor.regionserver.IndexRegionObserver; +import org.apache.hadoop.hbase.index.coprocessor.wal.IndexWALObserver; +import org.apache.hadoop.hbase.index.util.ByteArrayBuilder; +import org.apache.hadoop.hbase.index.util.IndexUtils; +import org.apache.hadoop.hbase.io.Reference; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; +import org.apache.hadoop.hbase.io.hfile.HFile; +import org.apache.hadoop.hbase.io.hfile.HFileContext; +import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder; +import org.apache.hadoop.hbase.io.hfile.HFileScanner; +import org.apache.hadoop.hbase.master.LoadBalancer; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestIndexHalfStoreFileReader { + + private static HBaseTestingUtility UTIL = new HBaseTestingUtility(); + private static HBaseAdmin admin = null; + private KeyValue seekToKeyVal; + private byte[] expectedRow; + + @BeforeClass + public static void setupBeforeClass() throws Exception { + Configuration conf = UTIL.getConfiguration(); + conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, IndexMasterObserver.class.getName()); + conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, IndexRegionObserver.class.getName()); + conf.set(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY, IndexWALObserver.class.getName()); + conf.setBoolean("hbase.use.secondary.index", true); + conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, SecIndexLoadBalancer.class, + LoadBalancer.class); + UTIL.startMiniCluster(1); + admin = new IndexAdmin(conf); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + if (admin != null) admin.close(); + UTIL.shutdownMiniCluster(); + } + + @Test + public void testIndexHalfStoreFileReaderWithSeekTo() throws Exception { + HBaseTestingUtility test_util = new HBaseTestingUtility(); + String root_dir = test_util.getDataTestDir("TestIndexHalfStoreFile").toString(); + Path p = new Path(root_dir, "test"); + Configuration conf = test_util.getConfiguration(); + FileSystem fs = FileSystem.get(conf); + CacheConfig cacheConf = new CacheConfig(conf); + HFileContext meta = new HFileContextBuilder().withBlockSize(1024).build(); + + HFile.Writer w = + HFile.getWriterFactory(conf, cacheConf).withPath(fs, p).withFileContext(meta) + .withComparator(KeyValue.COMPARATOR).create(); + String usertableName = "testIndexHalfStore"; + List items = genSomeKeys(usertableName); + for (KeyValue kv : items) { + w.append(kv); + } + w.close(); + HFile.Reader r = HFile.createReader(fs, p, cacheConf, conf); + r.loadFileInfo(); + byte[] midkey = "005".getBytes(); + Reference top = new Reference(midkey, Reference.Range.top); + doTestOfScanAndReseek(p, fs, top, cacheConf, conf); + r.close(); + } + + private void doTestOfScanAndReseek(Path p, FileSystem fs, Reference bottom, + CacheConfig cacheConf, Configuration conf) throws IOException { + final IndexHalfStoreFileReader halfreader = + new IndexHalfStoreFileReader(fs, p, cacheConf, bottom, conf); + halfreader.loadFileInfo(); + final HFileScanner scanner = halfreader.getScanner(false, false); + KeyValue getseekTorowKey3 = getSeekToRowKey(); + scanner.seekTo(getseekTorowKey3.getBuffer(), 8, 17); + boolean next = scanner.next(); + KeyValue keyValue = null; + if (next) { + keyValue = scanner.getKeyValue(); + } + byte[] expectedRow = getExpected(); + byte[] actualRow = keyValue.getRow(); + Assert.assertArrayEquals(expectedRow, actualRow); + halfreader.close(true); + } + + private List genSomeKeys(String userTableName) throws Exception { + List ret = new ArrayList(4); + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); + HColumnDescriptor hcd1 = new HColumnDescriptor("column1"); + HColumnDescriptor hcd2 = new HColumnDescriptor("column2"); + IndexSpecification iSpec1 = new IndexSpecification("Index"); + iSpec1.addIndexColumn(hcd1, "q", ValueType.String, 10); + iSpec1.addIndexColumn(hcd2, "q", ValueType.String, 10); + ihtd.addFamily(hcd1); + ihtd.addFamily(hcd2); + TableIndices indices = new TableIndices(); + indices.addIndex(iSpec1); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + ByteArrayBuilder indexColVal = ByteArrayBuilder.allocate(4); + indexColVal.put(Bytes.toBytes((short) 3)); + indexColVal.put(Bytes.toBytes((short) 32)); + + Put p1 = generatePuts("006".getBytes(), "05".getBytes()); + Put p2 = generatePuts("003".getBytes(), "06".getBytes()); + Put p3 = generatePuts("004".getBytes(), "06".getBytes()); + Put p4 = generatePuts("007".getBytes(), "06".getBytes()); + + byte[] seekToPut = + new byte[3 + 1 + IndexUtils.getMaxIndexNameLength() + 10 + "006".getBytes().length]; + System.arraycopy(p1.getRow(), 0, seekToPut, 0, p1.getRow().length); + byte[] seekToRow = "007".getBytes(); + System.arraycopy(seekToRow, 0, seekToPut, p1.getRow().length - 3, seekToRow.length); + System.arraycopy("005".getBytes(), 0, seekToPut, 0, 3); + setSeekToRowKey(seekToPut, indexColVal); + + byte[] expectedPut = + new byte[3 + 1 + IndexUtils.getMaxIndexNameLength() + 10 + "006".getBytes().length]; + System.arraycopy(p4.getRow(), 0, expectedPut, 0, p4.getRow().length); + // Copy first 3 bytes to splitKey since getKeyValue will replace the start key with splitKey. + // Just for assertion this is been added + System.arraycopy("005".getBytes(), 0, expectedPut, 0, 3); + setExpected(expectedPut); + + KeyValue kv = + new KeyValue(p1.getRow(), Constants.IDX_COL_FAMILY, Constants.IDX_COL_QUAL, 0, + indexColVal.array()); + ret.add(kv); + KeyValue kv1 = + new KeyValue(p2.getRow(), Constants.IDX_COL_FAMILY, Constants.IDX_COL_QUAL, 0, + indexColVal.array()); + ret.add(kv1); + KeyValue kv2 = + new KeyValue(p3.getRow(), Constants.IDX_COL_FAMILY, Constants.IDX_COL_QUAL, 0, + indexColVal.array()); + ret.add(kv2); + KeyValue kv3 = + new KeyValue(p4.getRow(), Constants.IDX_COL_FAMILY, Constants.IDX_COL_QUAL, 0, + indexColVal.array()); + ret.add(kv3); + return ret; + } + + private void setSeekToRowKey(byte[] seekTorowKey3, ByteArrayBuilder indexColVal) { + KeyValue kv = + new KeyValue(seekTorowKey3, Constants.IDX_COL_FAMILY, Constants.IDX_COL_QUAL, 0, + indexColVal.array()); + this.seekToKeyVal = kv; + } + + private KeyValue getSeekToRowKey() { + return this.seekToKeyVal; + } + + private void setExpected(byte[] expected) { + this.expectedRow = expected; + } + + private byte[] getExpected() { + return this.expectedRow; + } + + private Put generatePuts(byte[] rowKey, byte[] val) throws Exception { + Configuration conf = UTIL.getConfiguration(); + String usertableName = "testIndexHalfStore"; + HTable table = new HTable(conf, usertableName + "_idx"); + byte[] onebyte = new byte[1]; + String indexName = "Indexname"; + byte[] remIndex = new byte[IndexUtils.getMaxIndexNameLength() - indexName.length()]; + byte[] valPad = new byte[8]; + ByteArrayBuilder indexColVal = ByteArrayBuilder.allocate(4); + indexColVal.put(Bytes.toBytes((short) 3)); + indexColVal.put(Bytes.toBytes((short) 32)); + byte[] put = new byte[3 + 1 + IndexUtils.getMaxIndexNameLength() + 10 + rowKey.length]; + System.arraycopy("000".getBytes(), 0, put, 0, 3); + System.arraycopy(onebyte, 0, put, 3, onebyte.length); + System.arraycopy(indexName.getBytes(), 0, put, 3 + onebyte.length, indexName.length()); + System.arraycopy(remIndex, 0, put, 3 + onebyte.length + indexName.length(), remIndex.length); + System.arraycopy(val, 0, put, 3 + onebyte.length + indexName.length() + remIndex.length, + val.length); + System.arraycopy(valPad, 0, put, 3 + onebyte.length + indexName.length() + remIndex.length + + val.length, valPad.length); + System.arraycopy(rowKey, 0, put, 3 + onebyte.length + indexName.length() + remIndex.length + + val.length + valPad.length, rowKey.length); + Put p = new Put(put); + p.add(Constants.IDX_COL_FAMILY, Constants.IDX_COL_QUAL, 0, indexColVal.array()); + table.put(p); + return p; + } +} Index: hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/io/TestIndexHalfStoreFileReaderWithEncoding.java =================================================================== --- hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/io/TestIndexHalfStoreFileReaderWithEncoding.java (revision 0) +++ hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/io/TestIndexHalfStoreFileReaderWithEncoding.java (working copy) @@ -0,0 +1,124 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.io; + +import static org.junit.Assert.assertEquals; + +import java.util.Map.Entry; +import java.util.NavigableMap; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.apache.hadoop.hbase.index.SecIndexLoadBalancer; +import org.apache.hadoop.hbase.index.TestUtils; +import org.apache.hadoop.hbase.index.client.IndexAdmin; +import org.apache.hadoop.hbase.index.coprocessor.master.IndexMasterObserver; +import org.apache.hadoop.hbase.index.coprocessor.regionserver.IndexRegionObserver; +import org.apache.hadoop.hbase.index.coprocessor.wal.IndexWALObserver; +import org.apache.hadoop.hbase.master.LoadBalancer; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(LargeTests.class) +public class TestIndexHalfStoreFileReaderWithEncoding { + private static final String COL1 = "c1"; + private static final String CF1 = "cf1"; + private static HBaseTestingUtility UTIL = new HBaseTestingUtility(); + + @BeforeClass + public static void setupBeforeClass() throws Exception { + Configuration conf = UTIL.getConfiguration(); + conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, IndexMasterObserver.class.getName()); + conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, IndexRegionObserver.class.getName()); + conf.set(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY, IndexWALObserver.class.getName()); + conf.setBoolean("hbase.use.secondary.index", true); + conf.set("index.data.block.encoding.algo", "PREFIX"); + conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, SecIndexLoadBalancer.class, + LoadBalancer.class); + UTIL.startMiniCluster(1); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + UTIL.shutdownMiniCluster(); + } + + @Test(timeout = 180000) + public void testWithFastDiffEncoding() throws Exception { + HBaseAdmin admin = new IndexAdmin(UTIL.getConfiguration()); + String tableName = "testWithFastDiffEncoding"; + String idxTableName = "testWithFastDiffEncoding_idx"; + HTableDescriptor htd = TestUtils.createIndexedHTableDescriptor(tableName, CF1, "idx1", CF1, COL1); + admin.createTable(htd); + HTable ht = new HTable(UTIL.getConfiguration(), tableName); + HTable hti = new HTable(UTIL.getConfiguration(), idxTableName); + Put put = new Put("a".getBytes()); + put.add(CF1.getBytes(), COL1.getBytes(), "1".getBytes()); + ht.put(put); + put = new Put("d".getBytes()); + put.add(CF1.getBytes(), COL1.getBytes(), "1".getBytes()); + ht.put(put); + put = new Put("k".getBytes()); + put.add(CF1.getBytes(), COL1.getBytes(), "1".getBytes()); + ht.put(put); + put = new Put("z".getBytes()); + put.add(CF1.getBytes(), COL1.getBytes(), "1".getBytes()); + ht.put(put); + Delete delete = new Delete("z".getBytes()); + ht.delete(delete); + admin.flush(tableName); + admin.flush(idxTableName); + NavigableMap regionLocations = ht.getRegionLocations(); + byte[] regionName = null; + for (Entry e : regionLocations.entrySet()) { + regionName = e.getKey().getRegionName(); + break; + } + // Splitting the single region. + admin.split(regionName, "e".getBytes()); + // Sleeping so that the compaction can complete. + // Split will initiate a compaction. + Thread.sleep(5 * 1000); + Scan scan = new Scan(); + ResultScanner scanner = hti.getScanner(scan); + Result res = scanner.next(); + int count = 0; + while (res != null) { + count++; + res = scanner.next(); + } + assertEquals(3, count); + admin.close(); + } + +} Index: hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/manager/TestIndexManager.java =================================================================== --- hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/manager/TestIndexManager.java (revision 0) +++ hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/manager/TestIndexManager.java (working copy) @@ -0,0 +1,106 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.manager; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import junit.framework.TestCase; + +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.index.ColumnQualifier; +import org.apache.hadoop.hbase.index.IndexSpecification; +import org.junit.experimental.categories.Category; + +/** + * Test index manager functionality. + */ + +@Category(MediumTests.class) +public class TestIndexManager extends TestCase { + public void testAddIndexForTable() throws Exception { + + IndexManager im = IndexManager.getInstance(); + assertNotNull("Index Manager should not be null.", im); + + List indexList = new ArrayList(1); + IndexSpecification iSpec = new IndexSpecification("index_name"); + + iSpec.addIndexColumn(new HColumnDescriptor("cf"), "cq", null, 10); + indexList.add(iSpec); + im.addIndexForTable("index_name", indexList); + indexList = im.getIndicesForTable("index_name"); + assertEquals("Index name should be equal with actual value.", "index_name", indexList.get(0) + .getName()); + assertTrue("Column qualifier state mismatch.", + indexList.get(0).getIndexColumns().contains(new ColumnQualifier("cf", "cq", null, 10))); + + } + + public void testShouldNotThrowNPEIfValueTypeIsNull() throws Exception { + IndexManager im = IndexManager.getInstance(); + assertNotNull("Index Manager should not be null.", im); + + List indexList = new ArrayList(1); + IndexSpecification iSpec = new IndexSpecification("index_name"); + + iSpec.addIndexColumn(new HColumnDescriptor("cf"), "cq", null, 5); + indexList.add(iSpec); + im.addIndexForTable("index_name", indexList); + indexList = im.getIndicesForTable("index_name"); + + Set indexColumns = indexList.get(0).getIndexColumns(); + for (ColumnQualifier columnQualifier : indexColumns) { + assertNotNull(columnQualifier.getType()); + } + } + + public void testAddIndexForTableWhenStringAndValLengthIsZero() throws Exception { + IndexManager im = IndexManager.getInstance(); + assertNotNull("Index Manager should not be null.", im); + + List indexList = new ArrayList(1); + IndexSpecification iSpec = new IndexSpecification("index_name"); + + iSpec.addIndexColumn(new HColumnDescriptor("cf"), "cq", null, 0); + indexList.add(iSpec); + im.addIndexForTable("index_name", indexList); + indexList = im.getIndicesForTable("index_name"); + assertEquals("the total value length should be 2", 2, indexList.get(0).getTotalValueLength()); + } + + public void testRemoveIndicesForTable() throws Exception { + + IndexManager im = IndexManager.getInstance(); + assertNotNull("Index Manager should not be null.", im); + + List indexList = new ArrayList(1); + IndexSpecification iSpec = new IndexSpecification("index_name"); + + iSpec.addIndexColumn(new HColumnDescriptor("cf"), "cq", null, 10); + indexList.add(iSpec); + im.removeIndices("index_name"); + indexList = im.getIndicesForTable("index_name"); + assertNull("Index specification List should be null.", indexList); + + } + +} Index: hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/mapreduce/TestIndexImportTsv.java =================================================================== --- hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/mapreduce/TestIndexImportTsv.java (revision 0) +++ hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/mapreduce/TestIndexImportTsv.java (working copy) @@ -0,0 +1,353 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.mapreduce; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.UnsupportedEncodingException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.index.SecIndexLoadBalancer; +import org.apache.hadoop.hbase.mapreduce.ImportTsv.TsvParser; +import org.apache.hadoop.hbase.mapreduce.ImportTsv.TsvParser.BadTsvLineException; +import org.apache.hadoop.hbase.mapreduce.ImportTsv.TsvParser.ParsedLine; +import org.apache.hadoop.hbase.mapreduce.ImportTsv; +import org.apache.hadoop.hbase.master.LoadBalancer; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.util.GenericOptionsParser; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import com.google.common.base.Joiner; +import com.google.common.base.Splitter; +import com.google.common.collect.Iterables; + +@Category(MediumTests.class) +public class TestIndexImportTsv { + + @Test + public void testTsvParserSpecParsing() { + TsvParser parser; + + parser = new TsvParser("HBASE_ROW_KEY", "\t"); + assertNull(parser.getFamily(0)); + assertNull(parser.getQualifier(0)); + assertEquals(0, parser.getRowKeyColumnIndex()); + assertFalse(parser.hasTimestamp()); + + parser = new TsvParser("HBASE_ROW_KEY,col1:scol1", "\t"); + assertNull(parser.getFamily(0)); + assertNull(parser.getQualifier(0)); + assertBytesEquals(Bytes.toBytes("col1"), parser.getFamily(1)); + assertBytesEquals(Bytes.toBytes("scol1"), parser.getQualifier(1)); + assertEquals(0, parser.getRowKeyColumnIndex()); + assertFalse(parser.hasTimestamp()); + + parser = new TsvParser("HBASE_ROW_KEY,col1:scol1,col1:scol2", "\t"); + assertNull(parser.getFamily(0)); + assertNull(parser.getQualifier(0)); + assertBytesEquals(Bytes.toBytes("col1"), parser.getFamily(1)); + assertBytesEquals(Bytes.toBytes("scol1"), parser.getQualifier(1)); + assertBytesEquals(Bytes.toBytes("col1"), parser.getFamily(2)); + assertBytesEquals(Bytes.toBytes("scol2"), parser.getQualifier(2)); + assertEquals(0, parser.getRowKeyColumnIndex()); + assertFalse(parser.hasTimestamp()); + + parser = new TsvParser("HBASE_ROW_KEY,col1:scol1,HBASE_TS_KEY,col1:scol2", "\t"); + assertNull(parser.getFamily(0)); + assertNull(parser.getQualifier(0)); + assertBytesEquals(Bytes.toBytes("col1"), parser.getFamily(1)); + assertBytesEquals(Bytes.toBytes("scol1"), parser.getQualifier(1)); + assertBytesEquals(Bytes.toBytes("col1"), parser.getFamily(3)); + assertBytesEquals(Bytes.toBytes("scol2"), parser.getQualifier(3)); + assertEquals(0, parser.getRowKeyColumnIndex()); + assertTrue(parser.hasTimestamp()); + assertEquals(2, parser.getTimestampKeyColumnIndex()); + } + + @Test + public void testTsvParser() throws BadTsvLineException { + TsvParser parser = new TsvParser("col_a,col_b:qual,HBASE_ROW_KEY,col_d", "\t"); + assertBytesEquals(Bytes.toBytes("col_a"), parser.getFamily(0)); + assertBytesEquals(HConstants.EMPTY_BYTE_ARRAY, parser.getQualifier(0)); + assertBytesEquals(Bytes.toBytes("col_b"), parser.getFamily(1)); + assertBytesEquals(Bytes.toBytes("qual"), parser.getQualifier(1)); + assertNull(parser.getFamily(2)); + assertNull(parser.getQualifier(2)); + assertEquals(2, parser.getRowKeyColumnIndex()); + + assertEquals(ImportTsv.TsvParser.DEFAULT_TIMESTAMP_COLUMN_INDEX, + parser.getTimestampKeyColumnIndex()); + + byte[] line = Bytes.toBytes("val_a\tval_b\tval_c\tval_d"); + ParsedLine parsed = parser.parse(line, line.length); + checkParsing(parsed, Splitter.on("\t").split(Bytes.toString(line))); + } + + @Test + public void testTsvParserWithTimestamp() throws BadTsvLineException { + TsvParser parser = new TsvParser("HBASE_ROW_KEY,HBASE_TS_KEY,col_a,", "\t"); + assertNull(parser.getFamily(0)); + assertNull(parser.getQualifier(0)); + assertNull(parser.getFamily(1)); + assertNull(parser.getQualifier(1)); + assertBytesEquals(Bytes.toBytes("col_a"), parser.getFamily(2)); + assertBytesEquals(HConstants.EMPTY_BYTE_ARRAY, parser.getQualifier(2)); + assertEquals(0, parser.getRowKeyColumnIndex()); + assertEquals(1, parser.getTimestampKeyColumnIndex()); + + byte[] line = Bytes.toBytes("rowkey\t1234\tval_a"); + ParsedLine parsed = parser.parse(line, line.length); + assertEquals(1234l, parsed.getTimestamp(-1)); + checkParsing(parsed, Splitter.on("\t").split(Bytes.toString(line))); + } + + private void checkParsing(ParsedLine parsed, Iterable expected) { + ArrayList parsedCols = new ArrayList(); + for (int i = 0; i < parsed.getColumnCount(); i++) { + parsedCols.add(Bytes.toString(parsed.getLineBytes(), parsed.getColumnOffset(i), + parsed.getColumnLength(i))); + } + if (!Iterables.elementsEqual(parsedCols, expected)) { + fail("Expected: " + Joiner.on(",").join(expected) + "\n" + "Got:" + + Joiner.on(",").join(parsedCols)); + } + } + + private void assertBytesEquals(byte[] a, byte[] b) { + assertEquals(Bytes.toStringBinary(a), Bytes.toStringBinary(b)); + } + + /** + * Test cases that throw BadTsvLineException + */ + @Test(expected = BadTsvLineException.class) + public void testTsvParserBadTsvLineExcessiveColumns() throws BadTsvLineException { + TsvParser parser = new TsvParser("HBASE_ROW_KEY,col_a", "\t"); + byte[] line = Bytes.toBytes("val_a\tval_b\tval_c"); + parser.parse(line, line.length); + } + + @Test(expected = BadTsvLineException.class) + public void testTsvParserBadTsvLineZeroColumn() throws BadTsvLineException { + TsvParser parser = new TsvParser("HBASE_ROW_KEY,col_a", "\t"); + byte[] line = Bytes.toBytes(""); + parser.parse(line, line.length); + } + + @Test(expected = BadTsvLineException.class) + public void testTsvParserBadTsvLineOnlyKey() throws BadTsvLineException { + TsvParser parser = new TsvParser("HBASE_ROW_KEY,col_a", "\t"); + byte[] line = Bytes.toBytes("key_only"); + parser.parse(line, line.length); + } + + @Test(expected = BadTsvLineException.class) + public void testTsvParserBadTsvLineNoRowKey() throws BadTsvLineException { + TsvParser parser = new TsvParser("col_a,HBASE_ROW_KEY", "\t"); + byte[] line = Bytes.toBytes("only_cola_data_and_no_row_key"); + parser.parse(line, line.length); + } + + @Test(expected = BadTsvLineException.class) + public void testTsvParserInvalidTimestamp() throws BadTsvLineException { + TsvParser parser = new TsvParser("HBASE_ROW_KEY,HBASE_TS_KEY,col_a,", "\t"); + assertEquals(1, parser.getTimestampKeyColumnIndex()); + byte[] line = Bytes.toBytes("rowkey\ttimestamp\tval_a"); + ParsedLine parsed = parser.parse(line, line.length); + assertEquals(-1, parsed.getTimestamp(-1)); + checkParsing(parsed, Splitter.on("\t").split(Bytes.toString(line))); + } + + @Test(expected = BadTsvLineException.class) + public void testTsvParserNoTimestampValue() throws BadTsvLineException { + TsvParser parser = new TsvParser("HBASE_ROW_KEY,col_a,HBASE_TS_KEY", "\t"); + assertEquals(2, parser.getTimestampKeyColumnIndex()); + byte[] line = Bytes.toBytes("rowkey\tval_a"); + parser.parse(line, line.length); + } + + @Test + public void testMROnTable() throws Exception { + String TABLE_NAME = "TestTable"; + String FAMILY = "FAM"; + String INPUT_FILE = "InputFile.esv"; + + // Prepare the arguments required for the test. + String[] args = + new String[] { "-D" + ImportTsv.COLUMNS_CONF_KEY + "=HBASE_ROW_KEY,FAM:A,FAM:B", + "-D" + ImportTsv.SEPARATOR_CONF_KEY + "=\u001b", TABLE_NAME, INPUT_FILE }; + + doMROnTableTest(INPUT_FILE, FAMILY, TABLE_NAME, null, args, 1); + } + + @Test + public void testMROnTableWithTimestamp() throws Exception { + String TABLE_NAME = "TestTable"; + String FAMILY = "FAM"; + String INPUT_FILE = "InputFile1.csv"; + + // Prepare the arguments required for the test. + String[] args = + new String[] { + "-D" + ImportTsv.COLUMNS_CONF_KEY + "=HBASE_ROW_KEY,HBASE_TS_KEY,FAM:A,FAM:B", + "-D" + ImportTsv.SEPARATOR_CONF_KEY + "=,", TABLE_NAME, INPUT_FILE }; + + String data = "KEY,1234,VALUE1,VALUE2\n"; + doMROnTableTest(INPUT_FILE, FAMILY, TABLE_NAME, data, args, 1); + } + + @Test + public void testMROnTableWithCustomMapper() throws Exception { + String TABLE_NAME = "TestTable"; + String FAMILY = "FAM"; + String INPUT_FILE = "InputFile2.esv"; + + // Prepare the arguments required for the test. + String[] args = + new String[] { + "-D" + ImportTsv.MAPPER_CONF_KEY + + "=org.apache.hadoop.hbase.mapreduce.TsvImporterCustomTestMapper", TABLE_NAME, + INPUT_FILE }; + + doMROnTableTest(INPUT_FILE, FAMILY, TABLE_NAME, null, args, 3); + } + + private void doMROnTableTest(String inputFile, String family, String tableName, String data, + String[] args, int valueMultiplier) throws Exception { + + // Cluster + HBaseTestingUtility htu1 = new HBaseTestingUtility(); + htu1.getConfiguration().setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, + SecIndexLoadBalancer.class, LoadBalancer.class); + htu1.startMiniCluster(); + htu1.startMiniMapReduceCluster(); + + GenericOptionsParser opts = new GenericOptionsParser(htu1.getConfiguration(), args); + Configuration conf = opts.getConfiguration(); + args = opts.getRemainingArgs(); + + try { + + FileSystem fs = FileSystem.get(conf); + FSDataOutputStream op = fs.create(new Path(inputFile), true); + if (data == null) { + data = "KEY\u001bVALUE1\u001bVALUE2\n"; + } + op.write(Bytes.toBytes(data)); + op.close(); + + final byte[] FAM = Bytes.toBytes(family); + final byte[] TAB = Bytes.toBytes(tableName); + final byte[] QA = Bytes.toBytes("A"); + final byte[] QB = Bytes.toBytes("B"); + + if (conf.get(ImportTsv.BULK_OUTPUT_CONF_KEY) == null) { + HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(TAB)); + desc.addFamily(new HColumnDescriptor(FAM)); + new HBaseAdmin(conf).createTable(desc); + } + + IndexImportTsv.createHbaseAdmin(conf); + + Job job = IndexImportTsv.createSubmittableJob(conf, args); + job.waitForCompletion(false); + assertTrue(job.isSuccessful()); + + HTable table = new HTable(new Configuration(conf), TAB); + boolean verified = false; + long pause = conf.getLong("hbase.client.pause", 5 * 1000); + int numRetries = conf.getInt("hbase.client.retries.number", 5); + for (int i = 0; i < numRetries; i++) { + try { + Scan scan = new Scan(); + // Scan entire family. + scan.addFamily(FAM); + ResultScanner resScanner = table.getScanner(scan); + for (Result res : resScanner) { + assertTrue(res.size() == 2); + List kvs = res.list(); + assertEquals(toU8Str(kvs.get(0).getRow()), toU8Str(Bytes.toBytes("KEY"))); + assertEquals(toU8Str(kvs.get(1).getRow()), toU8Str(Bytes.toBytes("KEY"))); + assertEquals(toU8Str(kvs.get(0).getValue()), + toU8Str(Bytes.toBytes("VALUE" + valueMultiplier))); + assertEquals(toU8Str(kvs.get(1).getValue()), + toU8Str(Bytes.toBytes("VALUE" + 2 * valueMultiplier))); + // Only one result set is expected, so let it loop. + } + verified = true; + break; + } catch (NullPointerException e) { + // If here, a cell was empty. Presume its because updates came in + // after the scanner had been opened. Wait a while and retry. + } + try { + Thread.sleep(pause); + } catch (InterruptedException e) { + // continue + } + } + assertTrue(verified); + } finally { + htu1.shutdownMiniMapReduceCluster(); + htu1.shutdownMiniCluster(); + } + } + + @Test + public void testBulkOutputWithoutAnExistingTable() throws Exception { + String TABLE_NAME = "TestTable"; + String FAMILY = "FAM"; + String INPUT_FILE = "InputFile2.esv"; + + // Prepare the arguments required for the test. + String[] args = + new String[] { "-D" + ImportTsv.COLUMNS_CONF_KEY + "=HBASE_ROW_KEY,FAM:A,FAM:B", + "-D" + ImportTsv.SEPARATOR_CONF_KEY + "=\u001b", + "-D" + ImportTsv.BULK_OUTPUT_CONF_KEY + "=output", TABLE_NAME, INPUT_FILE }; + doMROnTableTest(INPUT_FILE, FAMILY, TABLE_NAME, null, args, 3); + } + + public static String toU8Str(byte[] bytes) throws UnsupportedEncodingException { + return Bytes.toString(bytes); + } + +} + Index: hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/mapreduce/TestIndexMapReduceUtil.java =================================================================== --- hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/mapreduce/TestIndexMapReduceUtil.java (revision 0) +++ hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/mapreduce/TestIndexMapReduceUtil.java (working copy) @@ -0,0 +1,227 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.mapreduce; + +import static org.junit.Assert.*; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; + + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionLocation; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.apache.hadoop.hbase.index.ColumnQualifier.ValueType; +import org.apache.hadoop.hbase.index.Constants; +import org.apache.hadoop.hbase.index.IndexSpecification; +import org.apache.hadoop.hbase.index.SecIndexLoadBalancer; +import org.apache.hadoop.hbase.index.TableIndices; +import org.apache.hadoop.hbase.index.TestUtils; +import org.apache.hadoop.hbase.index.client.IndexAdmin; +import org.apache.hadoop.hbase.index.coprocessor.master.IndexMasterObserver; +import org.apache.hadoop.hbase.index.coprocessor.regionserver.IndexRegionObserver; +import org.apache.hadoop.hbase.index.coprocessor.wal.IndexWALObserver; +import org.apache.hadoop.hbase.index.util.IndexUtils; +import org.apache.hadoop.hbase.mapreduce.TableInputFormat; +import org.apache.hadoop.hbase.master.LoadBalancer; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(LargeTests.class) +public class TestIndexMapReduceUtil { + + private static HBaseTestingUtility UTIL = new HBaseTestingUtility(); + private static HBaseAdmin admin; + private Configuration conf; + private String tableName; + + @BeforeClass + public static void setupBeforeClass() throws Exception { + Configuration conf = UTIL.getConfiguration(); + conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, IndexMasterObserver.class.getName()); + conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, IndexRegionObserver.class.getName()); + conf.set(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY, IndexWALObserver.class.getName()); + conf.setBoolean("hbase.use.secondary.index", true); + conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, SecIndexLoadBalancer.class, + LoadBalancer.class); + UTIL.startMiniCluster(1); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + if (admin != null) admin.close(); + UTIL.shutdownMiniCluster(); + } + + @Before + public void setUp() throws Exception { + conf = UTIL.getConfiguration(); + admin = new IndexAdmin(conf); + } + + @After + public void tearDown() throws Exception { + admin.disableTable(tableName); + admin.deleteTable(tableName); + } + + @Test(timeout = 180000) + public void testShouldAbleReturnTrueForIndexedTable() throws Exception { + tableName = "testShouldAbleReturnTrueForIndexedTable"; + HTableDescriptor ihtd = + TestUtils.createIndexedHTableDescriptor(tableName, "col", "ScanIndexf", "col", "ql"); + admin.createTable(ihtd); + assertTrue(IndexMapReduceUtil.isIndexedTable(tableName, conf)); + } + + @Test(timeout = 180000) + public void testShouldAbleReturnFalseForNonIndexedTable() throws Exception { + tableName = "testShouldAbleReturnFalseForNonIndexedTable"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(tableName)); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + ihtd.addFamily(hcd); + admin.createTable(ihtd); + assertFalse(IndexMapReduceUtil.isIndexedTable(tableName, conf)); + } + + @Test(timeout = 180000) + public void testShouldReturnStartKeyBesedOnTheRowKeyFromPreSplitRegion() throws Exception { + + tableName = "testShouldReturnStartKeyBesedOnTheRowKeyFromPreSplitRegion"; + HTable table = + UTIL.createTable(tableName.getBytes(), new byte[][] { "families".getBytes() }, 3, + "0".getBytes(), "9".getBytes(), 5); + + assertStartKey(conf, tableName, table, "3"); + assertStartKey(conf, tableName, table, "0"); + assertStartKey(conf, tableName, table, "25"); + assertStartKey(conf, tableName, table, "AAAAA123"); + assertStartKey(conf, tableName, table, "63"); + assertStartKey(conf, tableName, table, ""); + assertStartKey(conf, tableName, table, "9222"); + } + + @Test(timeout = 180000) + public void testShouldReturnStartKeyBesedOnTheRowKey() throws Exception { + + tableName = "testShouldReturnStartKeyBesedOnTheRowKey"; + HTable table = UTIL.createTable(tableName.getBytes(), new byte[][] { "families".getBytes() }); + + assertStartKey(conf, tableName, table, "3"); + assertStartKey(conf, tableName, table, "0"); + assertStartKey(conf, tableName, table, "25"); + assertStartKey(conf, tableName, table, "AAAAA123"); + assertStartKey(conf, tableName, table, ""); + } + + @Test(timeout = 180000) + public void testShouldFormIndexPutsAndIndexDeletes() throws Exception { + tableName = "testShouldFormIndexPutsAndIndexDeletes"; + HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(tableName)); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + IndexSpecification iSpec = new IndexSpecification("ScanIndexf"); + iSpec.addIndexColumn(hcd, "q1", ValueType.String, 10); + iSpec.addIndexColumn(hcd, "q2", ValueType.String, 10); + ihtd.addFamily(hcd); + admin.getConfiguration().set(TableInputFormat.INPUT_TABLE, tableName); + TableIndices indices = new TableIndices(); + indices.addIndex(iSpec); + ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); + admin.createTable(ihtd); + HTable mainTable = new HTable(conf, Bytes.toBytes(tableName)); + Put put = new Put(Bytes.toBytes("r1")); + put.add(hcd.getName(), Bytes.toBytes("q1"), Bytes.toBytes("v1")); + mainTable.put(put); + put = new Put(Bytes.toBytes("r2")); + put.add(hcd.getName(), Bytes.toBytes("q1"), Bytes.toBytes("v1")); + mainTable.put(put); + put = new Put(Bytes.toBytes("r3")); + put.add(hcd.getName(), Bytes.toBytes("q1"), Bytes.toBytes("v1")); + put.add(hcd.getName(), Bytes.toBytes("q2"), Bytes.toBytes("v2")); + mainTable.put(put); + put = new Put(Bytes.toBytes("r4")); + put.add(hcd.getName(), Bytes.toBytes("q1"), Bytes.toBytes("v1")); + mainTable.put(put); + put = new Put(Bytes.toBytes("r5")); + put.add(hcd.getName(), Bytes.toBytes("q1"), Bytes.toBytes("v1")); + mainTable.put(put); + mainTable.flushCommits(); + admin.flush(tableName); + Delete del = new Delete(Bytes.toBytes("r3")); + del.deleteFamily(hcd.getName()); + mainTable.delete(del); + HRegionLocation regionLocation = mainTable.getRegionLocation(del.getRow()); + List indexDeletes = new ArrayList(); + for (IndexSpecification index : indices.getIndices()) { + Delete indexDelete = + IndexUtils.prepareIndexDelete(del, index, regionLocation.getRegionInfo().getStartKey()); + if (indexDelete != null) { + indexDeletes.add(indexDelete); + } + } + assertTrue(indexDeletes.size() == 0); + admin.flush(tableName); + del = new Delete(Bytes.toBytes("r5")); + del.deleteColumns(hcd.getName(), Bytes.toBytes("q1")); + mainTable.delete(del); + indexDeletes = new ArrayList(); + for (IndexSpecification index : indices.getIndices()) { + Delete indexDelete = + IndexUtils.prepareIndexDelete(del, index, regionLocation.getRegionInfo().getStartKey()); + if (indexDelete != null) { + indexDeletes.add(indexDelete); + } + } + Map> familyMap = ((Delete) indexDeletes.get(0)).getFamilyCellMap(); + Set>> entrySet = familyMap.entrySet(); + for (Entry> entry : entrySet) { + List value = entry.getValue(); + assertTrue(!((KeyValue)value.get(0)).isDeleteFamily()); + } + + } + + private void assertStartKey(Configuration conf, String tableName, HTable table, String rowKey) + throws IOException { + byte[] startKey = IndexMapReduceUtil.getStartKey(conf, table.getStartKeys(), Bytes.toBytes(rowKey)); + assertEquals("Fetching wrong start key for " + rowKey, + Bytes.toString(table.getRegionLocation(rowKey).getRegionInfo().getStartKey()), + Bytes.toString(startKey)); + } +} Index: hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/utils/TestIndexUtils.java =================================================================== --- hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/utils/TestIndexUtils.java (revision 0) +++ hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/utils/TestIndexUtils.java (working copy) @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.utils; + +import static org.junit.Assert.assertEquals; + +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.index.util.IndexUtils; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestIndexUtils { + + @Test + public void testIncrementValue1() throws Exception { + byte[] val = new byte[] { 97, 97, 97, 97, 127 }; + byte[] incrementValue = IndexUtils.incrementValue(val, true); + assertEquals(Bytes.compareTo(incrementValue, val), 1); + } + + @Test + public void testIncrementValue2() throws Exception { + byte[] val = new byte[] { 1, 127, 127, 127, 127, 127 }; + byte[] incrementValue = IndexUtils.incrementValue(val, true); + assertEquals(Bytes.compareTo(incrementValue, val), 1); + } + + @Test + public void testIncrementValue3() throws Exception { + byte[] val = new byte[] { 127, 127, 127, 127, -128 }; + byte[] incrementValue = IndexUtils.incrementValue(val, true); + assertEquals(Bytes.compareTo(incrementValue, val), 1); + } + + @Test + public void testIncrementValue4() throws Exception { + byte[] val = new byte[] { -1, -1, -1, -1, -1 }; + byte[] incrementValue = IndexUtils.incrementValue(val, true); + assertEquals(Bytes.compareTo(incrementValue, new byte[] { 0, 0, 0, 0, 0 }), 0); + } + + @Test + public void testIncrementValue5() throws Exception { + byte[] val = new byte[] { 56, 57, 58, -1, 127 }; + byte[] incrementValue = IndexUtils.incrementValue(val, true); + assertEquals(Bytes.compareTo(incrementValue, new byte[] { 56, 57, 58, -1, -128 }), 0); + } + +} Index: hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/utils/TestSecIndexColocator.java =================================================================== --- hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/utils/TestSecIndexColocator.java (revision 0) +++ hbase-secondaryindex/src/test/java/org/apache/hadoop/hbase/index/utils/TestSecIndexColocator.java (working copy) @@ -0,0 +1,402 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.utils; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.catalog.MetaEditor; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.apache.hadoop.hbase.index.SecIndexLoadBalancer; +import org.apache.hadoop.hbase.index.TestUtils; +import org.apache.hadoop.hbase.index.client.IndexAdmin; +import org.apache.hadoop.hbase.index.coprocessor.master.IndexMasterObserver; +import org.apache.hadoop.hbase.index.coprocessor.regionserver.IndexRegionObserver; +import org.apache.hadoop.hbase.index.coprocessor.wal.IndexWALObserver; +import org.apache.hadoop.hbase.index.util.IndexUtils; +import org.apache.hadoop.hbase.index.util.SecondaryIndexColocator; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.master.LoadBalancer; +import org.apache.hadoop.hbase.master.MasterFileSystem; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.FSTableDescriptors; +import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread; +import org.apache.hadoop.hbase.zookeeper.ZKAssign; +import org.apache.hadoop.hbase.zookeeper.ZKTableReadOnly; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(LargeTests.class) +public class TestSecIndexColocator { + + private static HBaseTestingUtility UTIL = new HBaseTestingUtility(); + + private static HBaseAdmin admin = null; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + Configuration conf = UTIL.getConfiguration(); + conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, IndexMasterObserver.class.getName()); + conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, IndexRegionObserver.class.getName()); + conf.set(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY, IndexWALObserver.class.getName()); + conf.setBoolean("hbase.use.secondary.index", true); + conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, SecIndexLoadBalancer.class, + LoadBalancer.class); + UTIL.startMiniCluster(2); + admin = new IndexAdmin(conf); + } + + @AfterClass + public static void finishAfterClass() throws Exception { + if(admin != null) admin.close(); + UTIL.shutdownMiniCluster(); + } + + @Test(timeout = 180000) + public void testCoLocationFixing() throws Exception { + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + Configuration config = UTIL.getConfiguration(); + String userTableName = "testCoLocationFixing"; + HTableDescriptor ihtd = + TestUtils.createIndexedHTableDescriptor(userTableName, "cf1", "idx", "cf1", "q"); + char c = 'A'; + byte[][] splits = new byte[5][]; + for (int i = 0; i < 5; i++) { + byte[] b = { (byte) c }; + c++; + splits[i] = b; + } + admin.createTable(ihtd, splits); + + String userTableName1 = "testCoLocationFixing1"; + + ihtd = TestUtils.createIndexedHTableDescriptor(userTableName1, "cf1", "idx1", "cf1", "q"); + admin.createTable(ihtd, splits); + + String userTableName2 = "testCoLocationFixing2"; + ihtd = TestUtils.createIndexedHTableDescriptor(userTableName2, "cf1", "idx2", "cf1", "q"); + + admin.createTable(ihtd, splits); + TableName indexTableName = TableName.valueOf(IndexUtils.getIndexTableName(userTableName)); + List regions = UTIL.getMetaTableRows(indexTableName); + List regionsEncod = getEncodedNames(regions); + + List regions1 = UTIL.getMetaTableRows(indexTableName); + List regionsEncod1 = getEncodedNames(regions1); + + List regions2 = UTIL.getMetaTableRows(indexTableName); + List regionsEncod2 = getEncodedNames(regions2); + + for (int i = 0; i < 2; i++) { + admin.move(regionsEncod.get(i), null); + admin.move(regionsEncod1.get(i), null); + admin.move(regionsEncod2.get(i), null); + } + + ZKAssign.blockUntilNoRIT(zkw); + SecondaryIndexColocator colocator = new SecondaryIndexColocator(config); + colocator.setUp(); + boolean inconsistent = colocator.checkForCoLocationInconsistency(); + assertTrue("Inconsistency should be there before running the tool.", inconsistent); + colocator.fixCoLocationInconsistency(); + + ZKAssign.blockUntilNoRIT(zkw); + + colocator = new SecondaryIndexColocator(config); + colocator.setUp(); + inconsistent = colocator.checkForCoLocationInconsistency(); + assertFalse("No inconsistency should be there after running the tool", inconsistent); + } + + private List getEncodedNames(List regions) { + List regionsEncod = new ArrayList(); + for (byte[] r : regions) { + String rs = Bytes.toString(r); + int firstOcc = rs.indexOf('.'); + int lastOcc = rs.lastIndexOf('.'); + regionsEncod.add(Bytes.toBytes(rs.substring(firstOcc + 1, lastOcc))); + } + return regionsEncod; + } + + @Test(timeout = 180000) + public void testWhenUserTableIsDisabledButIndexTableIsInDisablingState() throws Exception { + String table = "testWhenUserTableIsDisabledButIndexTableIsInDisablingState"; + HTableDescriptor htd = + TestUtils.createIndexedHTableDescriptor(table, "cf", "index_name", "cf", "cq"); + byte[][] splits = new byte[10][]; + char c = 'A'; + for (int i = 0; i < 10; i++) { + byte[] b = { (byte) c }; + splits[i] = b; + c++; + } + admin.createTable(htd, splits); + admin.disableTable(table); + byte[][] splits2 = new byte[11][]; + c = 'A'; + splits2[0] = new byte[0]; + for (int i = 1; i < 11; i++) { + byte[] b = { (byte) c }; + splits2[i] = b; + c++; + } + + HMaster master = UTIL.getMiniHBaseCluster().getMasterThreads().get(0).getMaster(); + TableName tableName2 = TableName.valueOf("testWhenUserTableIsDisabledButIndexTableIsInDisablingState_idx"); + master.getAssignmentManager().getZKTable().setDisablingTable(tableName2); + + SecondaryIndexColocator colocator = new SecondaryIndexColocator(UTIL.getConfiguration()); + colocator.setUp(); + boolean inconsistent = colocator.checkForCoLocationInconsistency(); + assertTrue("The disabling table should be now disabled", + ZKTableReadOnly.isDisabledTable(HBaseTestingUtility.getZooKeeperWatcher(UTIL), tableName2)); + } + + @Test(timeout = 180000) + public void testWhenUserTableIsDisabledButIndexTableIsInEnabledState() throws Exception { + String table = "testWhenUserTableIsDisabledButIndexTableIsInEnabledState"; + HTableDescriptor htd = + TestUtils.createIndexedHTableDescriptor(table, "cf", "index_name", "cf", "cq"); + byte[][] splits = new byte[10][]; + char c = 'A'; + for (int i = 0; i < 10; i++) { + byte[] b = { (byte) c }; + splits[i] = b; + c++; + } + admin.createTable(htd, splits); + admin.disableTable(table); + admin.enableTable(IndexUtils.getIndexTableName(table)); + SecondaryIndexColocator colocator = new SecondaryIndexColocator(UTIL.getConfiguration()); + colocator.setUp(); + colocator.checkForCoLocationInconsistency(); + assertTrue( + "The enabled table should be now disabled", + ZKTableReadOnly.isDisabledTable(HBaseTestingUtility.getZooKeeperWatcher(UTIL), + TableName.valueOf(IndexUtils.getIndexTableName(table)))); + } + + @Test//(timeout = 180000) + public void testWhenAllUSerRegionsAreAssignedButNotSameForIndex() throws Exception { + String table = "testWhenAllUSerRegionsAreAssignedButNotSameForIndex"; + HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(table)); + htd.addFamily(new HColumnDescriptor(new String("cf"))); + byte[][] splits = new byte[10][]; + char c = 'A'; + for (int i = 0; i < 10; i++) { + byte[] b = { (byte) c }; + splits[i] = b; + c++; + } + admin.createTable(htd, splits); + ZKAssign.blockUntilNoRIT(HBaseTestingUtility.getZooKeeperWatcher(UTIL)); + + byte[][] splits2 = new byte[11][]; + c = 'A'; + splits2[0] = new byte[0]; + for (int i = 1; i < 11; i++) { + byte[] b = { (byte) c }; + splits2[i] = b; + c++; + } + + Configuration conf = UTIL.getConfiguration(); + HMaster master = UTIL.getMiniHBaseCluster().getMasterThreads().get(0).getMaster(); + String table2 = "testWhenAllUSerRegionsAreAssignedButNotSameForIndex_idx"; + TableName tableName2 = TableName.valueOf(table2); + HTableDescriptor htd2 = new HTableDescriptor(tableName2); + htd2.addFamily(new HColumnDescriptor(new String("cf1"))); + MasterFileSystem fileSystemManager = master.getMasterFileSystem(); + // 1. Create Table Descriptor + Path tempTableDir = FSUtils.getTableDir(fileSystemManager.getTempDir(), tableName2); + new FSTableDescriptors(conf).createTableDescriptorForTableDirectory( + tempTableDir, htd2, false); + Path tableDir = FSUtils.getTableDir(fileSystemManager.getRootDir(), tableName2); + if (!fileSystemManager.getFileSystem().rename(tempTableDir, tableDir)) { + throw new IOException("Unable to move table from temp=" + tempTableDir + + " to hbase root=" + tableDir); + } + List regionsInMeta = + UTIL.createMultiRegionsInMeta(UTIL.getConfiguration(), htd2, splits2); + List newRegions = new ArrayList(); + for (int i = 0; i < regionsInMeta.size() / 2; i++) { + admin.assign(regionsInMeta.get(i).getRegionName()); + } + ZKAssign.blockUntilNoRIT(HBaseTestingUtility.getZooKeeperWatcher(UTIL)); + master.getAssignmentManager().getZKTable().setEnabledTable(tableName2); + + SecondaryIndexColocator colocator = new SecondaryIndexColocator(UTIL.getConfiguration()); + colocator.setUp(); + colocator.checkForCoLocationInconsistency(); + List serverThreads = + UTIL.getMiniHBaseCluster().getLiveRegionServerThreads(); + List rs = new ArrayList(); + for (RegionServerThread regionServerThread : serverThreads) { + rs.add(regionServerThread.getRegionServer()); + } + + List onlineregions = new ArrayList(); + for (HRegionServer hrs : rs) { + List regions = hrs.getOnlineRegions(tableName2); + for (HRegion region : regions) { + onlineregions.add(region.getRegionInfo()); + } + } + + boolean regionOffline = false; + for (HRegionInfo hri : newRegions) { + if (!onlineregions.contains(hri)) { + regionOffline = true; + break; + } + } + assertFalse("All region from the index Table should be online.", regionOffline); + } + + @Test(timeout = 180000) + public void testWhenUserTableIsEabledButIndexTableIsDisabled() throws Exception { + String table = "testWhenUserTableIsEabledButIndexTableIsDisabled"; + HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(table)); + htd.addFamily(new HColumnDescriptor(new String("cf"))); + byte[][] splits = new byte[10][]; + char c = 'A'; + for (int i = 0; i < 10; i++) { + byte[] b = { (byte) c }; + splits[i] = b; + c++; + } + admin.createTable(htd, splits); + ZKAssign.blockUntilNoRIT(HBaseTestingUtility.getZooKeeperWatcher(UTIL)); + + String table2 = "testWhenUserTableIsEabledButIndexTableIsDisabled_idx"; + TableName tableName2 = TableName.valueOf(table2); + HTableDescriptor htd2 = new HTableDescriptor(tableName2); + htd2.addFamily(new HColumnDescriptor(new String("cf"))); + admin.createTable(htd2, splits); + ZKAssign.blockUntilNoRIT(HBaseTestingUtility.getZooKeeperWatcher(UTIL)); + admin.disableTable(table2); + ZKAssign.blockUntilNoRIT(HBaseTestingUtility.getZooKeeperWatcher(UTIL)); + + List tableRegions = admin.getTableRegions(Bytes.toBytes(table2)); + SecondaryIndexColocator colocator = new SecondaryIndexColocator(UTIL.getConfiguration()); + colocator.setUp(); + boolean inconsistent = colocator.checkForCoLocationInconsistency(); + List serverThreads = + UTIL.getMiniHBaseCluster().getLiveRegionServerThreads(); + + List rs = new ArrayList(); + for (RegionServerThread regionServerThread : serverThreads) { + rs.add(regionServerThread.getRegionServer()); + } + + List onlineregions = new ArrayList(); + for (HRegionServer hrs : rs) { + List regions = hrs.getOnlineRegions(tableName2); + for (HRegion region : regions) { + onlineregions.add(region.getRegionInfo()); + } + } + + boolean regionOffline = false; + for (HRegionInfo hri : tableRegions) { + if (!onlineregions.contains(hri)) { + regionOffline = true; + break; + } + } + assertFalse("All region from the disabledTable should be online.", regionOffline); + } + + @Test(timeout = 180000) + public void testWhenRegionsAreNotAssignedAccordingToMeta() throws Exception { + String table = "testWhenRegionsAreNotAssignedAccordingToMeta"; + TableName tableName = TableName.valueOf(table); + HTableDescriptor htd = new HTableDescriptor(tableName); + htd.addFamily(new HColumnDescriptor(new String("cf"))); + byte[][] splits = new byte[10][]; + char c = 'A'; + for (int i = 0; i < 10; i++) { + byte[] b = { (byte) c }; + splits[i] = b; + c++; + } + admin.createTable(htd, splits); + + ServerName sn = ServerName.valueOf("example.org", 1234, 5678); + HMaster master = UTIL.getMiniHBaseCluster().getMaster(0); + + List tableRegions = admin.getTableRegions(Bytes.toBytes(table)); + List hRegions = UTIL.getMiniHBaseCluster().getRegions(Bytes.toBytes(table)); + for (int i = 0; i < 5; i++) { + MetaEditor.updateRegionLocation(master.getCatalogTracker(), tableRegions.get(i), sn, hRegions + .get(i).getOpenSeqNum()); + } + + SecondaryIndexColocator colocator = new SecondaryIndexColocator(UTIL.getConfiguration()); + colocator.setUp(); + colocator.checkForCoLocationInconsistency(); + + List serverThreads = + UTIL.getMiniHBaseCluster().getLiveRegionServerThreads(); + List rs = new ArrayList(); + for (RegionServerThread regionServerThread : serverThreads) { + rs.add(regionServerThread.getRegionServer()); + } + + List onlineregions = new ArrayList(); + for (HRegionServer hrs : rs) { + List regions = hrs.getOnlineRegions(tableName); + for (HRegion region : regions) { + onlineregions.add(region.getRegionInfo()); + } + } + + boolean regionOffline = false; + for (HRegionInfo hri : tableRegions) { + if (!onlineregions.contains(hri)) { + regionOffline = true; + break; + } + } + assertFalse("All the regions with wrong META info should be assiged to some online server.", + regionOffline); + } +} Index: hbase-secondaryindex/src/test/resources/log4j.properties =================================================================== --- hbase-secondaryindex/src/test/resources/log4j.properties (revision 0) +++ hbase-secondaryindex/src/test/resources/log4j.properties (working copy) @@ -0,0 +1,66 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Define some default values that can be overridden by system properties +hbase.root.logger=INFO,console +hbase.log.dir=. +hbase.log.file=hbase.log + +# Define the root logger to the system property "hbase.root.logger". +log4j.rootLogger=${hbase.root.logger} + +# Logging Threshold +log4j.threshhold=ALL + +# +# Daily Rolling File Appender +# +log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender +log4j.appender.DRFA.File=${hbase.log.dir}/${hbase.log.file} + +# Rollver at midnight +log4j.appender.DRFA.DatePattern=.yyyy-MM-dd + +# 30-day backup +#log4j.appender.DRFA.MaxBackupIndex=30 +log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout +# Debugging Pattern format +log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %C{2}(%L): %m%n + + +# +# console +# Add "console" to rootlogger above if you want to use this +# +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.err +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %C{2}(%L): %m%n + +# Custom Logging levels + +#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG + +log4j.logger.org.apache.hadoop=WARN +log4j.logger.org.apache.zookeeper=ERROR +log4j.logger.org.apache.hadoop.hbase=DEBUG + +#These two settings are workarounds against spurious logs from the minicluster. +#See HBASE-4709 +log4j.org.apache.hadoop.metrics2.impl.MetricsSystemImpl=ERROR +log4j.org.apache.hadoop.metrics2.util.MBeans=ERROR +# Enable this to get detailed connection error/retry logging. +# log4j.logger.org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation=TRACE Index: hbase-server/src/main/java/org/apache/hadoop/hbase/io/Reference.java =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/io/Reference.java (revision 1552923) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/io/Reference.java (working copy) @@ -63,7 +63,7 @@ * For split HStoreFiles, it specifies if the file covers the lower half or * the upper half of the key range */ - static enum Range { + public static enum Range { /** HStoreFile contains upper half of key range */ top, /** HStoreFile contains lower half of key range */ @@ -91,7 +91,7 @@ * @param splitRow This is row we are splitting around. * @param fr */ - Reference(final byte [] splitRow, final Range fr) { + public Reference(final byte [] splitRow, final Range fr) { this.splitkey = splitRow == null? null: KeyValue.createFirstOnRow(splitRow).getKey(); this.region = fr; } Index: hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat.java =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat.java (revision 1552923) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat.java (working copy) @@ -388,7 +388,7 @@ * @return a map from column family to the name of the configured compression * algorithm */ - static Map createFamilyCompressionMap(Configuration conf) { + public static Map createFamilyCompressionMap(Configuration conf) { return createFamilyConfValueMap(conf, COMPRESSION_CONF_KEY); } @@ -396,7 +396,7 @@ return createFamilyConfValueMap(conf, BLOOM_TYPE_CONF_KEY); } - private static Map createFamilyBlockSizeMap(Configuration conf) { + public static Map createFamilyBlockSizeMap(Configuration conf) { return createFamilyConfValueMap(conf, BLOCK_SIZE_CONF_KEY); } Index: hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java (revision 1552923) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java (working copy) @@ -83,7 +83,7 @@ public final static String COLUMNS_CONF_KEY = "importtsv.columns"; public final static String SEPARATOR_CONF_KEY = "importtsv.separator"; public final static String ATTRIBUTE_SEPERATOR_CONF_KEY = "attributes.seperator"; - final static String DEFAULT_SEPARATOR = "\t"; + public final static String DEFAULT_SEPARATOR = "\t"; final static String DEFAULT_ATTRIBUTES_SEPERATOR = "=>"; final static String DEFAULT_MULTIPLE_ATTRIBUTES_SEPERATOR = ","; final static Class DEFAULT_MAPPER = TsvImporterMapper.class; @@ -233,7 +233,7 @@ return new ParsedLine(tabOffsets, lineBytes); } - class ParsedLine { + public class ParsedLine { private final ArrayList tabOffsets; private byte[] lineBytes; Index: hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java (revision 1552923) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java (working copy) @@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.master; import java.io.IOException; +import java.lang.reflect.Method; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -65,6 +66,7 @@ import org.apache.hadoop.hbase.master.RegionState.State; import org.apache.hadoop.hbase.master.balancer.FavoredNodeAssignmentHelper; import org.apache.hadoop.hbase.master.balancer.FavoredNodeLoadBalancer; +import org.apache.hadoop.hbase.master.balancer.LoadBalancerFactory; import org.apache.hadoop.hbase.master.handler.ClosedRegionHandler; import org.apache.hadoop.hbase.master.handler.DisableTableHandler; import org.apache.hadoop.hbase.master.handler.EnableTableHandler; @@ -704,8 +706,9 @@ // Region is opened, insert into RIT and handle it // This could be done asynchronously, we would need then to acquire the lock in the // handler. - regionStates.updateRegionState(rt, State.OPEN); - new OpenedRegionHandler(server, this, regionInfo, sn, expectedVersion).process(); + regionStates.updateRegionState(rt, State.OPEN); + this.balancer.updateRegionLocation(regionInfo, sn); + new OpenedRegionHandler(server, this, regionInfo, sn, expectedVersion).process(); break; case RS_ZK_REQUEST_REGION_SPLIT: case RS_ZK_REGION_SPLITTING: @@ -1272,6 +1275,7 @@ clearRegionPlan(regionInfo); // Add the server to serversInUpdatingTimer addToServersInUpdatingTimer(sn); + this.balancer.updateRegionLocation(regionInfo, sn); } /** @@ -2570,7 +2574,7 @@ * @return True if nothing in regions in transition. * @throws InterruptedException */ - boolean waitUntilNoRegionsInTransition(final long timeout) + public boolean waitUntilNoRegionsInTransition(final long timeout) throws InterruptedException { // Blocks until there are no regions in transition. It is possible that // there @@ -2663,6 +2667,7 @@ if (!disabledOrEnablingTables.contains(tableName)) { regionStates.updateRegionState(regionInfo, State.OPEN, regionLocation); regionStates.regionOnline(regionInfo, regionLocation); + this.balancer.updateRegionLocation(regionInfo, regionLocation); } // need to enable the table if not disabled or disabling or enabling // this will be used in rolling restarts @@ -3489,6 +3494,7 @@ removeClosedRegion(regionInfo); // remove the region plan as well just in case. clearRegionPlan(regionInfo); + this.balancer.removeRegionLocation(regionInfo); } /** Index: hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java (revision 1552923) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java (working copy) @@ -595,4 +595,14 @@ LOG.info("Load Balancer stop requested: "+why); stopped = true; } + + @Override + public void updateRegionLocation(HRegionInfo hri, ServerName sn) { + + } + + @Override + public void removeRegionLocation(HRegionInfo hri) { + + } } Index: hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java (revision 1552923) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java (working copy) @@ -1413,6 +1413,10 @@ LOG.debug("Master has not been initialized, don't run balancer."); return false; } + return balanceInternals(); + } + + public boolean balanceInternals() throws HBaseIOException { // If balance not true, don't run balancer. if (!this.loadBalancerTracker.isBalancerOn()) return false; // Do this call outside of synchronized block. @@ -1661,7 +1665,7 @@ return mrr; } - void move(final byte[] encodedRegionName, + public void move(final byte[] encodedRegionName, final byte[] destServerName) throws HBaseIOException { RegionState regionState = assignmentManager.getRegionStates(). getRegionState(Bytes.toString(encodedRegionName)); Index: hbase-server/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java (revision 1552923) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java (working copy) @@ -117,4 +117,9 @@ * @throws HBaseIOException */ void initialize() throws HBaseIOException; + + void updateRegionLocation(HRegionInfo hri, ServerName sn); + + void removeRegionLocation(HRegionInfo hri); + } Index: hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java (revision 1552923) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java (working copy) @@ -2145,7 +2145,46 @@ return batchOp.retCodeDetails; } + /** + * Not to be used by external users. This is only for the IndexRegionObserver. + * @param mutationsAndLocks + * @return + * @throws IOException + */ + public OperationStatus[] batchMutateForIndex(Mutation[] mutations) + throws IOException { + BatchOperationInProgress batchOp = + new MutationBatch(mutations, HConstants.NO_NONCE, HConstants.NO_NONCE); + + boolean initialized = false; + + while (!batchOp.isDone()) { + long newSize; + + try { + if (coprocessorHost != null) { + coprocessorHost.postStartRegionOperation(Operation.BATCH_MUTATE); + } + if (!initialized) { + this.writeRequestsCount.increment(); + doPreMutationHook(batchOp); + initialized = true; + } + long addedSize = doMiniBatchMutation(batchOp); + newSize = this.addAndGetGlobalMemstoreSize(addedSize); + } finally { + if (coprocessorHost != null) { + coprocessorHost.postCloseRegionOperation(Operation.BATCH_MUTATE); + } + } + if (isFlushSize(newSize)) { + requestFlush(); + } + } + return batchOp.retCodeDetails; + } + private void doPreMutationHook(BatchOperationInProgress batchOp) throws IOException { /* Run coprocessor pre hook outside of locks to avoid deadlock */ @@ -2741,7 +2780,7 @@ * We throw RegionTooBusyException if above memstore limit * and expect client to retry using some kind of backoff */ - private void checkResources() + public void checkResources() throws RegionTooBusyException { // If catalog region, do not impose resource constraints or block updates. if (this.getRegionInfo().isMetaRegion()) return; @@ -5468,7 +5507,7 @@ return this.explicitSplitPoint; } - void forceSplit(byte[] sp) { + public void forceSplit(byte[] sp) { // NOTE : this HRegion will go away after the forced split is successfull // therefore, no reason to clear this value this.splitRequest = true; @@ -5577,7 +5616,7 @@ * @param op The operation is about to be taken on the region * @throws IOException */ - protected void startRegionOperation(Operation op) throws IOException { + public void startRegionOperation(Operation op) throws IOException { switch (op) { case INCREMENT: case APPEND: Index: hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java (revision 1552923) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java (working copy) @@ -523,29 +523,31 @@ // Check whether the split row lies in the range of the store file // If it is outside the range, return directly. - if (top) { - //check if larger than last key. - KeyValue splitKey = KeyValue.createFirstOnRow(splitRow); - byte[] lastKey = f.createReader().getLastKey(); - // If lastKey is null means storefile is empty. - if (lastKey == null) return null; - if (f.getReader().getComparator().compareFlatKey(splitKey.getBuffer(), + if (!isIndexTable()) { + if (top) { + //check if larger than last key. + KeyValue splitKey = KeyValue.createFirstOnRow(splitRow); + byte[] lastKey = f.createReader().getLastKey(); + // If lastKey is null means storefile is empty. + if (lastKey == null) return null; + if (f.getReader().getComparator().compareFlatKey(splitKey.getBuffer(), splitKey.getKeyOffset(), splitKey.getKeyLength(), lastKey, 0, lastKey.length) > 0) { - return null; + return null; + } + } else { + //check if smaller than first key + KeyValue splitKey = KeyValue.createLastOnRow(splitRow); + byte[] firstKey = f.createReader().getFirstKey(); + // If firstKey is null means storefile is empty. + if (firstKey == null) return null; + if (f.getReader().getComparator().compareFlatKey(splitKey.getBuffer(), + splitKey.getKeyOffset(), splitKey.getKeyLength(), firstKey, 0, firstKey.length) < 0) { + return null; + } } - } else { - //check if smaller than first key - KeyValue splitKey = KeyValue.createLastOnRow(splitRow); - byte[] firstKey = f.createReader().getFirstKey(); - // If firstKey is null means storefile is empty. - if (firstKey == null) return null; - if (f.getReader().getComparator().compareFlatKey(splitKey.getBuffer(), - splitKey.getKeyOffset(), splitKey.getKeyLength(), firstKey, 0, firstKey.length) < 0) { - return null; - } + + f.getReader().close(true); } - - f.getReader().close(true); Path splitDir = new Path(getSplitsDir(hri), familyName); // A reference to the bottom half of the hsf store file. @@ -562,6 +564,11 @@ return r.write(fs, p); } + private boolean isIndexTable() { + String tablePath = this.tableDir.getName(); + return tablePath.endsWith("_idx"); + } + // =========================================================================== // Merge Helpers // =========================================================================== Index: hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java (revision 1552923) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java (working copy) @@ -686,7 +686,7 @@ } } - private void splitStoreFiles(final Map> hstoreFilesToSplit) + public void splitStoreFiles(final Map> hstoreFilesToSplit) throws IOException { if (hstoreFilesToSplit == null) { // Could be null because close didn't succeed -- for now consider it fatal Index: hbase-server/src/test/java/org/apache/hadoop/hbase/MultithreadedTestUtil.java =================================================================== --- hbase-server/src/test/java/org/apache/hadoop/hbase/MultithreadedTestUtil.java (revision 1552923) +++ hbase-server/src/test/java/org/apache/hadoop/hbase/MultithreadedTestUtil.java (working copy) @@ -44,7 +44,7 @@ this.conf = configuration; } - protected Configuration getConf() { + public Configuration getConf() { return conf; } Index: pom.xml =================================================================== --- pom.xml (revision 1552923) +++ pom.xml (working copy) @@ -60,6 +60,7 @@ hbase-examples hbase-prefix-tree hbase-assembly + hbase-secondaryindex hbase-testing-util @@ -1025,6 +1026,18 @@ test + hbase-secondaryindex + org.apache.hbase + ${project.version} + + + hbase-secondaryindex + org.apache.hbase + ${project.version} + test-jar + test + + hbase-shell org.apache.hbase ${project.version}