diff --git hcatalog/storage-handlers/hbase/if/transaction.thrift hcatalog/storage-handlers/hbase/if/transaction.thrift index a9e697e..a74ac3d 100644 --- hcatalog/storage-handlers/hbase/if/transaction.thrift +++ hcatalog/storage-handlers/hbase/if/transaction.thrift @@ -17,7 +17,7 @@ * under the License. */ -namespace java org.apache.hive.hcatalog.hbase.snapshot.transaction.thrift +namespace java org.apache.hcatalog.hbase.snapshot.transaction.thrift namespace cpp Apache.HCatalog.HBase struct StoreFamilyRevision { diff --git hcatalog/storage-handlers/hbase/src/gen-java/org/apache/hcatalog/hbase/snapshot/transaction/thrift/StoreFamilyRevision.java hcatalog/storage-handlers/hbase/src/gen-java/org/apache/hcatalog/hbase/snapshot/transaction/thrift/StoreFamilyRevision.java new file mode 100644 index 0000000..a5d8213 --- /dev/null +++ hcatalog/storage-handlers/hbase/src/gen-java/org/apache/hcatalog/hbase/snapshot/transaction/thrift/StoreFamilyRevision.java @@ -0,0 +1,416 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * This class is used to store the revision and timestamp of a column family + * in a transaction. + * + */ +/** + * Autogenerated by Thrift Compiler (0.7.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + */ +package org.apache.hcatalog.hbase.snapshot.transaction.thrift; + +import java.util.Map; +import java.util.HashMap; +import java.util.EnumMap; +import java.util.EnumSet; +import java.util.Collections; +import java.util.BitSet; + +public class StoreFamilyRevision implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("StoreFamilyRevision"); + + private static final org.apache.thrift.protocol.TField REVISION_FIELD_DESC = new org.apache.thrift.protocol.TField("revision", org.apache.thrift.protocol.TType.I64, (short) 1); + private static final org.apache.thrift.protocol.TField TIMESTAMP_FIELD_DESC = new org.apache.thrift.protocol.TField("timestamp", org.apache.thrift.protocol.TType.I64, (short) 2); + + public long revision; // required + public long timestamp; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + REVISION((short) 1, "revision"), + TIMESTAMP((short) 2, "timestamp"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch (fieldId) { + case 1: // REVISION + return REVISION; + case 2: // TIMESTAMP + return TIMESTAMP; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __REVISION_ISSET_ID = 0; + private static final int __TIMESTAMP_ISSET_ID = 1; + private BitSet __isset_bit_vector = new BitSet(2); + + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.REVISION, new org.apache.thrift.meta_data.FieldMetaData("revision", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); + tmpMap.put(_Fields.TIMESTAMP, new org.apache.thrift.meta_data.FieldMetaData("timestamp", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(StoreFamilyRevision.class, metaDataMap); + } + + public StoreFamilyRevision() { + } + + public StoreFamilyRevision( + long revision, + long timestamp) { + this(); + this.revision = revision; + setRevisionIsSet(true); + this.timestamp = timestamp; + setTimestampIsSet(true); + } + + /** + * Performs a deep copy on other. + */ + public StoreFamilyRevision(StoreFamilyRevision other) { + __isset_bit_vector.clear(); + __isset_bit_vector.or(other.__isset_bit_vector); + this.revision = other.revision; + this.timestamp = other.timestamp; + } + + public StoreFamilyRevision deepCopy() { + return new StoreFamilyRevision(this); + } + + @Override + public void clear() { + setRevisionIsSet(false); + this.revision = 0; + setTimestampIsSet(false); + this.timestamp = 0; + } + + public long getRevision() { + return this.revision; + } + + public StoreFamilyRevision setRevision(long revision) { + this.revision = revision; + setRevisionIsSet(true); + return this; + } + + public void unsetRevision() { + __isset_bit_vector.clear(__REVISION_ISSET_ID); + } + + /** Returns true if field revision is set (has been assigned a value) and false otherwise */ + public boolean isSetRevision() { + return __isset_bit_vector.get(__REVISION_ISSET_ID); + } + + public void setRevisionIsSet(boolean value) { + __isset_bit_vector.set(__REVISION_ISSET_ID, value); + } + + public long getTimestamp() { + return this.timestamp; + } + + public StoreFamilyRevision setTimestamp(long timestamp) { + this.timestamp = timestamp; + setTimestampIsSet(true); + return this; + } + + public void unsetTimestamp() { + __isset_bit_vector.clear(__TIMESTAMP_ISSET_ID); + } + + /** Returns true if field timestamp is set (has been assigned a value) and false otherwise */ + public boolean isSetTimestamp() { + return __isset_bit_vector.get(__TIMESTAMP_ISSET_ID); + } + + public void setTimestampIsSet(boolean value) { + __isset_bit_vector.set(__TIMESTAMP_ISSET_ID, value); + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case REVISION: + if (value == null) { + unsetRevision(); + } else { + setRevision((Long) value); + } + break; + + case TIMESTAMP: + if (value == null) { + unsetTimestamp(); + } else { + setTimestamp((Long) value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case REVISION: + return Long.valueOf(getRevision()); + + case TIMESTAMP: + return Long.valueOf(getTimestamp()); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case REVISION: + return isSetRevision(); + case TIMESTAMP: + return isSetTimestamp(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof StoreFamilyRevision) + return this.equals((StoreFamilyRevision) that); + return false; + } + + public boolean equals(StoreFamilyRevision that) { + if (that == null) + return false; + + boolean this_present_revision = true; + boolean that_present_revision = true; + if (this_present_revision || that_present_revision) { + if (!(this_present_revision && that_present_revision)) + return false; + if (this.revision != that.revision) + return false; + } + + boolean this_present_timestamp = true; + boolean that_present_timestamp = true; + if (this_present_timestamp || that_present_timestamp) { + if (!(this_present_timestamp && that_present_timestamp)) + return false; + if (this.timestamp != that.timestamp) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(StoreFamilyRevision other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + StoreFamilyRevision typedOther = (StoreFamilyRevision) other; + + lastComparison = Boolean.valueOf(isSetRevision()).compareTo(typedOther.isSetRevision()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetRevision()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.revision, typedOther.revision); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetTimestamp()).compareTo(typedOther.isSetTimestamp()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTimestamp()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.timestamp, typedOther.timestamp); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField field; + iprot.readStructBegin(); + while (true) { + field = iprot.readFieldBegin(); + if (field.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (field.id) { + case 1: // REVISION + if (field.type == org.apache.thrift.protocol.TType.I64) { + this.revision = iprot.readI64(); + setRevisionIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type); + } + break; + case 2: // TIMESTAMP + if (field.type == org.apache.thrift.protocol.TType.I64) { + this.timestamp = iprot.readI64(); + setTimestampIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + validate(); + + oprot.writeStructBegin(STRUCT_DESC); + oprot.writeFieldBegin(REVISION_FIELD_DESC); + oprot.writeI64(this.revision); + oprot.writeFieldEnd(); + oprot.writeFieldBegin(TIMESTAMP_FIELD_DESC); + oprot.writeI64(this.timestamp); + oprot.writeFieldEnd(); + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("StoreFamilyRevision("); + boolean first = true; + + sb.append("revision:"); + sb.append(this.revision); + first = false; + if (!first) sb.append(", "); + sb.append("timestamp:"); + sb.append(this.timestamp); + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bit_vector = new BitSet(1); + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + +} + diff --git hcatalog/storage-handlers/hbase/src/gen-java/org/apache/hcatalog/hbase/snapshot/transaction/thrift/StoreFamilyRevisionList.java hcatalog/storage-handlers/hbase/src/gen-java/org/apache/hcatalog/hbase/snapshot/transaction/thrift/StoreFamilyRevisionList.java new file mode 100644 index 0000000..0f661cb --- /dev/null +++ hcatalog/storage-handlers/hbase/src/gen-java/org/apache/hcatalog/hbase/snapshot/transaction/thrift/StoreFamilyRevisionList.java @@ -0,0 +1,369 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * This class is used to store a list of StoreFamilyRevision for a column + * family in zookeeper. + * + */ +/** + * Autogenerated by Thrift Compiler (0.7.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + */ +package org.apache.hcatalog.hbase.snapshot.transaction.thrift; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.EnumMap; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class StoreFamilyRevisionList implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("StoreFamilyRevisionList"); + + private static final org.apache.thrift.protocol.TField REVISION_LIST_FIELD_DESC = new org.apache.thrift.protocol.TField("revisionList", org.apache.thrift.protocol.TType.LIST, (short) 1); + + public List revisionList; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + REVISION_LIST((short) 1, "revisionList"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch (fieldId) { + case 1: // REVISION_LIST + return REVISION_LIST; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.REVISION_LIST, new org.apache.thrift.meta_data.FieldMetaData("revisionList", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, StoreFamilyRevision.class)))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(StoreFamilyRevisionList.class, metaDataMap); + } + + public StoreFamilyRevisionList() { + } + + public StoreFamilyRevisionList( + List revisionList) { + this(); + this.revisionList = revisionList; + } + + /** + * Performs a deep copy on other. + */ + public StoreFamilyRevisionList(StoreFamilyRevisionList other) { + if (other.isSetRevisionList()) { + List __this__revisionList = new ArrayList(); + for (StoreFamilyRevision other_element : other.revisionList) { + __this__revisionList.add(new StoreFamilyRevision(other_element)); + } + this.revisionList = __this__revisionList; + } + } + + public StoreFamilyRevisionList deepCopy() { + return new StoreFamilyRevisionList(this); + } + + @Override + public void clear() { + this.revisionList = null; + } + + public int getRevisionListSize() { + return (this.revisionList == null) ? 0 : this.revisionList.size(); + } + + public java.util.Iterator getRevisionListIterator() { + return (this.revisionList == null) ? null : this.revisionList.iterator(); + } + + public void addToRevisionList(StoreFamilyRevision elem) { + if (this.revisionList == null) { + this.revisionList = new ArrayList(); + } + this.revisionList.add(elem); + } + + public List getRevisionList() { + return this.revisionList; + } + + public StoreFamilyRevisionList setRevisionList(List revisionList) { + this.revisionList = revisionList; + return this; + } + + public void unsetRevisionList() { + this.revisionList = null; + } + + /** Returns true if field revisionList is set (has been assigned a value) and false otherwise */ + public boolean isSetRevisionList() { + return this.revisionList != null; + } + + public void setRevisionListIsSet(boolean value) { + if (!value) { + this.revisionList = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case REVISION_LIST: + if (value == null) { + unsetRevisionList(); + } else { + setRevisionList((List) value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case REVISION_LIST: + return getRevisionList(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case REVISION_LIST: + return isSetRevisionList(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof StoreFamilyRevisionList) + return this.equals((StoreFamilyRevisionList) that); + return false; + } + + public boolean equals(StoreFamilyRevisionList that) { + if (that == null) + return false; + + boolean this_present_revisionList = true && this.isSetRevisionList(); + boolean that_present_revisionList = true && that.isSetRevisionList(); + if (this_present_revisionList || that_present_revisionList) { + if (!(this_present_revisionList && that_present_revisionList)) + return false; + if (!this.revisionList.equals(that.revisionList)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(StoreFamilyRevisionList other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + StoreFamilyRevisionList typedOther = (StoreFamilyRevisionList) other; + + lastComparison = Boolean.valueOf(isSetRevisionList()).compareTo(typedOther.isSetRevisionList()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetRevisionList()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.revisionList, typedOther.revisionList); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField field; + iprot.readStructBegin(); + while (true) { + field = iprot.readFieldBegin(); + if (field.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (field.id) { + case 1: // REVISION_LIST + if (field.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list0 = iprot.readListBegin(); + this.revisionList = new ArrayList(_list0.size); + for (int _i1 = 0; _i1 < _list0.size; ++_i1) { + StoreFamilyRevision _elem2; // required + _elem2 = new StoreFamilyRevision(); + _elem2.read(iprot); + this.revisionList.add(_elem2); + } + iprot.readListEnd(); + } + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (this.revisionList != null) { + oprot.writeFieldBegin(REVISION_LIST_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, this.revisionList.size())); + for (StoreFamilyRevision _iter3 : this.revisionList) { + _iter3.write(oprot); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("StoreFamilyRevisionList("); + boolean first = true; + + sb.append("revisionList:"); + if (this.revisionList == null) { + sb.append("null"); + } else { + sb.append(this.revisionList); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + +} + diff --git hcatalog/storage-handlers/hbase/src/gen-java/org/apache/hive/hcatalog/hbase/snapshot/transaction/thrift/StoreFamilyRevision.java hcatalog/storage-handlers/hbase/src/gen-java/org/apache/hive/hcatalog/hbase/snapshot/transaction/thrift/StoreFamilyRevision.java deleted file mode 100644 index 6c72030..0000000 --- hcatalog/storage-handlers/hbase/src/gen-java/org/apache/hive/hcatalog/hbase/snapshot/transaction/thrift/StoreFamilyRevision.java +++ /dev/null @@ -1,416 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/** - * This class is used to store the revision and timestamp of a column family - * in a transaction. - * - */ -/** - * Autogenerated by Thrift Compiler (0.7.0) - * - * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING - */ -package org.apache.hive.hcatalog.hbase.snapshot.transaction.thrift; - -import java.util.Map; -import java.util.HashMap; -import java.util.EnumMap; -import java.util.EnumSet; -import java.util.Collections; -import java.util.BitSet; - -public class StoreFamilyRevision implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { - private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("StoreFamilyRevision"); - - private static final org.apache.thrift.protocol.TField REVISION_FIELD_DESC = new org.apache.thrift.protocol.TField("revision", org.apache.thrift.protocol.TType.I64, (short) 1); - private static final org.apache.thrift.protocol.TField TIMESTAMP_FIELD_DESC = new org.apache.thrift.protocol.TField("timestamp", org.apache.thrift.protocol.TType.I64, (short) 2); - - public long revision; // required - public long timestamp; // required - - /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ - public enum _Fields implements org.apache.thrift.TFieldIdEnum { - REVISION((short) 1, "revision"), - TIMESTAMP((short) 2, "timestamp"); - - private static final Map byName = new HashMap(); - - static { - for (_Fields field : EnumSet.allOf(_Fields.class)) { - byName.put(field.getFieldName(), field); - } - } - - /** - * Find the _Fields constant that matches fieldId, or null if its not found. - */ - public static _Fields findByThriftId(int fieldId) { - switch (fieldId) { - case 1: // REVISION - return REVISION; - case 2: // TIMESTAMP - return TIMESTAMP; - default: - return null; - } - } - - /** - * Find the _Fields constant that matches fieldId, throwing an exception - * if it is not found. - */ - public static _Fields findByThriftIdOrThrow(int fieldId) { - _Fields fields = findByThriftId(fieldId); - if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); - return fields; - } - - /** - * Find the _Fields constant that matches name, or null if its not found. - */ - public static _Fields findByName(String name) { - return byName.get(name); - } - - private final short _thriftId; - private final String _fieldName; - - _Fields(short thriftId, String fieldName) { - _thriftId = thriftId; - _fieldName = fieldName; - } - - public short getThriftFieldId() { - return _thriftId; - } - - public String getFieldName() { - return _fieldName; - } - } - - // isset id assignments - private static final int __REVISION_ISSET_ID = 0; - private static final int __TIMESTAMP_ISSET_ID = 1; - private BitSet __isset_bit_vector = new BitSet(2); - - public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; - - static { - Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); - tmpMap.put(_Fields.REVISION, new org.apache.thrift.meta_data.FieldMetaData("revision", org.apache.thrift.TFieldRequirementType.DEFAULT, - new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); - tmpMap.put(_Fields.TIMESTAMP, new org.apache.thrift.meta_data.FieldMetaData("timestamp", org.apache.thrift.TFieldRequirementType.DEFAULT, - new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); - metaDataMap = Collections.unmodifiableMap(tmpMap); - org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(StoreFamilyRevision.class, metaDataMap); - } - - public StoreFamilyRevision() { - } - - public StoreFamilyRevision( - long revision, - long timestamp) { - this(); - this.revision = revision; - setRevisionIsSet(true); - this.timestamp = timestamp; - setTimestampIsSet(true); - } - - /** - * Performs a deep copy on other. - */ - public StoreFamilyRevision(StoreFamilyRevision other) { - __isset_bit_vector.clear(); - __isset_bit_vector.or(other.__isset_bit_vector); - this.revision = other.revision; - this.timestamp = other.timestamp; - } - - public StoreFamilyRevision deepCopy() { - return new StoreFamilyRevision(this); - } - - @Override - public void clear() { - setRevisionIsSet(false); - this.revision = 0; - setTimestampIsSet(false); - this.timestamp = 0; - } - - public long getRevision() { - return this.revision; - } - - public StoreFamilyRevision setRevision(long revision) { - this.revision = revision; - setRevisionIsSet(true); - return this; - } - - public void unsetRevision() { - __isset_bit_vector.clear(__REVISION_ISSET_ID); - } - - /** Returns true if field revision is set (has been assigned a value) and false otherwise */ - public boolean isSetRevision() { - return __isset_bit_vector.get(__REVISION_ISSET_ID); - } - - public void setRevisionIsSet(boolean value) { - __isset_bit_vector.set(__REVISION_ISSET_ID, value); - } - - public long getTimestamp() { - return this.timestamp; - } - - public StoreFamilyRevision setTimestamp(long timestamp) { - this.timestamp = timestamp; - setTimestampIsSet(true); - return this; - } - - public void unsetTimestamp() { - __isset_bit_vector.clear(__TIMESTAMP_ISSET_ID); - } - - /** Returns true if field timestamp is set (has been assigned a value) and false otherwise */ - public boolean isSetTimestamp() { - return __isset_bit_vector.get(__TIMESTAMP_ISSET_ID); - } - - public void setTimestampIsSet(boolean value) { - __isset_bit_vector.set(__TIMESTAMP_ISSET_ID, value); - } - - public void setFieldValue(_Fields field, Object value) { - switch (field) { - case REVISION: - if (value == null) { - unsetRevision(); - } else { - setRevision((Long) value); - } - break; - - case TIMESTAMP: - if (value == null) { - unsetTimestamp(); - } else { - setTimestamp((Long) value); - } - break; - - } - } - - public Object getFieldValue(_Fields field) { - switch (field) { - case REVISION: - return Long.valueOf(getRevision()); - - case TIMESTAMP: - return Long.valueOf(getTimestamp()); - - } - throw new IllegalStateException(); - } - - /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ - public boolean isSet(_Fields field) { - if (field == null) { - throw new IllegalArgumentException(); - } - - switch (field) { - case REVISION: - return isSetRevision(); - case TIMESTAMP: - return isSetTimestamp(); - } - throw new IllegalStateException(); - } - - @Override - public boolean equals(Object that) { - if (that == null) - return false; - if (that instanceof StoreFamilyRevision) - return this.equals((StoreFamilyRevision) that); - return false; - } - - public boolean equals(StoreFamilyRevision that) { - if (that == null) - return false; - - boolean this_present_revision = true; - boolean that_present_revision = true; - if (this_present_revision || that_present_revision) { - if (!(this_present_revision && that_present_revision)) - return false; - if (this.revision != that.revision) - return false; - } - - boolean this_present_timestamp = true; - boolean that_present_timestamp = true; - if (this_present_timestamp || that_present_timestamp) { - if (!(this_present_timestamp && that_present_timestamp)) - return false; - if (this.timestamp != that.timestamp) - return false; - } - - return true; - } - - @Override - public int hashCode() { - return 0; - } - - public int compareTo(StoreFamilyRevision other) { - if (!getClass().equals(other.getClass())) { - return getClass().getName().compareTo(other.getClass().getName()); - } - - int lastComparison = 0; - StoreFamilyRevision typedOther = (StoreFamilyRevision) other; - - lastComparison = Boolean.valueOf(isSetRevision()).compareTo(typedOther.isSetRevision()); - if (lastComparison != 0) { - return lastComparison; - } - if (isSetRevision()) { - lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.revision, typedOther.revision); - if (lastComparison != 0) { - return lastComparison; - } - } - lastComparison = Boolean.valueOf(isSetTimestamp()).compareTo(typedOther.isSetTimestamp()); - if (lastComparison != 0) { - return lastComparison; - } - if (isSetTimestamp()) { - lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.timestamp, typedOther.timestamp); - if (lastComparison != 0) { - return lastComparison; - } - } - return 0; - } - - public _Fields fieldForId(int fieldId) { - return _Fields.findByThriftId(fieldId); - } - - public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { - org.apache.thrift.protocol.TField field; - iprot.readStructBegin(); - while (true) { - field = iprot.readFieldBegin(); - if (field.type == org.apache.thrift.protocol.TType.STOP) { - break; - } - switch (field.id) { - case 1: // REVISION - if (field.type == org.apache.thrift.protocol.TType.I64) { - this.revision = iprot.readI64(); - setRevisionIsSet(true); - } else { - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type); - } - break; - case 2: // TIMESTAMP - if (field.type == org.apache.thrift.protocol.TType.I64) { - this.timestamp = iprot.readI64(); - setTimestampIsSet(true); - } else { - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type); - } - break; - default: - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type); - } - iprot.readFieldEnd(); - } - iprot.readStructEnd(); - - // check for required fields of primitive type, which can't be checked in the validate method - validate(); - } - - public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { - validate(); - - oprot.writeStructBegin(STRUCT_DESC); - oprot.writeFieldBegin(REVISION_FIELD_DESC); - oprot.writeI64(this.revision); - oprot.writeFieldEnd(); - oprot.writeFieldBegin(TIMESTAMP_FIELD_DESC); - oprot.writeI64(this.timestamp); - oprot.writeFieldEnd(); - oprot.writeFieldStop(); - oprot.writeStructEnd(); - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder("StoreFamilyRevision("); - boolean first = true; - - sb.append("revision:"); - sb.append(this.revision); - first = false; - if (!first) sb.append(", "); - sb.append("timestamp:"); - sb.append(this.timestamp); - first = false; - sb.append(")"); - return sb.toString(); - } - - public void validate() throws org.apache.thrift.TException { - // check for required fields - } - - private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { - try { - write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); - } catch (org.apache.thrift.TException te) { - throw new java.io.IOException(te); - } - } - - private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { - try { - // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. - __isset_bit_vector = new BitSet(1); - read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); - } catch (org.apache.thrift.TException te) { - throw new java.io.IOException(te); - } - } - -} - diff --git hcatalog/storage-handlers/hbase/src/gen-java/org/apache/hive/hcatalog/hbase/snapshot/transaction/thrift/StoreFamilyRevisionList.java hcatalog/storage-handlers/hbase/src/gen-java/org/apache/hive/hcatalog/hbase/snapshot/transaction/thrift/StoreFamilyRevisionList.java deleted file mode 100644 index 9b04a64..0000000 --- hcatalog/storage-handlers/hbase/src/gen-java/org/apache/hive/hcatalog/hbase/snapshot/transaction/thrift/StoreFamilyRevisionList.java +++ /dev/null @@ -1,369 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/** - * This class is used to store a list of StoreFamilyRevision for a column - * family in zookeeper. - * - */ -/** - * Autogenerated by Thrift Compiler (0.7.0) - * - * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING - */ -package org.apache.hive.hcatalog.hbase.snapshot.transaction.thrift; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.EnumMap; -import java.util.EnumSet; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public class StoreFamilyRevisionList implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { - private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("StoreFamilyRevisionList"); - - private static final org.apache.thrift.protocol.TField REVISION_LIST_FIELD_DESC = new org.apache.thrift.protocol.TField("revisionList", org.apache.thrift.protocol.TType.LIST, (short) 1); - - public List revisionList; // required - - /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ - public enum _Fields implements org.apache.thrift.TFieldIdEnum { - REVISION_LIST((short) 1, "revisionList"); - - private static final Map byName = new HashMap(); - - static { - for (_Fields field : EnumSet.allOf(_Fields.class)) { - byName.put(field.getFieldName(), field); - } - } - - /** - * Find the _Fields constant that matches fieldId, or null if its not found. - */ - public static _Fields findByThriftId(int fieldId) { - switch (fieldId) { - case 1: // REVISION_LIST - return REVISION_LIST; - default: - return null; - } - } - - /** - * Find the _Fields constant that matches fieldId, throwing an exception - * if it is not found. - */ - public static _Fields findByThriftIdOrThrow(int fieldId) { - _Fields fields = findByThriftId(fieldId); - if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); - return fields; - } - - /** - * Find the _Fields constant that matches name, or null if its not found. - */ - public static _Fields findByName(String name) { - return byName.get(name); - } - - private final short _thriftId; - private final String _fieldName; - - _Fields(short thriftId, String fieldName) { - _thriftId = thriftId; - _fieldName = fieldName; - } - - public short getThriftFieldId() { - return _thriftId; - } - - public String getFieldName() { - return _fieldName; - } - } - - // isset id assignments - - public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; - - static { - Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); - tmpMap.put(_Fields.REVISION_LIST, new org.apache.thrift.meta_data.FieldMetaData("revisionList", org.apache.thrift.TFieldRequirementType.DEFAULT, - new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, - new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, StoreFamilyRevision.class)))); - metaDataMap = Collections.unmodifiableMap(tmpMap); - org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(StoreFamilyRevisionList.class, metaDataMap); - } - - public StoreFamilyRevisionList() { - } - - public StoreFamilyRevisionList( - List revisionList) { - this(); - this.revisionList = revisionList; - } - - /** - * Performs a deep copy on other. - */ - public StoreFamilyRevisionList(StoreFamilyRevisionList other) { - if (other.isSetRevisionList()) { - List __this__revisionList = new ArrayList(); - for (StoreFamilyRevision other_element : other.revisionList) { - __this__revisionList.add(new StoreFamilyRevision(other_element)); - } - this.revisionList = __this__revisionList; - } - } - - public StoreFamilyRevisionList deepCopy() { - return new StoreFamilyRevisionList(this); - } - - @Override - public void clear() { - this.revisionList = null; - } - - public int getRevisionListSize() { - return (this.revisionList == null) ? 0 : this.revisionList.size(); - } - - public java.util.Iterator getRevisionListIterator() { - return (this.revisionList == null) ? null : this.revisionList.iterator(); - } - - public void addToRevisionList(StoreFamilyRevision elem) { - if (this.revisionList == null) { - this.revisionList = new ArrayList(); - } - this.revisionList.add(elem); - } - - public List getRevisionList() { - return this.revisionList; - } - - public StoreFamilyRevisionList setRevisionList(List revisionList) { - this.revisionList = revisionList; - return this; - } - - public void unsetRevisionList() { - this.revisionList = null; - } - - /** Returns true if field revisionList is set (has been assigned a value) and false otherwise */ - public boolean isSetRevisionList() { - return this.revisionList != null; - } - - public void setRevisionListIsSet(boolean value) { - if (!value) { - this.revisionList = null; - } - } - - public void setFieldValue(_Fields field, Object value) { - switch (field) { - case REVISION_LIST: - if (value == null) { - unsetRevisionList(); - } else { - setRevisionList((List) value); - } - break; - - } - } - - public Object getFieldValue(_Fields field) { - switch (field) { - case REVISION_LIST: - return getRevisionList(); - - } - throw new IllegalStateException(); - } - - /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ - public boolean isSet(_Fields field) { - if (field == null) { - throw new IllegalArgumentException(); - } - - switch (field) { - case REVISION_LIST: - return isSetRevisionList(); - } - throw new IllegalStateException(); - } - - @Override - public boolean equals(Object that) { - if (that == null) - return false; - if (that instanceof StoreFamilyRevisionList) - return this.equals((StoreFamilyRevisionList) that); - return false; - } - - public boolean equals(StoreFamilyRevisionList that) { - if (that == null) - return false; - - boolean this_present_revisionList = true && this.isSetRevisionList(); - boolean that_present_revisionList = true && that.isSetRevisionList(); - if (this_present_revisionList || that_present_revisionList) { - if (!(this_present_revisionList && that_present_revisionList)) - return false; - if (!this.revisionList.equals(that.revisionList)) - return false; - } - - return true; - } - - @Override - public int hashCode() { - return 0; - } - - public int compareTo(StoreFamilyRevisionList other) { - if (!getClass().equals(other.getClass())) { - return getClass().getName().compareTo(other.getClass().getName()); - } - - int lastComparison = 0; - StoreFamilyRevisionList typedOther = (StoreFamilyRevisionList) other; - - lastComparison = Boolean.valueOf(isSetRevisionList()).compareTo(typedOther.isSetRevisionList()); - if (lastComparison != 0) { - return lastComparison; - } - if (isSetRevisionList()) { - lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.revisionList, typedOther.revisionList); - if (lastComparison != 0) { - return lastComparison; - } - } - return 0; - } - - public _Fields fieldForId(int fieldId) { - return _Fields.findByThriftId(fieldId); - } - - public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { - org.apache.thrift.protocol.TField field; - iprot.readStructBegin(); - while (true) { - field = iprot.readFieldBegin(); - if (field.type == org.apache.thrift.protocol.TType.STOP) { - break; - } - switch (field.id) { - case 1: // REVISION_LIST - if (field.type == org.apache.thrift.protocol.TType.LIST) { - { - org.apache.thrift.protocol.TList _list0 = iprot.readListBegin(); - this.revisionList = new ArrayList(_list0.size); - for (int _i1 = 0; _i1 < _list0.size; ++_i1) { - StoreFamilyRevision _elem2; // required - _elem2 = new StoreFamilyRevision(); - _elem2.read(iprot); - this.revisionList.add(_elem2); - } - iprot.readListEnd(); - } - } else { - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type); - } - break; - default: - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type); - } - iprot.readFieldEnd(); - } - iprot.readStructEnd(); - - // check for required fields of primitive type, which can't be checked in the validate method - validate(); - } - - public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { - validate(); - - oprot.writeStructBegin(STRUCT_DESC); - if (this.revisionList != null) { - oprot.writeFieldBegin(REVISION_LIST_FIELD_DESC); - { - oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, this.revisionList.size())); - for (StoreFamilyRevision _iter3 : this.revisionList) { - _iter3.write(oprot); - } - oprot.writeListEnd(); - } - oprot.writeFieldEnd(); - } - oprot.writeFieldStop(); - oprot.writeStructEnd(); - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder("StoreFamilyRevisionList("); - boolean first = true; - - sb.append("revisionList:"); - if (this.revisionList == null) { - sb.append("null"); - } else { - sb.append(this.revisionList); - } - first = false; - sb.append(")"); - return sb.toString(); - } - - public void validate() throws org.apache.thrift.TException { - // check for required fields - } - - private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { - try { - write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); - } catch (org.apache.thrift.TException te) { - throw new java.io.IOException(te); - } - } - - private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { - try { - read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); - } catch (org.apache.thrift.TException te) { - throw new java.io.IOException(te); - } - } - -} - diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseAuthorizationProvider.java hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseAuthorizationProvider.java new file mode 100644 index 0000000..ee80389 --- /dev/null +++ hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseAuthorizationProvider.java @@ -0,0 +1,144 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hcatalog.hbase; + +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.ql.metadata.AuthorizationException; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.metadata.Partition; +import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.ql.security.HiveAuthenticationProvider; +import org.apache.hadoop.hive.ql.security.authorization.HiveAuthorizationProvider; +import org.apache.hadoop.hive.ql.security.authorization.Privilege; + +/** + * This class is an implementation of HiveAuthorizationProvider to provide + * authorization functionality for HBase tables. + */ +class HBaseAuthorizationProvider implements HiveAuthorizationProvider { + + @Override + public Configuration getConf() { + return null; + } + + @Override + public void setConf(Configuration conf) { + } + + /* + * (non-Javadoc) + * + * @see + * org.apache.hadoop.hive.ql.security.authorization.HiveAuthorizationProvider + * #init(org.apache.hadoop.conf.Configuration) + */ + @Override + public void init(Configuration conf) throws HiveException { + } + + @Override + public HiveAuthenticationProvider getAuthenticator() { + return null; + } + + @Override + public void setAuthenticator(HiveAuthenticationProvider authenticator) { + } + + /* + * (non-Javadoc) + * + * @see + * org.apache.hadoop.hive.ql.security.authorization.HiveAuthorizationProvider + * #authorize(org.apache.hadoop.hive.ql.security.authorization.Privilege[], + * org.apache.hadoop.hive.ql.security.authorization.Privilege[]) + */ + @Override + public void authorize(Privilege[] readRequiredPriv, + Privilege[] writeRequiredPriv) throws HiveException, + AuthorizationException { + } + + /* + * (non-Javadoc) + * + * @see + * org.apache.hadoop.hive.ql.security.authorization.HiveAuthorizationProvider + * #authorize(org.apache.hadoop.hive.metastore.api.Database, + * org.apache.hadoop.hive.ql.security.authorization.Privilege[], + * org.apache.hadoop.hive.ql.security.authorization.Privilege[]) + */ + @Override + public void authorize(Database db, Privilege[] readRequiredPriv, + Privilege[] writeRequiredPriv) throws HiveException, + AuthorizationException { + } + + /* + * (non-Javadoc) + * + * @see + * org.apache.hadoop.hive.ql.security.authorization.HiveAuthorizationProvider + * #authorize(org.apache.hadoop.hive.ql.metadata.Table, + * org.apache.hadoop.hive.ql.security.authorization.Privilege[], + * org.apache.hadoop.hive.ql.security.authorization.Privilege[]) + */ + @Override + public void authorize(Table table, Privilege[] readRequiredPriv, + Privilege[] writeRequiredPriv) throws HiveException, + AuthorizationException { + } + + /* + * (non-Javadoc) + * + * @see + * org.apache.hadoop.hive.ql.security.authorization.HiveAuthorizationProvider + * #authorize(org.apache.hadoop.hive.ql.metadata.Partition, + * org.apache.hadoop.hive.ql.security.authorization.Privilege[], + * org.apache.hadoop.hive.ql.security.authorization.Privilege[]) + */ + @Override + public void authorize(Partition part, Privilege[] readRequiredPriv, + Privilege[] writeRequiredPriv) throws HiveException, + AuthorizationException { + } + + /* + * (non-Javadoc) + * + * @see + * org.apache.hadoop.hive.ql.security.authorization.HiveAuthorizationProvider + * #authorize(org.apache.hadoop.hive.ql.metadata.Table, + * org.apache.hadoop.hive.ql.metadata.Partition, java.util.List, + * org.apache.hadoop.hive.ql.security.authorization.Privilege[], + * org.apache.hadoop.hive.ql.security.authorization.Privilege[]) + */ + @Override + public void authorize(Table table, Partition part, List columns, + Privilege[] readRequiredPriv, Privilege[] writeRequiredPriv) + throws HiveException, AuthorizationException { + } + +} diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseBaseOutputFormat.java hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseBaseOutputFormat.java new file mode 100644 index 0000000..8cfedfd --- /dev/null +++ hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseBaseOutputFormat.java @@ -0,0 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hcatalog.hbase; + +import java.io.IOException; +import java.util.Properties; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hive.ql.io.HiveOutputFormat; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.WritableComparable; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.OutputFormat; +import org.apache.hadoop.mapred.RecordWriter; +import org.apache.hadoop.util.Progressable; +import org.apache.hive.hcatalog.common.HCatConstants; +import org.apache.hive.hcatalog.common.HCatUtil; +import org.apache.hive.hcatalog.mapreduce.OutputJobInfo; + +public class HBaseBaseOutputFormat implements OutputFormat, Put>, + HiveOutputFormat, Put> { + + @Override + public org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter getHiveRecordWriter( + JobConf jc, Path finalOutPath, + Class valueClass, boolean isCompressed, + Properties tableProperties, Progressable progress) + throws IOException { + throw new UnsupportedOperationException("Not implemented"); + } + + @Override + public void checkOutputSpecs(FileSystem ignored, JobConf job) throws IOException { + OutputFormat, Put> outputFormat = getOutputFormat(job); + outputFormat.checkOutputSpecs(ignored, job); + } + + @Override + public RecordWriter, Put> getRecordWriter(FileSystem ignored, + JobConf job, String name, Progressable progress) throws IOException { + OutputFormat, Put> outputFormat = getOutputFormat(job); + return outputFormat.getRecordWriter(ignored, job, name, progress); + } + + private OutputFormat, Put> getOutputFormat(JobConf job) + throws IOException { + String outputInfo = job.get(HCatConstants.HCAT_KEY_OUTPUT_INFO); + OutputJobInfo outputJobInfo = (OutputJobInfo) HCatUtil.deserialize(outputInfo); + OutputFormat, Put> outputFormat = null; + if (HBaseHCatStorageHandler.isBulkMode(outputJobInfo)) { + outputFormat = new HBaseBulkOutputFormat(); + } else { + outputFormat = new HBaseDirectOutputFormat(); + } + return outputFormat; + } +} diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseBulkOutputFormat.java hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseBulkOutputFormat.java new file mode 100644 index 0000000..4a188e0 --- /dev/null +++ hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseBulkOutputFormat.java @@ -0,0 +1,221 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hcatalog.hbase; + +import java.io.IOException; +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.WritableComparable; +import org.apache.hadoop.mapred.FileOutputCommitter; +import org.apache.hadoop.mapred.FileOutputFormat; +import org.apache.hadoop.mapred.JobClient; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.JobContext; +import org.apache.hadoop.mapred.OutputCommitter; +import org.apache.hadoop.mapred.RecordWriter; +import org.apache.hadoop.mapred.Reporter; +import org.apache.hadoop.mapred.SequenceFileOutputFormat; +import org.apache.hadoop.mapred.TaskAttemptContext; +import org.apache.hadoop.util.Progressable; +import org.apache.hcatalog.hbase.snapshot.RevisionManager; + +/** + * Class which imports data into HBase via it's "bulk load" feature. Wherein + * regions are created by the MR job using HFileOutputFormat and then later + * "moved" into the appropriate region server. + */ +class HBaseBulkOutputFormat extends HBaseBaseOutputFormat { + + private final static ImmutableBytesWritable EMPTY_LIST = new ImmutableBytesWritable( + new byte[0]); + private SequenceFileOutputFormat, Put> baseOutputFormat; + + public HBaseBulkOutputFormat() { + baseOutputFormat = new SequenceFileOutputFormat, Put>(); + } + + @Override + public void checkOutputSpecs(FileSystem ignored, JobConf job) + throws IOException { + baseOutputFormat.checkOutputSpecs(ignored, job); + HBaseUtil.addHBaseDelegationToken(job); + addJTDelegationToken(job); + } + + @Override + public RecordWriter, Put> getRecordWriter( + FileSystem ignored, JobConf job, String name, Progressable progress) + throws IOException { + job.setOutputKeyClass(ImmutableBytesWritable.class); + job.setOutputValueClass(Put.class); + long version = HBaseRevisionManagerUtil.getOutputRevision(job); + return new HBaseBulkRecordWriter(baseOutputFormat.getRecordWriter( + ignored, job, name, progress), version); + } + + private void addJTDelegationToken(JobConf job) throws IOException { + // Get jobTracker delegation token if security is enabled + // we need to launch the ImportSequenceFile job + if (User.isSecurityEnabled()) { + JobClient jobClient = new JobClient(new JobConf(job)); + try { + job.getCredentials().addToken(new Text("my mr token"), + jobClient.getDelegationToken(null)); + } catch (InterruptedException e) { + throw new IOException("Error while getting JT delegation token", e); + } + } + } + + private static class HBaseBulkRecordWriter implements + RecordWriter, Put> { + + private RecordWriter, Put> baseWriter; + private final Long outputVersion; + + public HBaseBulkRecordWriter( + RecordWriter, Put> baseWriter, + Long outputVersion) { + this.baseWriter = baseWriter; + this.outputVersion = outputVersion; + } + + @Override + public void write(WritableComparable key, Put value) + throws IOException { + Put put = value; + if (outputVersion != null) { + put = new Put(value.getRow(), outputVersion.longValue()); + for (List row : value.getFamilyMap().values()) { + for (KeyValue el : row) { + put.add(el.getFamily(), el.getQualifier(), el.getValue()); + } + } + } + // we ignore the key + baseWriter.write(EMPTY_LIST, put); + } + + @Override + public void close(Reporter reporter) throws IOException { + baseWriter.close(reporter); + } + } + + public static class HBaseBulkOutputCommitter extends OutputCommitter { + + private final OutputCommitter baseOutputCommitter; + + public HBaseBulkOutputCommitter() { + baseOutputCommitter = new FileOutputCommitter(); + } + + @Override + public void abortTask(TaskAttemptContext taskContext) + throws IOException { + baseOutputCommitter.abortTask(taskContext); + } + + @Override + public void commitTask(TaskAttemptContext taskContext) + throws IOException { + // baseOutputCommitter.commitTask(taskContext); + } + + @Override + public boolean needsTaskCommit(TaskAttemptContext taskContext) + throws IOException { + return baseOutputCommitter.needsTaskCommit(taskContext); + } + + @Override + public void setupJob(JobContext jobContext) throws IOException { + baseOutputCommitter.setupJob(jobContext); + } + + @Override + public void setupTask(TaskAttemptContext taskContext) + throws IOException { + baseOutputCommitter.setupTask(taskContext); + } + + @Override + public void abortJob(JobContext jobContext, int status) + throws IOException { + baseOutputCommitter.abortJob(jobContext, status); + RevisionManager rm = null; + try { + rm = HBaseRevisionManagerUtil + .getOpenedRevisionManager(jobContext.getConfiguration()); + rm.abortWriteTransaction(HBaseRevisionManagerUtil + .getWriteTransaction(jobContext.getConfiguration())); + } finally { + cleanIntermediate(jobContext); + if (rm != null) + rm.close(); + } + } + + @Override + public void commitJob(JobContext jobContext) throws IOException { + baseOutputCommitter.commitJob(jobContext); + RevisionManager rm = null; + try { + Configuration conf = jobContext.getConfiguration(); + Path srcPath = FileOutputFormat.getOutputPath(jobContext.getJobConf()); + if (!FileSystem.get(conf).exists(srcPath)) { + throw new IOException("Failed to bulk import hfiles. " + + "Intermediate data directory is cleaned up or missing. " + + "Please look at the bulk import job if it exists for failure reason"); + } + Path destPath = new Path(srcPath.getParent(), srcPath.getName() + "_hfiles"); + boolean success = ImportSequenceFile.runJob(jobContext, + conf.get(HBaseConstants.PROPERTY_OUTPUT_TABLE_NAME_KEY), + srcPath, + destPath); + if (!success) { + cleanIntermediate(jobContext); + throw new IOException("Failed to bulk import hfiles." + + " Please look at the bulk import job for failure reason"); + } + rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(conf); + rm.commitWriteTransaction(HBaseRevisionManagerUtil.getWriteTransaction(conf)); + cleanIntermediate(jobContext); + } finally { + if (rm != null) + rm.close(); + } + } + + private void cleanIntermediate(JobContext jobContext) + throws IOException { + FileSystem fs = FileSystem.get(jobContext.getConfiguration()); + fs.delete(FileOutputFormat.getOutputPath(jobContext.getJobConf()), true); + } + } +} diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseConstants.java hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseConstants.java new file mode 100644 index 0000000..066419a --- /dev/null +++ hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseConstants.java @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hcatalog.hbase; + +import org.apache.hive.hcatalog.common.HCatConstants; + +/** + * Constants class for constants used in HBase storage handler. + */ +class HBaseConstants { + + /** key used to store write transaction object */ + public static final String PROPERTY_WRITE_TXN_KEY = HCatConstants.HCAT_DEFAULT_TOPIC_PREFIX + ".hbase.mapreduce.writeTxn"; + + /** key used to define the name of the table to write to */ + public static final String PROPERTY_OUTPUT_TABLE_NAME_KEY = HCatConstants.HCAT_DEFAULT_TOPIC_PREFIX + ".hbase.mapreduce.outputTableName"; + + /** key used to define whether bulk storage output format will be used or not */ + public static final String PROPERTY_BULK_OUTPUT_MODE_KEY = HCatConstants.HCAT_DEFAULT_TOPIC_PREFIX + ".hbase.output.bulkMode"; + + /** key used to define the hbase table snapshot. */ + public static final String PROPERTY_TABLE_SNAPSHOT_KEY = HCatConstants.HCAT_DEFAULT_TOPIC_PREFIX + "hbase.table.snapshot"; + +} diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseDirectOutputFormat.java hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseDirectOutputFormat.java new file mode 100644 index 0000000..b7537d4 --- /dev/null +++ hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseDirectOutputFormat.java @@ -0,0 +1,167 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hcatalog.hbase; + +import java.io.IOException; +import java.util.List; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.mapred.TableOutputFormat; +import org.apache.hadoop.io.WritableComparable; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.JobContext; +import org.apache.hadoop.mapred.OutputCommitter; +import org.apache.hadoop.mapred.RecordWriter; +import org.apache.hadoop.mapred.Reporter; +import org.apache.hadoop.mapred.TaskAttemptContext; +import org.apache.hadoop.util.Progressable; +import org.apache.hcatalog.hbase.snapshot.RevisionManager; +import org.apache.hcatalog.hbase.snapshot.Transaction; + +/** + * "Direct" implementation of OutputFormat for HBase. Uses HTable client's put + * API to write each row to HBase one a time. Presently it is just using + * TableOutputFormat as the underlying implementation in the future we can tune + * this to make the writes faster such as permanently disabling WAL, caching, + * etc. + */ +class HBaseDirectOutputFormat extends HBaseBaseOutputFormat { + + private TableOutputFormat outputFormat; + + public HBaseDirectOutputFormat() { + this.outputFormat = new TableOutputFormat(); + } + + @Override + public RecordWriter, Put> getRecordWriter(FileSystem ignored, + JobConf job, String name, Progressable progress) + throws IOException { + long version = HBaseRevisionManagerUtil.getOutputRevision(job); + return new HBaseDirectRecordWriter(outputFormat.getRecordWriter(ignored, job, name, + progress), version); + } + + @Override + public void checkOutputSpecs(FileSystem ignored, JobConf job) + throws IOException { + outputFormat.checkOutputSpecs(ignored, job); + HBaseUtil.addHBaseDelegationToken(job); + } + + private static class HBaseDirectRecordWriter implements + RecordWriter, Put> { + + private RecordWriter, Put> baseWriter; + private final Long outputVersion; + + public HBaseDirectRecordWriter( + RecordWriter, Put> baseWriter, + Long outputVersion) { + this.baseWriter = baseWriter; + this.outputVersion = outputVersion; + } + + @Override + public void write(WritableComparable key, Put value) + throws IOException { + Put put = value; + if (outputVersion != null) { + put = new Put(value.getRow(), outputVersion.longValue()); + for (List row : value.getFamilyMap().values()) { + for (KeyValue el : row) { + put.add(el.getFamily(), el.getQualifier(), el.getValue()); + } + } + } + baseWriter.write(key, put); + } + + @Override + public void close(Reporter reporter) throws IOException { + baseWriter.close(reporter); + } + + } + + public static class HBaseDirectOutputCommitter extends OutputCommitter { + + public HBaseDirectOutputCommitter() throws IOException { + } + + @Override + public void abortTask(TaskAttemptContext taskContext) + throws IOException { + } + + @Override + public void commitTask(TaskAttemptContext taskContext) + throws IOException { + } + + @Override + public boolean needsTaskCommit(TaskAttemptContext taskContext) + throws IOException { + return false; + } + + @Override + public void setupJob(JobContext jobContext) throws IOException { + } + + @Override + public void setupTask(TaskAttemptContext taskContext) + throws IOException { + } + + @Override + public void abortJob(JobContext jobContext, int status) + throws IOException { + super.abortJob(jobContext, status); + RevisionManager rm = null; + try { + rm = HBaseRevisionManagerUtil + .getOpenedRevisionManager(jobContext.getConfiguration()); + Transaction writeTransaction = HBaseRevisionManagerUtil + .getWriteTransaction(jobContext.getConfiguration()); + rm.abortWriteTransaction(writeTransaction); + } finally { + if (rm != null) + rm.close(); + } + } + + @Override + public void commitJob(JobContext jobContext) throws IOException { + RevisionManager rm = null; + try { + rm = HBaseRevisionManagerUtil + .getOpenedRevisionManager(jobContext.getConfiguration()); + rm.commitWriteTransaction(HBaseRevisionManagerUtil.getWriteTransaction(jobContext + .getConfiguration())); + } finally { + if (rm != null) + rm.close(); + } + } + } +} diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseHCatStorageHandler.java hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseHCatStorageHandler.java new file mode 100644 index 0000000..6644d26 --- /dev/null +++ hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseHCatStorageHandler.java @@ -0,0 +1,610 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hcatalog.hbase; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; + +import org.apache.hadoop.conf.Configurable; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.MasterNotRunningException; +import org.apache.hadoop.hbase.ZooKeeperConnectionException; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.mapred.TableOutputFormat; +import org.apache.hadoop.hbase.mapreduce.TableInputFormat; +import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hive.hbase.HBaseSerDe; +import org.apache.hadoop.hive.metastore.HiveMetaHook; +import org.apache.hadoop.hive.metastore.MetaStoreUtils; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.plan.TableDesc; +import org.apache.hadoop.hive.ql.security.authorization.HiveAuthorizationProvider; +import org.apache.hadoop.hive.serde2.SerDe; +import org.apache.hadoop.mapred.InputFormat; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.OutputFormat; +import org.apache.hadoop.util.StringUtils; +import org.apache.hive.hcatalog.common.HCatConstants; +import org.apache.hive.hcatalog.common.HCatUtil; +import org.apache.hive.hcatalog.data.schema.HCatSchema; +import org.apache.hcatalog.hbase.HBaseBulkOutputFormat.HBaseBulkOutputCommitter; +import org.apache.hcatalog.hbase.HBaseDirectOutputFormat.HBaseDirectOutputCommitter; +import org.apache.hcatalog.hbase.snapshot.RevisionManager; +import org.apache.hcatalog.hbase.snapshot.RevisionManagerConfiguration; +import org.apache.hcatalog.hbase.snapshot.Transaction; +import org.apache.hive.hcatalog.mapreduce.HCatOutputFormat; +import org.apache.hive.hcatalog.mapreduce.HCatTableInfo; +import org.apache.hive.hcatalog.mapreduce.InputJobInfo; +import org.apache.hive.hcatalog.mapreduce.OutputJobInfo; +import org.apache.hive.hcatalog.mapreduce.HCatStorageHandler; +import org.apache.thrift.TBase; +import org.apache.zookeeper.ZooKeeper; + +import com.facebook.fb303.FacebookBase; +import com.google.common.util.concurrent.ThreadFactoryBuilder; + +/** + * This class HBaseHCatStorageHandler provides functionality to create HBase + * tables through HCatalog. The implementation is very similar to the + * HiveHBaseStorageHandler, with more details to suit HCatalog. + */ +public class HBaseHCatStorageHandler extends HCatStorageHandler implements HiveMetaHook, Configurable { + + public final static String DEFAULT_PREFIX = "default."; + private final static String PROPERTY_INT_OUTPUT_LOCATION = "hcat.hbase.mapreduce.intermediateOutputLocation"; + + private Configuration hbaseConf; + private Configuration jobConf; + private HBaseAdmin admin; + + @Override + public void configureInputJobProperties(TableDesc tableDesc, Map jobProperties) { + // Populate jobProperties with input table name, table columns, RM snapshot, + // hbase-default.xml and hbase-site.xml + Map tableJobProperties = tableDesc.getJobProperties(); + String jobString = tableJobProperties.get(HCatConstants.HCAT_KEY_JOB_INFO); + try { + InputJobInfo inputJobInfo = (InputJobInfo) HCatUtil.deserialize(jobString); + HCatTableInfo tableInfo = inputJobInfo.getTableInfo(); + String qualifiedTableName = HBaseHCatStorageHandler.getFullyQualifiedHBaseTableName(tableInfo); + jobProperties.put(TableInputFormat.INPUT_TABLE, qualifiedTableName); + + Configuration jobConf = getJobConf(); + addResources(jobConf, jobProperties); + JobConf copyOfConf = new JobConf(jobConf); + HBaseConfiguration.addHbaseResources(copyOfConf); + //Getting hbase delegation token in getInputSplits does not work with PIG. So need to + //do it here + if (jobConf instanceof JobConf) { //Should be the case + HBaseUtil.addHBaseDelegationToken(copyOfConf); + ((JobConf) jobConf).getCredentials().addAll(copyOfConf.getCredentials()); + } + + String outputSchema = jobConf.get(HCatConstants.HCAT_KEY_OUTPUT_SCHEMA); + jobProperties.put(TableInputFormat.SCAN_COLUMNS, getScanColumns(tableInfo, outputSchema)); + + String serSnapshot = (String) inputJobInfo.getProperties().get( + HBaseConstants.PROPERTY_TABLE_SNAPSHOT_KEY); + if (serSnapshot == null) { + HCatTableSnapshot snapshot = + HBaseRevisionManagerUtil.createSnapshot( + RevisionManagerConfiguration.create(copyOfConf), + qualifiedTableName, tableInfo); + jobProperties.put(HBaseConstants.PROPERTY_TABLE_SNAPSHOT_KEY, + HCatUtil.serialize(snapshot)); + } + + //This adds it directly to the jobConf. Setting in jobProperties does not get propagated + //to JobConf as of now as the jobProperties is maintained per partition + //TODO: Remove when HCAT-308 is fixed + addOutputDependencyJars(jobConf); + jobProperties.put("tmpjars", jobConf.get("tmpjars")); + + } catch (IOException e) { + throw new IllegalStateException("Error while configuring job properties", e); + } + } + + @Override + public void configureOutputJobProperties(TableDesc tableDesc, Map jobProperties) { + // Populate jobProperties with output table name, hbase-default.xml, hbase-site.xml, OutputJobInfo + // Populate RM transaction in OutputJobInfo + // In case of bulk mode, populate intermediate output location + Map tableJobProperties = tableDesc.getJobProperties(); + String jobString = tableJobProperties.get(HCatConstants.HCAT_KEY_OUTPUT_INFO); + try { + OutputJobInfo outputJobInfo = (OutputJobInfo) HCatUtil.deserialize(jobString); + HCatTableInfo tableInfo = outputJobInfo.getTableInfo(); + String qualifiedTableName = HBaseHCatStorageHandler.getFullyQualifiedHBaseTableName(tableInfo); + jobProperties.put(HBaseConstants.PROPERTY_OUTPUT_TABLE_NAME_KEY, qualifiedTableName); + jobProperties.put(TableOutputFormat.OUTPUT_TABLE, qualifiedTableName); + + Configuration jobConf = getJobConf(); + addResources(jobConf, jobProperties); + + Configuration copyOfConf = new Configuration(jobConf); + HBaseConfiguration.addHbaseResources(copyOfConf); + + String txnString = outputJobInfo.getProperties().getProperty( + HBaseConstants.PROPERTY_WRITE_TXN_KEY); + Transaction txn = null; + if (txnString == null) { + txn = HBaseRevisionManagerUtil.beginWriteTransaction(qualifiedTableName, tableInfo, + RevisionManagerConfiguration.create(copyOfConf)); + String serializedTxn = HCatUtil.serialize(txn); + outputJobInfo.getProperties().setProperty(HBaseConstants.PROPERTY_WRITE_TXN_KEY, + serializedTxn); + } else { + txn = (Transaction) HCatUtil.deserialize(txnString); + } + if (isBulkMode(outputJobInfo)) { + String tableLocation = tableInfo.getTableLocation(); + String location = new Path(tableLocation, "REVISION_" + txn.getRevisionNumber()) + .toString(); + outputJobInfo.getProperties().setProperty(PROPERTY_INT_OUTPUT_LOCATION, location); + // We are writing out an intermediate sequenceFile hence + // location is not passed in OutputJobInfo.getLocation() + // TODO replace this with a mapreduce constant when available + jobProperties.put("mapred.output.dir", location); + jobProperties.put("mapred.output.committer.class", HBaseBulkOutputCommitter.class.getName()); + } else { + jobProperties.put("mapred.output.committer.class", HBaseDirectOutputCommitter.class.getName()); + } + + jobProperties.put(HCatConstants.HCAT_KEY_OUTPUT_INFO, HCatUtil.serialize(outputJobInfo)); + addOutputDependencyJars(jobConf); + jobProperties.put("tmpjars", jobConf.get("tmpjars")); + + } catch (IOException e) { + throw new IllegalStateException("Error while configuring job properties", e); + } + } + + /* + * @return instance of HiveAuthorizationProvider + * + * @throws HiveException + * + * @see org.apache.hive.hcatalog.storagehandler.HCatStorageHandler# + * getAuthorizationProvider() + */ + @Override + public HiveAuthorizationProvider getAuthorizationProvider() + throws HiveException { + + HBaseAuthorizationProvider hbaseAuth = new HBaseAuthorizationProvider(); + hbaseAuth.init(getConf()); + return hbaseAuth; + } + + /* + * @param table + * + * @throws MetaException + * + * @see org.apache.hive.hcatalog.storagehandler.HCatStorageHandler + * #commitCreateTable(org.apache.hadoop.hive.metastore.api.Table) + */ + @Override + public void commitCreateTable(Table table) throws MetaException { + } + + /* + * @param instance of table + * + * @param deleteData + * + * @throws MetaException + * + * @see org.apache.hive.hcatalog.storagehandler.HCatStorageHandler + * #commitDropTable(org.apache.hadoop.hive.metastore.api.Table, boolean) + */ + @Override + public void commitDropTable(Table tbl, boolean deleteData) + throws MetaException { + checkDeleteTable(tbl); + + } + + /* + * @param instance of table + * + * @throws MetaException + * + * @see org.apache.hive.hcatalog.storagehandler.HCatStorageHandler + * #preCreateTable(org.apache.hadoop.hive.metastore.api.Table) + */ + @Override + public void preCreateTable(Table tbl) throws MetaException { + boolean isExternal = MetaStoreUtils.isExternalTable(tbl); + + hbaseConf = getConf(); + + if (tbl.getSd().getLocation() != null) { + throw new MetaException("LOCATION may not be specified for HBase."); + } + + try { + String tableName = getFullyQualifiedHBaseTableName(tbl); + String hbaseColumnsMapping = tbl.getParameters().get( + HBaseSerDe.HBASE_COLUMNS_MAPPING); + + if (hbaseColumnsMapping == null) { + throw new MetaException( + "No hbase.columns.mapping defined in table" + + " properties."); + } + + List hbaseColumnFamilies = new ArrayList(); + List hbaseColumnQualifiers = new ArrayList(); + List hbaseColumnFamiliesBytes = new ArrayList(); + int iKey = HBaseUtil.parseColumnMapping(hbaseColumnsMapping, + hbaseColumnFamilies, hbaseColumnFamiliesBytes, + hbaseColumnQualifiers, null); + + HTableDescriptor tableDesc; + Set uniqueColumnFamilies = new HashSet(); + if (!getHBaseAdmin().tableExists(tableName)) { + // if it is not an external table then create one + if (!isExternal) { + // Create the column descriptors + tableDesc = new HTableDescriptor(tableName); + uniqueColumnFamilies.addAll(hbaseColumnFamilies); + uniqueColumnFamilies.remove(hbaseColumnFamilies.get(iKey)); + + for (String columnFamily : uniqueColumnFamilies) { + HColumnDescriptor familyDesc = new HColumnDescriptor(Bytes + .toBytes(columnFamily)); + familyDesc.setMaxVersions(Integer.MAX_VALUE); + tableDesc.addFamily(familyDesc); + } + + getHBaseAdmin().createTable(tableDesc); + } else { + // an external table + throw new MetaException("HBase table " + tableName + + " doesn't exist while the table is " + + "declared as an external table."); + } + + } else { + if (!isExternal) { + throw new MetaException("Table " + tableName + + " already exists within HBase." + + " Use CREATE EXTERNAL TABLE instead to" + + " register it in HCatalog."); + } + // make sure the schema mapping is right + tableDesc = getHBaseAdmin().getTableDescriptor( + Bytes.toBytes(tableName)); + + for (int i = 0; i < hbaseColumnFamilies.size(); i++) { + if (i == iKey) { + continue; + } + + if (!tableDesc.hasFamily(hbaseColumnFamiliesBytes.get(i))) { + throw new MetaException("Column Family " + + hbaseColumnFamilies.get(i) + + " is not defined in hbase table " + tableName); + } + } + } + + // ensure the table is online + new HTable(hbaseConf, tableDesc.getName()); + + //Set up table in revision manager. + RevisionManager rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(hbaseConf); + rm.createTable(tableName, new ArrayList(uniqueColumnFamilies)); + + } catch (MasterNotRunningException mnre) { + throw new MetaException(StringUtils.stringifyException(mnre)); + } catch (IOException ie) { + throw new MetaException(StringUtils.stringifyException(ie)); + } catch (IllegalArgumentException iae) { + throw new MetaException(StringUtils.stringifyException(iae)); + } + + } + + /* + * @param table + * + * @throws MetaException + * + * @see org.apache.hive.hcatalog.storagehandler.HCatStorageHandler + * #preDropTable(org.apache.hadoop.hive.metastore.api.Table) + */ + @Override + public void preDropTable(Table table) throws MetaException { + } + + /* + * @param table + * + * @throws MetaException + * + * @see org.apache.hive.hcatalog.storagehandler.HCatStorageHandler + * #rollbackCreateTable(org.apache.hadoop.hive.metastore.api.Table) + */ + @Override + public void rollbackCreateTable(Table table) throws MetaException { + checkDeleteTable(table); + } + + /* + * @param table + * + * @throws MetaException + * + * @see org.apache.hive.hcatalog.storagehandler.HCatStorageHandler + * #rollbackDropTable(org.apache.hadoop.hive.metastore.api.Table) + */ + @Override + public void rollbackDropTable(Table table) throws MetaException { + } + + /* + * @return instance of HiveMetaHook + * + * @see org.apache.hive.hcatalog.storagehandler.HCatStorageHandler#getMetaHook() + */ + @Override + public HiveMetaHook getMetaHook() { + return this; + } + + private HBaseAdmin getHBaseAdmin() throws MetaException { + try { + if (admin == null) { + admin = new HBaseAdmin(this.getConf()); + } + return admin; + } catch (MasterNotRunningException mnre) { + throw new MetaException(StringUtils.stringifyException(mnre)); + } catch (ZooKeeperConnectionException zkce) { + throw new MetaException(StringUtils.stringifyException(zkce)); + } + } + + private String getFullyQualifiedHBaseTableName(Table tbl) { + String tableName = tbl.getParameters().get(HBaseSerDe.HBASE_TABLE_NAME); + if (tableName == null) { + tableName = tbl.getSd().getSerdeInfo().getParameters() + .get(HBaseSerDe.HBASE_TABLE_NAME); + } + if (tableName == null) { + if (tbl.getDbName().equals(MetaStoreUtils.DEFAULT_DATABASE_NAME)) { + tableName = tbl.getTableName(); + } else { + tableName = tbl.getDbName() + "." + tbl.getTableName(); + } + tableName = tableName.toLowerCase(); + } + return tableName; + } + + static String getFullyQualifiedHBaseTableName(HCatTableInfo tableInfo) { + String qualifiedName = tableInfo.getStorerInfo().getProperties() + .getProperty(HBaseSerDe.HBASE_TABLE_NAME); + if (qualifiedName == null) { + String databaseName = tableInfo.getDatabaseName(); + String tableName = tableInfo.getTableName(); + if ((databaseName == null) + || (databaseName.equals(MetaStoreUtils.DEFAULT_DATABASE_NAME))) { + qualifiedName = tableName; + } else { + qualifiedName = databaseName + "." + tableName; + } + qualifiedName = qualifiedName.toLowerCase(); + } + return qualifiedName; + } + + @Override + public Class getInputFormatClass() { + return HBaseInputFormat.class; + } + + @Override + public Class getOutputFormatClass() { + return HBaseBaseOutputFormat.class; + } + + /* + * @return subclass of SerDe + * + * @throws UnsupportedOperationException + * + * @see + * org.apache.hive.hcatalog.storagehandler.HCatStorageHandler#getSerDeClass() + */ + @Override + public Class getSerDeClass() + throws UnsupportedOperationException { + return HBaseSerDe.class; + } + + public Configuration getJobConf() { + return jobConf; + } + + @Override + public Configuration getConf() { + + if (hbaseConf == null) { + hbaseConf = HBaseConfiguration.create(); + } + return hbaseConf; + } + + @Override + public void setConf(Configuration conf) { + //setConf is called both during DDL operations and mapred read/write jobs. + //Creating a copy of conf for DDL and adding hbase-default and hbase-site.xml to it. + //For jobs, maintaining a reference instead of cloning as we need to + // 1) add hbase delegation token to the Credentials. + // 2) set tmpjars on it. Putting in jobProperties does not get propagated to JobConf + // in case of InputFormat as they are maintained per partition. + //Not adding hbase-default.xml and hbase-site.xml to jobConf as it will override any + //hbase properties set in the JobConf by the user. In configureInputJobProperties and + //configureOutputJobProperties, we take care of adding the default properties + //that are not already present. TODO: Change to a copy for jobs after HCAT-308 is fixed. + jobConf = conf; + hbaseConf = RevisionManagerConfiguration.create(HBaseConfiguration.create(conf)); + } + + private void checkDeleteTable(Table table) throws MetaException { + boolean isExternal = MetaStoreUtils.isExternalTable(table); + String tableName = getFullyQualifiedHBaseTableName(table); + RevisionManager rm = null; + try { + if (!isExternal && getHBaseAdmin().tableExists(tableName)) { + // we have created an HBase table, so we delete it to roll back; + if (getHBaseAdmin().isTableEnabled(tableName)) { + getHBaseAdmin().disableTable(tableName); + } + getHBaseAdmin().deleteTable(tableName); + + //Drop table in revision manager. + rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(hbaseConf); + rm.dropTable(tableName); + } + } catch (IOException ie) { + throw new MetaException(StringUtils.stringifyException(ie)); + } finally { + HBaseRevisionManagerUtil.closeRevisionManagerQuietly(rm); + } + } + + /** + * Helper method for users to add the required depedency jars to distributed cache. + * @param conf + * @throws IOException + */ + private void addOutputDependencyJars(Configuration conf) throws IOException { + TableMapReduceUtil.addDependencyJars(conf, + //ZK + ZooKeeper.class, + //HBase + HTable.class, + //Hive + HiveException.class, + //HCatalog jar + HCatOutputFormat.class, + //hcat hbase storage handler jar + HBaseHCatStorageHandler.class, + //hive hbase storage handler jar + HBaseSerDe.class, + //hive jar + Table.class, + //libthrift jar + TBase.class, + //hbase jar + Bytes.class, + //thrift-fb303 .jar + FacebookBase.class, + //guava jar + ThreadFactoryBuilder.class); + } + + /** + * Utility method to add hbase-default.xml and hbase-site.xml properties to a new map + * if they are not already present in the jobConf. + * @param jobConf Job configuration + * @param newJobProperties Map to which new properties should be added + */ + private void addResources(Configuration jobConf, + Map newJobProperties) { + Configuration conf = new Configuration(false); + HBaseConfiguration.addHbaseResources(conf); + RevisionManagerConfiguration.addResources(conf); + for (Entry entry : conf) { + if (jobConf.get(entry.getKey()) == null) + newJobProperties.put(entry.getKey(), entry.getValue()); + } + } + + public static boolean isBulkMode(OutputJobInfo outputJobInfo) { + //Default is false + String bulkMode = outputJobInfo.getTableInfo().getStorerInfo().getProperties() + .getProperty(HBaseConstants.PROPERTY_BULK_OUTPUT_MODE_KEY, + "false"); + return "true".equals(bulkMode); + } + + private String getScanColumns(HCatTableInfo tableInfo, String outputColSchema) throws IOException { + StringBuilder builder = new StringBuilder(); + String hbaseColumnMapping = tableInfo.getStorerInfo().getProperties() + .getProperty(HBaseSerDe.HBASE_COLUMNS_MAPPING); + if (outputColSchema == null) { + String[] splits = hbaseColumnMapping.split("[,]"); + for (int i = 0; i < splits.length; i++) { + if (!splits[i].equals(HBaseSerDe.HBASE_KEY_COL)) + builder.append(splits[i]).append(" "); + } + } else { + HCatSchema outputSchema = (HCatSchema) HCatUtil.deserialize(outputColSchema); + HCatSchema tableSchema = tableInfo.getDataColumns(); + List outputFieldNames = outputSchema.getFieldNames(); + List outputColumnMapping = new ArrayList(); + for (String fieldName : outputFieldNames) { + int position = tableSchema.getPosition(fieldName); + outputColumnMapping.add(position); + } + List columnFamilies = new ArrayList(); + List columnQualifiers = new ArrayList(); + HBaseUtil.parseColumnMapping(hbaseColumnMapping, columnFamilies, null, + columnQualifiers, null); + for (int i = 0; i < outputColumnMapping.size(); i++) { + int cfIndex = outputColumnMapping.get(i); + String cf = columnFamilies.get(cfIndex); + // We skip the key column. + if (cf.equals(HBaseSerDe.HBASE_KEY_COL) == false) { + String qualifier = columnQualifiers.get(i); + builder.append(cf); + builder.append(":"); + if (qualifier != null) { + builder.append(qualifier); + } + builder.append(" "); + } + } + } + //Remove the extra space delimiter + builder.deleteCharAt(builder.length() - 1); + return builder.toString(); + } + +} diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseInputFormat.java hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseInputFormat.java new file mode 100644 index 0000000..8c0816c --- /dev/null +++ hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseInputFormat.java @@ -0,0 +1,126 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hcatalog.hbase; + +import java.io.IOException; +import java.util.List; + +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.mapred.TableSplit; +import org.apache.hadoop.hbase.mapreduce.TableInputFormat; +import org.apache.hive.hcatalog.mapreduce.HCatMapRedUtil; +import org.apache.hadoop.mapred.InputFormat; +import org.apache.hadoop.mapred.InputSplit; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.RecordReader; +import org.apache.hadoop.mapred.Reporter; +import org.apache.hive.hcatalog.common.HCatConstants; +import org.apache.hive.hcatalog.common.HCatUtil; +import org.apache.hive.hcatalog.mapreduce.InputJobInfo; + +/** + * This class HBaseInputFormat is a wrapper class of TableInputFormat in HBase. + */ +class HBaseInputFormat implements InputFormat { + + private final TableInputFormat inputFormat; + + public HBaseInputFormat() { + inputFormat = new TableInputFormat(); + } + + /* + * @param instance of InputSplit + * + * @param instance of TaskAttemptContext + * + * @return RecordReader + * + * @throws IOException + * + * @throws InterruptedException + * + * @see + * org.apache.hadoop.mapreduce.InputFormat#createRecordReader(org.apache + * .hadoop.mapreduce.InputSplit, + * org.apache.hadoop.mapreduce.TaskAttemptContext) + */ + @Override + public RecordReader getRecordReader( + InputSplit split, JobConf job, Reporter reporter) + throws IOException { + String jobString = job.get(HCatConstants.HCAT_KEY_JOB_INFO); + InputJobInfo inputJobInfo = (InputJobInfo) HCatUtil.deserialize(jobString); + + String tableName = job.get(TableInputFormat.INPUT_TABLE); + TableSplit tSplit = (TableSplit) split; + HbaseSnapshotRecordReader recordReader = new HbaseSnapshotRecordReader(inputJobInfo, job); + inputFormat.setConf(job); + Scan inputScan = inputFormat.getScan(); + // TODO: Make the caching configurable by the user + inputScan.setCaching(200); + inputScan.setCacheBlocks(false); + Scan sc = new Scan(inputScan); + sc.setStartRow(tSplit.getStartRow()); + sc.setStopRow(tSplit.getEndRow()); + recordReader.setScan(sc); + recordReader.setHTable(new HTable(job, tableName)); + recordReader.init(); + return recordReader; + } + + /* + * @param jobContext + * + * @return List of InputSplit + * + * @throws IOException + * + * @throws InterruptedException + * + * @see + * org.apache.hadoop.mapreduce.InputFormat#getSplits(org.apache.hadoop.mapreduce + * .JobContext) + */ + @Override + public org.apache.hadoop.mapred.InputSplit[] getSplits(JobConf job, int numSplits) + throws IOException { + inputFormat.setConf(job); + return convertSplits(inputFormat.getSplits(HCatMapRedUtil.createJobContext(job, null, + Reporter.NULL))); + } + + private InputSplit[] convertSplits(List splits) { + InputSplit[] converted = new InputSplit[splits.size()]; + for (int i = 0; i < splits.size(); i++) { + org.apache.hadoop.hbase.mapreduce.TableSplit tableSplit = + (org.apache.hadoop.hbase.mapreduce.TableSplit) splits.get(i); + TableSplit newTableSplit = new TableSplit(tableSplit.getTableName(), + tableSplit.getStartRow(), + tableSplit.getEndRow(), tableSplit.getRegionLocation()); + converted[i] = newTableSplit; + } + return converted; + } + +} diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseRevisionManagerUtil.java hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseRevisionManagerUtil.java new file mode 100644 index 0000000..02d99df --- /dev/null +++ hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseRevisionManagerUtil.java @@ -0,0 +1,257 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hcatalog.hbase; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.hbase.HBaseSerDe; +import org.apache.hive.hcatalog.common.HCatConstants; +import org.apache.hive.hcatalog.common.HCatUtil; +import org.apache.hive.hcatalog.data.schema.HCatFieldSchema; +import org.apache.hive.hcatalog.data.schema.HCatSchema; +import org.apache.hcatalog.hbase.snapshot.RevisionManager; +import org.apache.hcatalog.hbase.snapshot.RevisionManagerFactory; +import org.apache.hcatalog.hbase.snapshot.TableSnapshot; +import org.apache.hcatalog.hbase.snapshot.Transaction; +import org.apache.hive.hcatalog.mapreduce.HCatTableInfo; +import org.apache.hive.hcatalog.mapreduce.InputJobInfo; +import org.apache.hive.hcatalog.mapreduce.OutputJobInfo; +import org.apache.hive.hcatalog.mapreduce.StorerInfo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +/** + * The Class HBaseRevisionManagerUtil has utility methods to interact with Revision Manager + * + */ +class HBaseRevisionManagerUtil { + + private final static Logger LOG = LoggerFactory.getLogger(HBaseRevisionManagerUtil.class); + + private HBaseRevisionManagerUtil() { + } + + /** + * Creates the latest snapshot of the table. + * + * @param jobConf The job configuration. + * @param hbaseTableName The fully qualified name of the HBase table. + * @param tableInfo HCat table information + * @return An instance of HCatTableSnapshot + * @throws IOException Signals that an I/O exception has occurred. + */ + static HCatTableSnapshot createSnapshot(Configuration jobConf, + String hbaseTableName, HCatTableInfo tableInfo) throws IOException { + + RevisionManager rm = null; + TableSnapshot snpt; + try { + rm = getOpenedRevisionManager(jobConf); + snpt = rm.createSnapshot(hbaseTableName); + } finally { + closeRevisionManagerQuietly(rm); + } + + HCatTableSnapshot hcatSnapshot = HBaseRevisionManagerUtil.convertSnapshot(snpt, tableInfo); + return hcatSnapshot; + } + + /** + * Creates the snapshot using the revision specified by the user. + * + * @param jobConf The job configuration. + * @param tableName The fully qualified name of the table whose snapshot is being taken. + * @param revision The revision number to use for the snapshot. + * @return An instance of HCatTableSnapshot. + * @throws IOException Signals that an I/O exception has occurred. + */ + static HCatTableSnapshot createSnapshot(Configuration jobConf, + String tableName, long revision) + throws IOException { + + TableSnapshot snpt; + RevisionManager rm = null; + try { + rm = getOpenedRevisionManager(jobConf); + snpt = rm.createSnapshot(tableName, revision); + } finally { + closeRevisionManagerQuietly(rm); + } + + String inputJobString = jobConf.get(HCatConstants.HCAT_KEY_JOB_INFO); + if (inputJobString == null) { + throw new IOException( + "InputJobInfo information not found in JobContext. " + + "HCatInputFormat.setInput() not called?"); + } + InputJobInfo inputInfo = (InputJobInfo) HCatUtil.deserialize(inputJobString); + HCatTableSnapshot hcatSnapshot = HBaseRevisionManagerUtil + .convertSnapshot(snpt, inputInfo.getTableInfo()); + + return hcatSnapshot; + } + + /** + * Gets an instance of revision manager which is opened. + * + * @param jobConf The job configuration. + * @return RevisionManager An instance of revision manager. + * @throws IOException + */ + static RevisionManager getOpenedRevisionManager(Configuration jobConf) throws IOException { + return RevisionManagerFactory.getOpenedRevisionManager(jobConf); + } + + static void closeRevisionManagerQuietly(RevisionManager rm) { + if (rm != null) { + try { + rm.close(); + } catch (IOException e) { + LOG.warn("Error while trying to close revision manager", e); + } + } + } + + + static HCatTableSnapshot convertSnapshot(TableSnapshot hbaseSnapshot, + HCatTableInfo hcatTableInfo) throws IOException { + + HCatSchema hcatTableSchema = hcatTableInfo.getDataColumns(); + Map hcatHbaseColMap = getHCatHBaseColumnMapping(hcatTableInfo); + HashMap revisionMap = new HashMap(); + + for (HCatFieldSchema fSchema : hcatTableSchema.getFields()) { + if (hcatHbaseColMap.containsKey(fSchema.getName())) { + String colFamily = hcatHbaseColMap.get(fSchema.getName()); + long revisionID = hbaseSnapshot.getRevision(colFamily); + revisionMap.put(fSchema.getName(), revisionID); + } + } + + HCatTableSnapshot hcatSnapshot = new HCatTableSnapshot( + hcatTableInfo.getDatabaseName(), hcatTableInfo.getTableName(), revisionMap, hbaseSnapshot.getLatestRevision()); + return hcatSnapshot; + } + + static TableSnapshot convertSnapshot(HCatTableSnapshot hcatSnapshot, + HCatTableInfo hcatTableInfo) throws IOException { + + HCatSchema hcatTableSchema = hcatTableInfo.getDataColumns(); + Map revisionMap = new HashMap(); + Map hcatHbaseColMap = getHCatHBaseColumnMapping(hcatTableInfo); + for (HCatFieldSchema fSchema : hcatTableSchema.getFields()) { + String colFamily = hcatHbaseColMap.get(fSchema.getName()); + if (hcatSnapshot.containsColumn(fSchema.getName())) { + long revision = hcatSnapshot.getRevision(fSchema.getName()); + revisionMap.put(colFamily, revision); + } + } + + String fullyQualifiedName = hcatSnapshot.getDatabaseName() + "." + + hcatSnapshot.getTableName(); + return new TableSnapshot(fullyQualifiedName, revisionMap, hcatSnapshot.getLatestRevision()); + + } + + /** + * Begins a transaction in the revision manager for the given table. + * @param qualifiedTableName Name of the table + * @param tableInfo HCat Table information + * @param jobConf Job Configuration + * @return The new transaction in revision manager + * @throws IOException + */ + static Transaction beginWriteTransaction(String qualifiedTableName, + HCatTableInfo tableInfo, Configuration jobConf) throws IOException { + Transaction txn; + RevisionManager rm = null; + try { + rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(jobConf); + String hBaseColumns = tableInfo.getStorerInfo().getProperties() + .getProperty(HBaseSerDe.HBASE_COLUMNS_MAPPING); + String[] splits = hBaseColumns.split("[,:]"); + Set families = new HashSet(); + for (int i = 0; i < splits.length; i += 2) { + if (!splits[i].isEmpty()) + families.add(splits[i]); + } + txn = rm.beginWriteTransaction(qualifiedTableName, new ArrayList(families)); + } finally { + HBaseRevisionManagerUtil.closeRevisionManagerQuietly(rm); + } + return txn; + } + + static Transaction getWriteTransaction(Configuration conf) throws IOException { + OutputJobInfo outputJobInfo = (OutputJobInfo) HCatUtil.deserialize(conf.get(HCatConstants.HCAT_KEY_OUTPUT_INFO)); + return (Transaction) HCatUtil.deserialize(outputJobInfo.getProperties() + .getProperty(HBaseConstants.PROPERTY_WRITE_TXN_KEY)); + } + + static void setWriteTransaction(Configuration conf, Transaction txn) throws IOException { + OutputJobInfo outputJobInfo = (OutputJobInfo) HCatUtil.deserialize(conf.get(HCatConstants.HCAT_KEY_OUTPUT_INFO)); + outputJobInfo.getProperties().setProperty(HBaseConstants.PROPERTY_WRITE_TXN_KEY, HCatUtil.serialize(txn)); + conf.set(HCatConstants.HCAT_KEY_OUTPUT_INFO, HCatUtil.serialize(outputJobInfo)); + } + + /** + * Get the Revision number that will be assigned to this job's output data + * @param conf configuration of the job + * @return the revision number used + * @throws IOException + */ + static long getOutputRevision(Configuration conf) throws IOException { + return getWriteTransaction(conf).getRevisionNumber(); + } + + private static Map getHCatHBaseColumnMapping(HCatTableInfo hcatTableInfo) + throws IOException { + + HCatSchema hcatTableSchema = hcatTableInfo.getDataColumns(); + StorerInfo storeInfo = hcatTableInfo.getStorerInfo(); + String hbaseColumnMapping = storeInfo.getProperties().getProperty( + HBaseSerDe.HBASE_COLUMNS_MAPPING); + + Map hcatHbaseColMap = new HashMap(); + List columnFamilies = new ArrayList(); + List columnQualifiers = new ArrayList(); + HBaseUtil.parseColumnMapping(hbaseColumnMapping, columnFamilies, + null, columnQualifiers, null); + + for (HCatFieldSchema column : hcatTableSchema.getFields()) { + int fieldPos = hcatTableSchema.getPosition(column.getName()); + String colFamily = columnFamilies.get(fieldPos); + if (colFamily.equals(HBaseSerDe.HBASE_KEY_COL) == false) { + hcatHbaseColMap.put(column.getName(), colFamily); + } + } + + return hcatHbaseColMap; + } + +} diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseUtil.java hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseUtil.java new file mode 100644 index 0000000..27e165f --- /dev/null +++ hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseUtil.java @@ -0,0 +1,159 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hcatalog.hbase; + +import java.io.IOException; +import java.util.List; + +import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hive.hbase.HBaseSerDe; +import org.apache.hadoop.mapred.JobConf; + +class HBaseUtil { + + private HBaseUtil() { + } + + /** + * Parses the HBase columns mapping to identify the column families, qualifiers + * and also caches the byte arrays corresponding to them. One of the HCat table + * columns maps to the HBase row key, by default the first column. + * + * @param columnMapping - the column mapping specification to be parsed + * @param colFamilies - the list of HBase column family names + * @param colFamiliesBytes - the corresponding byte array + * @param colQualifiers - the list of HBase column qualifier names + * @param colQualifiersBytes - the corresponding byte array + * @return the row key index in the column names list + * @throws IOException + */ + static int parseColumnMapping( + String columnMapping, + List colFamilies, + List colFamiliesBytes, + List colQualifiers, + List colQualifiersBytes) throws IOException { + + int rowKeyIndex = -1; + + if (colFamilies == null || colQualifiers == null) { + throw new IllegalArgumentException("Error: caller must pass in lists for the column families " + + "and qualifiers."); + } + + colFamilies.clear(); + colQualifiers.clear(); + + if (columnMapping == null) { + throw new IllegalArgumentException("Error: hbase.columns.mapping missing for this HBase table."); + } + + if (columnMapping.equals("") || columnMapping.equals(HBaseSerDe.HBASE_KEY_COL)) { + throw new IllegalArgumentException("Error: hbase.columns.mapping specifies only the HBase table" + + " row key. A valid Hive-HBase table must specify at least one additional column."); + } + + String[] mapping = columnMapping.split(","); + + for (int i = 0; i < mapping.length; i++) { + String elem = mapping[i]; + int idxFirst = elem.indexOf(":"); + int idxLast = elem.lastIndexOf(":"); + + if (idxFirst < 0 || !(idxFirst == idxLast)) { + throw new IllegalArgumentException("Error: the HBase columns mapping contains a badly formed " + + "column family, column qualifier specification."); + } + + if (elem.equals(HBaseSerDe.HBASE_KEY_COL)) { + rowKeyIndex = i; + colFamilies.add(elem); + colQualifiers.add(null); + } else { + String[] parts = elem.split(":"); + assert (parts.length > 0 && parts.length <= 2); + colFamilies.add(parts[0]); + + if (parts.length == 2) { + colQualifiers.add(parts[1]); + } else { + colQualifiers.add(null); + } + } + } + + if (rowKeyIndex == -1) { + colFamilies.add(0, HBaseSerDe.HBASE_KEY_COL); + colQualifiers.add(0, null); + rowKeyIndex = 0; + } + + if (colFamilies.size() != colQualifiers.size()) { + throw new IOException("Error in parsing the hbase columns mapping."); + } + + // populate the corresponding byte [] if the client has passed in a non-null list + if (colFamiliesBytes != null) { + colFamiliesBytes.clear(); + + for (String fam : colFamilies) { + colFamiliesBytes.add(Bytes.toBytes(fam)); + } + } + + if (colQualifiersBytes != null) { + colQualifiersBytes.clear(); + + for (String qual : colQualifiers) { + if (qual == null) { + colQualifiersBytes.add(null); + } else { + colQualifiersBytes.add(Bytes.toBytes(qual)); + } + } + } + + if (colFamiliesBytes != null && colQualifiersBytes != null) { + if (colFamiliesBytes.size() != colQualifiersBytes.size()) { + throw new IOException("Error in caching the bytes for the hbase column families " + + "and qualifiers."); + } + } + + return rowKeyIndex; + } + + /** + * Get delegation token from hbase and add it to JobConf + * @param job + * @throws IOException + */ + static void addHBaseDelegationToken(JobConf job) throws IOException { + if (User.isHBaseSecurityEnabled(job)) { + try { + User.getCurrent().obtainAuthTokenForJob(job); + } catch (InterruptedException e) { + throw new IOException("Error while obtaining hbase delegation token", e); + } + } + } + +} diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HCatTableSnapshot.java hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HCatTableSnapshot.java new file mode 100644 index 0000000..61a439e --- /dev/null +++ hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HCatTableSnapshot.java @@ -0,0 +1,92 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hcatalog.hbase; + +import java.io.Serializable; +import java.util.Map; + + +/** + * The class HCatTableSnapshot represents a snapshot of a hcatalog table. + * This class is intended to be opaque. This class would used only by the + * record readers to obtain knowledge about the revisions of a + * column to be filtered. + */ +public class HCatTableSnapshot implements Serializable { + + private static final long serialVersionUID = 1L; + private String tableName; + private String databaseName; + private Map columnMap; + private long latestRevision; + + HCatTableSnapshot(String databaseName, String tableName, Map columnMap, long latestRevision) { + this.tableName = tableName; + this.databaseName = databaseName; + this.columnMap = columnMap; + this.latestRevision = latestRevision; + } + + /** + * @return The name of the table in the snapshot. + */ + public String getTableName() { + return this.tableName; + } + + /** + * @return The name of the database to which the table snapshot belongs. + */ + public String getDatabaseName() { + return this.databaseName; + } + + /** + * @return The revision number of a column in a snapshot. + */ + long getRevision(String column) { + if (columnMap.containsKey(column)) + return this.columnMap.get(column); + return latestRevision; + } + + /** + * The method checks if the snapshot contains information about a data column. + * + * @param column The data column of the table + * @return true, if successful + */ + boolean containsColumn(String column) { + return this.columnMap.containsKey(column); + } + + /** + * @return latest committed revision when snapshot was taken + */ + long getLatestRevision() { + return latestRevision; + } + + @Override + public String toString() { + String snapshot = " Database Name: " + this.databaseName + " Table Name : " + tableName + + "Latest Revision: " + latestRevision + " Column revision : " + columnMap.toString(); + return snapshot; + } +} diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HbaseSnapshotRecordReader.java hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HbaseSnapshotRecordReader.java new file mode 100644 index 0000000..4f0e30c --- /dev/null +++ hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HbaseSnapshotRecordReader.java @@ -0,0 +1,255 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hcatalog.hbase; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.io.DataInputBuffer; +import org.apache.hadoop.io.DataOutputBuffer; +import org.apache.hadoop.mapred.RecordReader; +import org.apache.hive.hcatalog.common.HCatUtil; +import org.apache.hcatalog.hbase.snapshot.FamilyRevision; +import org.apache.hcatalog.hbase.snapshot.RevisionManager; +import org.apache.hcatalog.hbase.snapshot.TableSnapshot; +import org.apache.hive.hcatalog.mapreduce.InputJobInfo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The Class HbaseSnapshotRecordReader implements logic for filtering records + * based on snapshot. + */ +class HbaseSnapshotRecordReader implements RecordReader { + + static final Logger LOG = LoggerFactory.getLogger(HbaseSnapshotRecordReader.class); + private final InputJobInfo inpJobInfo; + private final Configuration conf; + private final int maxRevisions = 1; + private ResultScanner scanner; + private Scan scan; + private HTable htable; + private TableSnapshot snapshot; + private Iterator resultItr; + private Set allAbortedTransactions; + private DataOutputBuffer valueOut = new DataOutputBuffer(); + private DataInputBuffer valueIn = new DataInputBuffer(); + + HbaseSnapshotRecordReader(InputJobInfo inputJobInfo, Configuration conf) throws IOException { + this.inpJobInfo = inputJobInfo; + this.conf = conf; + String snapshotString = conf.get(HBaseConstants.PROPERTY_TABLE_SNAPSHOT_KEY); + HCatTableSnapshot hcatSnapshot = (HCatTableSnapshot) HCatUtil + .deserialize(snapshotString); + this.snapshot = HBaseRevisionManagerUtil.convertSnapshot(hcatSnapshot, + inpJobInfo.getTableInfo()); + } + + public void init() throws IOException { + restart(scan.getStartRow()); + } + + public void restart(byte[] firstRow) throws IOException { + allAbortedTransactions = getAbortedTransactions(Bytes.toString(htable.getTableName()), scan); + long maxValidRevision = getMaximumRevision(scan, snapshot); + while (allAbortedTransactions.contains(maxValidRevision)) { + maxValidRevision--; + } + Scan newScan = new Scan(scan); + newScan.setStartRow(firstRow); + //TODO: See if filters in 0.92 can be used to optimize the scan + //TODO: Consider create a custom snapshot filter + //TODO: Make min revision a constant in RM + newScan.setTimeRange(0, maxValidRevision + 1); + newScan.setMaxVersions(); + this.scanner = this.htable.getScanner(newScan); + resultItr = this.scanner.iterator(); + } + + private Set getAbortedTransactions(String tableName, Scan scan) throws IOException { + Set abortedTransactions = new HashSet(); + RevisionManager rm = null; + try { + rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(conf); + byte[][] families = scan.getFamilies(); + for (byte[] familyKey : families) { + String family = Bytes.toString(familyKey); + List abortedWriteTransactions = rm.getAbortedWriteTransactions( + tableName, family); + if (abortedWriteTransactions != null) { + for (FamilyRevision revision : abortedWriteTransactions) { + abortedTransactions.add(revision.getRevision()); + } + } + } + return abortedTransactions; + } finally { + HBaseRevisionManagerUtil.closeRevisionManagerQuietly(rm); + } + } + + private long getMaximumRevision(Scan scan, TableSnapshot snapshot) { + long maxRevision = 0; + byte[][] families = scan.getFamilies(); + for (byte[] familyKey : families) { + String family = Bytes.toString(familyKey); + long revision = snapshot.getRevision(family); + if (revision > maxRevision) + maxRevision = revision; + } + return maxRevision; + } + + /* + * @param htable The HTable ( of HBase) to use for the record reader. + * + */ + public void setHTable(HTable htable) { + this.htable = htable; + } + + /* + * @param scan The scan to be used for reading records. + * + */ + public void setScan(Scan scan) { + this.scan = scan; + } + + @Override + public ImmutableBytesWritable createKey() { + return new ImmutableBytesWritable(); + } + + @Override + public Result createValue() { + return new Result(); + } + + @Override + public long getPos() { + // This should be the ordinal tuple in the range; + // not clear how to calculate... + return 0; + } + + @Override + public float getProgress() throws IOException { + // Depends on the total number of tuples + return 0; + } + + @Override + public boolean next(ImmutableBytesWritable key, Result value) throws IOException { + if (this.resultItr == null) { + LOG.warn("The HBase result iterator is found null. It is possible" + + " that the record reader has already been closed."); + } else { + while (resultItr.hasNext()) { + Result temp = resultItr.next(); + Result hbaseRow = prepareResult(temp.list()); + if (hbaseRow != null) { + // Update key and value. Currently no way to avoid serialization/de-serialization + // as no setters are available. + key.set(hbaseRow.getRow()); + valueOut.reset(); + hbaseRow.write(valueOut); + valueIn.reset(valueOut.getData(), valueOut.getLength()); + value.readFields(valueIn); + return true; + } + + } + } + return false; + } + + private Result prepareResult(List keyvalues) { + + List finalKeyVals = new ArrayList(); + Map> qualValMap = new HashMap>(); + for (KeyValue kv : keyvalues) { + byte[] cf = kv.getFamily(); + byte[] qualifier = kv.getQualifier(); + String key = Bytes.toString(cf) + ":" + Bytes.toString(qualifier); + List kvs; + if (qualValMap.containsKey(key)) { + kvs = qualValMap.get(key); + } else { + kvs = new ArrayList(); + } + + String family = Bytes.toString(kv.getFamily()); + //Ignore aborted transactions + if (allAbortedTransactions.contains(kv.getTimestamp())) { + continue; + } + + long desiredTS = snapshot.getRevision(family); + if (kv.getTimestamp() <= desiredTS) { + kvs.add(kv); + } + qualValMap.put(key, kvs); + } + + Set keys = qualValMap.keySet(); + for (String cf : keys) { + List kvs = qualValMap.get(cf); + if (maxRevisions <= kvs.size()) { + for (int i = 0; i < maxRevisions; i++) { + finalKeyVals.add(kvs.get(i)); + } + } else { + finalKeyVals.addAll(kvs); + } + } + + if (finalKeyVals.size() == 0) { + return null; + } else { + KeyValue[] kvArray = new KeyValue[finalKeyVals.size()]; + finalKeyVals.toArray(kvArray); + return new Result(kvArray); + } + } + + /* + * @see org.apache.hadoop.hbase.mapred.TableRecordReader#close() + */ + @Override + public void close() { + this.resultItr = null; + this.scanner.close(); + } + +} diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/ImportSequenceFile.java hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/ImportSequenceFile.java new file mode 100644 index 0000000..b9f02af --- /dev/null +++ hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/ImportSequenceFile.java @@ -0,0 +1,252 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hcatalog.hbase; + +import org.apache.hadoop.filecache.DistributedCache; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hbase.mapreduce.HFileOutputFormat; +import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles; +import org.apache.hadoop.hbase.mapreduce.PutSortReducer; +import org.apache.hadoop.hbase.mapreduce.hadoopbackport.TotalOrderPartitioner; + +import java.io.IOException; +import java.net.URI; +import java.util.Map; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.JobContext; +import org.apache.hadoop.mapreduce.JobStatus; +import org.apache.hadoop.mapreduce.Mapper; +import org.apache.hadoop.mapreduce.OutputCommitter; +import org.apache.hadoop.mapreduce.TaskAttemptContext; +import org.apache.hadoop.mapreduce.TaskAttemptID; +import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; +import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat; +import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; +import org.apache.hive.hcatalog.mapreduce.HCatMapRedUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.hadoop.hbase.mapreduce.hadoopbackport.TotalOrderPartitioner.DEFAULT_PATH; + + +/** + * MapReduce job which reads a series of Puts stored in a sequence file + * and imports the data into HBase. It needs to create the necessary HBase + * regions using HFileOutputFormat and then notify the correct region servers + * to doBulkLoad(). This will be used After an MR job has written the SequenceFile + * and data needs to be bulk loaded onto HBase. + */ +class ImportSequenceFile { + private final static Logger LOG = LoggerFactory.getLogger(ImportSequenceFile.class); + private final static String NAME = "HCatImportSequenceFile"; + private final static String IMPORTER_WORK_DIR = "_IMPORTER_MR_WORK_DIR"; + + + private static class SequenceFileImporter extends Mapper { + + @Override + public void map(ImmutableBytesWritable rowKey, Put value, + Context context) + throws IOException { + try { + context.write(new ImmutableBytesWritable(value.getRow()), value); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } + + private static class ImporterOutputFormat extends HFileOutputFormat { + @Override + public OutputCommitter getOutputCommitter(TaskAttemptContext context) throws IOException { + final OutputCommitter baseOutputCommitter = super.getOutputCommitter(context); + + return new OutputCommitter() { + @Override + public void setupJob(JobContext jobContext) throws IOException { + baseOutputCommitter.setupJob(jobContext); + } + + @Override + public void setupTask(TaskAttemptContext taskContext) throws IOException { + baseOutputCommitter.setupTask(taskContext); + } + + @Override + public boolean needsTaskCommit(TaskAttemptContext taskContext) throws IOException { + return baseOutputCommitter.needsTaskCommit(taskContext); + } + + @Override + public void commitTask(TaskAttemptContext taskContext) throws IOException { + baseOutputCommitter.commitTask(taskContext); + } + + @Override + public void abortTask(TaskAttemptContext taskContext) throws IOException { + baseOutputCommitter.abortTask(taskContext); + } + + @Override + public void abortJob(JobContext jobContext, JobStatus.State state) throws IOException { + try { + baseOutputCommitter.abortJob(jobContext, state); + } finally { + cleanupScratch(jobContext); + } + } + + @Override + public void commitJob(JobContext jobContext) throws IOException { + try { + baseOutputCommitter.commitJob(jobContext); + Configuration conf = jobContext.getConfiguration(); + try { + //import hfiles + new LoadIncrementalHFiles(conf) + .doBulkLoad(HFileOutputFormat.getOutputPath(jobContext), + new HTable(conf, + conf.get(HBaseConstants.PROPERTY_OUTPUT_TABLE_NAME_KEY))); + } catch (Exception e) { + throw new IOException("BulkLoad failed.", e); + } + } finally { + cleanupScratch(jobContext); + } + } + + @Override + public void cleanupJob(JobContext context) throws IOException { + try { + baseOutputCommitter.cleanupJob(context); + } finally { + cleanupScratch(context); + } + } + + private void cleanupScratch(JobContext context) throws IOException { + FileSystem fs = FileSystem.get(context.getConfiguration()); + fs.delete(HFileOutputFormat.getOutputPath(context), true); + } + }; + } + } + + private static Job createSubmittableJob(Configuration conf, String tableName, Path inputDir, Path scratchDir, boolean localMode) + throws IOException { + Job job = new Job(conf, NAME + "_" + tableName); + job.setJarByClass(SequenceFileImporter.class); + FileInputFormat.setInputPaths(job, inputDir); + job.setInputFormatClass(SequenceFileInputFormat.class); + job.setMapperClass(SequenceFileImporter.class); + + HTable table = new HTable(conf, tableName); + job.setReducerClass(PutSortReducer.class); + FileOutputFormat.setOutputPath(job, scratchDir); + job.setMapOutputKeyClass(ImmutableBytesWritable.class); + job.setMapOutputValueClass(Put.class); + HFileOutputFormat.configureIncrementalLoad(job, table); + //override OutputFormatClass with our own so we can include cleanup in the committer + job.setOutputFormatClass(ImporterOutputFormat.class); + + //local mode doesn't support symbolic links so we have to manually set the actual path + if (localMode) { + String partitionFile = null; + for (URI uri : DistributedCache.getCacheFiles(job.getConfiguration())) { + if (DEFAULT_PATH.equals(uri.getFragment())) { + partitionFile = uri.toString(); + break; + } + } + partitionFile = partitionFile.substring(0, partitionFile.lastIndexOf("#")); + job.getConfiguration().set(TotalOrderPartitioner.PARTITIONER_PATH, partitionFile.toString()); + } + + return job; + } + + /** + * Method to run the Importer MapReduce Job. Normally will be called by another MR job + * during OutputCommitter.commitJob(). + * @param parentContext JobContext of the parent job + * @param tableName name of table to bulk load data into + * @param InputDir path of SequenceFile formatted data to read + * @param scratchDir temporary path for the Importer MR job to build the HFiles which will be imported + * @return + */ + static boolean runJob(JobContext parentContext, String tableName, Path InputDir, Path scratchDir) { + Configuration parentConf = parentContext.getConfiguration(); + Configuration conf = new Configuration(); + for (Map.Entry el : parentConf) { + if (el.getKey().startsWith("hbase.")) + conf.set(el.getKey(), el.getValue()); + if (el.getKey().startsWith("mapred.cache.archives")) + conf.set(el.getKey(), el.getValue()); + } + + //Inherit jar dependencies added to distributed cache loaded by parent job + conf.set("mapred.job.classpath.archives", parentConf.get("mapred.job.classpath.archives", "")); + conf.set("mapreduce.job.cache.archives.visibilities", parentConf.get("mapreduce.job.cache.archives.visibilities", "")); + + //Temporary fix until hbase security is ready + //We need the written HFile to be world readable so + //hbase regionserver user has the privileges to perform a hdfs move + if (parentConf.getBoolean("hadoop.security.authorization", false)) { + FsPermission.setUMask(conf, FsPermission.valueOf("----------")); + } + + conf.set(HBaseConstants.PROPERTY_OUTPUT_TABLE_NAME_KEY, tableName); + conf.setBoolean(JobContext.JOB_CANCEL_DELEGATION_TOKEN, false); + + boolean localMode = "local".equals(conf.get("mapred.job.tracker")); + + boolean success = false; + try { + FileSystem fs = FileSystem.get(parentConf); + Path workDir = new Path(new Job(parentConf).getWorkingDirectory(), IMPORTER_WORK_DIR); + if (!fs.mkdirs(workDir)) + throw new IOException("Importer work directory already exists: " + workDir); + Job job = createSubmittableJob(conf, tableName, InputDir, scratchDir, localMode); + job.setWorkingDirectory(workDir); + job.getCredentials().addAll(parentContext.getCredentials()); + success = job.waitForCompletion(true); + fs.delete(workDir, true); + //We only cleanup on success because failure might've been caused by existence of target directory + if (localMode && success) { + new ImporterOutputFormat().getOutputCommitter(HCatMapRedUtil.createTaskAttemptContext(conf, new TaskAttemptID())).commitJob(job); + } + } catch (InterruptedException e) { + LOG.error("ImportSequenceFile Failed", e); + } catch (ClassNotFoundException e) { + LOG.error("ImportSequenceFile Failed", e); + } catch (IOException e) { + LOG.error("ImportSequenceFile Failed", e); + } + return success; + } + +} diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/ResultConverter.java hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/ResultConverter.java new file mode 100644 index 0000000..f15bac7 --- /dev/null +++ hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/ResultConverter.java @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hcatalog.hbase; + +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hive.hcatalog.data.HCatRecord; + +import java.io.IOException; + +/** + * Interface used to define conversion of HCatRecord to and from Native HBase write (Put) and read (Result) objects. + * How the actual mapping is defined between an HBase Table's schema and an HCatalog Table's schema + * is up to the underlying implementation + */ +interface ResultConverter { + + /** + * convert HCatRecord instance to an HBase Put, used when writing out data. + * @param record instance to convert + * @return converted Put instance + * @throws IOException + */ + Put convert(HCatRecord record) throws IOException; + + /** + * convert HBase Result to HCatRecord instance, used when reading data. + * @param result instance to convert + * @return converted Result instance + * @throws IOException + */ + HCatRecord convert(Result result) throws IOException; + + /** + * Returns the hbase columns that are required for the scan. + * @return String containing hbase columns delimited by space. + * @throws IOException + */ + String getHBaseScanColumns() throws IOException; + +} diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/FamilyRevision.java hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/FamilyRevision.java new file mode 100644 index 0000000..b5103fc --- /dev/null +++ hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/FamilyRevision.java @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hcatalog.hbase.snapshot; + + +/** + * A FamiliyRevision class consists of a revision number and a expiration + * timestamp. When a write transaction starts, the transaction + * object is appended to the transaction list of the each column + * family and stored in the corresponding znode. When a write transaction is + * committed, the transaction object is removed from the list. + */ +public class FamilyRevision implements + Comparable { + + private long revision; + + private long timestamp; + + /** + * Create a FamilyRevision object + * @param rev revision number + * @param ts expiration timestamp + */ + FamilyRevision(long rev, long ts) { + this.revision = rev; + this.timestamp = ts; + } + + public long getRevision() { + return revision; + } + + public long getExpireTimestamp() { + return timestamp; + } + + void setExpireTimestamp(long ts) { + timestamp = ts; + } + + @Override + public String toString() { + String description = "revision: " + revision + " ts: " + timestamp; + return description; + } + + @Override + public int compareTo(FamilyRevision o) { + long d = revision - o.getRevision(); + return (d < 0) ? -1 : (d > 0) ? 1 : 0; + } + + +} diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/IDGenerator.java hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/IDGenerator.java new file mode 100644 index 0000000..a427544 --- /dev/null +++ hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/IDGenerator.java @@ -0,0 +1,145 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hcatalog.hbase.snapshot; + +import java.io.IOException; +import java.nio.charset.Charset; + +import org.apache.hcatalog.hbase.snapshot.lock.LockListener; +import org.apache.hcatalog.hbase.snapshot.lock.WriteLock; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.ZooDefs.Ids; +import org.apache.zookeeper.ZooKeeper; +import org.apache.zookeeper.data.Stat; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +/** + * This class generates revision id's for transactions. + */ +class IDGenerator implements LockListener { + + private ZooKeeper zookeeper; + private String zNodeDataLoc; + private String zNodeLockBasePath; + private long id; + private static final Logger LOG = LoggerFactory.getLogger(IDGenerator.class); + + IDGenerator(ZooKeeper zookeeper, String tableName, String idGenNode) + throws IOException { + this.zookeeper = zookeeper; + this.zNodeDataLoc = idGenNode; + this.zNodeLockBasePath = PathUtil.getLockManagementNode(idGenNode); + } + + /** + * This method obtains a revision id for a transaction. + * + * @return revision ID + * @throws IOException + */ + public long obtainID() throws IOException { + WriteLock wLock = new WriteLock(zookeeper, zNodeLockBasePath, Ids.OPEN_ACL_UNSAFE); + wLock.setLockListener(this); + try { + boolean lockGrabbed = wLock.lock(); + if (lockGrabbed == false) { + //TO DO : Let this request queue up and try obtaining lock. + throw new IOException("Unable to obtain lock to obtain id."); + } else { + id = incrementAndReadCounter(); + } + } catch (KeeperException e) { + LOG.warn("Exception while obtaining lock for ID.", e); + throw new IOException("Exception while obtaining lock for ID.", e); + } catch (InterruptedException e) { + LOG.warn("Exception while obtaining lock for ID.", e); + throw new IOException("Exception while obtaining lock for ID.", e); + } finally { + wLock.unlock(); + } + return id; + } + + /** + * This method reads the latest revision ID that has been used. The ID + * returned by this method cannot be used for transaction. + * @return revision ID + * @throws IOException + */ + public long readID() throws IOException { + long curId; + try { + Stat stat = new Stat(); + byte[] data = zookeeper.getData(this.zNodeDataLoc, false, stat); + curId = Long.parseLong(new String(data, Charset.forName("UTF-8"))); + } catch (KeeperException e) { + LOG.warn("Exception while reading current revision id.", e); + throw new IOException("Exception while reading current revision id.", e); + } catch (InterruptedException e) { + LOG.warn("Exception while reading current revision id.", e); + throw new IOException("Exception while reading current revision id.", e); + } + + return curId; + } + + + private long incrementAndReadCounter() throws IOException { + + long curId, usedId; + try { + Stat stat = new Stat(); + byte[] data = zookeeper.getData(this.zNodeDataLoc, false, stat); + usedId = Long.parseLong((new String(data, Charset.forName("UTF-8")))); + curId = usedId + 1; + String lastUsedID = String.valueOf(curId); + zookeeper.setData(this.zNodeDataLoc, lastUsedID.getBytes(Charset.forName("UTF-8")), -1); + + } catch (KeeperException e) { + LOG.warn("Exception while incrementing revision id.", e); + throw new IOException("Exception while incrementing revision id. ", e); + } catch (InterruptedException e) { + LOG.warn("Exception while incrementing revision id.", e); + throw new IOException("Exception while incrementing revision id. ", e); + } + + return curId; + } + + /* + * @see org.apache.hcatalog.hbase.snapshot.lock.LockListener#lockAcquired() + */ + @Override + public void lockAcquired() { + + + } + + /* + * @see org.apache.hcatalog.hbase.snapshot.lock.LockListener#lockReleased() + */ + @Override + public void lockReleased() { + + } + + +} diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/PathUtil.java hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/PathUtil.java new file mode 100644 index 0000000..993308b --- /dev/null +++ hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/PathUtil.java @@ -0,0 +1,132 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hcatalog.hbase.snapshot; + + +/** + * The PathUtil class is a utility class to provide information about various + * znode paths. The following is the znode structure used for storing information. + * baseDir/ClockNode + * baseDir/TrasactionBasePath + * baseDir/TrasactionBasePath/TableA/revisionID + * baseDir/TrasactionBasePath/TableA/columnFamily-1 + * baseDir/TrasactionBasePath/TableA/columnFamily-1/runnningTxns + * baseDir/TrasactionBasePath/TableA/columnFamily-1/abortedTxns + * baseDir/TrasactionBasePath/TableB/revisionID + * baseDir/TrasactionBasePath/TableB/columnFamily-1 + * baseDir/TrasactionBasePath/TableB/columnFamily-1/runnningTxns + * baseDir/TrasactionBasePath/TableB/columnFamily-1/abortedTxns + + */ +public class PathUtil { + + static final String DATA_DIR = "/data"; + static final String CLOCK_NODE = "/clock"; + + /** + * This method returns the data path associated with the currently + * running transactions of a given table and column/column family. + * @param baseDir + * @param tableName + * @param columnFamily + * @return The path of the running transactions data. + */ + static String getRunningTxnInfoPath(String baseDir, String tableName, + String columnFamily) { + String txnBasePath = getTransactionBasePath(baseDir); + String path = txnBasePath + "/" + tableName + "/" + columnFamily + + "/runningTxns"; + return path; + } + + /** + * This method returns the data path associated with the aborted + * transactions of a given table and column/column family. + * @param baseDir The base directory for revision management. + * @param tableName The name of the table. + * @param columnFamily + * @return The path of the aborted transactions data. + */ + static String getAbortInformationPath(String baseDir, String tableName, + String columnFamily) { + String txnBasePath = getTransactionBasePath(baseDir); + String path = txnBasePath + "/" + tableName + "/" + columnFamily + + "/abortData"; + return path; + } + + /** + * Gets the revision id node for a given table. + * + * @param baseDir the base dir for revision management. + * @param tableName the table name + * @return the revision id node path. + */ + static String getRevisionIDNode(String baseDir, String tableName) { + String rmBasePath = getTransactionBasePath(baseDir); + String revisionIDNode = rmBasePath + "/" + tableName + "/idgen"; + return revisionIDNode; + } + + /** + * Gets the lock management node for any znode that needs to be locked. + * + * @param path the path of the znode. + * @return the lock management node path. + */ + static String getLockManagementNode(String path) { + String lockNode = path + "_locknode_"; + return lockNode; + } + + /** + * This method returns the base path for the transaction data. + * + * @param baseDir The base dir for revision management. + * @return The base path for the transaction data. + */ + static String getTransactionBasePath(String baseDir) { + String txnBaseNode = baseDir + DATA_DIR; + return txnBaseNode; + } + + /** + * Gets the txn data path for a given table. + * + * @param baseDir the base dir for revision management. + * @param tableName the table name + * @return the txn data path for the table. + */ + static String getTxnDataPath(String baseDir, String tableName) { + String txnBasePath = getTransactionBasePath(baseDir); + String path = txnBasePath + "/" + tableName; + return path; + } + + /** + * This method returns the data path for clock node. + * + * @param baseDir + * @return The data path for clock. + */ + static String getClockPath(String baseDir) { + String clockNode = baseDir + CLOCK_NODE; + return clockNode; + } +} diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RMConstants.java hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RMConstants.java new file mode 100644 index 0000000..4d6fa80 --- /dev/null +++ hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RMConstants.java @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hcatalog.hbase.snapshot; + +public class RMConstants { + public static final String REVISION_MGR_ENDPOINT_IMPL_CLASS = "revision.manager.endpoint.impl.class"; + + public static final String WRITE_TRANSACTION_TIMEOUT = "revision.manager.writeTxn.timeout"; + + public static final String ZOOKEEPER_HOSTLIST = "revision.manager.zk.hostList"; + + public static final String ZOOKEEPER_DATADIR = "revision.manager.zk.dataDir"; +} diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManager.java hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManager.java new file mode 100644 index 0000000..4a6f842 --- /dev/null +++ hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManager.java @@ -0,0 +1,148 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hcatalog.hbase.snapshot; + +import org.apache.hadoop.conf.Configuration; + +import java.io.IOException; +import java.util.List; + +/** + * This interface provides APIs for implementing revision management. + */ +public interface RevisionManager { + /** + * Version property required by HBase to use this interface + * for CoprocessorProtocol / RPC. + */ + public static final long VERSION = 1L; // do not change + + /** + * Initialize the revision manager. + */ + public void initialize(Configuration conf); + + /** + * Opens the revision manager. + * + * @throws IOException + */ + public void open() throws IOException; + + /** + * Closes the revision manager. + * + * @throws IOException + */ + public void close() throws IOException; + + /** + * Setup revision management for a newly created hbase table. + * @param table the hbase table name + * @param columnFamilies the column families in the table + */ + public void createTable(String table, List columnFamilies) throws IOException; + + /** + * Remove table data from revision manager for a dropped table. + * @param table the hbase table name + */ + public void dropTable(String table) throws IOException; + + /** + * Start the write transaction. + * + * @param table + * @param families + * @return a new Transaction + * @throws IOException + */ + public Transaction beginWriteTransaction(String table, List families) + throws IOException; + + /** + * Start the write transaction. + * + * @param table + * @param families + * @param keepAlive + * @return a new Transaction + * @throws IOException + */ + public Transaction beginWriteTransaction(String table, + List families, long keepAlive) throws IOException; + + /** + * Commit the write transaction. + * + * @param transaction + * @throws IOException + */ + public void commitWriteTransaction(Transaction transaction) + throws IOException; + + /** + * Abort the write transaction. + * + * @param transaction + * @throws IOException + */ + public void abortWriteTransaction(Transaction transaction) + throws IOException; + + /** + * Get the list of aborted Transactions for a column family + * + * @param table the table name + * @param columnFamily the column family name + * @return a list of aborted WriteTransactions + * @throws java.io.IOException + */ + public List getAbortedWriteTransactions(String table, + String columnFamily) throws IOException; + + /** + * Create the latest snapshot of the table. + * + * @param tableName + * @return a new snapshot + * @throws IOException + */ + public TableSnapshot createSnapshot(String tableName) throws IOException; + + /** + * Create the snapshot of the table using the revision number. + * + * @param tableName + * @param revision + * @return a new snapshot + * @throws IOException + */ + public TableSnapshot createSnapshot(String tableName, long revision) + throws IOException; + + /** + * Extends the expiration of a transaction by the time indicated by keep alive. + * + * @param transaction + * @throws IOException + */ + public void keepAlive(Transaction transaction) throws IOException; + +} diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerConfiguration.java hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerConfiguration.java new file mode 100644 index 0000000..d5c4329 --- /dev/null +++ hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerConfiguration.java @@ -0,0 +1,59 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hcatalog.hbase.snapshot; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; + +public class RevisionManagerConfiguration { + + + public static Configuration addResources(Configuration conf) { + conf.addDefaultResource("revision-manager-default.xml"); + conf.addResource("revision-manager-site.xml"); + return conf; + } + + /** + * Creates a Configuration with Revision Manager resources + * @return a Configuration with Revision Manager resources + */ + public static Configuration create() { + Configuration conf = new Configuration(); + return addResources(conf); + } + + /** + * Creates a clone of passed configuration. + * @param that Configuration to clone. + * @return a Configuration created with the revision-manager-*.xml files plus + * the given configuration. + */ + public static Configuration create(final Configuration that) { + Configuration conf = create(); + //we need to merge things instead of doing new Configuration(that) + //because of a bug in Configuration wherein the config + //set on the MR fronted will get loaded on the backend as resouce called job.xml + //hence adding resources on the backed could potentially overwrite properties + //set on the frontend which we shouldn't be doing here + HBaseConfiguration.merge(conf, that); + return conf; + } +} diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerEndpoint.java hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerEndpoint.java new file mode 100644 index 0000000..49d9ad1 --- /dev/null +++ hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerEndpoint.java @@ -0,0 +1,141 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hcatalog.hbase.snapshot; + +import java.io.IOException; +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.CoprocessorEnvironment; +import org.apache.hadoop.hbase.coprocessor.BaseEndpointCoprocessor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Implementation of RevisionManager as HBase RPC endpoint. This class will control the lifecycle of + * and delegate to the actual RevisionManager implementation and make it available as a service + * hosted in the HBase region server (instead of running it in the client (storage handler). + * In the case of {@link ZKBasedRevisionManager} now only the region servers need write access to + * manage revision data. + */ +public class RevisionManagerEndpoint extends BaseEndpointCoprocessor implements RevisionManagerProtocol { + + private static final Logger LOGGER = + LoggerFactory.getLogger(RevisionManagerEndpoint.class.getName()); + + private RevisionManager rmImpl = null; + + @Override + public void start(CoprocessorEnvironment env) { + super.start(env); + try { + Configuration conf = RevisionManagerConfiguration.create(env.getConfiguration()); + String className = conf.get(RMConstants.REVISION_MGR_ENDPOINT_IMPL_CLASS, + ZKBasedRevisionManager.class.getName()); + LOGGER.debug("Using Revision Manager implementation: {}", className); + rmImpl = RevisionManagerFactory.getOpenedRevisionManager(className, conf); + } catch (IOException e) { + LOGGER.error("Failed to initialize revision manager", e); + } + } + + @Override + public void stop(CoprocessorEnvironment env) { + if (rmImpl != null) { + try { + rmImpl.close(); + } catch (IOException e) { + LOGGER.warn("Error closing revision manager.", e); + } + } + super.stop(env); + } + + @Override + public void initialize(Configuration conf) { + // do nothing, HBase controls life cycle + } + + @Override + public void open() throws IOException { + // do nothing, HBase controls life cycle + } + + @Override + public void close() throws IOException { + // do nothing, HBase controls life cycle + } + + @Override + public void createTable(String table, List columnFamilies) throws IOException { + rmImpl.createTable(table, columnFamilies); + } + + @Override + public void dropTable(String table) throws IOException { + rmImpl.dropTable(table); + } + + @Override + public Transaction beginWriteTransaction(String table, List families) + throws IOException { + return rmImpl.beginWriteTransaction(table, families); + } + + @Override + public Transaction beginWriteTransaction(String table, + List families, long keepAlive) throws IOException { + return rmImpl.beginWriteTransaction(table, families, keepAlive); + } + + @Override + public void commitWriteTransaction(Transaction transaction) + throws IOException { + rmImpl.commitWriteTransaction(transaction); + } + + @Override + public void abortWriteTransaction(Transaction transaction) + throws IOException { + rmImpl.abortWriteTransaction(transaction); + } + + @Override + public TableSnapshot createSnapshot(String tableName) throws IOException { + return rmImpl.createSnapshot(tableName); + } + + @Override + public TableSnapshot createSnapshot(String tableName, long revision) + throws IOException { + return rmImpl.createSnapshot(tableName, revision); + } + + @Override + public void keepAlive(Transaction transaction) throws IOException { + rmImpl.keepAlive(transaction); + } + + @Override + public List getAbortedWriteTransactions(String table, + String columnFamily) throws IOException { + return rmImpl.getAbortedWriteTransactions(table, columnFamily); + } + +} diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerEndpointClient.java hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerEndpointClient.java new file mode 100644 index 0000000..c6ee50e --- /dev/null +++ hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerEndpointClient.java @@ -0,0 +1,125 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hcatalog.hbase.snapshot; + +import java.io.IOException; +import java.util.List; + +import org.apache.hadoop.conf.Configurable; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.util.Bytes; + +/** + * This class is nothing but a delegate for the enclosed proxy, + * which is created upon setting the configuration. + */ +public class RevisionManagerEndpointClient implements RevisionManager, Configurable { + + private Configuration conf = null; + private RevisionManager rmProxy; + + @Override + public Configuration getConf() { + return this.conf; + } + + @Override + public void setConf(Configuration arg0) { + this.conf = arg0; + } + + @Override + public void initialize(Configuration conf) { + // do nothing + } + + @Override + public void open() throws IOException { + // clone to adjust RPC settings unique to proxy + Configuration clonedConf = new Configuration(conf); + // conf.set("hbase.ipc.client.connect.max.retries", "0"); + // conf.setInt(HConstants.HBASE_CLIENT_RPC_MAXATTEMPTS, 1); + clonedConf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1); // do not retry RPC + HTable table = new HTable(clonedConf, HConstants.ROOT_TABLE_NAME); + rmProxy = table.coprocessorProxy(RevisionManagerProtocol.class, + Bytes.toBytes("anyRow")); + rmProxy.open(); + } + + @Override + public void close() throws IOException { + rmProxy.close(); + } + + @Override + public void createTable(String table, List columnFamilies) throws IOException { + rmProxy.createTable(table, columnFamilies); + } + + @Override + public void dropTable(String table) throws IOException { + rmProxy.dropTable(table); + } + + @Override + public Transaction beginWriteTransaction(String table, List families) throws IOException { + return rmProxy.beginWriteTransaction(table, families); + } + + @Override + public Transaction beginWriteTransaction(String table, List families, long keepAlive) + throws IOException { + return rmProxy.beginWriteTransaction(table, families, keepAlive); + } + + @Override + public void commitWriteTransaction(Transaction transaction) throws IOException { + rmProxy.commitWriteTransaction(transaction); + } + + @Override + public void abortWriteTransaction(Transaction transaction) throws IOException { + rmProxy.abortWriteTransaction(transaction); + } + + @Override + public List getAbortedWriteTransactions(String table, String columnFamily) + throws IOException { + return rmProxy.getAbortedWriteTransactions(table, columnFamily); + } + + @Override + public TableSnapshot createSnapshot(String tableName) throws IOException { + return rmProxy.createSnapshot(tableName); + } + + @Override + public TableSnapshot createSnapshot(String tableName, long revision) throws IOException { + return rmProxy.createSnapshot(tableName, revision); + } + + @Override + public void keepAlive(Transaction transaction) throws IOException { + rmProxy.keepAlive(transaction); + } + +} diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerFactory.java hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerFactory.java new file mode 100644 index 0000000..2d3d3e4 --- /dev/null +++ hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerFactory.java @@ -0,0 +1,105 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hcatalog.hbase.snapshot; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configurable; +import org.apache.hadoop.conf.Configuration; + +/** + * Utility to instantiate the revision manager (not a true factory actually). + * Depends on HBase configuration to resolve ZooKeeper connection (when ZK is used). + */ +public class RevisionManagerFactory { + + public static final String REVISION_MGR_IMPL_CLASS = "revision.manager.impl.class"; + + /** + * Gets an instance of revision manager. + * + * @param conf The configuration required to created the revision manager. + * @return the revision manager An instance of revision manager. + * @throws IOException Signals that an I/O exception has occurred. + */ + private static RevisionManager getRevisionManager(String className, Configuration conf) throws IOException { + + RevisionManager revisionMgr; + ClassLoader classLoader = Thread.currentThread() + .getContextClassLoader(); + if (classLoader == null) { + classLoader = RevisionManagerFactory.class.getClassLoader(); + } + try { + Class revisionMgrClass = Class + .forName(className, true, classLoader).asSubclass(RevisionManager.class); + revisionMgr = (RevisionManager) revisionMgrClass.newInstance(); + revisionMgr.initialize(conf); + } catch (ClassNotFoundException e) { + throw new IOException( + "The implementation class of revision manager not found.", + e); + } catch (InstantiationException e) { + throw new IOException( + "Exception encountered during instantiating revision manager implementation.", + e); + } catch (IllegalAccessException e) { + throw new IOException( + "IllegalAccessException encountered during instantiating revision manager implementation.", + e); + } catch (IllegalArgumentException e) { + throw new IOException( + "IllegalArgumentException encountered during instantiating revision manager implementation.", + e); + } + return revisionMgr; + } + + /** + * Internally used by endpoint implementation to instantiate from different configuration setting. + * @param className + * @param conf + * @return the opened revision manager + * @throws IOException + */ + static RevisionManager getOpenedRevisionManager(String className, Configuration conf) throws IOException { + + RevisionManager revisionMgr = RevisionManagerFactory.getRevisionManager(className, conf); + if (revisionMgr instanceof Configurable) { + ((Configurable) revisionMgr).setConf(conf); + } + revisionMgr.open(); + return revisionMgr; + } + + /** + * Gets an instance of revision manager which is opened. + * The revision manager implementation can be specified as {@link #REVISION_MGR_IMPL_CLASS}, + * default is {@link ZKBasedRevisionManager}. + * @param conf revision manager configuration + * @return RevisionManager An instance of revision manager. + * @throws IOException + */ + public static RevisionManager getOpenedRevisionManager(Configuration conf) throws IOException { + String className = conf.get(RevisionManagerFactory.REVISION_MGR_IMPL_CLASS, + ZKBasedRevisionManager.class.getName()); + return getOpenedRevisionManager(className, conf); + } + +} diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerProtocol.java hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerProtocol.java new file mode 100644 index 0000000..4cbde74 --- /dev/null +++ hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerProtocol.java @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hcatalog.hbase.snapshot; + +import org.apache.hadoop.hbase.ipc.CoprocessorProtocol; + +/** + * Interface marker to implement RevisionManager as Coprocessor. + * (needs to extend CoprocessorProtocol) + */ +public interface RevisionManagerProtocol extends RevisionManager, + CoprocessorProtocol { + +} diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/TableSnapshot.java hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/TableSnapshot.java new file mode 100644 index 0000000..fa94157 --- /dev/null +++ hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/TableSnapshot.java @@ -0,0 +1,90 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hcatalog.hbase.snapshot; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * The snapshot for a table and a list of column families. + */ +public class TableSnapshot implements Serializable { + + private String name; + + private Map cfRevisionMap; + + private long latestRevision; + + + public TableSnapshot(String name, Map cfRevMap, long latestRevision) { + this.name = name; + if (cfRevMap == null) { + throw new IllegalArgumentException("revision map cannot be null"); + } + this.cfRevisionMap = cfRevMap; + this.latestRevision = latestRevision; + } + + /** + * Gets the table name. + * + * @return String The name of the table. + */ + public String getTableName() { + return name; + } + + /** + * Gets the column families. + * + * @return List A list of column families associated with the snapshot. + */ + public List getColumnFamilies(){ + return new ArrayList(this.cfRevisionMap.keySet()); + } + + /** + * Gets the revision. + * + * @param familyName The name of the column family. + * @return the revision + */ + public long getRevision(String familyName){ + if(cfRevisionMap.containsKey(familyName)) + return cfRevisionMap.get(familyName); + return latestRevision; + } + + /** + * @return the latest committed revision when this snapshot was taken + */ + public long getLatestRevision() { + return latestRevision; + } + + @Override + public String toString() { + String snapshot = "Table Name : " + name +" Latest Revision: " + latestRevision + + " Column Familiy revision : " + cfRevisionMap.toString(); + return snapshot; + } +} diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/Transaction.java hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/Transaction.java new file mode 100644 index 0000000..1d17ca5 --- /dev/null +++ hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/Transaction.java @@ -0,0 +1,116 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hcatalog.hbase.snapshot; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +/** + * This class is responsible for storing information related to + * transactions. + */ +public class Transaction implements Serializable { + + private String tableName; + private List columnFamilies = new ArrayList(); + private long timeStamp; + private long keepAlive; + private long revision; + + + Transaction(String tableName, List columnFamilies, long revision, long timestamp) { + this.tableName = tableName; + this.columnFamilies = columnFamilies; + this.timeStamp = timestamp; + this.revision = revision; + } + + /** + * @return The revision number associated with a transaction. + */ + public long getRevisionNumber() { + return this.revision; + } + + /** + * @return The table name associated with a transaction. + */ + public String getTableName() { + return tableName; + } + + /** + * @return The column families associated with a transaction. + */ + public List getColumnFamilies() { + return columnFamilies; + } + + /** + * @return The expire timestamp associated with a transaction. + */ + long getTransactionExpireTimeStamp() { + return this.timeStamp + this.keepAlive; + } + + void setKeepAlive(long seconds) { + this.keepAlive = seconds; + } + + /** + * Gets the keep alive value. + * + * @return long The keep alive value for the transaction. + */ + public long getKeepAliveValue() { + return this.keepAlive; + } + + /** + * Gets the family revision info. + * + * @return FamilyRevision An instance of FamilyRevision associated with the transaction. + */ + FamilyRevision getFamilyRevisionInfo() { + return new FamilyRevision(revision, getTransactionExpireTimeStamp()); + } + + /** + * Keep alive transaction. This methods extends the expire timestamp of a + * transaction by the "keep alive" amount. + */ + void keepAliveTransaction() { + this.timeStamp = this.timeStamp + this.keepAlive; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("Revision : "); + sb.append(this.getRevisionNumber()); + sb.append(" Timestamp : "); + sb.append(this.getTransactionExpireTimeStamp()); + sb.append("\n").append("Table : "); + sb.append(this.tableName).append("\n"); + sb.append("Column Families : "); + sb.append(this.columnFamilies.toString()); + return sb.toString(); + } +} diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/ZKBasedRevisionManager.java hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/ZKBasedRevisionManager.java new file mode 100644 index 0000000..f4556d1 --- /dev/null +++ hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/ZKBasedRevisionManager.java @@ -0,0 +1,461 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hcatalog.hbase.snapshot; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hcatalog.hbase.snapshot.lock.LockListener; +import org.apache.hcatalog.hbase.snapshot.lock.WriteLock; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.ZooDefs.Ids; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The service for providing revision management to Hbase tables. + */ +public class ZKBasedRevisionManager implements RevisionManager { + + private static final Logger LOG = LoggerFactory.getLogger(ZKBasedRevisionManager.class); + private String zkHostList; + private String baseDir; + private ZKUtil zkUtil; + private long writeTxnTimeout; + + + /* + * @see org.apache.hcatalog.hbase.snapshot.RevisionManager#initialize() + */ + @Override + public void initialize(Configuration conf) { + conf = new Configuration(conf); + if (conf.get(RMConstants.ZOOKEEPER_HOSTLIST) == null) { + String zkHostList = conf.get(HConstants.ZOOKEEPER_QUORUM); + int port = conf.getInt(HConstants.ZOOKEEPER_CLIENT_PORT, + HConstants.DEFAULT_ZOOKEPER_CLIENT_PORT); + String[] splits = zkHostList.split(","); + StringBuffer sb = new StringBuffer(); + for (String split : splits) { + sb.append(split); + sb.append(':'); + sb.append(port); + sb.append(','); + } + sb.deleteCharAt(sb.length() - 1); + conf.set(RMConstants.ZOOKEEPER_HOSTLIST, sb.toString()); + } + this.zkHostList = conf.get(RMConstants.ZOOKEEPER_HOSTLIST); + this.baseDir = conf.get(RMConstants.ZOOKEEPER_DATADIR); + this.writeTxnTimeout = Long.parseLong(conf.get(RMConstants.WRITE_TRANSACTION_TIMEOUT)); + } + + /** + * Open a ZooKeeper connection + * @throws java.io.IOException + */ + + public void open() throws IOException { + zkUtil = new ZKUtil(zkHostList, this.baseDir); + zkUtil.createRootZNodes(); + LOG.info("Created root znodes for revision manager."); + } + + /** + * Close Zookeeper connection + */ + public void close() { + zkUtil.closeZKConnection(); + } + + private void checkInputParams(String table, List families) { + if (table == null) { + throw new IllegalArgumentException( + "The table name must be specified for reading."); + } + if (families == null || families.isEmpty()) { + throw new IllegalArgumentException( + "At least one column family should be specified for reading."); + } + } + + @Override + public void createTable(String table, List columnFamilies) throws IOException { + zkUtil.createRootZNodes(); + zkUtil.setUpZnodesForTable(table, columnFamilies); + } + + @Override + public void dropTable(String table) throws IOException { + zkUtil.deleteZNodes(table); + } + + /* @param table + /* @param families + /* @param keepAlive + /* @return + /* @throws IOException + * @see org.apache.hcatalog.hbase.snapshot.RevisionManager#beginWriteTransaction(java.lang.String, java.util.List, long) + */ + public Transaction beginWriteTransaction(String table, + List families, long keepAlive) throws IOException { + + checkInputParams(table, families); + zkUtil.setUpZnodesForTable(table, families); + long nextId = zkUtil.nextId(table); + long expireTimestamp = zkUtil.getTimeStamp(); + Transaction transaction = new Transaction(table, families, nextId, + expireTimestamp); + if (keepAlive != -1) { + transaction.setKeepAlive(keepAlive); + } else { + transaction.setKeepAlive(writeTxnTimeout); + } + + refreshTransactionList(transaction.getTableName()); + String lockPath = prepareLockNode(table); + WriteLock wLock = new WriteLock(zkUtil.getSession(), lockPath, + Ids.OPEN_ACL_UNSAFE); + RMLockListener myLockListener = new RMLockListener(); + wLock.setLockListener(myLockListener); + try { + boolean lockGrabbed = wLock.lock(); + if (lockGrabbed == false) { + //TO DO : Let this request queue up and try obtaining lock. + throw new IOException( + "Unable to obtain lock while beginning transaction. " + + transaction.toString()); + } else { + List colFamilies = transaction.getColumnFamilies(); + FamilyRevision revisionData = transaction.getFamilyRevisionInfo(); + for (String cfamily : colFamilies) { + String path = PathUtil.getRunningTxnInfoPath( + baseDir, table, cfamily); + zkUtil.updateData(path, revisionData, + ZKUtil.UpdateMode.APPEND); + } + } + } catch (KeeperException e) { + throw new IOException("Exception while obtaining lock.", e); + } catch (InterruptedException e) { + throw new IOException("Exception while obtaining lock.", e); + } finally { + wLock.unlock(); + } + + return transaction; + } + + /* @param table The table name. + /* @param families The column families involved in the transaction. + /* @return transaction The transaction which was started. + /* @throws IOException + * @see org.apache.hcatalog.hbase.snapshot.RevisionManager#beginWriteTransaction(java.lang.String, java.util.List) + */ + public Transaction beginWriteTransaction(String table, List families) + throws IOException { + return beginWriteTransaction(table, families, -1); + } + + /** + * This method commits a write transaction. + * @param transaction The revision information associated with transaction. + * @throws java.io.IOException + */ + public void commitWriteTransaction(Transaction transaction) throws IOException { + refreshTransactionList(transaction.getTableName()); + + String lockPath = prepareLockNode(transaction.getTableName()); + WriteLock wLock = new WriteLock(zkUtil.getSession(), lockPath, + Ids.OPEN_ACL_UNSAFE); + RMLockListener myLockListener = new RMLockListener(); + wLock.setLockListener(myLockListener); + try { + boolean lockGrabbed = wLock.lock(); + if (lockGrabbed == false) { + //TO DO : Let this request queue up and try obtaining lock. + throw new IOException( + "Unable to obtain lock while commiting transaction. " + + transaction.toString()); + } else { + String tableName = transaction.getTableName(); + List colFamilies = transaction.getColumnFamilies(); + FamilyRevision revisionData = transaction.getFamilyRevisionInfo(); + for (String cfamily : colFamilies) { + String path = PathUtil.getRunningTxnInfoPath( + baseDir, tableName, cfamily); + zkUtil.updateData(path, revisionData, + ZKUtil.UpdateMode.REMOVE); + } + + } + } catch (KeeperException e) { + throw new IOException("Exception while obtaining lock.", e); + } catch (InterruptedException e) { + throw new IOException("Exception while obtaining lock.", e); + } finally { + wLock.unlock(); + } + LOG.info("Write Transaction committed: " + transaction.toString()); + } + + /** + * This method aborts a write transaction. + * @param transaction + * @throws java.io.IOException + */ + public void abortWriteTransaction(Transaction transaction) throws IOException { + + refreshTransactionList(transaction.getTableName()); + String lockPath = prepareLockNode(transaction.getTableName()); + WriteLock wLock = new WriteLock(zkUtil.getSession(), lockPath, + Ids.OPEN_ACL_UNSAFE); + RMLockListener myLockListener = new RMLockListener(); + wLock.setLockListener(myLockListener); + try { + boolean lockGrabbed = wLock.lock(); + if (lockGrabbed == false) { + //TO DO : Let this request queue up and try obtaining lock. + throw new IOException( + "Unable to obtain lock while aborting transaction. " + + transaction.toString()); + } else { + String tableName = transaction.getTableName(); + List colFamilies = transaction.getColumnFamilies(); + FamilyRevision revisionData = transaction + .getFamilyRevisionInfo(); + for (String cfamily : colFamilies) { + String path = PathUtil.getRunningTxnInfoPath( + baseDir, tableName, cfamily); + zkUtil.updateData(path, revisionData, + ZKUtil.UpdateMode.REMOVE); + path = PathUtil.getAbortInformationPath(baseDir, + tableName, cfamily); + zkUtil.updateData(path, revisionData, + ZKUtil.UpdateMode.APPEND); + } + + } + } catch (KeeperException e) { + throw new IOException("Exception while obtaining lock.", e); + } catch (InterruptedException e) { + throw new IOException("Exception while obtaining lock.", e); + } finally { + wLock.unlock(); + } + LOG.info("Write Transaction aborted: " + transaction.toString()); + } + + + /* @param transaction + /* @throws IOException + * @see org.apache.hcatalog.hbase.snapshot.RevsionManager#keepAlive(org.apache.hcatalog.hbase.snapshot.Transaction) + */ + public void keepAlive(Transaction transaction) + throws IOException { + + refreshTransactionList(transaction.getTableName()); + transaction.keepAliveTransaction(); + String lockPath = prepareLockNode(transaction.getTableName()); + WriteLock wLock = new WriteLock(zkUtil.getSession(), lockPath, + Ids.OPEN_ACL_UNSAFE); + RMLockListener myLockListener = new RMLockListener(); + wLock.setLockListener(myLockListener); + try { + boolean lockGrabbed = wLock.lock(); + if (lockGrabbed == false) { + //TO DO : Let this request queue up and try obtaining lock. + throw new IOException( + "Unable to obtain lock for keep alive of transaction. " + + transaction.toString()); + } else { + String tableName = transaction.getTableName(); + List colFamilies = transaction.getColumnFamilies(); + FamilyRevision revisionData = transaction.getFamilyRevisionInfo(); + for (String cfamily : colFamilies) { + String path = PathUtil.getRunningTxnInfoPath( + baseDir, tableName, cfamily); + zkUtil.updateData(path, revisionData, + ZKUtil.UpdateMode.KEEP_ALIVE); + } + + } + } catch (KeeperException e) { + throw new IOException("Exception while obtaining lock.", e); + } catch (InterruptedException e) { + throw new IOException("Exception while obtaining lock.", e); + } finally { + wLock.unlock(); + } + + } + + /* This method allows the user to create latest snapshot of a + /* table. + /* @param tableName The table whose snapshot is being created. + /* @return TableSnapshot An instance of TableSnaphot + /* @throws IOException + * @see org.apache.hcatalog.hbase.snapshot.RevsionManager#createSnapshot(java.lang.String) + */ + public TableSnapshot createSnapshot(String tableName) throws IOException { + refreshTransactionList(tableName); + long latestID = zkUtil.currentID(tableName); + HashMap cfMap = new HashMap(); + List columnFamilyNames = zkUtil.getColumnFamiliesOfTable(tableName); + + for (String cfName : columnFamilyNames) { + String cfPath = PathUtil.getRunningTxnInfoPath(baseDir, tableName, cfName); + List tranxList = zkUtil.getTransactionList(cfPath); + long version; + if (!tranxList.isEmpty()) { + Collections.sort(tranxList); + // get the smallest running Transaction ID + long runningVersion = tranxList.get(0).getRevision(); + version = runningVersion - 1; + } else { + version = latestID; + } + cfMap.put(cfName, version); + } + + TableSnapshot snapshot = new TableSnapshot(tableName, cfMap, latestID); + LOG.debug("Created snapshot For table: " + tableName + " snapshot: " + snapshot); + return snapshot; + } + + /* This method allows the user to create snapshot of a + /* table with a given revision number. + /* @param tableName + /* @param revision + /* @return TableSnapshot + /* @throws IOException + * @see org.apache.hcatalog.hbase.snapshot.RevsionManager#createSnapshot(java.lang.String, long) + */ + public TableSnapshot createSnapshot(String tableName, long revision) throws IOException { + + long currentID = zkUtil.currentID(tableName); + if (revision > currentID) { + throw new IOException( + "The revision specified in the snapshot is higher than the current revision of the table."); + } + refreshTransactionList(tableName); + HashMap cfMap = new HashMap(); + List columnFamilies = zkUtil.getColumnFamiliesOfTable(tableName); + + for (String cf : columnFamilies) { + cfMap.put(cf, revision); + } + + return new TableSnapshot(tableName, cfMap, revision); + } + + /** + * Get the list of in-progress Transactions for a column family + * @param table the table name + * @param columnFamily the column family name + * @return a list of in-progress WriteTransactions + * @throws java.io.IOException + */ + List getRunningTransactions(String table, + String columnFamily) throws IOException { + String path = PathUtil.getRunningTxnInfoPath(baseDir, table, + columnFamily); + return zkUtil.getTransactionList(path); + } + + @Override + public List getAbortedWriteTransactions(String table, + String columnFamily) throws IOException { + String path = PathUtil.getAbortInformationPath(baseDir, table, columnFamily); + return zkUtil.getTransactionList(path); + } + + private void refreshTransactionList(String tableName) throws IOException { + String lockPath = prepareLockNode(tableName); + WriteLock wLock = new WriteLock(zkUtil.getSession(), lockPath, + Ids.OPEN_ACL_UNSAFE); + RMLockListener myLockListener = new RMLockListener(); + wLock.setLockListener(myLockListener); + try { + boolean lockGrabbed = wLock.lock(); + if (lockGrabbed == false) { + //TO DO : Let this request queue up and try obtaining lock. + throw new IOException( + "Unable to obtain lock while refreshing transactions of table " + + tableName + "."); + } else { + List cfPaths = zkUtil + .getColumnFamiliesOfTable(tableName); + for (String cf : cfPaths) { + String runningDataPath = PathUtil.getRunningTxnInfoPath( + baseDir, tableName, cf); + zkUtil.refreshTransactions(runningDataPath); + } + + } + } catch (KeeperException e) { + throw new IOException("Exception while obtaining lock.", e); + } catch (InterruptedException e) { + throw new IOException("Exception while obtaining lock.", e); + } finally { + wLock.unlock(); + } + + } + + private String prepareLockNode(String tableName) throws IOException { + String txnDataPath = PathUtil.getTxnDataPath(this.baseDir, tableName); + String lockPath = PathUtil.getLockManagementNode(txnDataPath); + zkUtil.ensurePathExists(lockPath, null, Ids.OPEN_ACL_UNSAFE, + CreateMode.PERSISTENT); + return lockPath; + } + + /* + * This class is a listener class for the locks used in revision management. + * TBD: Use the following class to signal that that the lock is actually + * been granted. + */ + class RMLockListener implements LockListener { + + /* + * @see org.apache.hcatalog.hbase.snapshot.lock.LockListener#lockAcquired() + */ + @Override + public void lockAcquired() { + + } + + /* + * @see org.apache.hcatalog.hbase.snapshot.lock.LockListener#lockReleased() + */ + @Override + public void lockReleased() { + + } + + } + + +} diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/ZKUtil.java hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/ZKUtil.java new file mode 100644 index 0000000..58fd435 --- /dev/null +++ hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/ZKUtil.java @@ -0,0 +1,525 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hcatalog.hbase.snapshot; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hcatalog.hbase.snapshot.transaction.thrift.StoreFamilyRevision; +import org.apache.hcatalog.hbase.snapshot.transaction.thrift.StoreFamilyRevisionList; +import org.apache.thrift.TBase; +import org.apache.thrift.TDeserializer; +import org.apache.thrift.TSerializer; +import org.apache.thrift.protocol.TBinaryProtocol; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.WatchedEvent; +import org.apache.zookeeper.Watcher; +import org.apache.zookeeper.ZooDefs.Ids; +import org.apache.zookeeper.ZooKeeper; +import org.apache.zookeeper.ZooKeeper.States; +import org.apache.zookeeper.data.ACL; +import org.apache.zookeeper.data.Stat; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +class ZKUtil { + + private int DEFAULT_SESSION_TIMEOUT = 1000000; + private ZooKeeper zkSession; + private String baseDir; + private String connectString; + private static final Logger LOG = LoggerFactory.getLogger(ZKUtil.class); + + static enum UpdateMode { + APPEND, REMOVE, KEEP_ALIVE + } + + ; + + ZKUtil(String connection, String baseDir) { + this.connectString = connection; + this.baseDir = baseDir; + } + + /** + * This method creates znodes related to table. + * + * @param table The name of the table. + * @param families The list of column families of the table. + * @throws IOException + */ + void setUpZnodesForTable(String table, List families) + throws IOException { + + String transactionDataTablePath = PathUtil.getTxnDataPath(baseDir, table); + ensurePathExists(transactionDataTablePath, null, Ids.OPEN_ACL_UNSAFE, + CreateMode.PERSISTENT); + for (String cf : families) { + String runningDataPath = PathUtil.getRunningTxnInfoPath( + this.baseDir, table, cf); + ensurePathExists(runningDataPath, null, Ids.OPEN_ACL_UNSAFE, + CreateMode.PERSISTENT); + String abortDataPath = PathUtil.getAbortInformationPath( + this.baseDir, table, cf); + ensurePathExists(abortDataPath, null, Ids.OPEN_ACL_UNSAFE, + CreateMode.PERSISTENT); + } + + } + + /** + * This method ensures that a given path exists in zookeeper. If the path + * does not exists, it creates one. + * + * @param path The path of znode that is required to exist. + * @param data The data to be associated with the znode. + * @param acl The ACLs required. + * @param flags The CreateMode for the znode. + * @throws IOException + */ + void ensurePathExists(String path, byte[] data, List acl, + CreateMode flags) throws IOException { + String[] dirs = path.split("/"); + String parentPath = ""; + for (String subDir : dirs) { + if (subDir.equals("") == false) { + parentPath = parentPath + "/" + subDir; + try { + Stat stat = getSession().exists(parentPath, false); + if (stat == null) { + getSession().create(parentPath, data, acl, flags); + } + } catch (Exception e) { + throw new IOException("Exception while creating path " + + parentPath, e); + } + } + } + + } + + /** + * This method returns a list of columns of a table which were used in any + * of the transactions. + * + * @param tableName The name of table. + * @return List The list of column families in table. + * @throws IOException + */ + List getColumnFamiliesOfTable(String tableName) throws IOException { + String path = PathUtil.getTxnDataPath(baseDir, tableName); + List children = null; + List columnFamlies = new ArrayList(); + try { + children = getSession().getChildren(path, false); + } catch (KeeperException e) { + LOG.warn("Caught: ", e); + throw new IOException("Exception while obtaining columns of table.", e); + } catch (InterruptedException e) { + LOG.warn("Caught: ", e); + throw new IOException("Exception while obtaining columns of table.", e); + } + + for (String child : children) { + if ((child.contains("idgen") == false) + && (child.contains("_locknode_") == false)) { + columnFamlies.add(child); + } + } + return columnFamlies; + } + + /** + * This method returns a time stamp for use by the transactions. + * + * @return long The current timestamp in zookeeper. + * @throws IOException + */ + long getTimeStamp() throws IOException { + long timeStamp; + Stat stat; + String clockPath = PathUtil.getClockPath(this.baseDir); + ensurePathExists(clockPath, null, Ids.OPEN_ACL_UNSAFE, + CreateMode.PERSISTENT); + try { + getSession().exists(clockPath, false); + stat = getSession().setData(clockPath, null, -1); + + } catch (KeeperException e) { + LOG.warn("Caught: ", e); + throw new IOException("Exception while obtaining timestamp ", e); + } catch (InterruptedException e) { + LOG.warn("Caught: ", e); + throw new IOException("Exception while obtaining timestamp ", e); + } + timeStamp = stat.getMtime(); + return timeStamp; + } + + /** + * This method returns the next revision number to be used for any + * transaction purposes. + * + * @param tableName The name of the table. + * @return revision number The revision number last used by any transaction. + * @throws IOException + */ + long nextId(String tableName) throws IOException { + String idNode = PathUtil.getRevisionIDNode(this.baseDir, tableName); + ensurePathExists(idNode, Bytes.toBytes("0"), Ids.OPEN_ACL_UNSAFE, + CreateMode.PERSISTENT); + String lockNode = PathUtil.getLockManagementNode(idNode); + ensurePathExists(lockNode, null, Ids.OPEN_ACL_UNSAFE, + CreateMode.PERSISTENT); + IDGenerator idf = new IDGenerator(getSession(), tableName, idNode); + long id = idf.obtainID(); + return id; + } + + /** + * The latest used revision id of the table. + * + * @param tableName The name of the table. + * @return the long The revision number to use by any transaction. + * @throws IOException Signals that an I/O exception has occurred. + */ + long currentID(String tableName) throws IOException { + String idNode = PathUtil.getRevisionIDNode(this.baseDir, tableName); + ensurePathExists(idNode, Bytes.toBytes("0"), Ids.OPEN_ACL_UNSAFE, + CreateMode.PERSISTENT); + String lockNode = PathUtil.getLockManagementNode(idNode); + ensurePathExists(lockNode, null, Ids.OPEN_ACL_UNSAFE, + CreateMode.PERSISTENT); + IDGenerator idf = new IDGenerator(getSession(), tableName, idNode); + long id = idf.readID(); + return id; + } + + /** + * This methods retrieves the list of transaction information associated + * with each column/column family of a table. + * + * @param path The znode path + * @return List of FamilyRevision The list of transactions in the given path. + * @throws IOException + */ + List getTransactionList(String path) + throws IOException { + + byte[] data = getRawData(path, new Stat()); + ArrayList wtxnList = new ArrayList(); + if (data == null) { + return wtxnList; + } + StoreFamilyRevisionList txnList = new StoreFamilyRevisionList(); + deserialize(txnList, data); + Iterator itr = txnList.getRevisionListIterator(); + + while (itr.hasNext()) { + StoreFamilyRevision wtxn = itr.next(); + wtxnList.add(new FamilyRevision(wtxn.getRevision(), wtxn + .getTimestamp())); + } + + return wtxnList; + } + + /** + * This method returns the data associated with the path in zookeeper. + * + * @param path The znode path + * @param stat Zookeeper stat + * @return byte array The data stored in the znode. + * @throws IOException + */ + byte[] getRawData(String path, Stat stat) throws IOException { + byte[] data = null; + try { + data = getSession().getData(path, false, stat); + } catch (Exception e) { + throw new IOException( + "Exception while obtaining raw data from zookeeper path " + + path, e); + } + return data; + } + + /** + * This method created the basic znodes in zookeeper for revision + * management. + * + * @throws IOException + */ + void createRootZNodes() throws IOException { + String txnBaseNode = PathUtil.getTransactionBasePath(this.baseDir); + String clockNode = PathUtil.getClockPath(this.baseDir); + ensurePathExists(txnBaseNode, null, Ids.OPEN_ACL_UNSAFE, + CreateMode.PERSISTENT); + ensurePathExists(clockNode, null, Ids.OPEN_ACL_UNSAFE, + CreateMode.PERSISTENT); + } + + /** + * This method closes the zookeeper session. + */ + void closeZKConnection() { + if (zkSession != null) { + try { + zkSession.close(); + } catch (InterruptedException e) { + LOG.warn("Close failed: ", e); + } + zkSession = null; + LOG.info("Disconnected to ZooKeeper"); + } + } + + /** + * This method returns a zookeeper session. If the current session is closed, + * then a new session is created. + * + * @return ZooKeeper An instance of zookeeper client. + * @throws IOException + */ + ZooKeeper getSession() throws IOException { + if (zkSession == null || zkSession.getState() == States.CLOSED) { + synchronized (this) { + if (zkSession == null || zkSession.getState() == States.CLOSED) { + zkSession = new ZooKeeper(this.connectString, + this.DEFAULT_SESSION_TIMEOUT, new ZKWatcher()); + while (zkSession.getState() == States.CONNECTING) { + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + } + } + } + } + } + return zkSession; + } + + /** + * This method updates the transaction data related to a znode. + * + * @param path The path to the transaction data. + * @param updateTx The FamilyRevision to be updated. + * @param mode The mode to update like append, update, remove. + * @throws IOException + */ + void updateData(String path, FamilyRevision updateTx, UpdateMode mode) + throws IOException { + + if (updateTx == null) { + throw new IOException( + "The transaction to be updated found to be null."); + } + List currentData = getTransactionList(path); + List newData = new ArrayList(); + boolean dataFound = false; + long updateVersion = updateTx.getRevision(); + for (FamilyRevision tranx : currentData) { + if (tranx.getRevision() != updateVersion) { + newData.add(tranx); + } else { + dataFound = true; + } + } + switch (mode) { + case REMOVE: + if (dataFound == false) { + throw new IOException( + "The transaction to be removed not found in the data."); + } + LOG.info("Removed trasaction : " + updateTx.toString()); + break; + case KEEP_ALIVE: + if (dataFound == false) { + throw new IOException( + "The transaction to be kept alove not found in the data. It might have been expired."); + } + newData.add(updateTx); + LOG.info("keep alive of transaction : " + updateTx.toString()); + break; + case APPEND: + if (dataFound == true) { + throw new IOException( + "The data to be appended already exists."); + } + newData.add(updateTx); + LOG.info("Added transaction : " + updateTx.toString()); + break; + } + + // For serialization purposes. + List newTxnList = new ArrayList(); + for (FamilyRevision wtxn : newData) { + StoreFamilyRevision newTxn = new StoreFamilyRevision(wtxn.getRevision(), + wtxn.getExpireTimestamp()); + newTxnList.add(newTxn); + } + StoreFamilyRevisionList wtxnList = new StoreFamilyRevisionList(newTxnList); + byte[] newByteData = serialize(wtxnList); + + Stat stat = null; + try { + stat = zkSession.setData(path, newByteData, -1); + } catch (KeeperException e) { + throw new IOException( + "Exception while updating trasactional data. ", e); + } catch (InterruptedException e) { + throw new IOException( + "Exception while updating trasactional data. ", e); + } + + if (stat != null) { + LOG.info("Transaction list stored at " + path + "."); + } + + } + + /** + * Refresh transactions on a given transaction data path. + * + * @param path The path to the transaction data. + * @throws IOException Signals that an I/O exception has occurred. + */ + void refreshTransactions(String path) throws IOException { + List currentData = getTransactionList(path); + List newData = new ArrayList(); + + for (FamilyRevision tranx : currentData) { + if (tranx.getExpireTimestamp() > getTimeStamp()) { + newData.add(tranx); + } + } + + if (newData.equals(currentData) == false) { + List newTxnList = new ArrayList(); + for (FamilyRevision wtxn : newData) { + StoreFamilyRevision newTxn = new StoreFamilyRevision(wtxn.getRevision(), + wtxn.getExpireTimestamp()); + newTxnList.add(newTxn); + } + StoreFamilyRevisionList wtxnList = new StoreFamilyRevisionList(newTxnList); + byte[] newByteData = serialize(wtxnList); + + try { + zkSession.setData(path, newByteData, -1); + } catch (KeeperException e) { + throw new IOException( + "Exception while updating trasactional data. ", e); + } catch (InterruptedException e) { + throw new IOException( + "Exception while updating trasactional data. ", e); + } + + } + + } + + /** + * Delete table znodes. + * + * @param tableName the hbase table name + * @throws IOException Signals that an I/O exception has occurred. + */ + void deleteZNodes(String tableName) throws IOException { + String transactionDataTablePath = PathUtil.getTxnDataPath(baseDir, + tableName); + deleteRecursively(transactionDataTablePath); + } + + void deleteRecursively(String path) throws IOException { + try { + List children = getSession().getChildren(path, false); + if (children.size() != 0) { + for (String child : children) { + deleteRecursively(path + "/" + child); + } + } + getSession().delete(path, -1); + } catch (KeeperException e) { + throw new IOException( + "Exception while deleting path " + path + ".", e); + } catch (InterruptedException e) { + throw new IOException( + "Exception while deleting path " + path + ".", e); + } + } + + /** + * This method serializes a given instance of TBase object. + * + * @param obj An instance of TBase + * @return byte array The serialized data. + * @throws IOException + */ + static byte[] serialize(TBase obj) throws IOException { + if (obj == null) + return new byte[0]; + try { + TSerializer serializer = new TSerializer( + new TBinaryProtocol.Factory()); + byte[] bytes = serializer.serialize(obj); + return bytes; + } catch (Exception e) { + throw new IOException("Serialization error: ", e); + } + } + + + /** + * This method deserializes the given byte array into the TBase object. + * + * @param obj An instance of TBase + * @param data Output of deserialization. + * @throws IOException + */ + static void deserialize(TBase obj, byte[] data) throws IOException { + if (data == null || data.length == 0) + return; + try { + TDeserializer deserializer = new TDeserializer( + new TBinaryProtocol.Factory()); + deserializer.deserialize(obj, data); + } catch (Exception e) { + throw new IOException("Deserialization error: " + e.getMessage(), e); + } + } + + private class ZKWatcher implements Watcher { + public void process(WatchedEvent event) { + switch (event.getState()) { + case Expired: + LOG.info("The client session has expired. Try opening a new " + + "session and connecting again."); + zkSession = null; + break; + default: + + } + } + } + +} diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/lock/LockListener.java hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/lock/LockListener.java new file mode 100644 index 0000000..3c5f95b --- /dev/null +++ hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/lock/LockListener.java @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hcatalog.hbase.snapshot.lock; + +/** + * This class has two methods which are call + * back methods when a lock is acquired and + * when the lock is released. + * This class has been used as-is from the zookeeper 3.3.4 recipes minor changes + * in the package name. + */ +public interface LockListener { + /** + * call back called when the lock + * is acquired + */ + public void lockAcquired(); + + /** + * call back called when the lock is + * released. + */ + public void lockReleased(); +} diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/lock/ProtocolSupport.java hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/lock/ProtocolSupport.java new file mode 100644 index 0000000..0f97589 --- /dev/null +++ hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/lock/ProtocolSupport.java @@ -0,0 +1,195 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hcatalog.hbase.snapshot.lock; + +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.ZooDefs; +import org.apache.zookeeper.ZooKeeper; +import org.apache.zookeeper.data.ACL; +import org.apache.zookeeper.data.Stat; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * A base class for protocol implementations which provides a number of higher + * level helper methods for working with ZooKeeper along with retrying synchronous + * operations if the connection to ZooKeeper closes such as + * {@link #retryOperation(ZooKeeperOperation)} + * This class has been used as-is from the zookeeper 3.4.0 recipes with + * changes in the retry delay, retry count values and package name. + */ +class ProtocolSupport { + private static final Logger LOG = LoggerFactory.getLogger(ProtocolSupport.class); + + protected final ZooKeeper zookeeper; + private AtomicBoolean closed = new AtomicBoolean(false); + private long retryDelay = 500L; + private int retryCount = 3; + private List acl = ZooDefs.Ids.OPEN_ACL_UNSAFE; + + public ProtocolSupport(ZooKeeper zookeeper) { + this.zookeeper = zookeeper; + } + + /** + * Closes this strategy and releases any ZooKeeper resources; but keeps the + * ZooKeeper instance open + */ + public void close() { + if (closed.compareAndSet(false, true)) { + doClose(); + } + } + + /** + * return zookeeper client instance + * @return zookeeper client instance + */ + public ZooKeeper getZookeeper() { + return zookeeper; + } + + /** + * return the acl its using + * @return the acl. + */ + public List getAcl() { + return acl; + } + + /** + * set the acl + * @param acl the acl to set to + */ + public void setAcl(List acl) { + this.acl = acl; + } + + /** + * get the retry delay in milliseconds + * @return the retry delay + */ + public long getRetryDelay() { + return retryDelay; + } + + /** + * Sets the time waited between retry delays + * @param retryDelay the retry delay + */ + public void setRetryDelay(long retryDelay) { + this.retryDelay = retryDelay; + } + + /** + * Allow derived classes to perform + * some custom closing operations to release resources + */ + protected void doClose() { + } + + + /** + * Perform the given operation, retrying if the connection fails + * @return object. it needs to be cast to the callee's expected + * return type. + */ + protected Object retryOperation(ZooKeeperOperation operation) + throws KeeperException, InterruptedException { + KeeperException exception = null; + for (int i = 0; i < retryCount; i++) { + try { + return operation.execute(); + } catch (KeeperException.SessionExpiredException e) { + LOG.warn("Session expired for: " + zookeeper + " so reconnecting due to: " + e, e); + throw e; + } catch (KeeperException.ConnectionLossException e) { + if (exception == null) { + exception = e; + } + LOG.debug("Attempt " + i + " failed with connection loss so " + + "attempting to reconnect: " + e, e); + retryDelay(i); + } + } + throw exception; + } + + /** + * Ensures that the given path exists with no data, the current + * ACL and no flags + * @param path + */ + protected void ensurePathExists(String path) { + ensureExists(path, null, acl, CreateMode.PERSISTENT); + } + + /** + * Ensures that the given path exists with the given data, ACL and flags + * @param path + * @param acl + * @param flags + */ + protected void ensureExists(final String path, final byte[] data, + final List acl, final CreateMode flags) { + try { + retryOperation(new ZooKeeperOperation() { + public boolean execute() throws KeeperException, InterruptedException { + Stat stat = zookeeper.exists(path, false); + if (stat != null) { + return true; + } + zookeeper.create(path, data, acl, flags); + return true; + } + }); + } catch (KeeperException e) { + LOG.warn("Caught: " + e, e); + } catch (InterruptedException e) { + LOG.warn("Caught: " + e, e); + } + } + + /** + * Returns true if this protocol has been closed + * @return true if this protocol is closed + */ + protected boolean isClosed() { + return closed.get(); + } + + /** + * Performs a retry delay if this is not the first attempt + * @param attemptCount the number of the attempts performed so far + */ + protected void retryDelay(int attemptCount) { + if (attemptCount > 0) { + try { + Thread.sleep(attemptCount * retryDelay); + } catch (InterruptedException e) { + LOG.debug("Failed to sleep: " + e, e); + } + } + } +} diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/lock/WriteLock.java hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/lock/WriteLock.java new file mode 100644 index 0000000..6838fe9 --- /dev/null +++ hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/lock/WriteLock.java @@ -0,0 +1,303 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hcatalog.hbase.snapshot.lock; + +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.WatchedEvent; +import org.apache.zookeeper.Watcher; + +import static org.apache.zookeeper.CreateMode.EPHEMERAL_SEQUENTIAL; + +import org.apache.zookeeper.ZooKeeper; +import org.apache.zookeeper.data.ACL; +import org.apache.zookeeper.data.Stat; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.SortedSet; +import java.util.TreeSet; + +/** + * A protocol to implement an exclusive + * write lock or to elect a leader.

You invoke {@link #lock()} to + * start the process of grabbing the lock; you may get the lock then or it may be + * some time later.

You can register a listener so that you are invoked + * when you get the lock; otherwise you can ask if you have the lock + * by calling {@link #isOwner()} + * This class has been used as-is from the zookeeper 3.4.0 recipes. The only change + * made is a TODO for sorting using suffixes and the package name. + */ +public class WriteLock extends ProtocolSupport { + private static final Logger LOG = LoggerFactory.getLogger(WriteLock.class); + + private final String dir; + private String id; + private ZNodeName idName; + private String ownerId; + private String lastChildId; + private byte[] data = {0x12, 0x34}; + private LockListener callback; + private LockZooKeeperOperation zop; + + /** + * zookeeper contructor for writelock + * @param zookeeper zookeeper client instance + * @param dir the parent path you want to use for locking + * @param acl the acls that you want to use for all the paths, + * if null world read/write is used. + */ + public WriteLock(ZooKeeper zookeeper, String dir, List acl) { + super(zookeeper); + this.dir = dir; + if (acl != null) { + setAcl(acl); + } + this.zop = new LockZooKeeperOperation(); + } + + /** + * zookeeper contructor for writelock with callback + * @param zookeeper the zookeeper client instance + * @param dir the parent path you want to use for locking + * @param acl the acls that you want to use for all the paths + * @param callback the call back instance + */ + public WriteLock(ZooKeeper zookeeper, String dir, List acl, + LockListener callback) { + this(zookeeper, dir, acl); + this.callback = callback; + } + + /** + * return the current locklistener + * @return the locklistener + */ + public LockListener getLockListener() { + return this.callback; + } + + /** + * register a different call back listener + * @param callback the call back instance + */ + public void setLockListener(LockListener callback) { + this.callback = callback; + } + + /** + * Removes the lock or associated znode if + * you no longer require the lock. this also + * removes your request in the queue for locking + * in case you do not already hold the lock. + * @throws RuntimeException throws a runtime exception + * if it cannot connect to zookeeper. + */ + public synchronized void unlock() throws RuntimeException { + + if (!isClosed() && id != null) { + // we don't need to retry this operation in the case of failure + // as ZK will remove ephemeral files and we don't wanna hang + // this process when closing if we cannot reconnect to ZK + try { + + ZooKeeperOperation zopdel = new ZooKeeperOperation() { + public boolean execute() throws KeeperException, + InterruptedException { + zookeeper.delete(id, -1); + return Boolean.TRUE; + } + }; + zopdel.execute(); + } catch (InterruptedException e) { + LOG.warn("Caught: " + e, e); + //set that we have been interrupted. + Thread.currentThread().interrupt(); + } catch (KeeperException.NoNodeException e) { + // do nothing + } catch (KeeperException e) { + LOG.warn("Caught: " + e, e); + throw (RuntimeException) new RuntimeException(e.getMessage()). + initCause(e); + } finally { + if (callback != null) { + callback.lockReleased(); + } + id = null; + } + } + } + + /** + * the watcher called on + * getting watch while watching + * my predecessor + */ + private class LockWatcher implements Watcher { + public void process(WatchedEvent event) { + // lets either become the leader or watch the new/updated node + LOG.debug("Watcher fired on path: " + event.getPath() + " state: " + + event.getState() + " type " + event.getType()); + try { + lock(); + } catch (Exception e) { + LOG.warn("Failed to acquire lock: " + e, e); + } + } + } + + /** + * a zoookeeper operation that is mainly responsible + * for all the magic required for locking. + */ + private class LockZooKeeperOperation implements ZooKeeperOperation { + + /** find if we have been created earler if not create our node + * + * @param prefix the prefix node + * @param zookeeper teh zookeeper client + * @param dir the dir paretn + * @throws KeeperException + * @throws InterruptedException + */ + private void findPrefixInChildren(String prefix, ZooKeeper zookeeper, String dir) + throws KeeperException, InterruptedException { + List names = zookeeper.getChildren(dir, false); + for (String name : names) { + if (name.startsWith(prefix)) { + id = name; + if (LOG.isDebugEnabled()) { + LOG.debug("Found id created last time: " + id); + } + break; + } + } + if (id == null) { + id = zookeeper.create(dir + "/" + prefix, data, + getAcl(), EPHEMERAL_SEQUENTIAL); + + if (LOG.isDebugEnabled()) { + LOG.debug("Created id: " + id); + } + } + + } + + /** + * the command that is run and retried for actually + * obtaining the lock + * @return if the command was successful or not + */ + public boolean execute() throws KeeperException, InterruptedException { + do { + if (id == null) { + long sessionId = zookeeper.getSessionId(); + String prefix = "x-" + sessionId + "-"; + // lets try look up the current ID if we failed + // in the middle of creating the znode + findPrefixInChildren(prefix, zookeeper, dir); + idName = new ZNodeName(id); + } + if (id != null) { + List names = zookeeper.getChildren(dir, false); + if (names.isEmpty()) { + LOG.warn("No children in: " + dir + " when we've just " + + "created one! Lets recreate it..."); + // lets force the recreation of the id + id = null; + } else { + // lets sort them explicitly (though they do seem to come back in order ususally :) + SortedSet sortedNames = new TreeSet(); + for (String name : names) { + //TODO: Just use the suffix to sort. + sortedNames.add(new ZNodeName(dir + "/" + name)); + } + ownerId = sortedNames.first().getName(); + SortedSet lessThanMe = sortedNames.headSet(idName); + if (!lessThanMe.isEmpty()) { + ZNodeName lastChildName = lessThanMe.last(); + lastChildId = lastChildName.getName(); + if (LOG.isDebugEnabled()) { + LOG.debug("watching less than me node: " + lastChildId); + } + Stat stat = zookeeper.exists(lastChildId, new LockWatcher()); + if (stat != null) { + return Boolean.FALSE; + } else { + LOG.warn("Could not find the" + + " stats for less than me: " + lastChildName.getName()); + } + } else { + if (isOwner()) { + if (callback != null) { + callback.lockAcquired(); + } + return Boolean.TRUE; + } + } + } + } + } + while (id == null); + return Boolean.FALSE; + } + } + + ; + + /** + * Attempts to acquire the exclusive write lock returning whether or not it was + * acquired. Note that the exclusive lock may be acquired some time later after + * this method has been invoked due to the current lock owner going away. + */ + public synchronized boolean lock() throws KeeperException, InterruptedException { + if (isClosed()) { + return false; + } + ensurePathExists(dir); + + return (Boolean) retryOperation(zop); + } + + /** + * return the parent dir for lock + * @return the parent dir used for locks. + */ + public String getDir() { + return dir; + } + + /** + * Returns true if this node is the owner of the + * lock (or the leader) + */ + public boolean isOwner() { + return id != null && ownerId != null && id.equals(ownerId); + } + + /** + * return the id for this lock + * @return the id for this lock + */ + public String getId() { + return this.id; + } +} + diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/lock/ZNodeName.java hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/lock/ZNodeName.java new file mode 100644 index 0000000..51f0f18 --- /dev/null +++ hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/lock/ZNodeName.java @@ -0,0 +1,113 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hcatalog.hbase.snapshot.lock; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Represents an ephemeral znode name which has an ordered sequence number + * and can be sorted in order + * This class has been used as-is from the zookeeper 3.4.0 recipes with a + * change in package name. + */ +public class ZNodeName implements Comparable { + private final String name; + private String prefix; + private int sequence = -1; + private static final Logger LOG = LoggerFactory.getLogger(ZNodeName.class); + + public ZNodeName(String name) { + if (name == null) { + throw new NullPointerException("id cannot be null"); + } + this.name = name; + this.prefix = name; + int idx = name.lastIndexOf('-'); + if (idx >= 0) { + this.prefix = name.substring(0, idx); + try { + this.sequence = Integer.parseInt(name.substring(idx + 1)); + // If an exception occurred we misdetected a sequence suffix, + // so return -1. + } catch (NumberFormatException e) { + LOG.info("Number format exception for " + idx, e); + } catch (ArrayIndexOutOfBoundsException e) { + LOG.info("Array out of bounds for " + idx, e); + } + } + } + + @Override + public String toString() { + return name.toString(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + ZNodeName sequence = (ZNodeName) o; + + if (!name.equals(sequence.name)) return false; + + return true; + } + + @Override + public int hashCode() { + return name.hashCode() + 37; + } + + public int compareTo(ZNodeName that) { + int answer = this.prefix.compareTo(that.prefix); + if (answer == 0) { + int s1 = this.sequence; + int s2 = that.sequence; + if (s1 == -1 && s2 == -1) { + return this.name.compareTo(that.name); + } + answer = s1 == -1 ? 1 : s2 == -1 ? -1 : s1 - s2; + } + return answer; + } + + /** + * Returns the name of the znode + */ + public String getName() { + return name; + } + + /** + * Returns the sequence number + */ + public int getZNodeName() { + return sequence; + } + + /** + * Returns the text prefix before the sequence number + */ + public String getPrefix() { + return prefix; + } +} diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/lock/ZooKeeperOperation.java hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/lock/ZooKeeperOperation.java new file mode 100644 index 0000000..9291125 --- /dev/null +++ hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/lock/ZooKeeperOperation.java @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hcatalog.hbase.snapshot.lock; + +import org.apache.zookeeper.KeeperException; + +/** + * A callback object which can be used for implementing retry-able operations in the + * {@link org.apache.hcatalog.hbase.snapshot.lock.ProtocolSupport} class + * This class has been used as-is from the zookeeper 3.4.0 with change in the + * package name . + */ +public interface ZooKeeperOperation { + + /** + * Performs the operation - which may be involved multiple times if the connection + * to ZooKeeper closes during this operation + * + * @return the result of the operation or null + * @throws KeeperException + * @throws InterruptedException + */ + public boolean execute() throws KeeperException, InterruptedException; +} diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/package-info.java hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/package-info.java new file mode 100644 index 0000000..822b2b6 --- /dev/null +++ hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/package-info.java @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/** + * Provides a revision manager for data stored in HBase that can be used to implement repeatable reads. + * The component is designed to be usable for revision management of data stored in HBase in general, + * independent and not limited to HCatalog. It is used by the HCatalog HBase storage handler, implementation depends on HBase 0.92+. + *

+ * For more information please see + * Snapshots and Repeatable reads for HBase Tables. + * @since 0.4 + */ +package org.apache.hcatalog.hbase.snapshot; diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/HBaseAuthorizationProvider.java hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/HBaseAuthorizationProvider.java deleted file mode 100644 index 0778b40..0000000 --- hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/HBaseAuthorizationProvider.java +++ /dev/null @@ -1,144 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hive.hcatalog.hbase; - -import java.util.List; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.metastore.api.Database; -import org.apache.hadoop.hive.ql.metadata.AuthorizationException; -import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.ql.metadata.Partition; -import org.apache.hadoop.hive.ql.metadata.Table; -import org.apache.hadoop.hive.ql.security.HiveAuthenticationProvider; -import org.apache.hadoop.hive.ql.security.authorization.HiveAuthorizationProvider; -import org.apache.hadoop.hive.ql.security.authorization.Privilege; - -/** - * This class is an implementation of HiveAuthorizationProvider to provide - * authorization functionality for HBase tables. - */ -class HBaseAuthorizationProvider implements HiveAuthorizationProvider { - - @Override - public Configuration getConf() { - return null; - } - - @Override - public void setConf(Configuration conf) { - } - - /* - * (non-Javadoc) - * - * @see - * org.apache.hadoop.hive.ql.security.authorization.HiveAuthorizationProvider - * #init(org.apache.hadoop.conf.Configuration) - */ - @Override - public void init(Configuration conf) throws HiveException { - } - - @Override - public HiveAuthenticationProvider getAuthenticator() { - return null; - } - - @Override - public void setAuthenticator(HiveAuthenticationProvider authenticator) { - } - - /* - * (non-Javadoc) - * - * @see - * org.apache.hadoop.hive.ql.security.authorization.HiveAuthorizationProvider - * #authorize(org.apache.hadoop.hive.ql.security.authorization.Privilege[], - * org.apache.hadoop.hive.ql.security.authorization.Privilege[]) - */ - @Override - public void authorize(Privilege[] readRequiredPriv, - Privilege[] writeRequiredPriv) throws HiveException, - AuthorizationException { - } - - /* - * (non-Javadoc) - * - * @see - * org.apache.hadoop.hive.ql.security.authorization.HiveAuthorizationProvider - * #authorize(org.apache.hadoop.hive.metastore.api.Database, - * org.apache.hadoop.hive.ql.security.authorization.Privilege[], - * org.apache.hadoop.hive.ql.security.authorization.Privilege[]) - */ - @Override - public void authorize(Database db, Privilege[] readRequiredPriv, - Privilege[] writeRequiredPriv) throws HiveException, - AuthorizationException { - } - - /* - * (non-Javadoc) - * - * @see - * org.apache.hadoop.hive.ql.security.authorization.HiveAuthorizationProvider - * #authorize(org.apache.hadoop.hive.ql.metadata.Table, - * org.apache.hadoop.hive.ql.security.authorization.Privilege[], - * org.apache.hadoop.hive.ql.security.authorization.Privilege[]) - */ - @Override - public void authorize(Table table, Privilege[] readRequiredPriv, - Privilege[] writeRequiredPriv) throws HiveException, - AuthorizationException { - } - - /* - * (non-Javadoc) - * - * @see - * org.apache.hadoop.hive.ql.security.authorization.HiveAuthorizationProvider - * #authorize(org.apache.hadoop.hive.ql.metadata.Partition, - * org.apache.hadoop.hive.ql.security.authorization.Privilege[], - * org.apache.hadoop.hive.ql.security.authorization.Privilege[]) - */ - @Override - public void authorize(Partition part, Privilege[] readRequiredPriv, - Privilege[] writeRequiredPriv) throws HiveException, - AuthorizationException { - } - - /* - * (non-Javadoc) - * - * @see - * org.apache.hadoop.hive.ql.security.authorization.HiveAuthorizationProvider - * #authorize(org.apache.hadoop.hive.ql.metadata.Table, - * org.apache.hadoop.hive.ql.metadata.Partition, java.util.List, - * org.apache.hadoop.hive.ql.security.authorization.Privilege[], - * org.apache.hadoop.hive.ql.security.authorization.Privilege[]) - */ - @Override - public void authorize(Table table, Partition part, List columns, - Privilege[] readRequiredPriv, Privilege[] writeRequiredPriv) - throws HiveException, AuthorizationException { - } - -} diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/HBaseBaseOutputFormat.java hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/HBaseBaseOutputFormat.java deleted file mode 100644 index 895e100..0000000 --- hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/HBaseBaseOutputFormat.java +++ /dev/null @@ -1,76 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hive.hcatalog.hbase; - -import java.io.IOException; -import java.util.Properties; - -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hive.ql.io.HiveOutputFormat; -import org.apache.hadoop.io.Writable; -import org.apache.hadoop.io.WritableComparable; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.OutputFormat; -import org.apache.hadoop.mapred.RecordWriter; -import org.apache.hadoop.util.Progressable; -import org.apache.hive.hcatalog.common.HCatConstants; -import org.apache.hive.hcatalog.common.HCatUtil; -import org.apache.hive.hcatalog.mapreduce.OutputJobInfo; - -public class HBaseBaseOutputFormat implements OutputFormat, Put>, - HiveOutputFormat, Put> { - - @Override - public org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter getHiveRecordWriter( - JobConf jc, Path finalOutPath, - Class valueClass, boolean isCompressed, - Properties tableProperties, Progressable progress) - throws IOException { - throw new UnsupportedOperationException("Not implemented"); - } - - @Override - public void checkOutputSpecs(FileSystem ignored, JobConf job) throws IOException { - OutputFormat, Put> outputFormat = getOutputFormat(job); - outputFormat.checkOutputSpecs(ignored, job); - } - - @Override - public RecordWriter, Put> getRecordWriter(FileSystem ignored, - JobConf job, String name, Progressable progress) throws IOException { - OutputFormat, Put> outputFormat = getOutputFormat(job); - return outputFormat.getRecordWriter(ignored, job, name, progress); - } - - private OutputFormat, Put> getOutputFormat(JobConf job) - throws IOException { - String outputInfo = job.get(HCatConstants.HCAT_KEY_OUTPUT_INFO); - OutputJobInfo outputJobInfo = (OutputJobInfo) HCatUtil.deserialize(outputInfo); - OutputFormat, Put> outputFormat = null; - if (HBaseHCatStorageHandler.isBulkMode(outputJobInfo)) { - outputFormat = new HBaseBulkOutputFormat(); - } else { - outputFormat = new HBaseDirectOutputFormat(); - } - return outputFormat; - } -} diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/HBaseBulkOutputFormat.java hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/HBaseBulkOutputFormat.java deleted file mode 100644 index 6ba7839..0000000 --- hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/HBaseBulkOutputFormat.java +++ /dev/null @@ -1,221 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hive.hcatalog.hbase; - -import java.io.IOException; -import java.util.List; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.hadoop.hbase.security.User; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.io.WritableComparable; -import org.apache.hadoop.mapred.FileOutputCommitter; -import org.apache.hadoop.mapred.FileOutputFormat; -import org.apache.hadoop.mapred.JobClient; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.JobContext; -import org.apache.hadoop.mapred.OutputCommitter; -import org.apache.hadoop.mapred.RecordWriter; -import org.apache.hadoop.mapred.Reporter; -import org.apache.hadoop.mapred.SequenceFileOutputFormat; -import org.apache.hadoop.mapred.TaskAttemptContext; -import org.apache.hadoop.util.Progressable; -import org.apache.hive.hcatalog.hbase.snapshot.RevisionManager; - -/** - * Class which imports data into HBase via it's "bulk load" feature. Wherein - * regions are created by the MR job using HFileOutputFormat and then later - * "moved" into the appropriate region server. - */ -class HBaseBulkOutputFormat extends HBaseBaseOutputFormat { - - private final static ImmutableBytesWritable EMPTY_LIST = new ImmutableBytesWritable( - new byte[0]); - private SequenceFileOutputFormat, Put> baseOutputFormat; - - public HBaseBulkOutputFormat() { - baseOutputFormat = new SequenceFileOutputFormat, Put>(); - } - - @Override - public void checkOutputSpecs(FileSystem ignored, JobConf job) - throws IOException { - baseOutputFormat.checkOutputSpecs(ignored, job); - HBaseUtil.addHBaseDelegationToken(job); - addJTDelegationToken(job); - } - - @Override - public RecordWriter, Put> getRecordWriter( - FileSystem ignored, JobConf job, String name, Progressable progress) - throws IOException { - job.setOutputKeyClass(ImmutableBytesWritable.class); - job.setOutputValueClass(Put.class); - long version = HBaseRevisionManagerUtil.getOutputRevision(job); - return new HBaseBulkRecordWriter(baseOutputFormat.getRecordWriter( - ignored, job, name, progress), version); - } - - private void addJTDelegationToken(JobConf job) throws IOException { - // Get jobTracker delegation token if security is enabled - // we need to launch the ImportSequenceFile job - if (User.isSecurityEnabled()) { - JobClient jobClient = new JobClient(new JobConf(job)); - try { - job.getCredentials().addToken(new Text("my mr token"), - jobClient.getDelegationToken(null)); - } catch (InterruptedException e) { - throw new IOException("Error while getting JT delegation token", e); - } - } - } - - private static class HBaseBulkRecordWriter implements - RecordWriter, Put> { - - private RecordWriter, Put> baseWriter; - private final Long outputVersion; - - public HBaseBulkRecordWriter( - RecordWriter, Put> baseWriter, - Long outputVersion) { - this.baseWriter = baseWriter; - this.outputVersion = outputVersion; - } - - @Override - public void write(WritableComparable key, Put value) - throws IOException { - Put put = value; - if (outputVersion != null) { - put = new Put(value.getRow(), outputVersion.longValue()); - for (List row : value.getFamilyMap().values()) { - for (KeyValue el : row) { - put.add(el.getFamily(), el.getQualifier(), el.getValue()); - } - } - } - // we ignore the key - baseWriter.write(EMPTY_LIST, put); - } - - @Override - public void close(Reporter reporter) throws IOException { - baseWriter.close(reporter); - } - } - - public static class HBaseBulkOutputCommitter extends OutputCommitter { - - private final OutputCommitter baseOutputCommitter; - - public HBaseBulkOutputCommitter() { - baseOutputCommitter = new FileOutputCommitter(); - } - - @Override - public void abortTask(TaskAttemptContext taskContext) - throws IOException { - baseOutputCommitter.abortTask(taskContext); - } - - @Override - public void commitTask(TaskAttemptContext taskContext) - throws IOException { - // baseOutputCommitter.commitTask(taskContext); - } - - @Override - public boolean needsTaskCommit(TaskAttemptContext taskContext) - throws IOException { - return baseOutputCommitter.needsTaskCommit(taskContext); - } - - @Override - public void setupJob(JobContext jobContext) throws IOException { - baseOutputCommitter.setupJob(jobContext); - } - - @Override - public void setupTask(TaskAttemptContext taskContext) - throws IOException { - baseOutputCommitter.setupTask(taskContext); - } - - @Override - public void abortJob(JobContext jobContext, int status) - throws IOException { - baseOutputCommitter.abortJob(jobContext, status); - RevisionManager rm = null; - try { - rm = HBaseRevisionManagerUtil - .getOpenedRevisionManager(jobContext.getConfiguration()); - rm.abortWriteTransaction(HBaseRevisionManagerUtil - .getWriteTransaction(jobContext.getConfiguration())); - } finally { - cleanIntermediate(jobContext); - if (rm != null) - rm.close(); - } - } - - @Override - public void commitJob(JobContext jobContext) throws IOException { - baseOutputCommitter.commitJob(jobContext); - RevisionManager rm = null; - try { - Configuration conf = jobContext.getConfiguration(); - Path srcPath = FileOutputFormat.getOutputPath(jobContext.getJobConf()); - if (!FileSystem.get(conf).exists(srcPath)) { - throw new IOException("Failed to bulk import hfiles. " + - "Intermediate data directory is cleaned up or missing. " + - "Please look at the bulk import job if it exists for failure reason"); - } - Path destPath = new Path(srcPath.getParent(), srcPath.getName() + "_hfiles"); - boolean success = ImportSequenceFile.runJob(jobContext, - conf.get(HBaseConstants.PROPERTY_OUTPUT_TABLE_NAME_KEY), - srcPath, - destPath); - if (!success) { - cleanIntermediate(jobContext); - throw new IOException("Failed to bulk import hfiles." + - " Please look at the bulk import job for failure reason"); - } - rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(conf); - rm.commitWriteTransaction(HBaseRevisionManagerUtil.getWriteTransaction(conf)); - cleanIntermediate(jobContext); - } finally { - if (rm != null) - rm.close(); - } - } - - private void cleanIntermediate(JobContext jobContext) - throws IOException { - FileSystem fs = FileSystem.get(jobContext.getConfiguration()); - fs.delete(FileOutputFormat.getOutputPath(jobContext.getJobConf()), true); - } - } -} diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/HBaseConstants.java hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/HBaseConstants.java deleted file mode 100644 index d966881..0000000 --- hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/HBaseConstants.java +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hive.hcatalog.hbase; - -import org.apache.hive.hcatalog.common.HCatConstants; - -/** - * Constants class for constants used in HBase storage handler. - */ -class HBaseConstants { - - /** key used to store write transaction object */ - public static final String PROPERTY_WRITE_TXN_KEY = HCatConstants.HCAT_DEFAULT_TOPIC_PREFIX + ".hbase.mapreduce.writeTxn"; - - /** key used to define the name of the table to write to */ - public static final String PROPERTY_OUTPUT_TABLE_NAME_KEY = HCatConstants.HCAT_DEFAULT_TOPIC_PREFIX + ".hbase.mapreduce.outputTableName"; - - /** key used to define whether bulk storage output format will be used or not */ - public static final String PROPERTY_BULK_OUTPUT_MODE_KEY = HCatConstants.HCAT_DEFAULT_TOPIC_PREFIX + ".hbase.output.bulkMode"; - - /** key used to define the hbase table snapshot. */ - public static final String PROPERTY_TABLE_SNAPSHOT_KEY = HCatConstants.HCAT_DEFAULT_TOPIC_PREFIX + "hbase.table.snapshot"; - -} diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/HBaseDirectOutputFormat.java hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/HBaseDirectOutputFormat.java deleted file mode 100644 index 797378b..0000000 --- hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/HBaseDirectOutputFormat.java +++ /dev/null @@ -1,167 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hive.hcatalog.hbase; - -import java.io.IOException; -import java.util.List; - -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.mapred.TableOutputFormat; -import org.apache.hadoop.io.WritableComparable; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.JobContext; -import org.apache.hadoop.mapred.OutputCommitter; -import org.apache.hadoop.mapred.RecordWriter; -import org.apache.hadoop.mapred.Reporter; -import org.apache.hadoop.mapred.TaskAttemptContext; -import org.apache.hadoop.util.Progressable; -import org.apache.hive.hcatalog.hbase.snapshot.RevisionManager; -import org.apache.hive.hcatalog.hbase.snapshot.Transaction; - -/** - * "Direct" implementation of OutputFormat for HBase. Uses HTable client's put - * API to write each row to HBase one a time. Presently it is just using - * TableOutputFormat as the underlying implementation in the future we can tune - * this to make the writes faster such as permanently disabling WAL, caching, - * etc. - */ -class HBaseDirectOutputFormat extends HBaseBaseOutputFormat { - - private TableOutputFormat outputFormat; - - public HBaseDirectOutputFormat() { - this.outputFormat = new TableOutputFormat(); - } - - @Override - public RecordWriter, Put> getRecordWriter(FileSystem ignored, - JobConf job, String name, Progressable progress) - throws IOException { - long version = HBaseRevisionManagerUtil.getOutputRevision(job); - return new HBaseDirectRecordWriter(outputFormat.getRecordWriter(ignored, job, name, - progress), version); - } - - @Override - public void checkOutputSpecs(FileSystem ignored, JobConf job) - throws IOException { - outputFormat.checkOutputSpecs(ignored, job); - HBaseUtil.addHBaseDelegationToken(job); - } - - private static class HBaseDirectRecordWriter implements - RecordWriter, Put> { - - private RecordWriter, Put> baseWriter; - private final Long outputVersion; - - public HBaseDirectRecordWriter( - RecordWriter, Put> baseWriter, - Long outputVersion) { - this.baseWriter = baseWriter; - this.outputVersion = outputVersion; - } - - @Override - public void write(WritableComparable key, Put value) - throws IOException { - Put put = value; - if (outputVersion != null) { - put = new Put(value.getRow(), outputVersion.longValue()); - for (List row : value.getFamilyMap().values()) { - for (KeyValue el : row) { - put.add(el.getFamily(), el.getQualifier(), el.getValue()); - } - } - } - baseWriter.write(key, put); - } - - @Override - public void close(Reporter reporter) throws IOException { - baseWriter.close(reporter); - } - - } - - public static class HBaseDirectOutputCommitter extends OutputCommitter { - - public HBaseDirectOutputCommitter() throws IOException { - } - - @Override - public void abortTask(TaskAttemptContext taskContext) - throws IOException { - } - - @Override - public void commitTask(TaskAttemptContext taskContext) - throws IOException { - } - - @Override - public boolean needsTaskCommit(TaskAttemptContext taskContext) - throws IOException { - return false; - } - - @Override - public void setupJob(JobContext jobContext) throws IOException { - } - - @Override - public void setupTask(TaskAttemptContext taskContext) - throws IOException { - } - - @Override - public void abortJob(JobContext jobContext, int status) - throws IOException { - super.abortJob(jobContext, status); - RevisionManager rm = null; - try { - rm = HBaseRevisionManagerUtil - .getOpenedRevisionManager(jobContext.getConfiguration()); - Transaction writeTransaction = HBaseRevisionManagerUtil - .getWriteTransaction(jobContext.getConfiguration()); - rm.abortWriteTransaction(writeTransaction); - } finally { - if (rm != null) - rm.close(); - } - } - - @Override - public void commitJob(JobContext jobContext) throws IOException { - RevisionManager rm = null; - try { - rm = HBaseRevisionManagerUtil - .getOpenedRevisionManager(jobContext.getConfiguration()); - rm.commitWriteTransaction(HBaseRevisionManagerUtil.getWriteTransaction(jobContext - .getConfiguration())); - } finally { - if (rm != null) - rm.close(); - } - } - } -} diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/HBaseHCatStorageHandler.java hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/HBaseHCatStorageHandler.java deleted file mode 100644 index d886a61..0000000 --- hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/HBaseHCatStorageHandler.java +++ /dev/null @@ -1,610 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hive.hcatalog.hbase; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; - -import org.apache.hadoop.conf.Configurable; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.MasterNotRunningException; -import org.apache.hadoop.hbase.ZooKeeperConnectionException; -import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.hbase.client.HTable; -import org.apache.hadoop.hbase.mapred.TableOutputFormat; -import org.apache.hadoop.hbase.mapreduce.TableInputFormat; -import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hive.hbase.HBaseSerDe; -import org.apache.hadoop.hive.metastore.HiveMetaHook; -import org.apache.hadoop.hive.metastore.MetaStoreUtils; -import org.apache.hadoop.hive.metastore.api.MetaException; -import org.apache.hadoop.hive.metastore.api.Table; -import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.ql.plan.TableDesc; -import org.apache.hadoop.hive.ql.security.authorization.HiveAuthorizationProvider; -import org.apache.hadoop.hive.serde2.SerDe; -import org.apache.hadoop.mapred.InputFormat; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.OutputFormat; -import org.apache.hadoop.util.StringUtils; -import org.apache.hive.hcatalog.common.HCatConstants; -import org.apache.hive.hcatalog.common.HCatUtil; -import org.apache.hive.hcatalog.data.schema.HCatSchema; -import org.apache.hive.hcatalog.hbase.HBaseBulkOutputFormat.HBaseBulkOutputCommitter; -import org.apache.hive.hcatalog.hbase.HBaseDirectOutputFormat.HBaseDirectOutputCommitter; -import org.apache.hive.hcatalog.hbase.snapshot.RevisionManager; -import org.apache.hive.hcatalog.hbase.snapshot.RevisionManagerConfiguration; -import org.apache.hive.hcatalog.hbase.snapshot.Transaction; -import org.apache.hive.hcatalog.mapreduce.HCatOutputFormat; -import org.apache.hive.hcatalog.mapreduce.HCatTableInfo; -import org.apache.hive.hcatalog.mapreduce.InputJobInfo; -import org.apache.hive.hcatalog.mapreduce.OutputJobInfo; -import org.apache.hive.hcatalog.mapreduce.HCatStorageHandler; -import org.apache.thrift.TBase; -import org.apache.zookeeper.ZooKeeper; - -import com.facebook.fb303.FacebookBase; -import com.google.common.util.concurrent.ThreadFactoryBuilder; - -/** - * This class HBaseHCatStorageHandler provides functionality to create HBase - * tables through HCatalog. The implementation is very similar to the - * HiveHBaseStorageHandler, with more details to suit HCatalog. - */ -public class HBaseHCatStorageHandler extends HCatStorageHandler implements HiveMetaHook, Configurable { - - public final static String DEFAULT_PREFIX = "default."; - private final static String PROPERTY_INT_OUTPUT_LOCATION = "hcat.hbase.mapreduce.intermediateOutputLocation"; - - private Configuration hbaseConf; - private Configuration jobConf; - private HBaseAdmin admin; - - @Override - public void configureInputJobProperties(TableDesc tableDesc, Map jobProperties) { - // Populate jobProperties with input table name, table columns, RM snapshot, - // hbase-default.xml and hbase-site.xml - Map tableJobProperties = tableDesc.getJobProperties(); - String jobString = tableJobProperties.get(HCatConstants.HCAT_KEY_JOB_INFO); - try { - InputJobInfo inputJobInfo = (InputJobInfo) HCatUtil.deserialize(jobString); - HCatTableInfo tableInfo = inputJobInfo.getTableInfo(); - String qualifiedTableName = HBaseHCatStorageHandler.getFullyQualifiedHBaseTableName(tableInfo); - jobProperties.put(TableInputFormat.INPUT_TABLE, qualifiedTableName); - - Configuration jobConf = getJobConf(); - addResources(jobConf, jobProperties); - JobConf copyOfConf = new JobConf(jobConf); - HBaseConfiguration.addHbaseResources(copyOfConf); - //Getting hbase delegation token in getInputSplits does not work with PIG. So need to - //do it here - if (jobConf instanceof JobConf) { //Should be the case - HBaseUtil.addHBaseDelegationToken(copyOfConf); - ((JobConf) jobConf).getCredentials().addAll(copyOfConf.getCredentials()); - } - - String outputSchema = jobConf.get(HCatConstants.HCAT_KEY_OUTPUT_SCHEMA); - jobProperties.put(TableInputFormat.SCAN_COLUMNS, getScanColumns(tableInfo, outputSchema)); - - String serSnapshot = (String) inputJobInfo.getProperties().get( - HBaseConstants.PROPERTY_TABLE_SNAPSHOT_KEY); - if (serSnapshot == null) { - HCatTableSnapshot snapshot = - HBaseRevisionManagerUtil.createSnapshot( - RevisionManagerConfiguration.create(copyOfConf), - qualifiedTableName, tableInfo); - jobProperties.put(HBaseConstants.PROPERTY_TABLE_SNAPSHOT_KEY, - HCatUtil.serialize(snapshot)); - } - - //This adds it directly to the jobConf. Setting in jobProperties does not get propagated - //to JobConf as of now as the jobProperties is maintained per partition - //TODO: Remove when HCAT-308 is fixed - addOutputDependencyJars(jobConf); - jobProperties.put("tmpjars", jobConf.get("tmpjars")); - - } catch (IOException e) { - throw new IllegalStateException("Error while configuring job properties", e); - } - } - - @Override - public void configureOutputJobProperties(TableDesc tableDesc, Map jobProperties) { - // Populate jobProperties with output table name, hbase-default.xml, hbase-site.xml, OutputJobInfo - // Populate RM transaction in OutputJobInfo - // In case of bulk mode, populate intermediate output location - Map tableJobProperties = tableDesc.getJobProperties(); - String jobString = tableJobProperties.get(HCatConstants.HCAT_KEY_OUTPUT_INFO); - try { - OutputJobInfo outputJobInfo = (OutputJobInfo) HCatUtil.deserialize(jobString); - HCatTableInfo tableInfo = outputJobInfo.getTableInfo(); - String qualifiedTableName = HBaseHCatStorageHandler.getFullyQualifiedHBaseTableName(tableInfo); - jobProperties.put(HBaseConstants.PROPERTY_OUTPUT_TABLE_NAME_KEY, qualifiedTableName); - jobProperties.put(TableOutputFormat.OUTPUT_TABLE, qualifiedTableName); - - Configuration jobConf = getJobConf(); - addResources(jobConf, jobProperties); - - Configuration copyOfConf = new Configuration(jobConf); - HBaseConfiguration.addHbaseResources(copyOfConf); - - String txnString = outputJobInfo.getProperties().getProperty( - HBaseConstants.PROPERTY_WRITE_TXN_KEY); - Transaction txn = null; - if (txnString == null) { - txn = HBaseRevisionManagerUtil.beginWriteTransaction(qualifiedTableName, tableInfo, - RevisionManagerConfiguration.create(copyOfConf)); - String serializedTxn = HCatUtil.serialize(txn); - outputJobInfo.getProperties().setProperty(HBaseConstants.PROPERTY_WRITE_TXN_KEY, - serializedTxn); - } else { - txn = (Transaction) HCatUtil.deserialize(txnString); - } - if (isBulkMode(outputJobInfo)) { - String tableLocation = tableInfo.getTableLocation(); - String location = new Path(tableLocation, "REVISION_" + txn.getRevisionNumber()) - .toString(); - outputJobInfo.getProperties().setProperty(PROPERTY_INT_OUTPUT_LOCATION, location); - // We are writing out an intermediate sequenceFile hence - // location is not passed in OutputJobInfo.getLocation() - // TODO replace this with a mapreduce constant when available - jobProperties.put("mapred.output.dir", location); - jobProperties.put("mapred.output.committer.class", HBaseBulkOutputCommitter.class.getName()); - } else { - jobProperties.put("mapred.output.committer.class", HBaseDirectOutputCommitter.class.getName()); - } - - jobProperties.put(HCatConstants.HCAT_KEY_OUTPUT_INFO, HCatUtil.serialize(outputJobInfo)); - addOutputDependencyJars(jobConf); - jobProperties.put("tmpjars", jobConf.get("tmpjars")); - - } catch (IOException e) { - throw new IllegalStateException("Error while configuring job properties", e); - } - } - - /* - * @return instance of HiveAuthorizationProvider - * - * @throws HiveException - * - * @see org.apache.hive.hcatalog.storagehandler.HCatStorageHandler# - * getAuthorizationProvider() - */ - @Override - public HiveAuthorizationProvider getAuthorizationProvider() - throws HiveException { - - HBaseAuthorizationProvider hbaseAuth = new HBaseAuthorizationProvider(); - hbaseAuth.init(getConf()); - return hbaseAuth; - } - - /* - * @param table - * - * @throws MetaException - * - * @see org.apache.hive.hcatalog.storagehandler.HCatStorageHandler - * #commitCreateTable(org.apache.hadoop.hive.metastore.api.Table) - */ - @Override - public void commitCreateTable(Table table) throws MetaException { - } - - /* - * @param instance of table - * - * @param deleteData - * - * @throws MetaException - * - * @see org.apache.hive.hcatalog.storagehandler.HCatStorageHandler - * #commitDropTable(org.apache.hadoop.hive.metastore.api.Table, boolean) - */ - @Override - public void commitDropTable(Table tbl, boolean deleteData) - throws MetaException { - checkDeleteTable(tbl); - - } - - /* - * @param instance of table - * - * @throws MetaException - * - * @see org.apache.hive.hcatalog.storagehandler.HCatStorageHandler - * #preCreateTable(org.apache.hadoop.hive.metastore.api.Table) - */ - @Override - public void preCreateTable(Table tbl) throws MetaException { - boolean isExternal = MetaStoreUtils.isExternalTable(tbl); - - hbaseConf = getConf(); - - if (tbl.getSd().getLocation() != null) { - throw new MetaException("LOCATION may not be specified for HBase."); - } - - try { - String tableName = getFullyQualifiedHBaseTableName(tbl); - String hbaseColumnsMapping = tbl.getParameters().get( - HBaseSerDe.HBASE_COLUMNS_MAPPING); - - if (hbaseColumnsMapping == null) { - throw new MetaException( - "No hbase.columns.mapping defined in table" - + " properties."); - } - - List hbaseColumnFamilies = new ArrayList(); - List hbaseColumnQualifiers = new ArrayList(); - List hbaseColumnFamiliesBytes = new ArrayList(); - int iKey = HBaseUtil.parseColumnMapping(hbaseColumnsMapping, - hbaseColumnFamilies, hbaseColumnFamiliesBytes, - hbaseColumnQualifiers, null); - - HTableDescriptor tableDesc; - Set uniqueColumnFamilies = new HashSet(); - if (!getHBaseAdmin().tableExists(tableName)) { - // if it is not an external table then create one - if (!isExternal) { - // Create the column descriptors - tableDesc = new HTableDescriptor(tableName); - uniqueColumnFamilies.addAll(hbaseColumnFamilies); - uniqueColumnFamilies.remove(hbaseColumnFamilies.get(iKey)); - - for (String columnFamily : uniqueColumnFamilies) { - HColumnDescriptor familyDesc = new HColumnDescriptor(Bytes - .toBytes(columnFamily)); - familyDesc.setMaxVersions(Integer.MAX_VALUE); - tableDesc.addFamily(familyDesc); - } - - getHBaseAdmin().createTable(tableDesc); - } else { - // an external table - throw new MetaException("HBase table " + tableName - + " doesn't exist while the table is " - + "declared as an external table."); - } - - } else { - if (!isExternal) { - throw new MetaException("Table " + tableName - + " already exists within HBase." - + " Use CREATE EXTERNAL TABLE instead to" - + " register it in HCatalog."); - } - // make sure the schema mapping is right - tableDesc = getHBaseAdmin().getTableDescriptor( - Bytes.toBytes(tableName)); - - for (int i = 0; i < hbaseColumnFamilies.size(); i++) { - if (i == iKey) { - continue; - } - - if (!tableDesc.hasFamily(hbaseColumnFamiliesBytes.get(i))) { - throw new MetaException("Column Family " - + hbaseColumnFamilies.get(i) - + " is not defined in hbase table " + tableName); - } - } - } - - // ensure the table is online - new HTable(hbaseConf, tableDesc.getName()); - - //Set up table in revision manager. - RevisionManager rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(hbaseConf); - rm.createTable(tableName, new ArrayList(uniqueColumnFamilies)); - - } catch (MasterNotRunningException mnre) { - throw new MetaException(StringUtils.stringifyException(mnre)); - } catch (IOException ie) { - throw new MetaException(StringUtils.stringifyException(ie)); - } catch (IllegalArgumentException iae) { - throw new MetaException(StringUtils.stringifyException(iae)); - } - - } - - /* - * @param table - * - * @throws MetaException - * - * @see org.apache.hive.hcatalog.storagehandler.HCatStorageHandler - * #preDropTable(org.apache.hadoop.hive.metastore.api.Table) - */ - @Override - public void preDropTable(Table table) throws MetaException { - } - - /* - * @param table - * - * @throws MetaException - * - * @see org.apache.hive.hcatalog.storagehandler.HCatStorageHandler - * #rollbackCreateTable(org.apache.hadoop.hive.metastore.api.Table) - */ - @Override - public void rollbackCreateTable(Table table) throws MetaException { - checkDeleteTable(table); - } - - /* - * @param table - * - * @throws MetaException - * - * @see org.apache.hive.hcatalog.storagehandler.HCatStorageHandler - * #rollbackDropTable(org.apache.hadoop.hive.metastore.api.Table) - */ - @Override - public void rollbackDropTable(Table table) throws MetaException { - } - - /* - * @return instance of HiveMetaHook - * - * @see org.apache.hive.hcatalog.storagehandler.HCatStorageHandler#getMetaHook() - */ - @Override - public HiveMetaHook getMetaHook() { - return this; - } - - private HBaseAdmin getHBaseAdmin() throws MetaException { - try { - if (admin == null) { - admin = new HBaseAdmin(this.getConf()); - } - return admin; - } catch (MasterNotRunningException mnre) { - throw new MetaException(StringUtils.stringifyException(mnre)); - } catch (ZooKeeperConnectionException zkce) { - throw new MetaException(StringUtils.stringifyException(zkce)); - } - } - - private String getFullyQualifiedHBaseTableName(Table tbl) { - String tableName = tbl.getParameters().get(HBaseSerDe.HBASE_TABLE_NAME); - if (tableName == null) { - tableName = tbl.getSd().getSerdeInfo().getParameters() - .get(HBaseSerDe.HBASE_TABLE_NAME); - } - if (tableName == null) { - if (tbl.getDbName().equals(MetaStoreUtils.DEFAULT_DATABASE_NAME)) { - tableName = tbl.getTableName(); - } else { - tableName = tbl.getDbName() + "." + tbl.getTableName(); - } - tableName = tableName.toLowerCase(); - } - return tableName; - } - - static String getFullyQualifiedHBaseTableName(HCatTableInfo tableInfo) { - String qualifiedName = tableInfo.getStorerInfo().getProperties() - .getProperty(HBaseSerDe.HBASE_TABLE_NAME); - if (qualifiedName == null) { - String databaseName = tableInfo.getDatabaseName(); - String tableName = tableInfo.getTableName(); - if ((databaseName == null) - || (databaseName.equals(MetaStoreUtils.DEFAULT_DATABASE_NAME))) { - qualifiedName = tableName; - } else { - qualifiedName = databaseName + "." + tableName; - } - qualifiedName = qualifiedName.toLowerCase(); - } - return qualifiedName; - } - - @Override - public Class getInputFormatClass() { - return HBaseInputFormat.class; - } - - @Override - public Class getOutputFormatClass() { - return HBaseBaseOutputFormat.class; - } - - /* - * @return subclass of SerDe - * - * @throws UnsupportedOperationException - * - * @see - * org.apache.hive.hcatalog.storagehandler.HCatStorageHandler#getSerDeClass() - */ - @Override - public Class getSerDeClass() - throws UnsupportedOperationException { - return HBaseSerDe.class; - } - - public Configuration getJobConf() { - return jobConf; - } - - @Override - public Configuration getConf() { - - if (hbaseConf == null) { - hbaseConf = HBaseConfiguration.create(); - } - return hbaseConf; - } - - @Override - public void setConf(Configuration conf) { - //setConf is called both during DDL operations and mapred read/write jobs. - //Creating a copy of conf for DDL and adding hbase-default and hbase-site.xml to it. - //For jobs, maintaining a reference instead of cloning as we need to - // 1) add hbase delegation token to the Credentials. - // 2) set tmpjars on it. Putting in jobProperties does not get propagated to JobConf - // in case of InputFormat as they are maintained per partition. - //Not adding hbase-default.xml and hbase-site.xml to jobConf as it will override any - //hbase properties set in the JobConf by the user. In configureInputJobProperties and - //configureOutputJobProperties, we take care of adding the default properties - //that are not already present. TODO: Change to a copy for jobs after HCAT-308 is fixed. - jobConf = conf; - hbaseConf = RevisionManagerConfiguration.create(HBaseConfiguration.create(conf)); - } - - private void checkDeleteTable(Table table) throws MetaException { - boolean isExternal = MetaStoreUtils.isExternalTable(table); - String tableName = getFullyQualifiedHBaseTableName(table); - RevisionManager rm = null; - try { - if (!isExternal && getHBaseAdmin().tableExists(tableName)) { - // we have created an HBase table, so we delete it to roll back; - if (getHBaseAdmin().isTableEnabled(tableName)) { - getHBaseAdmin().disableTable(tableName); - } - getHBaseAdmin().deleteTable(tableName); - - //Drop table in revision manager. - rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(hbaseConf); - rm.dropTable(tableName); - } - } catch (IOException ie) { - throw new MetaException(StringUtils.stringifyException(ie)); - } finally { - HBaseRevisionManagerUtil.closeRevisionManagerQuietly(rm); - } - } - - /** - * Helper method for users to add the required depedency jars to distributed cache. - * @param conf - * @throws IOException - */ - private void addOutputDependencyJars(Configuration conf) throws IOException { - TableMapReduceUtil.addDependencyJars(conf, - //ZK - ZooKeeper.class, - //HBase - HTable.class, - //Hive - HiveException.class, - //HCatalog jar - HCatOutputFormat.class, - //hcat hbase storage handler jar - HBaseHCatStorageHandler.class, - //hive hbase storage handler jar - HBaseSerDe.class, - //hive jar - Table.class, - //libthrift jar - TBase.class, - //hbase jar - Bytes.class, - //thrift-fb303 .jar - FacebookBase.class, - //guava jar - ThreadFactoryBuilder.class); - } - - /** - * Utility method to add hbase-default.xml and hbase-site.xml properties to a new map - * if they are not already present in the jobConf. - * @param jobConf Job configuration - * @param newJobProperties Map to which new properties should be added - */ - private void addResources(Configuration jobConf, - Map newJobProperties) { - Configuration conf = new Configuration(false); - HBaseConfiguration.addHbaseResources(conf); - RevisionManagerConfiguration.addResources(conf); - for (Entry entry : conf) { - if (jobConf.get(entry.getKey()) == null) - newJobProperties.put(entry.getKey(), entry.getValue()); - } - } - - public static boolean isBulkMode(OutputJobInfo outputJobInfo) { - //Default is false - String bulkMode = outputJobInfo.getTableInfo().getStorerInfo().getProperties() - .getProperty(HBaseConstants.PROPERTY_BULK_OUTPUT_MODE_KEY, - "false"); - return "true".equals(bulkMode); - } - - private String getScanColumns(HCatTableInfo tableInfo, String outputColSchema) throws IOException { - StringBuilder builder = new StringBuilder(); - String hbaseColumnMapping = tableInfo.getStorerInfo().getProperties() - .getProperty(HBaseSerDe.HBASE_COLUMNS_MAPPING); - if (outputColSchema == null) { - String[] splits = hbaseColumnMapping.split("[,]"); - for (int i = 0; i < splits.length; i++) { - if (!splits[i].equals(HBaseSerDe.HBASE_KEY_COL)) - builder.append(splits[i]).append(" "); - } - } else { - HCatSchema outputSchema = (HCatSchema) HCatUtil.deserialize(outputColSchema); - HCatSchema tableSchema = tableInfo.getDataColumns(); - List outputFieldNames = outputSchema.getFieldNames(); - List outputColumnMapping = new ArrayList(); - for (String fieldName : outputFieldNames) { - int position = tableSchema.getPosition(fieldName); - outputColumnMapping.add(position); - } - List columnFamilies = new ArrayList(); - List columnQualifiers = new ArrayList(); - HBaseUtil.parseColumnMapping(hbaseColumnMapping, columnFamilies, null, - columnQualifiers, null); - for (int i = 0; i < outputColumnMapping.size(); i++) { - int cfIndex = outputColumnMapping.get(i); - String cf = columnFamilies.get(cfIndex); - // We skip the key column. - if (cf.equals(HBaseSerDe.HBASE_KEY_COL) == false) { - String qualifier = columnQualifiers.get(i); - builder.append(cf); - builder.append(":"); - if (qualifier != null) { - builder.append(qualifier); - } - builder.append(" "); - } - } - } - //Remove the extra space delimiter - builder.deleteCharAt(builder.length() - 1); - return builder.toString(); - } - -} diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/HBaseInputFormat.java hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/HBaseInputFormat.java deleted file mode 100644 index 79959cc..0000000 --- hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/HBaseInputFormat.java +++ /dev/null @@ -1,126 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hive.hcatalog.hbase; - -import java.io.IOException; -import java.util.List; - -import org.apache.hadoop.hbase.client.HTable; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.hadoop.hbase.mapred.TableSplit; -import org.apache.hadoop.hbase.mapreduce.TableInputFormat; -import org.apache.hive.hcatalog.mapreduce.HCatMapRedUtil; -import org.apache.hadoop.mapred.InputFormat; -import org.apache.hadoop.mapred.InputSplit; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.RecordReader; -import org.apache.hadoop.mapred.Reporter; -import org.apache.hive.hcatalog.common.HCatConstants; -import org.apache.hive.hcatalog.common.HCatUtil; -import org.apache.hive.hcatalog.mapreduce.InputJobInfo; - -/** - * This class HBaseInputFormat is a wrapper class of TableInputFormat in HBase. - */ -class HBaseInputFormat implements InputFormat { - - private final TableInputFormat inputFormat; - - public HBaseInputFormat() { - inputFormat = new TableInputFormat(); - } - - /* - * @param instance of InputSplit - * - * @param instance of TaskAttemptContext - * - * @return RecordReader - * - * @throws IOException - * - * @throws InterruptedException - * - * @see - * org.apache.hadoop.mapreduce.InputFormat#createRecordReader(org.apache - * .hadoop.mapreduce.InputSplit, - * org.apache.hadoop.mapreduce.TaskAttemptContext) - */ - @Override - public RecordReader getRecordReader( - InputSplit split, JobConf job, Reporter reporter) - throws IOException { - String jobString = job.get(HCatConstants.HCAT_KEY_JOB_INFO); - InputJobInfo inputJobInfo = (InputJobInfo) HCatUtil.deserialize(jobString); - - String tableName = job.get(TableInputFormat.INPUT_TABLE); - TableSplit tSplit = (TableSplit) split; - HbaseSnapshotRecordReader recordReader = new HbaseSnapshotRecordReader(inputJobInfo, job); - inputFormat.setConf(job); - Scan inputScan = inputFormat.getScan(); - // TODO: Make the caching configurable by the user - inputScan.setCaching(200); - inputScan.setCacheBlocks(false); - Scan sc = new Scan(inputScan); - sc.setStartRow(tSplit.getStartRow()); - sc.setStopRow(tSplit.getEndRow()); - recordReader.setScan(sc); - recordReader.setHTable(new HTable(job, tableName)); - recordReader.init(); - return recordReader; - } - - /* - * @param jobContext - * - * @return List of InputSplit - * - * @throws IOException - * - * @throws InterruptedException - * - * @see - * org.apache.hadoop.mapreduce.InputFormat#getSplits(org.apache.hadoop.mapreduce - * .JobContext) - */ - @Override - public org.apache.hadoop.mapred.InputSplit[] getSplits(JobConf job, int numSplits) - throws IOException { - inputFormat.setConf(job); - return convertSplits(inputFormat.getSplits(HCatMapRedUtil.createJobContext(job, null, - Reporter.NULL))); - } - - private InputSplit[] convertSplits(List splits) { - InputSplit[] converted = new InputSplit[splits.size()]; - for (int i = 0; i < splits.size(); i++) { - org.apache.hadoop.hbase.mapreduce.TableSplit tableSplit = - (org.apache.hadoop.hbase.mapreduce.TableSplit) splits.get(i); - TableSplit newTableSplit = new TableSplit(tableSplit.getTableName(), - tableSplit.getStartRow(), - tableSplit.getEndRow(), tableSplit.getRegionLocation()); - converted[i] = newTableSplit; - } - return converted; - } - -} diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/HBaseRevisionManagerUtil.java hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/HBaseRevisionManagerUtil.java deleted file mode 100644 index 979d4ce..0000000 --- hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/HBaseRevisionManagerUtil.java +++ /dev/null @@ -1,257 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hive.hcatalog.hbase; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.hbase.HBaseSerDe; -import org.apache.hive.hcatalog.common.HCatConstants; -import org.apache.hive.hcatalog.common.HCatUtil; -import org.apache.hive.hcatalog.data.schema.HCatFieldSchema; -import org.apache.hive.hcatalog.data.schema.HCatSchema; -import org.apache.hive.hcatalog.hbase.snapshot.RevisionManager; -import org.apache.hive.hcatalog.hbase.snapshot.RevisionManagerFactory; -import org.apache.hive.hcatalog.hbase.snapshot.TableSnapshot; -import org.apache.hive.hcatalog.hbase.snapshot.Transaction; -import org.apache.hive.hcatalog.mapreduce.HCatTableInfo; -import org.apache.hive.hcatalog.mapreduce.InputJobInfo; -import org.apache.hive.hcatalog.mapreduce.OutputJobInfo; -import org.apache.hive.hcatalog.mapreduce.StorerInfo; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - - -/** - * The Class HBaseRevisionManagerUtil has utility methods to interact with Revision Manager - * - */ -class HBaseRevisionManagerUtil { - - private final static Logger LOG = LoggerFactory.getLogger(HBaseRevisionManagerUtil.class); - - private HBaseRevisionManagerUtil() { - } - - /** - * Creates the latest snapshot of the table. - * - * @param jobConf The job configuration. - * @param hbaseTableName The fully qualified name of the HBase table. - * @param tableInfo HCat table information - * @return An instance of HCatTableSnapshot - * @throws IOException Signals that an I/O exception has occurred. - */ - static HCatTableSnapshot createSnapshot(Configuration jobConf, - String hbaseTableName, HCatTableInfo tableInfo) throws IOException { - - RevisionManager rm = null; - TableSnapshot snpt; - try { - rm = getOpenedRevisionManager(jobConf); - snpt = rm.createSnapshot(hbaseTableName); - } finally { - closeRevisionManagerQuietly(rm); - } - - HCatTableSnapshot hcatSnapshot = HBaseRevisionManagerUtil.convertSnapshot(snpt, tableInfo); - return hcatSnapshot; - } - - /** - * Creates the snapshot using the revision specified by the user. - * - * @param jobConf The job configuration. - * @param tableName The fully qualified name of the table whose snapshot is being taken. - * @param revision The revision number to use for the snapshot. - * @return An instance of HCatTableSnapshot. - * @throws IOException Signals that an I/O exception has occurred. - */ - static HCatTableSnapshot createSnapshot(Configuration jobConf, - String tableName, long revision) - throws IOException { - - TableSnapshot snpt; - RevisionManager rm = null; - try { - rm = getOpenedRevisionManager(jobConf); - snpt = rm.createSnapshot(tableName, revision); - } finally { - closeRevisionManagerQuietly(rm); - } - - String inputJobString = jobConf.get(HCatConstants.HCAT_KEY_JOB_INFO); - if (inputJobString == null) { - throw new IOException( - "InputJobInfo information not found in JobContext. " - + "HCatInputFormat.setInput() not called?"); - } - InputJobInfo inputInfo = (InputJobInfo) HCatUtil.deserialize(inputJobString); - HCatTableSnapshot hcatSnapshot = HBaseRevisionManagerUtil - .convertSnapshot(snpt, inputInfo.getTableInfo()); - - return hcatSnapshot; - } - - /** - * Gets an instance of revision manager which is opened. - * - * @param jobConf The job configuration. - * @return RevisionManager An instance of revision manager. - * @throws IOException - */ - static RevisionManager getOpenedRevisionManager(Configuration jobConf) throws IOException { - return RevisionManagerFactory.getOpenedRevisionManager(jobConf); - } - - static void closeRevisionManagerQuietly(RevisionManager rm) { - if (rm != null) { - try { - rm.close(); - } catch (IOException e) { - LOG.warn("Error while trying to close revision manager", e); - } - } - } - - - static HCatTableSnapshot convertSnapshot(TableSnapshot hbaseSnapshot, - HCatTableInfo hcatTableInfo) throws IOException { - - HCatSchema hcatTableSchema = hcatTableInfo.getDataColumns(); - Map hcatHbaseColMap = getHCatHBaseColumnMapping(hcatTableInfo); - HashMap revisionMap = new HashMap(); - - for (HCatFieldSchema fSchema : hcatTableSchema.getFields()) { - if (hcatHbaseColMap.containsKey(fSchema.getName())) { - String colFamily = hcatHbaseColMap.get(fSchema.getName()); - long revisionID = hbaseSnapshot.getRevision(colFamily); - revisionMap.put(fSchema.getName(), revisionID); - } - } - - HCatTableSnapshot hcatSnapshot = new HCatTableSnapshot( - hcatTableInfo.getDatabaseName(), hcatTableInfo.getTableName(), revisionMap, hbaseSnapshot.getLatestRevision()); - return hcatSnapshot; - } - - static TableSnapshot convertSnapshot(HCatTableSnapshot hcatSnapshot, - HCatTableInfo hcatTableInfo) throws IOException { - - HCatSchema hcatTableSchema = hcatTableInfo.getDataColumns(); - Map revisionMap = new HashMap(); - Map hcatHbaseColMap = getHCatHBaseColumnMapping(hcatTableInfo); - for (HCatFieldSchema fSchema : hcatTableSchema.getFields()) { - String colFamily = hcatHbaseColMap.get(fSchema.getName()); - if (hcatSnapshot.containsColumn(fSchema.getName())) { - long revision = hcatSnapshot.getRevision(fSchema.getName()); - revisionMap.put(colFamily, revision); - } - } - - String fullyQualifiedName = hcatSnapshot.getDatabaseName() + "." - + hcatSnapshot.getTableName(); - return new TableSnapshot(fullyQualifiedName, revisionMap, hcatSnapshot.getLatestRevision()); - - } - - /** - * Begins a transaction in the revision manager for the given table. - * @param qualifiedTableName Name of the table - * @param tableInfo HCat Table information - * @param jobConf Job Configuration - * @return The new transaction in revision manager - * @throws IOException - */ - static Transaction beginWriteTransaction(String qualifiedTableName, - HCatTableInfo tableInfo, Configuration jobConf) throws IOException { - Transaction txn; - RevisionManager rm = null; - try { - rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(jobConf); - String hBaseColumns = tableInfo.getStorerInfo().getProperties() - .getProperty(HBaseSerDe.HBASE_COLUMNS_MAPPING); - String[] splits = hBaseColumns.split("[,:]"); - Set families = new HashSet(); - for (int i = 0; i < splits.length; i += 2) { - if (!splits[i].isEmpty()) - families.add(splits[i]); - } - txn = rm.beginWriteTransaction(qualifiedTableName, new ArrayList(families)); - } finally { - HBaseRevisionManagerUtil.closeRevisionManagerQuietly(rm); - } - return txn; - } - - static Transaction getWriteTransaction(Configuration conf) throws IOException { - OutputJobInfo outputJobInfo = (OutputJobInfo) HCatUtil.deserialize(conf.get(HCatConstants.HCAT_KEY_OUTPUT_INFO)); - return (Transaction) HCatUtil.deserialize(outputJobInfo.getProperties() - .getProperty(HBaseConstants.PROPERTY_WRITE_TXN_KEY)); - } - - static void setWriteTransaction(Configuration conf, Transaction txn) throws IOException { - OutputJobInfo outputJobInfo = (OutputJobInfo) HCatUtil.deserialize(conf.get(HCatConstants.HCAT_KEY_OUTPUT_INFO)); - outputJobInfo.getProperties().setProperty(HBaseConstants.PROPERTY_WRITE_TXN_KEY, HCatUtil.serialize(txn)); - conf.set(HCatConstants.HCAT_KEY_OUTPUT_INFO, HCatUtil.serialize(outputJobInfo)); - } - - /** - * Get the Revision number that will be assigned to this job's output data - * @param conf configuration of the job - * @return the revision number used - * @throws IOException - */ - static long getOutputRevision(Configuration conf) throws IOException { - return getWriteTransaction(conf).getRevisionNumber(); - } - - private static Map getHCatHBaseColumnMapping(HCatTableInfo hcatTableInfo) - throws IOException { - - HCatSchema hcatTableSchema = hcatTableInfo.getDataColumns(); - StorerInfo storeInfo = hcatTableInfo.getStorerInfo(); - String hbaseColumnMapping = storeInfo.getProperties().getProperty( - HBaseSerDe.HBASE_COLUMNS_MAPPING); - - Map hcatHbaseColMap = new HashMap(); - List columnFamilies = new ArrayList(); - List columnQualifiers = new ArrayList(); - HBaseUtil.parseColumnMapping(hbaseColumnMapping, columnFamilies, - null, columnQualifiers, null); - - for (HCatFieldSchema column : hcatTableSchema.getFields()) { - int fieldPos = hcatTableSchema.getPosition(column.getName()); - String colFamily = columnFamilies.get(fieldPos); - if (colFamily.equals(HBaseSerDe.HBASE_KEY_COL) == false) { - hcatHbaseColMap.put(column.getName(), colFamily); - } - } - - return hcatHbaseColMap; - } - -} diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/HBaseUtil.java hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/HBaseUtil.java deleted file mode 100644 index 0651bc6..0000000 --- hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/HBaseUtil.java +++ /dev/null @@ -1,159 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hive.hcatalog.hbase; - -import java.io.IOException; -import java.util.List; - -import org.apache.hadoop.hbase.security.User; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hive.hbase.HBaseSerDe; -import org.apache.hadoop.mapred.JobConf; - -class HBaseUtil { - - private HBaseUtil() { - } - - /** - * Parses the HBase columns mapping to identify the column families, qualifiers - * and also caches the byte arrays corresponding to them. One of the HCat table - * columns maps to the HBase row key, by default the first column. - * - * @param columnMapping - the column mapping specification to be parsed - * @param colFamilies - the list of HBase column family names - * @param colFamiliesBytes - the corresponding byte array - * @param colQualifiers - the list of HBase column qualifier names - * @param colQualifiersBytes - the corresponding byte array - * @return the row key index in the column names list - * @throws IOException - */ - static int parseColumnMapping( - String columnMapping, - List colFamilies, - List colFamiliesBytes, - List colQualifiers, - List colQualifiersBytes) throws IOException { - - int rowKeyIndex = -1; - - if (colFamilies == null || colQualifiers == null) { - throw new IllegalArgumentException("Error: caller must pass in lists for the column families " + - "and qualifiers."); - } - - colFamilies.clear(); - colQualifiers.clear(); - - if (columnMapping == null) { - throw new IllegalArgumentException("Error: hbase.columns.mapping missing for this HBase table."); - } - - if (columnMapping.equals("") || columnMapping.equals(HBaseSerDe.HBASE_KEY_COL)) { - throw new IllegalArgumentException("Error: hbase.columns.mapping specifies only the HBase table" - + " row key. A valid Hive-HBase table must specify at least one additional column."); - } - - String[] mapping = columnMapping.split(","); - - for (int i = 0; i < mapping.length; i++) { - String elem = mapping[i]; - int idxFirst = elem.indexOf(":"); - int idxLast = elem.lastIndexOf(":"); - - if (idxFirst < 0 || !(idxFirst == idxLast)) { - throw new IllegalArgumentException("Error: the HBase columns mapping contains a badly formed " + - "column family, column qualifier specification."); - } - - if (elem.equals(HBaseSerDe.HBASE_KEY_COL)) { - rowKeyIndex = i; - colFamilies.add(elem); - colQualifiers.add(null); - } else { - String[] parts = elem.split(":"); - assert (parts.length > 0 && parts.length <= 2); - colFamilies.add(parts[0]); - - if (parts.length == 2) { - colQualifiers.add(parts[1]); - } else { - colQualifiers.add(null); - } - } - } - - if (rowKeyIndex == -1) { - colFamilies.add(0, HBaseSerDe.HBASE_KEY_COL); - colQualifiers.add(0, null); - rowKeyIndex = 0; - } - - if (colFamilies.size() != colQualifiers.size()) { - throw new IOException("Error in parsing the hbase columns mapping."); - } - - // populate the corresponding byte [] if the client has passed in a non-null list - if (colFamiliesBytes != null) { - colFamiliesBytes.clear(); - - for (String fam : colFamilies) { - colFamiliesBytes.add(Bytes.toBytes(fam)); - } - } - - if (colQualifiersBytes != null) { - colQualifiersBytes.clear(); - - for (String qual : colQualifiers) { - if (qual == null) { - colQualifiersBytes.add(null); - } else { - colQualifiersBytes.add(Bytes.toBytes(qual)); - } - } - } - - if (colFamiliesBytes != null && colQualifiersBytes != null) { - if (colFamiliesBytes.size() != colQualifiersBytes.size()) { - throw new IOException("Error in caching the bytes for the hbase column families " + - "and qualifiers."); - } - } - - return rowKeyIndex; - } - - /** - * Get delegation token from hbase and add it to JobConf - * @param job - * @throws IOException - */ - static void addHBaseDelegationToken(JobConf job) throws IOException { - if (User.isHBaseSecurityEnabled(job)) { - try { - User.getCurrent().obtainAuthTokenForJob(job); - } catch (InterruptedException e) { - throw new IOException("Error while obtaining hbase delegation token", e); - } - } - } - -} diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/HCatTableSnapshot.java hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/HCatTableSnapshot.java deleted file mode 100644 index 63d7414..0000000 --- hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/HCatTableSnapshot.java +++ /dev/null @@ -1,92 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.hive.hcatalog.hbase; - -import java.io.Serializable; -import java.util.Map; - - -/** - * The class HCatTableSnapshot represents a snapshot of a hcatalog table. - * This class is intended to be opaque. This class would used only by the - * record readers to obtain knowledge about the revisions of a - * column to be filtered. - */ -public class HCatTableSnapshot implements Serializable { - - private static final long serialVersionUID = 1L; - private String tableName; - private String databaseName; - private Map columnMap; - private long latestRevision; - - HCatTableSnapshot(String databaseName, String tableName, Map columnMap, long latestRevision) { - this.tableName = tableName; - this.databaseName = databaseName; - this.columnMap = columnMap; - this.latestRevision = latestRevision; - } - - /** - * @return The name of the table in the snapshot. - */ - public String getTableName() { - return this.tableName; - } - - /** - * @return The name of the database to which the table snapshot belongs. - */ - public String getDatabaseName() { - return this.databaseName; - } - - /** - * @return The revision number of a column in a snapshot. - */ - long getRevision(String column) { - if (columnMap.containsKey(column)) - return this.columnMap.get(column); - return latestRevision; - } - - /** - * The method checks if the snapshot contains information about a data column. - * - * @param column The data column of the table - * @return true, if successful - */ - boolean containsColumn(String column) { - return this.columnMap.containsKey(column); - } - - /** - * @return latest committed revision when snapshot was taken - */ - long getLatestRevision() { - return latestRevision; - } - - @Override - public String toString() { - String snapshot = " Database Name: " + this.databaseName + " Table Name : " + tableName + - "Latest Revision: " + latestRevision + " Column revision : " + columnMap.toString(); - return snapshot; - } -} diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/HbaseSnapshotRecordReader.java hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/HbaseSnapshotRecordReader.java deleted file mode 100644 index 5450a6f..0000000 --- hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/HbaseSnapshotRecordReader.java +++ /dev/null @@ -1,255 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.hive.hcatalog.hbase; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.client.HTable; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.ResultScanner; -import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.io.DataInputBuffer; -import org.apache.hadoop.io.DataOutputBuffer; -import org.apache.hadoop.mapred.RecordReader; -import org.apache.hive.hcatalog.common.HCatUtil; -import org.apache.hive.hcatalog.hbase.snapshot.FamilyRevision; -import org.apache.hive.hcatalog.hbase.snapshot.RevisionManager; -import org.apache.hive.hcatalog.hbase.snapshot.TableSnapshot; -import org.apache.hive.hcatalog.mapreduce.InputJobInfo; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * The Class HbaseSnapshotRecordReader implements logic for filtering records - * based on snapshot. - */ -class HbaseSnapshotRecordReader implements RecordReader { - - static final Logger LOG = LoggerFactory.getLogger(HbaseSnapshotRecordReader.class); - private final InputJobInfo inpJobInfo; - private final Configuration conf; - private final int maxRevisions = 1; - private ResultScanner scanner; - private Scan scan; - private HTable htable; - private TableSnapshot snapshot; - private Iterator resultItr; - private Set allAbortedTransactions; - private DataOutputBuffer valueOut = new DataOutputBuffer(); - private DataInputBuffer valueIn = new DataInputBuffer(); - - HbaseSnapshotRecordReader(InputJobInfo inputJobInfo, Configuration conf) throws IOException { - this.inpJobInfo = inputJobInfo; - this.conf = conf; - String snapshotString = conf.get(HBaseConstants.PROPERTY_TABLE_SNAPSHOT_KEY); - HCatTableSnapshot hcatSnapshot = (HCatTableSnapshot) HCatUtil - .deserialize(snapshotString); - this.snapshot = HBaseRevisionManagerUtil.convertSnapshot(hcatSnapshot, - inpJobInfo.getTableInfo()); - } - - public void init() throws IOException { - restart(scan.getStartRow()); - } - - public void restart(byte[] firstRow) throws IOException { - allAbortedTransactions = getAbortedTransactions(Bytes.toString(htable.getTableName()), scan); - long maxValidRevision = getMaximumRevision(scan, snapshot); - while (allAbortedTransactions.contains(maxValidRevision)) { - maxValidRevision--; - } - Scan newScan = new Scan(scan); - newScan.setStartRow(firstRow); - //TODO: See if filters in 0.92 can be used to optimize the scan - //TODO: Consider create a custom snapshot filter - //TODO: Make min revision a constant in RM - newScan.setTimeRange(0, maxValidRevision + 1); - newScan.setMaxVersions(); - this.scanner = this.htable.getScanner(newScan); - resultItr = this.scanner.iterator(); - } - - private Set getAbortedTransactions(String tableName, Scan scan) throws IOException { - Set abortedTransactions = new HashSet(); - RevisionManager rm = null; - try { - rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(conf); - byte[][] families = scan.getFamilies(); - for (byte[] familyKey : families) { - String family = Bytes.toString(familyKey); - List abortedWriteTransactions = rm.getAbortedWriteTransactions( - tableName, family); - if (abortedWriteTransactions != null) { - for (FamilyRevision revision : abortedWriteTransactions) { - abortedTransactions.add(revision.getRevision()); - } - } - } - return abortedTransactions; - } finally { - HBaseRevisionManagerUtil.closeRevisionManagerQuietly(rm); - } - } - - private long getMaximumRevision(Scan scan, TableSnapshot snapshot) { - long maxRevision = 0; - byte[][] families = scan.getFamilies(); - for (byte[] familyKey : families) { - String family = Bytes.toString(familyKey); - long revision = snapshot.getRevision(family); - if (revision > maxRevision) - maxRevision = revision; - } - return maxRevision; - } - - /* - * @param htable The HTable ( of HBase) to use for the record reader. - * - */ - public void setHTable(HTable htable) { - this.htable = htable; - } - - /* - * @param scan The scan to be used for reading records. - * - */ - public void setScan(Scan scan) { - this.scan = scan; - } - - @Override - public ImmutableBytesWritable createKey() { - return new ImmutableBytesWritable(); - } - - @Override - public Result createValue() { - return new Result(); - } - - @Override - public long getPos() { - // This should be the ordinal tuple in the range; - // not clear how to calculate... - return 0; - } - - @Override - public float getProgress() throws IOException { - // Depends on the total number of tuples - return 0; - } - - @Override - public boolean next(ImmutableBytesWritable key, Result value) throws IOException { - if (this.resultItr == null) { - LOG.warn("The HBase result iterator is found null. It is possible" - + " that the record reader has already been closed."); - } else { - while (resultItr.hasNext()) { - Result temp = resultItr.next(); - Result hbaseRow = prepareResult(temp.list()); - if (hbaseRow != null) { - // Update key and value. Currently no way to avoid serialization/de-serialization - // as no setters are available. - key.set(hbaseRow.getRow()); - valueOut.reset(); - hbaseRow.write(valueOut); - valueIn.reset(valueOut.getData(), valueOut.getLength()); - value.readFields(valueIn); - return true; - } - - } - } - return false; - } - - private Result prepareResult(List keyvalues) { - - List finalKeyVals = new ArrayList(); - Map> qualValMap = new HashMap>(); - for (KeyValue kv : keyvalues) { - byte[] cf = kv.getFamily(); - byte[] qualifier = kv.getQualifier(); - String key = Bytes.toString(cf) + ":" + Bytes.toString(qualifier); - List kvs; - if (qualValMap.containsKey(key)) { - kvs = qualValMap.get(key); - } else { - kvs = new ArrayList(); - } - - String family = Bytes.toString(kv.getFamily()); - //Ignore aborted transactions - if (allAbortedTransactions.contains(kv.getTimestamp())) { - continue; - } - - long desiredTS = snapshot.getRevision(family); - if (kv.getTimestamp() <= desiredTS) { - kvs.add(kv); - } - qualValMap.put(key, kvs); - } - - Set keys = qualValMap.keySet(); - for (String cf : keys) { - List kvs = qualValMap.get(cf); - if (maxRevisions <= kvs.size()) { - for (int i = 0; i < maxRevisions; i++) { - finalKeyVals.add(kvs.get(i)); - } - } else { - finalKeyVals.addAll(kvs); - } - } - - if (finalKeyVals.size() == 0) { - return null; - } else { - KeyValue[] kvArray = new KeyValue[finalKeyVals.size()]; - finalKeyVals.toArray(kvArray); - return new Result(kvArray); - } - } - - /* - * @see org.apache.hadoop.hbase.mapred.TableRecordReader#close() - */ - @Override - public void close() { - this.resultItr = null; - this.scanner.close(); - } - -} diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/ImportSequenceFile.java hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/ImportSequenceFile.java deleted file mode 100644 index a84bf7f..0000000 --- hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/ImportSequenceFile.java +++ /dev/null @@ -1,252 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hive.hcatalog.hbase; - -import org.apache.hadoop.filecache.DistributedCache; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.hbase.mapreduce.HFileOutputFormat; -import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles; -import org.apache.hadoop.hbase.mapreduce.PutSortReducer; -import org.apache.hadoop.hbase.mapreduce.hadoopbackport.TotalOrderPartitioner; - -import java.io.IOException; -import java.net.URI; -import java.util.Map; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.client.HTable; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.mapreduce.JobContext; -import org.apache.hadoop.mapreduce.JobStatus; -import org.apache.hadoop.mapreduce.Mapper; -import org.apache.hadoop.mapreduce.OutputCommitter; -import org.apache.hadoop.mapreduce.TaskAttemptContext; -import org.apache.hadoop.mapreduce.TaskAttemptID; -import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; -import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat; -import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; -import org.apache.hive.hcatalog.mapreduce.HCatMapRedUtil; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static org.apache.hadoop.hbase.mapreduce.hadoopbackport.TotalOrderPartitioner.DEFAULT_PATH; - - -/** - * MapReduce job which reads a series of Puts stored in a sequence file - * and imports the data into HBase. It needs to create the necessary HBase - * regions using HFileOutputFormat and then notify the correct region servers - * to doBulkLoad(). This will be used After an MR job has written the SequenceFile - * and data needs to be bulk loaded onto HBase. - */ -class ImportSequenceFile { - private final static Logger LOG = LoggerFactory.getLogger(ImportSequenceFile.class); - private final static String NAME = "HCatImportSequenceFile"; - private final static String IMPORTER_WORK_DIR = "_IMPORTER_MR_WORK_DIR"; - - - private static class SequenceFileImporter extends Mapper { - - @Override - public void map(ImmutableBytesWritable rowKey, Put value, - Context context) - throws IOException { - try { - context.write(new ImmutableBytesWritable(value.getRow()), value); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - } - - private static class ImporterOutputFormat extends HFileOutputFormat { - @Override - public OutputCommitter getOutputCommitter(TaskAttemptContext context) throws IOException { - final OutputCommitter baseOutputCommitter = super.getOutputCommitter(context); - - return new OutputCommitter() { - @Override - public void setupJob(JobContext jobContext) throws IOException { - baseOutputCommitter.setupJob(jobContext); - } - - @Override - public void setupTask(TaskAttemptContext taskContext) throws IOException { - baseOutputCommitter.setupTask(taskContext); - } - - @Override - public boolean needsTaskCommit(TaskAttemptContext taskContext) throws IOException { - return baseOutputCommitter.needsTaskCommit(taskContext); - } - - @Override - public void commitTask(TaskAttemptContext taskContext) throws IOException { - baseOutputCommitter.commitTask(taskContext); - } - - @Override - public void abortTask(TaskAttemptContext taskContext) throws IOException { - baseOutputCommitter.abortTask(taskContext); - } - - @Override - public void abortJob(JobContext jobContext, JobStatus.State state) throws IOException { - try { - baseOutputCommitter.abortJob(jobContext, state); - } finally { - cleanupScratch(jobContext); - } - } - - @Override - public void commitJob(JobContext jobContext) throws IOException { - try { - baseOutputCommitter.commitJob(jobContext); - Configuration conf = jobContext.getConfiguration(); - try { - //import hfiles - new LoadIncrementalHFiles(conf) - .doBulkLoad(HFileOutputFormat.getOutputPath(jobContext), - new HTable(conf, - conf.get(HBaseConstants.PROPERTY_OUTPUT_TABLE_NAME_KEY))); - } catch (Exception e) { - throw new IOException("BulkLoad failed.", e); - } - } finally { - cleanupScratch(jobContext); - } - } - - @Override - public void cleanupJob(JobContext context) throws IOException { - try { - baseOutputCommitter.cleanupJob(context); - } finally { - cleanupScratch(context); - } - } - - private void cleanupScratch(JobContext context) throws IOException { - FileSystem fs = FileSystem.get(context.getConfiguration()); - fs.delete(HFileOutputFormat.getOutputPath(context), true); - } - }; - } - } - - private static Job createSubmittableJob(Configuration conf, String tableName, Path inputDir, Path scratchDir, boolean localMode) - throws IOException { - Job job = new Job(conf, NAME + "_" + tableName); - job.setJarByClass(SequenceFileImporter.class); - FileInputFormat.setInputPaths(job, inputDir); - job.setInputFormatClass(SequenceFileInputFormat.class); - job.setMapperClass(SequenceFileImporter.class); - - HTable table = new HTable(conf, tableName); - job.setReducerClass(PutSortReducer.class); - FileOutputFormat.setOutputPath(job, scratchDir); - job.setMapOutputKeyClass(ImmutableBytesWritable.class); - job.setMapOutputValueClass(Put.class); - HFileOutputFormat.configureIncrementalLoad(job, table); - //override OutputFormatClass with our own so we can include cleanup in the committer - job.setOutputFormatClass(ImporterOutputFormat.class); - - //local mode doesn't support symbolic links so we have to manually set the actual path - if (localMode) { - String partitionFile = null; - for (URI uri : DistributedCache.getCacheFiles(job.getConfiguration())) { - if (DEFAULT_PATH.equals(uri.getFragment())) { - partitionFile = uri.toString(); - break; - } - } - partitionFile = partitionFile.substring(0, partitionFile.lastIndexOf("#")); - job.getConfiguration().set(TotalOrderPartitioner.PARTITIONER_PATH, partitionFile.toString()); - } - - return job; - } - - /** - * Method to run the Importer MapReduce Job. Normally will be called by another MR job - * during OutputCommitter.commitJob(). - * @param parentContext JobContext of the parent job - * @param tableName name of table to bulk load data into - * @param InputDir path of SequenceFile formatted data to read - * @param scratchDir temporary path for the Importer MR job to build the HFiles which will be imported - * @return - */ - static boolean runJob(JobContext parentContext, String tableName, Path InputDir, Path scratchDir) { - Configuration parentConf = parentContext.getConfiguration(); - Configuration conf = new Configuration(); - for (Map.Entry el : parentConf) { - if (el.getKey().startsWith("hbase.")) - conf.set(el.getKey(), el.getValue()); - if (el.getKey().startsWith("mapred.cache.archives")) - conf.set(el.getKey(), el.getValue()); - } - - //Inherit jar dependencies added to distributed cache loaded by parent job - conf.set("mapred.job.classpath.archives", parentConf.get("mapred.job.classpath.archives", "")); - conf.set("mapreduce.job.cache.archives.visibilities", parentConf.get("mapreduce.job.cache.archives.visibilities", "")); - - //Temporary fix until hbase security is ready - //We need the written HFile to be world readable so - //hbase regionserver user has the privileges to perform a hdfs move - if (parentConf.getBoolean("hadoop.security.authorization", false)) { - FsPermission.setUMask(conf, FsPermission.valueOf("----------")); - } - - conf.set(HBaseConstants.PROPERTY_OUTPUT_TABLE_NAME_KEY, tableName); - conf.setBoolean(JobContext.JOB_CANCEL_DELEGATION_TOKEN, false); - - boolean localMode = "local".equals(conf.get("mapred.job.tracker")); - - boolean success = false; - try { - FileSystem fs = FileSystem.get(parentConf); - Path workDir = new Path(new Job(parentConf).getWorkingDirectory(), IMPORTER_WORK_DIR); - if (!fs.mkdirs(workDir)) - throw new IOException("Importer work directory already exists: " + workDir); - Job job = createSubmittableJob(conf, tableName, InputDir, scratchDir, localMode); - job.setWorkingDirectory(workDir); - job.getCredentials().addAll(parentContext.getCredentials()); - success = job.waitForCompletion(true); - fs.delete(workDir, true); - //We only cleanup on success because failure might've been caused by existence of target directory - if (localMode && success) { - new ImporterOutputFormat().getOutputCommitter(HCatMapRedUtil.createTaskAttemptContext(conf, new TaskAttemptID())).commitJob(job); - } - } catch (InterruptedException e) { - LOG.error("ImportSequenceFile Failed", e); - } catch (ClassNotFoundException e) { - LOG.error("ImportSequenceFile Failed", e); - } catch (IOException e) { - LOG.error("ImportSequenceFile Failed", e); - } - return success; - } - -} diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/ResultConverter.java hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/ResultConverter.java deleted file mode 100644 index 4a7f0cb..0000000 --- hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/ResultConverter.java +++ /dev/null @@ -1,58 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hive.hcatalog.hbase; - -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hive.hcatalog.data.HCatRecord; - -import java.io.IOException; - -/** - * Interface used to define conversion of HCatRecord to and from Native HBase write (Put) and read (Result) objects. - * How the actual mapping is defined between an HBase Table's schema and an HCatalog Table's schema - * is up to the underlying implementation - */ -interface ResultConverter { - - /** - * convert HCatRecord instance to an HBase Put, used when writing out data. - * @param record instance to convert - * @return converted Put instance - * @throws IOException - */ - Put convert(HCatRecord record) throws IOException; - - /** - * convert HBase Result to HCatRecord instance, used when reading data. - * @param result instance to convert - * @return converted Result instance - * @throws IOException - */ - HCatRecord convert(Result result) throws IOException; - - /** - * Returns the hbase columns that are required for the scan. - * @return String containing hbase columns delimited by space. - * @throws IOException - */ - String getHBaseScanColumns() throws IOException; - -} diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/snapshot/FamilyRevision.java hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/snapshot/FamilyRevision.java deleted file mode 100644 index a4c4350..0000000 --- hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/snapshot/FamilyRevision.java +++ /dev/null @@ -1,71 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.hive.hcatalog.hbase.snapshot; - - -/** - * A FamiliyRevision class consists of a revision number and a expiration - * timestamp. When a write transaction starts, the transaction - * object is appended to the transaction list of the each column - * family and stored in the corresponding znode. When a write transaction is - * committed, the transaction object is removed from the list. - */ -public class FamilyRevision implements - Comparable { - - private long revision; - - private long timestamp; - - /** - * Create a FamilyRevision object - * @param rev revision number - * @param ts expiration timestamp - */ - FamilyRevision(long rev, long ts) { - this.revision = rev; - this.timestamp = ts; - } - - public long getRevision() { - return revision; - } - - public long getExpireTimestamp() { - return timestamp; - } - - void setExpireTimestamp(long ts) { - timestamp = ts; - } - - @Override - public String toString() { - String description = "revision: " + revision + " ts: " + timestamp; - return description; - } - - @Override - public int compareTo(FamilyRevision o) { - long d = revision - o.getRevision(); - return (d < 0) ? -1 : (d > 0) ? 1 : 0; - } - - -} diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/snapshot/IDGenerator.java hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/snapshot/IDGenerator.java deleted file mode 100644 index 8881f03..0000000 --- hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/snapshot/IDGenerator.java +++ /dev/null @@ -1,145 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.hive.hcatalog.hbase.snapshot; - -import java.io.IOException; -import java.nio.charset.Charset; - -import org.apache.hive.hcatalog.hbase.snapshot.lock.LockListener; -import org.apache.hive.hcatalog.hbase.snapshot.lock.WriteLock; -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.ZooDefs.Ids; -import org.apache.zookeeper.ZooKeeper; -import org.apache.zookeeper.data.Stat; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - - -/** - * This class generates revision id's for transactions. - */ -class IDGenerator implements LockListener { - - private ZooKeeper zookeeper; - private String zNodeDataLoc; - private String zNodeLockBasePath; - private long id; - private static final Logger LOG = LoggerFactory.getLogger(IDGenerator.class); - - IDGenerator(ZooKeeper zookeeper, String tableName, String idGenNode) - throws IOException { - this.zookeeper = zookeeper; - this.zNodeDataLoc = idGenNode; - this.zNodeLockBasePath = PathUtil.getLockManagementNode(idGenNode); - } - - /** - * This method obtains a revision id for a transaction. - * - * @return revision ID - * @throws IOException - */ - public long obtainID() throws IOException { - WriteLock wLock = new WriteLock(zookeeper, zNodeLockBasePath, Ids.OPEN_ACL_UNSAFE); - wLock.setLockListener(this); - try { - boolean lockGrabbed = wLock.lock(); - if (lockGrabbed == false) { - //TO DO : Let this request queue up and try obtaining lock. - throw new IOException("Unable to obtain lock to obtain id."); - } else { - id = incrementAndReadCounter(); - } - } catch (KeeperException e) { - LOG.warn("Exception while obtaining lock for ID.", e); - throw new IOException("Exception while obtaining lock for ID.", e); - } catch (InterruptedException e) { - LOG.warn("Exception while obtaining lock for ID.", e); - throw new IOException("Exception while obtaining lock for ID.", e); - } finally { - wLock.unlock(); - } - return id; - } - - /** - * This method reads the latest revision ID that has been used. The ID - * returned by this method cannot be used for transaction. - * @return revision ID - * @throws IOException - */ - public long readID() throws IOException { - long curId; - try { - Stat stat = new Stat(); - byte[] data = zookeeper.getData(this.zNodeDataLoc, false, stat); - curId = Long.parseLong(new String(data, Charset.forName("UTF-8"))); - } catch (KeeperException e) { - LOG.warn("Exception while reading current revision id.", e); - throw new IOException("Exception while reading current revision id.", e); - } catch (InterruptedException e) { - LOG.warn("Exception while reading current revision id.", e); - throw new IOException("Exception while reading current revision id.", e); - } - - return curId; - } - - - private long incrementAndReadCounter() throws IOException { - - long curId, usedId; - try { - Stat stat = new Stat(); - byte[] data = zookeeper.getData(this.zNodeDataLoc, false, stat); - usedId = Long.parseLong((new String(data, Charset.forName("UTF-8")))); - curId = usedId + 1; - String lastUsedID = String.valueOf(curId); - zookeeper.setData(this.zNodeDataLoc, lastUsedID.getBytes(Charset.forName("UTF-8")), -1); - - } catch (KeeperException e) { - LOG.warn("Exception while incrementing revision id.", e); - throw new IOException("Exception while incrementing revision id. ", e); - } catch (InterruptedException e) { - LOG.warn("Exception while incrementing revision id.", e); - throw new IOException("Exception while incrementing revision id. ", e); - } - - return curId; - } - - /* - * @see org.apache.hive.hcatalog.hbase.snapshot.lock.LockListener#lockAcquired() - */ - @Override - public void lockAcquired() { - - - } - - /* - * @see org.apache.hive.hcatalog.hbase.snapshot.lock.LockListener#lockReleased() - */ - @Override - public void lockReleased() { - - } - - -} diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/snapshot/PathUtil.java hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/snapshot/PathUtil.java deleted file mode 100644 index 94e9975..0000000 --- hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/snapshot/PathUtil.java +++ /dev/null @@ -1,132 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.hive.hcatalog.hbase.snapshot; - - -/** - * The PathUtil class is a utility class to provide information about various - * znode paths. The following is the znode structure used for storing information. - * baseDir/ClockNode - * baseDir/TrasactionBasePath - * baseDir/TrasactionBasePath/TableA/revisionID - * baseDir/TrasactionBasePath/TableA/columnFamily-1 - * baseDir/TrasactionBasePath/TableA/columnFamily-1/runnningTxns - * baseDir/TrasactionBasePath/TableA/columnFamily-1/abortedTxns - * baseDir/TrasactionBasePath/TableB/revisionID - * baseDir/TrasactionBasePath/TableB/columnFamily-1 - * baseDir/TrasactionBasePath/TableB/columnFamily-1/runnningTxns - * baseDir/TrasactionBasePath/TableB/columnFamily-1/abortedTxns - - */ -public class PathUtil { - - static final String DATA_DIR = "/data"; - static final String CLOCK_NODE = "/clock"; - - /** - * This method returns the data path associated with the currently - * running transactions of a given table and column/column family. - * @param baseDir - * @param tableName - * @param columnFamily - * @return The path of the running transactions data. - */ - static String getRunningTxnInfoPath(String baseDir, String tableName, - String columnFamily) { - String txnBasePath = getTransactionBasePath(baseDir); - String path = txnBasePath + "/" + tableName + "/" + columnFamily - + "/runningTxns"; - return path; - } - - /** - * This method returns the data path associated with the aborted - * transactions of a given table and column/column family. - * @param baseDir The base directory for revision management. - * @param tableName The name of the table. - * @param columnFamily - * @return The path of the aborted transactions data. - */ - static String getAbortInformationPath(String baseDir, String tableName, - String columnFamily) { - String txnBasePath = getTransactionBasePath(baseDir); - String path = txnBasePath + "/" + tableName + "/" + columnFamily - + "/abortData"; - return path; - } - - /** - * Gets the revision id node for a given table. - * - * @param baseDir the base dir for revision management. - * @param tableName the table name - * @return the revision id node path. - */ - static String getRevisionIDNode(String baseDir, String tableName) { - String rmBasePath = getTransactionBasePath(baseDir); - String revisionIDNode = rmBasePath + "/" + tableName + "/idgen"; - return revisionIDNode; - } - - /** - * Gets the lock management node for any znode that needs to be locked. - * - * @param path the path of the znode. - * @return the lock management node path. - */ - static String getLockManagementNode(String path) { - String lockNode = path + "_locknode_"; - return lockNode; - } - - /** - * This method returns the base path for the transaction data. - * - * @param baseDir The base dir for revision management. - * @return The base path for the transaction data. - */ - static String getTransactionBasePath(String baseDir) { - String txnBaseNode = baseDir + DATA_DIR; - return txnBaseNode; - } - - /** - * Gets the txn data path for a given table. - * - * @param baseDir the base dir for revision management. - * @param tableName the table name - * @return the txn data path for the table. - */ - static String getTxnDataPath(String baseDir, String tableName) { - String txnBasePath = getTransactionBasePath(baseDir); - String path = txnBasePath + "/" + tableName; - return path; - } - - /** - * This method returns the data path for clock node. - * - * @param baseDir - * @return The data path for clock. - */ - static String getClockPath(String baseDir) { - String clockNode = baseDir + CLOCK_NODE; - return clockNode; - } -} diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/snapshot/RMConstants.java hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/snapshot/RMConstants.java deleted file mode 100644 index 494457e..0000000 --- hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/snapshot/RMConstants.java +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hive.hcatalog.hbase.snapshot; - -public class RMConstants { - public static final String REVISION_MGR_ENDPOINT_IMPL_CLASS = "revision.manager.endpoint.impl.class"; - - public static final String WRITE_TRANSACTION_TIMEOUT = "revision.manager.writeTxn.timeout"; - - public static final String ZOOKEEPER_HOSTLIST = "revision.manager.zk.hostList"; - - public static final String ZOOKEEPER_DATADIR = "revision.manager.zk.dataDir"; -} diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/snapshot/RevisionManager.java hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/snapshot/RevisionManager.java deleted file mode 100644 index d147d56..0000000 --- hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/snapshot/RevisionManager.java +++ /dev/null @@ -1,148 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.hive.hcatalog.hbase.snapshot; - -import org.apache.hadoop.conf.Configuration; - -import java.io.IOException; -import java.util.List; - -/** - * This interface provides APIs for implementing revision management. - */ -public interface RevisionManager { - /** - * Version property required by HBase to use this interface - * for CoprocessorProtocol / RPC. - */ - public static final long VERSION = 1L; // do not change - - /** - * Initialize the revision manager. - */ - public void initialize(Configuration conf); - - /** - * Opens the revision manager. - * - * @throws IOException - */ - public void open() throws IOException; - - /** - * Closes the revision manager. - * - * @throws IOException - */ - public void close() throws IOException; - - /** - * Setup revision management for a newly created hbase table. - * @param table the hbase table name - * @param columnFamilies the column families in the table - */ - public void createTable(String table, List columnFamilies) throws IOException; - - /** - * Remove table data from revision manager for a dropped table. - * @param table the hbase table name - */ - public void dropTable(String table) throws IOException; - - /** - * Start the write transaction. - * - * @param table - * @param families - * @return a new Transaction - * @throws IOException - */ - public Transaction beginWriteTransaction(String table, List families) - throws IOException; - - /** - * Start the write transaction. - * - * @param table - * @param families - * @param keepAlive - * @return a new Transaction - * @throws IOException - */ - public Transaction beginWriteTransaction(String table, - List families, long keepAlive) throws IOException; - - /** - * Commit the write transaction. - * - * @param transaction - * @throws IOException - */ - public void commitWriteTransaction(Transaction transaction) - throws IOException; - - /** - * Abort the write transaction. - * - * @param transaction - * @throws IOException - */ - public void abortWriteTransaction(Transaction transaction) - throws IOException; - - /** - * Get the list of aborted Transactions for a column family - * - * @param table the table name - * @param columnFamily the column family name - * @return a list of aborted WriteTransactions - * @throws java.io.IOException - */ - public List getAbortedWriteTransactions(String table, - String columnFamily) throws IOException; - - /** - * Create the latest snapshot of the table. - * - * @param tableName - * @return a new snapshot - * @throws IOException - */ - public TableSnapshot createSnapshot(String tableName) throws IOException; - - /** - * Create the snapshot of the table using the revision number. - * - * @param tableName - * @param revision - * @return a new snapshot - * @throws IOException - */ - public TableSnapshot createSnapshot(String tableName, long revision) - throws IOException; - - /** - * Extends the expiration of a transaction by the time indicated by keep alive. - * - * @param transaction - * @throws IOException - */ - public void keepAlive(Transaction transaction) throws IOException; - -} diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/snapshot/RevisionManagerConfiguration.java hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/snapshot/RevisionManagerConfiguration.java deleted file mode 100644 index 93ae8ca..0000000 --- hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/snapshot/RevisionManagerConfiguration.java +++ /dev/null @@ -1,59 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hive.hcatalog.hbase.snapshot; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseConfiguration; - -public class RevisionManagerConfiguration { - - - public static Configuration addResources(Configuration conf) { - conf.addDefaultResource("revision-manager-default.xml"); - conf.addResource("revision-manager-site.xml"); - return conf; - } - - /** - * Creates a Configuration with Revision Manager resources - * @return a Configuration with Revision Manager resources - */ - public static Configuration create() { - Configuration conf = new Configuration(); - return addResources(conf); - } - - /** - * Creates a clone of passed configuration. - * @param that Configuration to clone. - * @return a Configuration created with the revision-manager-*.xml files plus - * the given configuration. - */ - public static Configuration create(final Configuration that) { - Configuration conf = create(); - //we need to merge things instead of doing new Configuration(that) - //because of a bug in Configuration wherein the config - //set on the MR fronted will get loaded on the backend as resouce called job.xml - //hence adding resources on the backed could potentially overwrite properties - //set on the frontend which we shouldn't be doing here - HBaseConfiguration.merge(conf, that); - return conf; - } -} diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/snapshot/RevisionManagerEndpoint.java hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/snapshot/RevisionManagerEndpoint.java deleted file mode 100644 index a761566..0000000 --- hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/snapshot/RevisionManagerEndpoint.java +++ /dev/null @@ -1,141 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.hive.hcatalog.hbase.snapshot; - -import java.io.IOException; -import java.util.List; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.CoprocessorEnvironment; -import org.apache.hadoop.hbase.coprocessor.BaseEndpointCoprocessor; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Implementation of RevisionManager as HBase RPC endpoint. This class will control the lifecycle of - * and delegate to the actual RevisionManager implementation and make it available as a service - * hosted in the HBase region server (instead of running it in the client (storage handler). - * In the case of {@link ZKBasedRevisionManager} now only the region servers need write access to - * manage revision data. - */ -public class RevisionManagerEndpoint extends BaseEndpointCoprocessor implements RevisionManagerProtocol { - - private static final Logger LOGGER = - LoggerFactory.getLogger(RevisionManagerEndpoint.class.getName()); - - private RevisionManager rmImpl = null; - - @Override - public void start(CoprocessorEnvironment env) { - super.start(env); - try { - Configuration conf = RevisionManagerConfiguration.create(env.getConfiguration()); - String className = conf.get(RMConstants.REVISION_MGR_ENDPOINT_IMPL_CLASS, - ZKBasedRevisionManager.class.getName()); - LOGGER.debug("Using Revision Manager implementation: {}", className); - rmImpl = RevisionManagerFactory.getOpenedRevisionManager(className, conf); - } catch (IOException e) { - LOGGER.error("Failed to initialize revision manager", e); - } - } - - @Override - public void stop(CoprocessorEnvironment env) { - if (rmImpl != null) { - try { - rmImpl.close(); - } catch (IOException e) { - LOGGER.warn("Error closing revision manager.", e); - } - } - super.stop(env); - } - - @Override - public void initialize(Configuration conf) { - // do nothing, HBase controls life cycle - } - - @Override - public void open() throws IOException { - // do nothing, HBase controls life cycle - } - - @Override - public void close() throws IOException { - // do nothing, HBase controls life cycle - } - - @Override - public void createTable(String table, List columnFamilies) throws IOException { - rmImpl.createTable(table, columnFamilies); - } - - @Override - public void dropTable(String table) throws IOException { - rmImpl.dropTable(table); - } - - @Override - public Transaction beginWriteTransaction(String table, List families) - throws IOException { - return rmImpl.beginWriteTransaction(table, families); - } - - @Override - public Transaction beginWriteTransaction(String table, - List families, long keepAlive) throws IOException { - return rmImpl.beginWriteTransaction(table, families, keepAlive); - } - - @Override - public void commitWriteTransaction(Transaction transaction) - throws IOException { - rmImpl.commitWriteTransaction(transaction); - } - - @Override - public void abortWriteTransaction(Transaction transaction) - throws IOException { - rmImpl.abortWriteTransaction(transaction); - } - - @Override - public TableSnapshot createSnapshot(String tableName) throws IOException { - return rmImpl.createSnapshot(tableName); - } - - @Override - public TableSnapshot createSnapshot(String tableName, long revision) - throws IOException { - return rmImpl.createSnapshot(tableName, revision); - } - - @Override - public void keepAlive(Transaction transaction) throws IOException { - rmImpl.keepAlive(transaction); - } - - @Override - public List getAbortedWriteTransactions(String table, - String columnFamily) throws IOException { - return rmImpl.getAbortedWriteTransactions(table, columnFamily); - } - -} diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/snapshot/RevisionManagerEndpointClient.java hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/snapshot/RevisionManagerEndpointClient.java deleted file mode 100644 index fd7e6ae..0000000 --- hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/snapshot/RevisionManagerEndpointClient.java +++ /dev/null @@ -1,125 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hive.hcatalog.hbase.snapshot; - -import java.io.IOException; -import java.util.List; - -import org.apache.hadoop.conf.Configurable; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.client.HTable; -import org.apache.hadoop.hbase.util.Bytes; - -/** - * This class is nothing but a delegate for the enclosed proxy, - * which is created upon setting the configuration. - */ -public class RevisionManagerEndpointClient implements RevisionManager, Configurable { - - private Configuration conf = null; - private RevisionManager rmProxy; - - @Override - public Configuration getConf() { - return this.conf; - } - - @Override - public void setConf(Configuration arg0) { - this.conf = arg0; - } - - @Override - public void initialize(Configuration conf) { - // do nothing - } - - @Override - public void open() throws IOException { - // clone to adjust RPC settings unique to proxy - Configuration clonedConf = new Configuration(conf); - // conf.set("hbase.ipc.client.connect.max.retries", "0"); - // conf.setInt(HConstants.HBASE_CLIENT_RPC_MAXATTEMPTS, 1); - clonedConf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1); // do not retry RPC - HTable table = new HTable(clonedConf, HConstants.ROOT_TABLE_NAME); - rmProxy = table.coprocessorProxy(RevisionManagerProtocol.class, - Bytes.toBytes("anyRow")); - rmProxy.open(); - } - - @Override - public void close() throws IOException { - rmProxy.close(); - } - - @Override - public void createTable(String table, List columnFamilies) throws IOException { - rmProxy.createTable(table, columnFamilies); - } - - @Override - public void dropTable(String table) throws IOException { - rmProxy.dropTable(table); - } - - @Override - public Transaction beginWriteTransaction(String table, List families) throws IOException { - return rmProxy.beginWriteTransaction(table, families); - } - - @Override - public Transaction beginWriteTransaction(String table, List families, long keepAlive) - throws IOException { - return rmProxy.beginWriteTransaction(table, families, keepAlive); - } - - @Override - public void commitWriteTransaction(Transaction transaction) throws IOException { - rmProxy.commitWriteTransaction(transaction); - } - - @Override - public void abortWriteTransaction(Transaction transaction) throws IOException { - rmProxy.abortWriteTransaction(transaction); - } - - @Override - public List getAbortedWriteTransactions(String table, String columnFamily) - throws IOException { - return rmProxy.getAbortedWriteTransactions(table, columnFamily); - } - - @Override - public TableSnapshot createSnapshot(String tableName) throws IOException { - return rmProxy.createSnapshot(tableName); - } - - @Override - public TableSnapshot createSnapshot(String tableName, long revision) throws IOException { - return rmProxy.createSnapshot(tableName, revision); - } - - @Override - public void keepAlive(Transaction transaction) throws IOException { - rmProxy.keepAlive(transaction); - } - -} diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/snapshot/RevisionManagerFactory.java hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/snapshot/RevisionManagerFactory.java deleted file mode 100644 index 9a64c58..0000000 --- hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/snapshot/RevisionManagerFactory.java +++ /dev/null @@ -1,105 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.hive.hcatalog.hbase.snapshot; - -import java.io.IOException; - -import org.apache.hadoop.conf.Configurable; -import org.apache.hadoop.conf.Configuration; - -/** - * Utility to instantiate the revision manager (not a true factory actually). - * Depends on HBase configuration to resolve ZooKeeper connection (when ZK is used). - */ -public class RevisionManagerFactory { - - public static final String REVISION_MGR_IMPL_CLASS = "revision.manager.impl.class"; - - /** - * Gets an instance of revision manager. - * - * @param conf The configuration required to created the revision manager. - * @return the revision manager An instance of revision manager. - * @throws IOException Signals that an I/O exception has occurred. - */ - private static RevisionManager getRevisionManager(String className, Configuration conf) throws IOException { - - RevisionManager revisionMgr; - ClassLoader classLoader = Thread.currentThread() - .getContextClassLoader(); - if (classLoader == null) { - classLoader = RevisionManagerFactory.class.getClassLoader(); - } - try { - Class revisionMgrClass = Class - .forName(className, true, classLoader).asSubclass(RevisionManager.class); - revisionMgr = (RevisionManager) revisionMgrClass.newInstance(); - revisionMgr.initialize(conf); - } catch (ClassNotFoundException e) { - throw new IOException( - "The implementation class of revision manager not found.", - e); - } catch (InstantiationException e) { - throw new IOException( - "Exception encountered during instantiating revision manager implementation.", - e); - } catch (IllegalAccessException e) { - throw new IOException( - "IllegalAccessException encountered during instantiating revision manager implementation.", - e); - } catch (IllegalArgumentException e) { - throw new IOException( - "IllegalArgumentException encountered during instantiating revision manager implementation.", - e); - } - return revisionMgr; - } - - /** - * Internally used by endpoint implementation to instantiate from different configuration setting. - * @param className - * @param conf - * @return the opened revision manager - * @throws IOException - */ - static RevisionManager getOpenedRevisionManager(String className, Configuration conf) throws IOException { - - RevisionManager revisionMgr = RevisionManagerFactory.getRevisionManager(className, conf); - if (revisionMgr instanceof Configurable) { - ((Configurable) revisionMgr).setConf(conf); - } - revisionMgr.open(); - return revisionMgr; - } - - /** - * Gets an instance of revision manager which is opened. - * The revision manager implementation can be specified as {@link #REVISION_MGR_IMPL_CLASS}, - * default is {@link ZKBasedRevisionManager}. - * @param conf revision manager configuration - * @return RevisionManager An instance of revision manager. - * @throws IOException - */ - public static RevisionManager getOpenedRevisionManager(Configuration conf) throws IOException { - String className = conf.get(RevisionManagerFactory.REVISION_MGR_IMPL_CLASS, - ZKBasedRevisionManager.class.getName()); - return getOpenedRevisionManager(className, conf); - } - -} diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/snapshot/RevisionManagerProtocol.java hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/snapshot/RevisionManagerProtocol.java deleted file mode 100644 index a34e277..0000000 --- hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/snapshot/RevisionManagerProtocol.java +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.hive.hcatalog.hbase.snapshot; - -import org.apache.hadoop.hbase.ipc.CoprocessorProtocol; - -/** - * Interface marker to implement RevisionManager as Coprocessor. - * (needs to extend CoprocessorProtocol) - */ -public interface RevisionManagerProtocol extends RevisionManager, - CoprocessorProtocol { - -} diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/snapshot/TableSnapshot.java hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/snapshot/TableSnapshot.java deleted file mode 100644 index 1c500b5..0000000 --- hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/snapshot/TableSnapshot.java +++ /dev/null @@ -1,90 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.hive.hcatalog.hbase.snapshot; - -import java.io.Serializable; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -/** - * The snapshot for a table and a list of column families. - */ -public class TableSnapshot implements Serializable { - - private String name; - - private Map cfRevisionMap; - - private long latestRevision; - - - public TableSnapshot(String name, Map cfRevMap, long latestRevision) { - this.name = name; - if (cfRevMap == null) { - throw new IllegalArgumentException("revision map cannot be null"); - } - this.cfRevisionMap = cfRevMap; - this.latestRevision = latestRevision; - } - - /** - * Gets the table name. - * - * @return String The name of the table. - */ - public String getTableName() { - return name; - } - - /** - * Gets the column families. - * - * @return List A list of column families associated with the snapshot. - */ - public List getColumnFamilies(){ - return new ArrayList(this.cfRevisionMap.keySet()); - } - - /** - * Gets the revision. - * - * @param familyName The name of the column family. - * @return the revision - */ - public long getRevision(String familyName){ - if(cfRevisionMap.containsKey(familyName)) - return cfRevisionMap.get(familyName); - return latestRevision; - } - - /** - * @return the latest committed revision when this snapshot was taken - */ - public long getLatestRevision() { - return latestRevision; - } - - @Override - public String toString() { - String snapshot = "Table Name : " + name +" Latest Revision: " + latestRevision - + " Column Familiy revision : " + cfRevisionMap.toString(); - return snapshot; - } -} diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/snapshot/Transaction.java hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/snapshot/Transaction.java deleted file mode 100644 index ccd5001..0000000 --- hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/snapshot/Transaction.java +++ /dev/null @@ -1,116 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.hive.hcatalog.hbase.snapshot; - -import java.io.Serializable; -import java.util.ArrayList; -import java.util.List; - -/** - * This class is responsible for storing information related to - * transactions. - */ -public class Transaction implements Serializable { - - private String tableName; - private List columnFamilies = new ArrayList(); - private long timeStamp; - private long keepAlive; - private long revision; - - - Transaction(String tableName, List columnFamilies, long revision, long timestamp) { - this.tableName = tableName; - this.columnFamilies = columnFamilies; - this.timeStamp = timestamp; - this.revision = revision; - } - - /** - * @return The revision number associated with a transaction. - */ - public long getRevisionNumber() { - return this.revision; - } - - /** - * @return The table name associated with a transaction. - */ - public String getTableName() { - return tableName; - } - - /** - * @return The column families associated with a transaction. - */ - public List getColumnFamilies() { - return columnFamilies; - } - - /** - * @return The expire timestamp associated with a transaction. - */ - long getTransactionExpireTimeStamp() { - return this.timeStamp + this.keepAlive; - } - - void setKeepAlive(long seconds) { - this.keepAlive = seconds; - } - - /** - * Gets the keep alive value. - * - * @return long The keep alive value for the transaction. - */ - public long getKeepAliveValue() { - return this.keepAlive; - } - - /** - * Gets the family revision info. - * - * @return FamilyRevision An instance of FamilyRevision associated with the transaction. - */ - FamilyRevision getFamilyRevisionInfo() { - return new FamilyRevision(revision, getTransactionExpireTimeStamp()); - } - - /** - * Keep alive transaction. This methods extends the expire timestamp of a - * transaction by the "keep alive" amount. - */ - void keepAliveTransaction() { - this.timeStamp = this.timeStamp + this.keepAlive; - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append("Revision : "); - sb.append(this.getRevisionNumber()); - sb.append(" Timestamp : "); - sb.append(this.getTransactionExpireTimeStamp()); - sb.append("\n").append("Table : "); - sb.append(this.tableName).append("\n"); - sb.append("Column Families : "); - sb.append(this.columnFamilies.toString()); - return sb.toString(); - } -} diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/snapshot/ZKBasedRevisionManager.java hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/snapshot/ZKBasedRevisionManager.java deleted file mode 100644 index b93e75d..0000000 --- hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/snapshot/ZKBasedRevisionManager.java +++ /dev/null @@ -1,461 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.hive.hcatalog.hbase.snapshot; - -import java.io.IOException; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hive.hcatalog.hbase.snapshot.lock.LockListener; -import org.apache.hive.hcatalog.hbase.snapshot.lock.WriteLock; -import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.ZooDefs.Ids; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * The service for providing revision management to Hbase tables. - */ -public class ZKBasedRevisionManager implements RevisionManager { - - private static final Logger LOG = LoggerFactory.getLogger(ZKBasedRevisionManager.class); - private String zkHostList; - private String baseDir; - private ZKUtil zkUtil; - private long writeTxnTimeout; - - - /* - * @see org.apache.hive.hcatalog.hbase.snapshot.RevisionManager#initialize() - */ - @Override - public void initialize(Configuration conf) { - conf = new Configuration(conf); - if (conf.get(RMConstants.ZOOKEEPER_HOSTLIST) == null) { - String zkHostList = conf.get(HConstants.ZOOKEEPER_QUORUM); - int port = conf.getInt(HConstants.ZOOKEEPER_CLIENT_PORT, - HConstants.DEFAULT_ZOOKEPER_CLIENT_PORT); - String[] splits = zkHostList.split(","); - StringBuffer sb = new StringBuffer(); - for (String split : splits) { - sb.append(split); - sb.append(':'); - sb.append(port); - sb.append(','); - } - sb.deleteCharAt(sb.length() - 1); - conf.set(RMConstants.ZOOKEEPER_HOSTLIST, sb.toString()); - } - this.zkHostList = conf.get(RMConstants.ZOOKEEPER_HOSTLIST); - this.baseDir = conf.get(RMConstants.ZOOKEEPER_DATADIR); - this.writeTxnTimeout = Long.parseLong(conf.get(RMConstants.WRITE_TRANSACTION_TIMEOUT)); - } - - /** - * Open a ZooKeeper connection - * @throws java.io.IOException - */ - - public void open() throws IOException { - zkUtil = new ZKUtil(zkHostList, this.baseDir); - zkUtil.createRootZNodes(); - LOG.info("Created root znodes for revision manager."); - } - - /** - * Close Zookeeper connection - */ - public void close() { - zkUtil.closeZKConnection(); - } - - private void checkInputParams(String table, List families) { - if (table == null) { - throw new IllegalArgumentException( - "The table name must be specified for reading."); - } - if (families == null || families.isEmpty()) { - throw new IllegalArgumentException( - "At least one column family should be specified for reading."); - } - } - - @Override - public void createTable(String table, List columnFamilies) throws IOException { - zkUtil.createRootZNodes(); - zkUtil.setUpZnodesForTable(table, columnFamilies); - } - - @Override - public void dropTable(String table) throws IOException { - zkUtil.deleteZNodes(table); - } - - /* @param table - /* @param families - /* @param keepAlive - /* @return - /* @throws IOException - * @see org.apache.hive.hcatalog.hbase.snapshot.RevisionManager#beginWriteTransaction(java.lang.String, java.util.List, long) - */ - public Transaction beginWriteTransaction(String table, - List families, long keepAlive) throws IOException { - - checkInputParams(table, families); - zkUtil.setUpZnodesForTable(table, families); - long nextId = zkUtil.nextId(table); - long expireTimestamp = zkUtil.getTimeStamp(); - Transaction transaction = new Transaction(table, families, nextId, - expireTimestamp); - if (keepAlive != -1) { - transaction.setKeepAlive(keepAlive); - } else { - transaction.setKeepAlive(writeTxnTimeout); - } - - refreshTransactionList(transaction.getTableName()); - String lockPath = prepareLockNode(table); - WriteLock wLock = new WriteLock(zkUtil.getSession(), lockPath, - Ids.OPEN_ACL_UNSAFE); - RMLockListener myLockListener = new RMLockListener(); - wLock.setLockListener(myLockListener); - try { - boolean lockGrabbed = wLock.lock(); - if (lockGrabbed == false) { - //TO DO : Let this request queue up and try obtaining lock. - throw new IOException( - "Unable to obtain lock while beginning transaction. " - + transaction.toString()); - } else { - List colFamilies = transaction.getColumnFamilies(); - FamilyRevision revisionData = transaction.getFamilyRevisionInfo(); - for (String cfamily : colFamilies) { - String path = PathUtil.getRunningTxnInfoPath( - baseDir, table, cfamily); - zkUtil.updateData(path, revisionData, - ZKUtil.UpdateMode.APPEND); - } - } - } catch (KeeperException e) { - throw new IOException("Exception while obtaining lock.", e); - } catch (InterruptedException e) { - throw new IOException("Exception while obtaining lock.", e); - } finally { - wLock.unlock(); - } - - return transaction; - } - - /* @param table The table name. - /* @param families The column families involved in the transaction. - /* @return transaction The transaction which was started. - /* @throws IOException - * @see org.apache.hive.hcatalog.hbase.snapshot.RevisionManager#beginWriteTransaction(java.lang.String, java.util.List) - */ - public Transaction beginWriteTransaction(String table, List families) - throws IOException { - return beginWriteTransaction(table, families, -1); - } - - /** - * This method commits a write transaction. - * @param transaction The revision information associated with transaction. - * @throws java.io.IOException - */ - public void commitWriteTransaction(Transaction transaction) throws IOException { - refreshTransactionList(transaction.getTableName()); - - String lockPath = prepareLockNode(transaction.getTableName()); - WriteLock wLock = new WriteLock(zkUtil.getSession(), lockPath, - Ids.OPEN_ACL_UNSAFE); - RMLockListener myLockListener = new RMLockListener(); - wLock.setLockListener(myLockListener); - try { - boolean lockGrabbed = wLock.lock(); - if (lockGrabbed == false) { - //TO DO : Let this request queue up and try obtaining lock. - throw new IOException( - "Unable to obtain lock while commiting transaction. " - + transaction.toString()); - } else { - String tableName = transaction.getTableName(); - List colFamilies = transaction.getColumnFamilies(); - FamilyRevision revisionData = transaction.getFamilyRevisionInfo(); - for (String cfamily : colFamilies) { - String path = PathUtil.getRunningTxnInfoPath( - baseDir, tableName, cfamily); - zkUtil.updateData(path, revisionData, - ZKUtil.UpdateMode.REMOVE); - } - - } - } catch (KeeperException e) { - throw new IOException("Exception while obtaining lock.", e); - } catch (InterruptedException e) { - throw new IOException("Exception while obtaining lock.", e); - } finally { - wLock.unlock(); - } - LOG.info("Write Transaction committed: " + transaction.toString()); - } - - /** - * This method aborts a write transaction. - * @param transaction - * @throws java.io.IOException - */ - public void abortWriteTransaction(Transaction transaction) throws IOException { - - refreshTransactionList(transaction.getTableName()); - String lockPath = prepareLockNode(transaction.getTableName()); - WriteLock wLock = new WriteLock(zkUtil.getSession(), lockPath, - Ids.OPEN_ACL_UNSAFE); - RMLockListener myLockListener = new RMLockListener(); - wLock.setLockListener(myLockListener); - try { - boolean lockGrabbed = wLock.lock(); - if (lockGrabbed == false) { - //TO DO : Let this request queue up and try obtaining lock. - throw new IOException( - "Unable to obtain lock while aborting transaction. " - + transaction.toString()); - } else { - String tableName = transaction.getTableName(); - List colFamilies = transaction.getColumnFamilies(); - FamilyRevision revisionData = transaction - .getFamilyRevisionInfo(); - for (String cfamily : colFamilies) { - String path = PathUtil.getRunningTxnInfoPath( - baseDir, tableName, cfamily); - zkUtil.updateData(path, revisionData, - ZKUtil.UpdateMode.REMOVE); - path = PathUtil.getAbortInformationPath(baseDir, - tableName, cfamily); - zkUtil.updateData(path, revisionData, - ZKUtil.UpdateMode.APPEND); - } - - } - } catch (KeeperException e) { - throw new IOException("Exception while obtaining lock.", e); - } catch (InterruptedException e) { - throw new IOException("Exception while obtaining lock.", e); - } finally { - wLock.unlock(); - } - LOG.info("Write Transaction aborted: " + transaction.toString()); - } - - - /* @param transaction - /* @throws IOException - * @see org.apache.hive.hcatalog.hbase.snapshot.RevsionManager#keepAlive(org.apache.hive.hcatalog.hbase.snapshot.Transaction) - */ - public void keepAlive(Transaction transaction) - throws IOException { - - refreshTransactionList(transaction.getTableName()); - transaction.keepAliveTransaction(); - String lockPath = prepareLockNode(transaction.getTableName()); - WriteLock wLock = new WriteLock(zkUtil.getSession(), lockPath, - Ids.OPEN_ACL_UNSAFE); - RMLockListener myLockListener = new RMLockListener(); - wLock.setLockListener(myLockListener); - try { - boolean lockGrabbed = wLock.lock(); - if (lockGrabbed == false) { - //TO DO : Let this request queue up and try obtaining lock. - throw new IOException( - "Unable to obtain lock for keep alive of transaction. " - + transaction.toString()); - } else { - String tableName = transaction.getTableName(); - List colFamilies = transaction.getColumnFamilies(); - FamilyRevision revisionData = transaction.getFamilyRevisionInfo(); - for (String cfamily : colFamilies) { - String path = PathUtil.getRunningTxnInfoPath( - baseDir, tableName, cfamily); - zkUtil.updateData(path, revisionData, - ZKUtil.UpdateMode.KEEP_ALIVE); - } - - } - } catch (KeeperException e) { - throw new IOException("Exception while obtaining lock.", e); - } catch (InterruptedException e) { - throw new IOException("Exception while obtaining lock.", e); - } finally { - wLock.unlock(); - } - - } - - /* This method allows the user to create latest snapshot of a - /* table. - /* @param tableName The table whose snapshot is being created. - /* @return TableSnapshot An instance of TableSnaphot - /* @throws IOException - * @see org.apache.hive.hcatalog.hbase.snapshot.RevsionManager#createSnapshot(java.lang.String) - */ - public TableSnapshot createSnapshot(String tableName) throws IOException { - refreshTransactionList(tableName); - long latestID = zkUtil.currentID(tableName); - HashMap cfMap = new HashMap(); - List columnFamilyNames = zkUtil.getColumnFamiliesOfTable(tableName); - - for (String cfName : columnFamilyNames) { - String cfPath = PathUtil.getRunningTxnInfoPath(baseDir, tableName, cfName); - List tranxList = zkUtil.getTransactionList(cfPath); - long version; - if (!tranxList.isEmpty()) { - Collections.sort(tranxList); - // get the smallest running Transaction ID - long runningVersion = tranxList.get(0).getRevision(); - version = runningVersion - 1; - } else { - version = latestID; - } - cfMap.put(cfName, version); - } - - TableSnapshot snapshot = new TableSnapshot(tableName, cfMap, latestID); - LOG.debug("Created snapshot For table: " + tableName + " snapshot: " + snapshot); - return snapshot; - } - - /* This method allows the user to create snapshot of a - /* table with a given revision number. - /* @param tableName - /* @param revision - /* @return TableSnapshot - /* @throws IOException - * @see org.apache.hive.hcatalog.hbase.snapshot.RevsionManager#createSnapshot(java.lang.String, long) - */ - public TableSnapshot createSnapshot(String tableName, long revision) throws IOException { - - long currentID = zkUtil.currentID(tableName); - if (revision > currentID) { - throw new IOException( - "The revision specified in the snapshot is higher than the current revision of the table."); - } - refreshTransactionList(tableName); - HashMap cfMap = new HashMap(); - List columnFamilies = zkUtil.getColumnFamiliesOfTable(tableName); - - for (String cf : columnFamilies) { - cfMap.put(cf, revision); - } - - return new TableSnapshot(tableName, cfMap, revision); - } - - /** - * Get the list of in-progress Transactions for a column family - * @param table the table name - * @param columnFamily the column family name - * @return a list of in-progress WriteTransactions - * @throws java.io.IOException - */ - List getRunningTransactions(String table, - String columnFamily) throws IOException { - String path = PathUtil.getRunningTxnInfoPath(baseDir, table, - columnFamily); - return zkUtil.getTransactionList(path); - } - - @Override - public List getAbortedWriteTransactions(String table, - String columnFamily) throws IOException { - String path = PathUtil.getAbortInformationPath(baseDir, table, columnFamily); - return zkUtil.getTransactionList(path); - } - - private void refreshTransactionList(String tableName) throws IOException { - String lockPath = prepareLockNode(tableName); - WriteLock wLock = new WriteLock(zkUtil.getSession(), lockPath, - Ids.OPEN_ACL_UNSAFE); - RMLockListener myLockListener = new RMLockListener(); - wLock.setLockListener(myLockListener); - try { - boolean lockGrabbed = wLock.lock(); - if (lockGrabbed == false) { - //TO DO : Let this request queue up and try obtaining lock. - throw new IOException( - "Unable to obtain lock while refreshing transactions of table " - + tableName + "."); - } else { - List cfPaths = zkUtil - .getColumnFamiliesOfTable(tableName); - for (String cf : cfPaths) { - String runningDataPath = PathUtil.getRunningTxnInfoPath( - baseDir, tableName, cf); - zkUtil.refreshTransactions(runningDataPath); - } - - } - } catch (KeeperException e) { - throw new IOException("Exception while obtaining lock.", e); - } catch (InterruptedException e) { - throw new IOException("Exception while obtaining lock.", e); - } finally { - wLock.unlock(); - } - - } - - private String prepareLockNode(String tableName) throws IOException { - String txnDataPath = PathUtil.getTxnDataPath(this.baseDir, tableName); - String lockPath = PathUtil.getLockManagementNode(txnDataPath); - zkUtil.ensurePathExists(lockPath, null, Ids.OPEN_ACL_UNSAFE, - CreateMode.PERSISTENT); - return lockPath; - } - - /* - * This class is a listener class for the locks used in revision management. - * TBD: Use the following class to signal that that the lock is actually - * been granted. - */ - class RMLockListener implements LockListener { - - /* - * @see org.apache.hive.hcatalog.hbase.snapshot.lock.LockListener#lockAcquired() - */ - @Override - public void lockAcquired() { - - } - - /* - * @see org.apache.hive.hcatalog.hbase.snapshot.lock.LockListener#lockReleased() - */ - @Override - public void lockReleased() { - - } - - } - - -} diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/snapshot/ZKUtil.java hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/snapshot/ZKUtil.java deleted file mode 100644 index 100dcf0..0000000 --- hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/snapshot/ZKUtil.java +++ /dev/null @@ -1,525 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.hive.hcatalog.hbase.snapshot; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; - -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hive.hcatalog.hbase.snapshot.transaction.thrift.StoreFamilyRevision; -import org.apache.hive.hcatalog.hbase.snapshot.transaction.thrift.StoreFamilyRevisionList; -import org.apache.thrift.TBase; -import org.apache.thrift.TDeserializer; -import org.apache.thrift.TSerializer; -import org.apache.thrift.protocol.TBinaryProtocol; -import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.WatchedEvent; -import org.apache.zookeeper.Watcher; -import org.apache.zookeeper.ZooDefs.Ids; -import org.apache.zookeeper.ZooKeeper; -import org.apache.zookeeper.ZooKeeper.States; -import org.apache.zookeeper.data.ACL; -import org.apache.zookeeper.data.Stat; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -class ZKUtil { - - private int DEFAULT_SESSION_TIMEOUT = 1000000; - private ZooKeeper zkSession; - private String baseDir; - private String connectString; - private static final Logger LOG = LoggerFactory.getLogger(ZKUtil.class); - - static enum UpdateMode { - APPEND, REMOVE, KEEP_ALIVE - } - - ; - - ZKUtil(String connection, String baseDir) { - this.connectString = connection; - this.baseDir = baseDir; - } - - /** - * This method creates znodes related to table. - * - * @param table The name of the table. - * @param families The list of column families of the table. - * @throws IOException - */ - void setUpZnodesForTable(String table, List families) - throws IOException { - - String transactionDataTablePath = PathUtil.getTxnDataPath(baseDir, table); - ensurePathExists(transactionDataTablePath, null, Ids.OPEN_ACL_UNSAFE, - CreateMode.PERSISTENT); - for (String cf : families) { - String runningDataPath = PathUtil.getRunningTxnInfoPath( - this.baseDir, table, cf); - ensurePathExists(runningDataPath, null, Ids.OPEN_ACL_UNSAFE, - CreateMode.PERSISTENT); - String abortDataPath = PathUtil.getAbortInformationPath( - this.baseDir, table, cf); - ensurePathExists(abortDataPath, null, Ids.OPEN_ACL_UNSAFE, - CreateMode.PERSISTENT); - } - - } - - /** - * This method ensures that a given path exists in zookeeper. If the path - * does not exists, it creates one. - * - * @param path The path of znode that is required to exist. - * @param data The data to be associated with the znode. - * @param acl The ACLs required. - * @param flags The CreateMode for the znode. - * @throws IOException - */ - void ensurePathExists(String path, byte[] data, List acl, - CreateMode flags) throws IOException { - String[] dirs = path.split("/"); - String parentPath = ""; - for (String subDir : dirs) { - if (subDir.equals("") == false) { - parentPath = parentPath + "/" + subDir; - try { - Stat stat = getSession().exists(parentPath, false); - if (stat == null) { - getSession().create(parentPath, data, acl, flags); - } - } catch (Exception e) { - throw new IOException("Exception while creating path " - + parentPath, e); - } - } - } - - } - - /** - * This method returns a list of columns of a table which were used in any - * of the transactions. - * - * @param tableName The name of table. - * @return List The list of column families in table. - * @throws IOException - */ - List getColumnFamiliesOfTable(String tableName) throws IOException { - String path = PathUtil.getTxnDataPath(baseDir, tableName); - List children = null; - List columnFamlies = new ArrayList(); - try { - children = getSession().getChildren(path, false); - } catch (KeeperException e) { - LOG.warn("Caught: ", e); - throw new IOException("Exception while obtaining columns of table.", e); - } catch (InterruptedException e) { - LOG.warn("Caught: ", e); - throw new IOException("Exception while obtaining columns of table.", e); - } - - for (String child : children) { - if ((child.contains("idgen") == false) - && (child.contains("_locknode_") == false)) { - columnFamlies.add(child); - } - } - return columnFamlies; - } - - /** - * This method returns a time stamp for use by the transactions. - * - * @return long The current timestamp in zookeeper. - * @throws IOException - */ - long getTimeStamp() throws IOException { - long timeStamp; - Stat stat; - String clockPath = PathUtil.getClockPath(this.baseDir); - ensurePathExists(clockPath, null, Ids.OPEN_ACL_UNSAFE, - CreateMode.PERSISTENT); - try { - getSession().exists(clockPath, false); - stat = getSession().setData(clockPath, null, -1); - - } catch (KeeperException e) { - LOG.warn("Caught: ", e); - throw new IOException("Exception while obtaining timestamp ", e); - } catch (InterruptedException e) { - LOG.warn("Caught: ", e); - throw new IOException("Exception while obtaining timestamp ", e); - } - timeStamp = stat.getMtime(); - return timeStamp; - } - - /** - * This method returns the next revision number to be used for any - * transaction purposes. - * - * @param tableName The name of the table. - * @return revision number The revision number last used by any transaction. - * @throws IOException - */ - long nextId(String tableName) throws IOException { - String idNode = PathUtil.getRevisionIDNode(this.baseDir, tableName); - ensurePathExists(idNode, Bytes.toBytes("0"), Ids.OPEN_ACL_UNSAFE, - CreateMode.PERSISTENT); - String lockNode = PathUtil.getLockManagementNode(idNode); - ensurePathExists(lockNode, null, Ids.OPEN_ACL_UNSAFE, - CreateMode.PERSISTENT); - IDGenerator idf = new IDGenerator(getSession(), tableName, idNode); - long id = idf.obtainID(); - return id; - } - - /** - * The latest used revision id of the table. - * - * @param tableName The name of the table. - * @return the long The revision number to use by any transaction. - * @throws IOException Signals that an I/O exception has occurred. - */ - long currentID(String tableName) throws IOException { - String idNode = PathUtil.getRevisionIDNode(this.baseDir, tableName); - ensurePathExists(idNode, Bytes.toBytes("0"), Ids.OPEN_ACL_UNSAFE, - CreateMode.PERSISTENT); - String lockNode = PathUtil.getLockManagementNode(idNode); - ensurePathExists(lockNode, null, Ids.OPEN_ACL_UNSAFE, - CreateMode.PERSISTENT); - IDGenerator idf = new IDGenerator(getSession(), tableName, idNode); - long id = idf.readID(); - return id; - } - - /** - * This methods retrieves the list of transaction information associated - * with each column/column family of a table. - * - * @param path The znode path - * @return List of FamilyRevision The list of transactions in the given path. - * @throws IOException - */ - List getTransactionList(String path) - throws IOException { - - byte[] data = getRawData(path, new Stat()); - ArrayList wtxnList = new ArrayList(); - if (data == null) { - return wtxnList; - } - StoreFamilyRevisionList txnList = new StoreFamilyRevisionList(); - deserialize(txnList, data); - Iterator itr = txnList.getRevisionListIterator(); - - while (itr.hasNext()) { - StoreFamilyRevision wtxn = itr.next(); - wtxnList.add(new FamilyRevision(wtxn.getRevision(), wtxn - .getTimestamp())); - } - - return wtxnList; - } - - /** - * This method returns the data associated with the path in zookeeper. - * - * @param path The znode path - * @param stat Zookeeper stat - * @return byte array The data stored in the znode. - * @throws IOException - */ - byte[] getRawData(String path, Stat stat) throws IOException { - byte[] data = null; - try { - data = getSession().getData(path, false, stat); - } catch (Exception e) { - throw new IOException( - "Exception while obtaining raw data from zookeeper path " - + path, e); - } - return data; - } - - /** - * This method created the basic znodes in zookeeper for revision - * management. - * - * @throws IOException - */ - void createRootZNodes() throws IOException { - String txnBaseNode = PathUtil.getTransactionBasePath(this.baseDir); - String clockNode = PathUtil.getClockPath(this.baseDir); - ensurePathExists(txnBaseNode, null, Ids.OPEN_ACL_UNSAFE, - CreateMode.PERSISTENT); - ensurePathExists(clockNode, null, Ids.OPEN_ACL_UNSAFE, - CreateMode.PERSISTENT); - } - - /** - * This method closes the zookeeper session. - */ - void closeZKConnection() { - if (zkSession != null) { - try { - zkSession.close(); - } catch (InterruptedException e) { - LOG.warn("Close failed: ", e); - } - zkSession = null; - LOG.info("Disconnected to ZooKeeper"); - } - } - - /** - * This method returns a zookeeper session. If the current session is closed, - * then a new session is created. - * - * @return ZooKeeper An instance of zookeeper client. - * @throws IOException - */ - ZooKeeper getSession() throws IOException { - if (zkSession == null || zkSession.getState() == States.CLOSED) { - synchronized (this) { - if (zkSession == null || zkSession.getState() == States.CLOSED) { - zkSession = new ZooKeeper(this.connectString, - this.DEFAULT_SESSION_TIMEOUT, new ZKWatcher()); - while (zkSession.getState() == States.CONNECTING) { - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - } - } - } - } - } - return zkSession; - } - - /** - * This method updates the transaction data related to a znode. - * - * @param path The path to the transaction data. - * @param updateTx The FamilyRevision to be updated. - * @param mode The mode to update like append, update, remove. - * @throws IOException - */ - void updateData(String path, FamilyRevision updateTx, UpdateMode mode) - throws IOException { - - if (updateTx == null) { - throw new IOException( - "The transaction to be updated found to be null."); - } - List currentData = getTransactionList(path); - List newData = new ArrayList(); - boolean dataFound = false; - long updateVersion = updateTx.getRevision(); - for (FamilyRevision tranx : currentData) { - if (tranx.getRevision() != updateVersion) { - newData.add(tranx); - } else { - dataFound = true; - } - } - switch (mode) { - case REMOVE: - if (dataFound == false) { - throw new IOException( - "The transaction to be removed not found in the data."); - } - LOG.info("Removed trasaction : " + updateTx.toString()); - break; - case KEEP_ALIVE: - if (dataFound == false) { - throw new IOException( - "The transaction to be kept alove not found in the data. It might have been expired."); - } - newData.add(updateTx); - LOG.info("keep alive of transaction : " + updateTx.toString()); - break; - case APPEND: - if (dataFound == true) { - throw new IOException( - "The data to be appended already exists."); - } - newData.add(updateTx); - LOG.info("Added transaction : " + updateTx.toString()); - break; - } - - // For serialization purposes. - List newTxnList = new ArrayList(); - for (FamilyRevision wtxn : newData) { - StoreFamilyRevision newTxn = new StoreFamilyRevision(wtxn.getRevision(), - wtxn.getExpireTimestamp()); - newTxnList.add(newTxn); - } - StoreFamilyRevisionList wtxnList = new StoreFamilyRevisionList(newTxnList); - byte[] newByteData = serialize(wtxnList); - - Stat stat = null; - try { - stat = zkSession.setData(path, newByteData, -1); - } catch (KeeperException e) { - throw new IOException( - "Exception while updating trasactional data. ", e); - } catch (InterruptedException e) { - throw new IOException( - "Exception while updating trasactional data. ", e); - } - - if (stat != null) { - LOG.info("Transaction list stored at " + path + "."); - } - - } - - /** - * Refresh transactions on a given transaction data path. - * - * @param path The path to the transaction data. - * @throws IOException Signals that an I/O exception has occurred. - */ - void refreshTransactions(String path) throws IOException { - List currentData = getTransactionList(path); - List newData = new ArrayList(); - - for (FamilyRevision tranx : currentData) { - if (tranx.getExpireTimestamp() > getTimeStamp()) { - newData.add(tranx); - } - } - - if (newData.equals(currentData) == false) { - List newTxnList = new ArrayList(); - for (FamilyRevision wtxn : newData) { - StoreFamilyRevision newTxn = new StoreFamilyRevision(wtxn.getRevision(), - wtxn.getExpireTimestamp()); - newTxnList.add(newTxn); - } - StoreFamilyRevisionList wtxnList = new StoreFamilyRevisionList(newTxnList); - byte[] newByteData = serialize(wtxnList); - - try { - zkSession.setData(path, newByteData, -1); - } catch (KeeperException e) { - throw new IOException( - "Exception while updating trasactional data. ", e); - } catch (InterruptedException e) { - throw new IOException( - "Exception while updating trasactional data. ", e); - } - - } - - } - - /** - * Delete table znodes. - * - * @param tableName the hbase table name - * @throws IOException Signals that an I/O exception has occurred. - */ - void deleteZNodes(String tableName) throws IOException { - String transactionDataTablePath = PathUtil.getTxnDataPath(baseDir, - tableName); - deleteRecursively(transactionDataTablePath); - } - - void deleteRecursively(String path) throws IOException { - try { - List children = getSession().getChildren(path, false); - if (children.size() != 0) { - for (String child : children) { - deleteRecursively(path + "/" + child); - } - } - getSession().delete(path, -1); - } catch (KeeperException e) { - throw new IOException( - "Exception while deleting path " + path + ".", e); - } catch (InterruptedException e) { - throw new IOException( - "Exception while deleting path " + path + ".", e); - } - } - - /** - * This method serializes a given instance of TBase object. - * - * @param obj An instance of TBase - * @return byte array The serialized data. - * @throws IOException - */ - static byte[] serialize(TBase obj) throws IOException { - if (obj == null) - return new byte[0]; - try { - TSerializer serializer = new TSerializer( - new TBinaryProtocol.Factory()); - byte[] bytes = serializer.serialize(obj); - return bytes; - } catch (Exception e) { - throw new IOException("Serialization error: ", e); - } - } - - - /** - * This method deserializes the given byte array into the TBase object. - * - * @param obj An instance of TBase - * @param data Output of deserialization. - * @throws IOException - */ - static void deserialize(TBase obj, byte[] data) throws IOException { - if (data == null || data.length == 0) - return; - try { - TDeserializer deserializer = new TDeserializer( - new TBinaryProtocol.Factory()); - deserializer.deserialize(obj, data); - } catch (Exception e) { - throw new IOException("Deserialization error: " + e.getMessage(), e); - } - } - - private class ZKWatcher implements Watcher { - public void process(WatchedEvent event) { - switch (event.getState()) { - case Expired: - LOG.info("The client session has expired. Try opening a new " - + "session and connecting again."); - zkSession = null; - break; - default: - - } - } - } - -} diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/snapshot/lock/LockListener.java hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/snapshot/lock/LockListener.java deleted file mode 100644 index f53ef97..0000000 --- hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/snapshot/lock/LockListener.java +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hive.hcatalog.hbase.snapshot.lock; - -/** - * This class has two methods which are call - * back methods when a lock is acquired and - * when the lock is released. - * This class has been used as-is from the zookeeper 3.3.4 recipes minor changes - * in the package name. - */ -public interface LockListener { - /** - * call back called when the lock - * is acquired - */ - public void lockAcquired(); - - /** - * call back called when the lock is - * released. - */ - public void lockReleased(); -} diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/snapshot/lock/ProtocolSupport.java hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/snapshot/lock/ProtocolSupport.java deleted file mode 100644 index 818f36a..0000000 --- hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/snapshot/lock/ProtocolSupport.java +++ /dev/null @@ -1,195 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hive.hcatalog.hbase.snapshot.lock; - -import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.ZooDefs; -import org.apache.zookeeper.ZooKeeper; -import org.apache.zookeeper.data.ACL; -import org.apache.zookeeper.data.Stat; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.List; -import java.util.concurrent.atomic.AtomicBoolean; - -/** - * A base class for protocol implementations which provides a number of higher - * level helper methods for working with ZooKeeper along with retrying synchronous - * operations if the connection to ZooKeeper closes such as - * {@link #retryOperation(ZooKeeperOperation)} - * This class has been used as-is from the zookeeper 3.4.0 recipes with - * changes in the retry delay, retry count values and package name. - */ -class ProtocolSupport { - private static final Logger LOG = LoggerFactory.getLogger(ProtocolSupport.class); - - protected final ZooKeeper zookeeper; - private AtomicBoolean closed = new AtomicBoolean(false); - private long retryDelay = 500L; - private int retryCount = 3; - private List acl = ZooDefs.Ids.OPEN_ACL_UNSAFE; - - public ProtocolSupport(ZooKeeper zookeeper) { - this.zookeeper = zookeeper; - } - - /** - * Closes this strategy and releases any ZooKeeper resources; but keeps the - * ZooKeeper instance open - */ - public void close() { - if (closed.compareAndSet(false, true)) { - doClose(); - } - } - - /** - * return zookeeper client instance - * @return zookeeper client instance - */ - public ZooKeeper getZookeeper() { - return zookeeper; - } - - /** - * return the acl its using - * @return the acl. - */ - public List getAcl() { - return acl; - } - - /** - * set the acl - * @param acl the acl to set to - */ - public void setAcl(List acl) { - this.acl = acl; - } - - /** - * get the retry delay in milliseconds - * @return the retry delay - */ - public long getRetryDelay() { - return retryDelay; - } - - /** - * Sets the time waited between retry delays - * @param retryDelay the retry delay - */ - public void setRetryDelay(long retryDelay) { - this.retryDelay = retryDelay; - } - - /** - * Allow derived classes to perform - * some custom closing operations to release resources - */ - protected void doClose() { - } - - - /** - * Perform the given operation, retrying if the connection fails - * @return object. it needs to be cast to the callee's expected - * return type. - */ - protected Object retryOperation(ZooKeeperOperation operation) - throws KeeperException, InterruptedException { - KeeperException exception = null; - for (int i = 0; i < retryCount; i++) { - try { - return operation.execute(); - } catch (KeeperException.SessionExpiredException e) { - LOG.warn("Session expired for: " + zookeeper + " so reconnecting due to: " + e, e); - throw e; - } catch (KeeperException.ConnectionLossException e) { - if (exception == null) { - exception = e; - } - LOG.debug("Attempt " + i + " failed with connection loss so " + - "attempting to reconnect: " + e, e); - retryDelay(i); - } - } - throw exception; - } - - /** - * Ensures that the given path exists with no data, the current - * ACL and no flags - * @param path - */ - protected void ensurePathExists(String path) { - ensureExists(path, null, acl, CreateMode.PERSISTENT); - } - - /** - * Ensures that the given path exists with the given data, ACL and flags - * @param path - * @param acl - * @param flags - */ - protected void ensureExists(final String path, final byte[] data, - final List acl, final CreateMode flags) { - try { - retryOperation(new ZooKeeperOperation() { - public boolean execute() throws KeeperException, InterruptedException { - Stat stat = zookeeper.exists(path, false); - if (stat != null) { - return true; - } - zookeeper.create(path, data, acl, flags); - return true; - } - }); - } catch (KeeperException e) { - LOG.warn("Caught: " + e, e); - } catch (InterruptedException e) { - LOG.warn("Caught: " + e, e); - } - } - - /** - * Returns true if this protocol has been closed - * @return true if this protocol is closed - */ - protected boolean isClosed() { - return closed.get(); - } - - /** - * Performs a retry delay if this is not the first attempt - * @param attemptCount the number of the attempts performed so far - */ - protected void retryDelay(int attemptCount) { - if (attemptCount > 0) { - try { - Thread.sleep(attemptCount * retryDelay); - } catch (InterruptedException e) { - LOG.debug("Failed to sleep: " + e, e); - } - } - } -} diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/snapshot/lock/WriteLock.java hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/snapshot/lock/WriteLock.java deleted file mode 100644 index 4a66ca5..0000000 --- hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/snapshot/lock/WriteLock.java +++ /dev/null @@ -1,303 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hive.hcatalog.hbase.snapshot.lock; - -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.WatchedEvent; -import org.apache.zookeeper.Watcher; - -import static org.apache.zookeeper.CreateMode.EPHEMERAL_SEQUENTIAL; - -import org.apache.zookeeper.ZooKeeper; -import org.apache.zookeeper.data.ACL; -import org.apache.zookeeper.data.Stat; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.List; -import java.util.SortedSet; -import java.util.TreeSet; - -/** - * A protocol to implement an exclusive - * write lock or to elect a leader.

You invoke {@link #lock()} to - * start the process of grabbing the lock; you may get the lock then or it may be - * some time later.

You can register a listener so that you are invoked - * when you get the lock; otherwise you can ask if you have the lock - * by calling {@link #isOwner()} - * This class has been used as-is from the zookeeper 3.4.0 recipes. The only change - * made is a TODO for sorting using suffixes and the package name. - */ -public class WriteLock extends ProtocolSupport { - private static final Logger LOG = LoggerFactory.getLogger(WriteLock.class); - - private final String dir; - private String id; - private ZNodeName idName; - private String ownerId; - private String lastChildId; - private byte[] data = {0x12, 0x34}; - private LockListener callback; - private LockZooKeeperOperation zop; - - /** - * zookeeper contructor for writelock - * @param zookeeper zookeeper client instance - * @param dir the parent path you want to use for locking - * @param acl the acls that you want to use for all the paths, - * if null world read/write is used. - */ - public WriteLock(ZooKeeper zookeeper, String dir, List acl) { - super(zookeeper); - this.dir = dir; - if (acl != null) { - setAcl(acl); - } - this.zop = new LockZooKeeperOperation(); - } - - /** - * zookeeper contructor for writelock with callback - * @param zookeeper the zookeeper client instance - * @param dir the parent path you want to use for locking - * @param acl the acls that you want to use for all the paths - * @param callback the call back instance - */ - public WriteLock(ZooKeeper zookeeper, String dir, List acl, - LockListener callback) { - this(zookeeper, dir, acl); - this.callback = callback; - } - - /** - * return the current locklistener - * @return the locklistener - */ - public LockListener getLockListener() { - return this.callback; - } - - /** - * register a different call back listener - * @param callback the call back instance - */ - public void setLockListener(LockListener callback) { - this.callback = callback; - } - - /** - * Removes the lock or associated znode if - * you no longer require the lock. this also - * removes your request in the queue for locking - * in case you do not already hold the lock. - * @throws RuntimeException throws a runtime exception - * if it cannot connect to zookeeper. - */ - public synchronized void unlock() throws RuntimeException { - - if (!isClosed() && id != null) { - // we don't need to retry this operation in the case of failure - // as ZK will remove ephemeral files and we don't wanna hang - // this process when closing if we cannot reconnect to ZK - try { - - ZooKeeperOperation zopdel = new ZooKeeperOperation() { - public boolean execute() throws KeeperException, - InterruptedException { - zookeeper.delete(id, -1); - return Boolean.TRUE; - } - }; - zopdel.execute(); - } catch (InterruptedException e) { - LOG.warn("Caught: " + e, e); - //set that we have been interrupted. - Thread.currentThread().interrupt(); - } catch (KeeperException.NoNodeException e) { - // do nothing - } catch (KeeperException e) { - LOG.warn("Caught: " + e, e); - throw (RuntimeException) new RuntimeException(e.getMessage()). - initCause(e); - } finally { - if (callback != null) { - callback.lockReleased(); - } - id = null; - } - } - } - - /** - * the watcher called on - * getting watch while watching - * my predecessor - */ - private class LockWatcher implements Watcher { - public void process(WatchedEvent event) { - // lets either become the leader or watch the new/updated node - LOG.debug("Watcher fired on path: " + event.getPath() + " state: " + - event.getState() + " type " + event.getType()); - try { - lock(); - } catch (Exception e) { - LOG.warn("Failed to acquire lock: " + e, e); - } - } - } - - /** - * a zoookeeper operation that is mainly responsible - * for all the magic required for locking. - */ - private class LockZooKeeperOperation implements ZooKeeperOperation { - - /** find if we have been created earler if not create our node - * - * @param prefix the prefix node - * @param zookeeper teh zookeeper client - * @param dir the dir paretn - * @throws KeeperException - * @throws InterruptedException - */ - private void findPrefixInChildren(String prefix, ZooKeeper zookeeper, String dir) - throws KeeperException, InterruptedException { - List names = zookeeper.getChildren(dir, false); - for (String name : names) { - if (name.startsWith(prefix)) { - id = name; - if (LOG.isDebugEnabled()) { - LOG.debug("Found id created last time: " + id); - } - break; - } - } - if (id == null) { - id = zookeeper.create(dir + "/" + prefix, data, - getAcl(), EPHEMERAL_SEQUENTIAL); - - if (LOG.isDebugEnabled()) { - LOG.debug("Created id: " + id); - } - } - - } - - /** - * the command that is run and retried for actually - * obtaining the lock - * @return if the command was successful or not - */ - public boolean execute() throws KeeperException, InterruptedException { - do { - if (id == null) { - long sessionId = zookeeper.getSessionId(); - String prefix = "x-" + sessionId + "-"; - // lets try look up the current ID if we failed - // in the middle of creating the znode - findPrefixInChildren(prefix, zookeeper, dir); - idName = new ZNodeName(id); - } - if (id != null) { - List names = zookeeper.getChildren(dir, false); - if (names.isEmpty()) { - LOG.warn("No children in: " + dir + " when we've just " + - "created one! Lets recreate it..."); - // lets force the recreation of the id - id = null; - } else { - // lets sort them explicitly (though they do seem to come back in order ususally :) - SortedSet sortedNames = new TreeSet(); - for (String name : names) { - //TODO: Just use the suffix to sort. - sortedNames.add(new ZNodeName(dir + "/" + name)); - } - ownerId = sortedNames.first().getName(); - SortedSet lessThanMe = sortedNames.headSet(idName); - if (!lessThanMe.isEmpty()) { - ZNodeName lastChildName = lessThanMe.last(); - lastChildId = lastChildName.getName(); - if (LOG.isDebugEnabled()) { - LOG.debug("watching less than me node: " + lastChildId); - } - Stat stat = zookeeper.exists(lastChildId, new LockWatcher()); - if (stat != null) { - return Boolean.FALSE; - } else { - LOG.warn("Could not find the" + - " stats for less than me: " + lastChildName.getName()); - } - } else { - if (isOwner()) { - if (callback != null) { - callback.lockAcquired(); - } - return Boolean.TRUE; - } - } - } - } - } - while (id == null); - return Boolean.FALSE; - } - } - - ; - - /** - * Attempts to acquire the exclusive write lock returning whether or not it was - * acquired. Note that the exclusive lock may be acquired some time later after - * this method has been invoked due to the current lock owner going away. - */ - public synchronized boolean lock() throws KeeperException, InterruptedException { - if (isClosed()) { - return false; - } - ensurePathExists(dir); - - return (Boolean) retryOperation(zop); - } - - /** - * return the parent dir for lock - * @return the parent dir used for locks. - */ - public String getDir() { - return dir; - } - - /** - * Returns true if this node is the owner of the - * lock (or the leader) - */ - public boolean isOwner() { - return id != null && ownerId != null && id.equals(ownerId); - } - - /** - * return the id for this lock - * @return the id for this lock - */ - public String getId() { - return this.id; - } -} - diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/snapshot/lock/ZNodeName.java hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/snapshot/lock/ZNodeName.java deleted file mode 100644 index 9eeef3f..0000000 --- hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/snapshot/lock/ZNodeName.java +++ /dev/null @@ -1,113 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hive.hcatalog.hbase.snapshot.lock; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Represents an ephemeral znode name which has an ordered sequence number - * and can be sorted in order - * This class has been used as-is from the zookeeper 3.4.0 recipes with a - * change in package name. - */ -public class ZNodeName implements Comparable { - private final String name; - private String prefix; - private int sequence = -1; - private static final Logger LOG = LoggerFactory.getLogger(ZNodeName.class); - - public ZNodeName(String name) { - if (name == null) { - throw new NullPointerException("id cannot be null"); - } - this.name = name; - this.prefix = name; - int idx = name.lastIndexOf('-'); - if (idx >= 0) { - this.prefix = name.substring(0, idx); - try { - this.sequence = Integer.parseInt(name.substring(idx + 1)); - // If an exception occurred we misdetected a sequence suffix, - // so return -1. - } catch (NumberFormatException e) { - LOG.info("Number format exception for " + idx, e); - } catch (ArrayIndexOutOfBoundsException e) { - LOG.info("Array out of bounds for " + idx, e); - } - } - } - - @Override - public String toString() { - return name.toString(); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - ZNodeName sequence = (ZNodeName) o; - - if (!name.equals(sequence.name)) return false; - - return true; - } - - @Override - public int hashCode() { - return name.hashCode() + 37; - } - - public int compareTo(ZNodeName that) { - int answer = this.prefix.compareTo(that.prefix); - if (answer == 0) { - int s1 = this.sequence; - int s2 = that.sequence; - if (s1 == -1 && s2 == -1) { - return this.name.compareTo(that.name); - } - answer = s1 == -1 ? 1 : s2 == -1 ? -1 : s1 - s2; - } - return answer; - } - - /** - * Returns the name of the znode - */ - public String getName() { - return name; - } - - /** - * Returns the sequence number - */ - public int getZNodeName() { - return sequence; - } - - /** - * Returns the text prefix before the sequence number - */ - public String getPrefix() { - return prefix; - } -} diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/snapshot/lock/ZooKeeperOperation.java hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/snapshot/lock/ZooKeeperOperation.java deleted file mode 100644 index fc77994..0000000 --- hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/snapshot/lock/ZooKeeperOperation.java +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hive.hcatalog.hbase.snapshot.lock; - -import org.apache.zookeeper.KeeperException; - -/** - * A callback object which can be used for implementing retry-able operations in the - * {@link org.apache.hive.hcatalog.hbase.snapshot.lock.ProtocolSupport} class - * This class has been used as-is from the zookeeper 3.4.0 with change in the - * package name . - */ -public interface ZooKeeperOperation { - - /** - * Performs the operation - which may be involved multiple times if the connection - * to ZooKeeper closes during this operation - * - * @return the result of the operation or null - * @throws KeeperException - * @throws InterruptedException - */ - public boolean execute() throws KeeperException, InterruptedException; -} diff --git hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/snapshot/package-info.java hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/snapshot/package-info.java deleted file mode 100644 index 24c9e2d..0000000 --- hcatalog/storage-handlers/hbase/src/java/org/apache/hive/hcatalog/hbase/snapshot/package-info.java +++ /dev/null @@ -1,28 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -/** - * Provides a revision manager for data stored in HBase that can be used to implement repeatable reads. - * The component is designed to be usable for revision management of data stored in HBase in general, - * independent and not limited to HCatalog. It is used by the HCatalog HBase storage handler, implementation depends on HBase 0.92+. - *

- * For more information please see - * Snapshots and Repeatable reads for HBase Tables. - * @since 0.4 - */ -package org.apache.hive.hcatalog.hbase.snapshot; diff --git hcatalog/storage-handlers/hbase/src/resources/revision-manager-default.xml hcatalog/storage-handlers/hbase/src/resources/revision-manager-default.xml index 316a4a1..f5eca58 100644 --- hcatalog/storage-handlers/hbase/src/resources/revision-manager-default.xml +++ hcatalog/storage-handlers/hbase/src/resources/revision-manager-default.xml @@ -23,14 +23,14 @@ revision.manager.impl.class - org.apache.hive.hcatalog.hbase.snapshot.ZKBasedRevisionManager + org.apache.hcatalog.hbase.snapshot.ZKBasedRevisionManager Which revision manager implementation to use. revision.manager.endpoint.impl.class - org.apache.hive.hcatalog.hbase.snapshot.ZKBasedRevisionManager + org.apache.hcatalog.hbase.snapshot.ZKBasedRevisionManager diff --git hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/ManyMiniCluster.java hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/ManyMiniCluster.java new file mode 100644 index 0000000..8557cda --- /dev/null +++ hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/ManyMiniCluster.java @@ -0,0 +1,370 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hcatalog.hbase; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.MiniHBaseCluster; +import org.apache.hadoop.hbase.client.HConnectionManager; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.MiniMRCluster; + +import java.io.File; +import java.io.IOException; +import java.net.ServerSocket; + +/** + * MiniCluster class composed of a number of Hadoop Minicluster implementations + * and other necessary daemons needed for testing (HBase, Hive MetaStore, Zookeeper, MiniMRCluster) + */ +public class ManyMiniCluster { + + //MR stuff + private boolean miniMRClusterEnabled; + private MiniMRCluster mrCluster; + private int numTaskTrackers; + private JobConf jobConf; + + //HBase stuff + private boolean miniHBaseClusterEnabled; + private MiniHBaseCluster hbaseCluster; + private String hbaseRoot; + private Configuration hbaseConf; + private String hbaseDir; + + //ZK Stuff + private boolean miniZookeeperClusterEnabled; + private MiniZooKeeperCluster zookeeperCluster; + private int zookeeperPort; + private String zookeeperDir; + + //DFS Stuff + private MiniDFSCluster dfsCluster; + + //Hive Stuff + private boolean miniHiveMetastoreEnabled; + private HiveConf hiveConf; + private HiveMetaStoreClient hiveMetaStoreClient; + + private final File workDir; + private boolean started = false; + + + /** + * create a cluster instance using a builder which will expose configurable options + * @param workDir working directory ManyMiniCluster will use for all of it's *Minicluster instances + * @return a Builder instance + */ + public static Builder create(File workDir) { + return new Builder(workDir); + } + + private ManyMiniCluster(Builder b) { + workDir = b.workDir; + numTaskTrackers = b.numTaskTrackers; + hiveConf = b.hiveConf; + jobConf = b.jobConf; + hbaseConf = b.hbaseConf; + miniMRClusterEnabled = b.miniMRClusterEnabled; + miniHBaseClusterEnabled = b.miniHBaseClusterEnabled; + miniHiveMetastoreEnabled = b.miniHiveMetastoreEnabled; + miniZookeeperClusterEnabled = b.miniZookeeperClusterEnabled; + } + + protected synchronized void start() { + try { + if (!started) { + FileUtil.fullyDelete(workDir); + if (miniMRClusterEnabled) { + setupMRCluster(); + } + if (miniZookeeperClusterEnabled || miniHBaseClusterEnabled) { + miniZookeeperClusterEnabled = true; + setupZookeeper(); + } + if (miniHBaseClusterEnabled) { + setupHBaseCluster(); + } + if (miniHiveMetastoreEnabled) { + setUpMetastore(); + } + } + } catch (Exception e) { + throw new IllegalStateException("Failed to setup cluster", e); + } + } + + protected synchronized void stop() { + if (hbaseCluster != null) { + HConnectionManager.deleteAllConnections(true); + try { + hbaseCluster.shutdown(); + } catch (Exception e) { + e.printStackTrace(); + } + hbaseCluster = null; + } + if (zookeeperCluster != null) { + try { + zookeeperCluster.shutdown(); + } catch (Exception e) { + e.printStackTrace(); + } + zookeeperCluster = null; + } + if (mrCluster != null) { + try { + mrCluster.shutdown(); + } catch (Exception e) { + e.printStackTrace(); + } + mrCluster = null; + } + if (dfsCluster != null) { + try { + dfsCluster.getFileSystem().close(); + dfsCluster.shutdown(); + } catch (Exception e) { + e.printStackTrace(); + } + dfsCluster = null; + } + try { + FileSystem.closeAll(); + } catch (IOException e) { + e.printStackTrace(); + } + started = false; + } + + /** + * @return Configuration of mini HBase cluster + */ + public Configuration getHBaseConf() { + return HBaseConfiguration.create(hbaseConf); + } + + /** + * @return Configuration of mini MR cluster + */ + public Configuration getJobConf() { + return new Configuration(jobConf); + } + + /** + * @return Configuration of Hive Metastore, this is a standalone not a daemon + */ + public HiveConf getHiveConf() { + return new HiveConf(hiveConf); + } + + /** + * @return Filesystem used by MiniMRCluster and MiniHBaseCluster + */ + public FileSystem getFileSystem() { + try { + return FileSystem.get(jobConf); + } catch (IOException e) { + throw new IllegalStateException("Failed to get FileSystem", e); + } + } + + /** + * @return Metastore client instance + */ + public HiveMetaStoreClient getHiveMetaStoreClient() { + return hiveMetaStoreClient; + } + + private void setupMRCluster() { + try { + final int jobTrackerPort = findFreePort(); + final int taskTrackerPort = findFreePort(); + + if (jobConf == null) + jobConf = new JobConf(); + + jobConf.setInt("mapred.submit.replication", 1); + jobConf.set("yarn.scheduler.capacity.root.queues", "default"); + jobConf.set("yarn.scheduler.capacity.root.default.capacity", "100"); + //conf.set("hadoop.job.history.location",new File(workDir).getAbsolutePath()+"/history"); + System.setProperty("hadoop.log.dir", new File(workDir, "/logs").getAbsolutePath()); + + mrCluster = new MiniMRCluster(jobTrackerPort, + taskTrackerPort, + numTaskTrackers, + getFileSystem().getUri().toString(), + numTaskTrackers, + null, + null, + null, + jobConf); + + jobConf = mrCluster.createJobConf(); + } catch (IOException e) { + throw new IllegalStateException("Failed to Setup MR Cluster", e); + } + } + + private void setupZookeeper() { + try { + zookeeperDir = new File(workDir, "zk").getAbsolutePath(); + zookeeperPort = findFreePort(); + zookeeperCluster = new MiniZooKeeperCluster(); + zookeeperCluster.setDefaultClientPort(zookeeperPort); + zookeeperCluster.startup(new File(zookeeperDir)); + } catch (Exception e) { + throw new IllegalStateException("Failed to Setup Zookeeper Cluster", e); + } + } + + private void setupHBaseCluster() { + final int numRegionServers = 1; + + try { + hbaseDir = new File(workDir, "hbase").toString(); + hbaseDir = hbaseDir.replaceAll("\\\\", "/"); + hbaseRoot = "file://" + hbaseDir; + + if (hbaseConf == null) + hbaseConf = HBaseConfiguration.create(); + + hbaseConf.set("hbase.rootdir", hbaseRoot); + hbaseConf.set("hbase.master", "local"); + hbaseConf.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, zookeeperPort); + hbaseConf.set(HConstants.ZOOKEEPER_QUORUM, "127.0.0.1"); + hbaseConf.setInt("hbase.master.port", findFreePort()); + hbaseConf.setInt("hbase.master.info.port", -1); + hbaseConf.setInt("hbase.regionserver.port", findFreePort()); + hbaseConf.setInt("hbase.regionserver.info.port", -1); + + hbaseCluster = new MiniHBaseCluster(hbaseConf, numRegionServers); + hbaseConf.set("hbase.master", hbaseCluster.getMaster().getServerName().getHostAndPort()); + //opening the META table ensures that cluster is running + new HTable(hbaseConf, HConstants.META_TABLE_NAME); + } catch (Exception e) { + throw new IllegalStateException("Failed to setup HBase Cluster", e); + } + } + + private void setUpMetastore() throws Exception { + if (hiveConf == null) + hiveConf = new HiveConf(this.getClass()); + + //The default org.apache.hadoop.hive.ql.hooks.PreExecutePrinter hook + //is present only in the ql/test directory + hiveConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, ""); + hiveConf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, ""); + hiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false"); + hiveConf.set(HiveConf.ConfVars.METASTORECONNECTURLKEY.varname, + "jdbc:derby:" + new File(workDir + "/metastore_db") + ";create=true"); + hiveConf.set(HiveConf.ConfVars.METASTOREWAREHOUSE.toString(), + new File(workDir, "warehouse").toString()); + //set where derby logs + File derbyLogFile = new File(workDir + "/derby.log"); + derbyLogFile.createNewFile(); + System.setProperty("derby.stream.error.file", derbyLogFile.getPath()); + + +// Driver driver = new Driver(hiveConf); +// SessionState.start(new CliSessionState(hiveConf)); + + hiveMetaStoreClient = new HiveMetaStoreClient(hiveConf); + } + + private static int findFreePort() throws IOException { + ServerSocket server = new ServerSocket(0); + int port = server.getLocalPort(); + server.close(); + return port; + } + + public static class Builder { + private File workDir; + private int numTaskTrackers = 1; + private JobConf jobConf; + private Configuration hbaseConf; + private HiveConf hiveConf; + + private boolean miniMRClusterEnabled = true; + private boolean miniHBaseClusterEnabled = true; + private boolean miniHiveMetastoreEnabled = true; + private boolean miniZookeeperClusterEnabled = true; + + + private Builder(File workDir) { + this.workDir = workDir; + } + + public Builder numTaskTrackers(int num) { + numTaskTrackers = num; + return this; + } + + public Builder jobConf(JobConf jobConf) { + this.jobConf = jobConf; + return this; + } + + public Builder hbaseConf(Configuration hbaseConf) { + this.hbaseConf = hbaseConf; + return this; + } + + public Builder hiveConf(HiveConf hiveConf) { + this.hiveConf = hiveConf; + return this; + } + + public Builder miniMRClusterEnabled(boolean enabled) { + this.miniMRClusterEnabled = enabled; + return this; + } + + public Builder miniHBaseClusterEnabled(boolean enabled) { + this.miniHBaseClusterEnabled = enabled; + return this; + } + + public Builder miniZookeeperClusterEnabled(boolean enabled) { + this.miniZookeeperClusterEnabled = enabled; + return this; + } + + public Builder miniHiveMetastoreEnabled(boolean enabled) { + this.miniHiveMetastoreEnabled = enabled; + return this; + } + + + public ManyMiniCluster build() { + return new ManyMiniCluster(this); + } + + } +} diff --git hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/SkeletonHBaseTest.java hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/SkeletonHBaseTest.java new file mode 100644 index 0000000..aefe450 --- /dev/null +++ hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/SkeletonHBaseTest.java @@ -0,0 +1,237 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hcatalog.hbase; + +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Random; +import java.util.Set; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hive.conf.HiveConf; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +/** + * Base class for HBase Tests which need a mini cluster instance + */ +public abstract class SkeletonHBaseTest { + + protected static String TEST_DIR = "/tmp/build/test/data/"; + + protected final static String DEFAULT_CONTEXT_HANDLE = "default"; + + protected static Map contextMap = new HashMap(); + protected static Set tableNames = new HashSet(); + + /** + * Allow tests to alter the default MiniCluster configuration. + * (requires static initializer block as all setup here is static) + */ + protected static Configuration testConf = null; + + protected void createTable(String tableName, String[] families) { + try { + HBaseAdmin admin = new HBaseAdmin(getHbaseConf()); + HTableDescriptor tableDesc = new HTableDescriptor(tableName); + for (String family : families) { + HColumnDescriptor columnDescriptor = new HColumnDescriptor(family); + tableDesc.addFamily(columnDescriptor); + } + admin.createTable(tableDesc); + } catch (Exception e) { + e.printStackTrace(); + throw new IllegalStateException(e); + } + + } + + protected String newTableName(String prefix) { + String name = null; + int tries = 100; + do { + name = prefix + "_" + Math.abs(new Random().nextLong()); + } while (tableNames.contains(name) && --tries > 0); + if (tableNames.contains(name)) + throw new IllegalStateException("Couldn't find a unique table name, tableNames size: " + tableNames.size()); + tableNames.add(name); + return name; + } + + + /** + * startup an hbase cluster instance before a test suite runs + */ + @BeforeClass + public static void setup() { + if (!contextMap.containsKey(getContextHandle())) + contextMap.put(getContextHandle(), new Context(getContextHandle())); + + contextMap.get(getContextHandle()).start(); + } + + /** + * shutdown an hbase cluster instance ant the end of the test suite + */ + @AfterClass + public static void tearDown() { + contextMap.get(getContextHandle()).stop(); + } + + /** + * override this with a different context handle if tests suites are run simultaneously + * and ManyMiniCluster instances shouldn't be shared + * @return + */ + public static String getContextHandle() { + return DEFAULT_CONTEXT_HANDLE; + } + + /** + * @return working directory for a given test context, which normally is a test suite + */ + public String getTestDir() { + return contextMap.get(getContextHandle()).getTestDir(); + } + + /** + * @return ManyMiniCluster instance + */ + public ManyMiniCluster getCluster() { + return contextMap.get(getContextHandle()).getCluster(); + } + + /** + * @return configuration of MiniHBaseCluster + */ + public Configuration getHbaseConf() { + return contextMap.get(getContextHandle()).getHbaseConf(); + } + + /** + * @return configuration of MiniMRCluster + */ + public Configuration getJobConf() { + return contextMap.get(getContextHandle()).getJobConf(); + } + + /** + * @return configuration of Hive Metastore + */ + public HiveConf getHiveConf() { + return contextMap.get(getContextHandle()).getHiveConf(); + } + + /** + * @return filesystem used by ManyMiniCluster daemons + */ + public FileSystem getFileSystem() { + return contextMap.get(getContextHandle()).getFileSystem(); + } + + /** + * class used to encapsulate a context which is normally used by + * a single TestSuite or across TestSuites when multi-threaded testing is turned on + */ + public static class Context { + protected String testDir; + protected ManyMiniCluster cluster; + + protected Configuration hbaseConf; + protected Configuration jobConf; + protected HiveConf hiveConf; + + protected FileSystem fileSystem; + + protected int usageCount = 0; + + public Context(String handle) { + testDir = new File(TEST_DIR + "/test_" + handle + "_" + Math.abs(new Random().nextLong()) + "/").getPath(); + System.out.println("Cluster work directory: " + testDir); + } + + public void start() { + if (usageCount++ == 0) { + ManyMiniCluster.Builder b = ManyMiniCluster.create(new File(testDir)); + if (testConf != null) { + b.hbaseConf(HBaseConfiguration.create(testConf)); + } + cluster = b.build(); + cluster.start(); + this.hbaseConf = cluster.getHBaseConf(); + jobConf = cluster.getJobConf(); + fileSystem = cluster.getFileSystem(); + hiveConf = cluster.getHiveConf(); + } + } + + public void stop() { + if (--usageCount == 0) { + try { + cluster.stop(); + cluster = null; + } finally { + System.out.println("Trying to cleanup: " + testDir); + try { + FileSystem fs = FileSystem.get(jobConf); + fs.delete(new Path(testDir), true); + } catch (IOException e) { + throw new IllegalStateException("Failed to cleanup test dir", e); + } + + } + } + } + + public String getTestDir() { + return testDir; + } + + public ManyMiniCluster getCluster() { + return cluster; + } + + public Configuration getHbaseConf() { + return hbaseConf; + } + + public Configuration getJobConf() { + return jobConf; + } + + public HiveConf getHiveConf() { + return hiveConf; + } + + public FileSystem getFileSystem() { + return fileSystem; + } + } + +} diff --git hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseBulkOutputFormat.java hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseBulkOutputFormat.java new file mode 100644 index 0000000..4c7cc04 --- /dev/null +++ hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseBulkOutputFormat.java @@ -0,0 +1,631 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hcatalog.hbase; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hive.cli.CliSessionState; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.session.SessionState; +import org.apache.hadoop.io.BytesWritable; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapred.JobClient; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.OutputCollector; +import org.apache.hadoop.mapred.Reporter; +import org.apache.hadoop.mapred.RunningJob; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.Mapper; +import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; +import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat; +import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; +import org.apache.hive.hcatalog.cli.HCatDriver; +import org.apache.hive.hcatalog.cli.SemanticAnalysis.HCatSemanticAnalyzer; +import org.apache.hive.hcatalog.common.HCatConstants; +import org.apache.hive.hcatalog.common.HCatUtil; +import org.apache.hive.hcatalog.data.DefaultHCatRecord; +import org.apache.hive.hcatalog.data.HCatRecord; +import org.apache.hive.hcatalog.data.schema.HCatSchema; +import org.apache.hcatalog.hbase.HBaseBulkOutputFormat.HBaseBulkOutputCommitter; +import org.apache.hcatalog.hbase.TestHBaseDirectOutputFormat.MapReadAbortedTransaction; +import org.apache.hcatalog.hbase.TestHBaseDirectOutputFormat.MapWriteAbortTransaction; +import org.apache.hcatalog.hbase.snapshot.FamilyRevision; +import org.apache.hcatalog.hbase.snapshot.RevisionManager; +import org.apache.hcatalog.hbase.snapshot.RevisionManagerConfiguration; +import org.apache.hcatalog.hbase.snapshot.TableSnapshot; +import org.apache.hcatalog.hbase.snapshot.Transaction; +import org.apache.hive.hcatalog.mapreduce.HCatInputFormat; +import org.apache.hive.hcatalog.mapreduce.HCatOutputFormat; +import org.apache.hive.hcatalog.mapreduce.OutputJobInfo; + +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * Tests components of HBaseHCatStorageHandler using ManyMiniCluster. + * Including ImprtSequenceFile and HBaseBulkOutputFormat + */ +public class TestHBaseBulkOutputFormat extends SkeletonHBaseTest { + private final static Logger LOG = LoggerFactory.getLogger(TestHBaseBulkOutputFormat.class); + + private final HiveConf allConf; + private final HCatDriver hcatDriver; + + public TestHBaseBulkOutputFormat() { + allConf = getHiveConf(); + allConf.set(HiveConf.ConfVars.SEMANTIC_ANALYZER_HOOK.varname, + HCatSemanticAnalyzer.class.getName()); + allConf.set(HiveConf.ConfVars.HADOOPFS.varname, getFileSystem().getUri().toString()); + allConf.set(HiveConf.ConfVars.METASTOREWAREHOUSE.varname, new Path(getTestDir(), "warehouse").toString()); + + //Add hbase properties + for (Map.Entry el : getHbaseConf()) + allConf.set(el.getKey(), el.getValue()); + for (Map.Entry el : getJobConf()) + allConf.set(el.getKey(), el.getValue()); + + HBaseConfiguration.merge( + allConf, + RevisionManagerConfiguration.create()); + SessionState.start(new CliSessionState(allConf)); + hcatDriver = new HCatDriver(); + } + + public static class MapWriteOldMapper implements org.apache.hadoop.mapred.Mapper { + + @Override + public void close() throws IOException { + } + + @Override + public void configure(JobConf job) { + } + + @Override + public void map(LongWritable key, Text value, + OutputCollector output, + Reporter reporter) throws IOException { + String vals[] = value.toString().split(","); + Put put = new Put(Bytes.toBytes(vals[0])); + for (int i = 1; i < vals.length; i++) { + String pair[] = vals[i].split(":"); + put.add(Bytes.toBytes("my_family"), + Bytes.toBytes(pair[0]), + Bytes.toBytes(pair[1])); + } + output.collect(new ImmutableBytesWritable(Bytes.toBytes(vals[0])), put); + } + + } + + public static class MapWrite extends Mapper { + + @Override + public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { + String vals[] = value.toString().split(","); + Put put = new Put(Bytes.toBytes(vals[0])); + for (int i = 1; i < vals.length; i++) { + String pair[] = vals[i].split(":"); + put.add(Bytes.toBytes("my_family"), + Bytes.toBytes(pair[0]), + Bytes.toBytes(pair[1])); + } + context.write(new ImmutableBytesWritable(Bytes.toBytes(vals[0])), put); + } + } + + public static class MapHCatWrite extends Mapper { + @Override + public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { + OutputJobInfo jobInfo = (OutputJobInfo) HCatUtil.deserialize(context.getConfiguration().get(HCatConstants.HCAT_KEY_OUTPUT_INFO)); + HCatRecord record = new DefaultHCatRecord(3); + HCatSchema schema = jobInfo.getOutputSchema(); + String vals[] = value.toString().split(","); + record.setInteger("key", schema, Integer.parseInt(vals[0])); + for (int i = 1; i < vals.length; i++) { + String pair[] = vals[i].split(":"); + record.set(pair[0], schema, pair[1]); + } + context.write(null, record); + } + } + + @Test + public void hbaseBulkOutputFormatTest() throws IOException, ClassNotFoundException, InterruptedException { + String testName = "hbaseBulkOutputFormatTest"; + Path methodTestDir = new Path(getTestDir(), testName); + LOG.info("starting: " + testName); + + String tableName = newTableName(testName).toLowerCase(); + String familyName = "my_family"; + byte[] familyNameBytes = Bytes.toBytes(familyName); + + //include hbase config in conf file + Configuration conf = new Configuration(allConf); + + //create table + conf.set(HBaseConstants.PROPERTY_OUTPUT_TABLE_NAME_KEY, tableName); + conf.set("yarn.scheduler.capacity.root.queues", "default"); + conf.set("yarn.scheduler.capacity.root.default.capacity", "100"); + createTable(tableName, new String[]{familyName}); + + String data[] = {"1,english:one,spanish:uno", + "2,english:two,spanish:dos", + "3,english:three,spanish:tres"}; + + + // input/output settings + Path inputPath = new Path(methodTestDir, "mr_input"); + FSDataOutputStream os = getFileSystem().create(new Path(inputPath, "inputFile.txt")); + for (String line : data) + os.write(Bytes.toBytes(line + "\n")); + os.close(); + Path interPath = new Path(methodTestDir, "inter"); + //create job + JobConf job = new JobConf(conf); + job.setWorkingDirectory(new Path(methodTestDir, "mr_work")); + job.setJarByClass(this.getClass()); + job.setMapperClass(MapWriteOldMapper.class); + + job.setInputFormat(org.apache.hadoop.mapred.TextInputFormat.class); + org.apache.hadoop.mapred.TextInputFormat.setInputPaths(job, inputPath); + + job.setOutputFormat(HBaseBulkOutputFormat.class); + org.apache.hadoop.mapred.SequenceFileOutputFormat.setOutputPath(job, interPath); + job.setOutputCommitter(HBaseBulkOutputCommitter.class); + + //manually create transaction + RevisionManager rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(conf); + try { + OutputJobInfo outputJobInfo = OutputJobInfo.create("default", tableName, null); + Transaction txn = rm.beginWriteTransaction(tableName, Arrays.asList(familyName)); + outputJobInfo.getProperties().setProperty(HBaseConstants.PROPERTY_WRITE_TXN_KEY, + HCatUtil.serialize(txn)); + job.set(HCatConstants.HCAT_KEY_OUTPUT_INFO, + HCatUtil.serialize(outputJobInfo)); + } finally { + rm.close(); + } + + job.setMapOutputKeyClass(ImmutableBytesWritable.class); + job.setMapOutputValueClass(HCatRecord.class); + + job.setOutputKeyClass(ImmutableBytesWritable.class); + job.setOutputValueClass(HCatRecord.class); + + job.setNumReduceTasks(0); + + RunningJob runJob = JobClient.runJob(job); + runJob.waitForCompletion(); + assertTrue(runJob.isSuccessful()); + + //verify + HTable table = new HTable(conf, tableName); + Scan scan = new Scan(); + scan.addFamily(familyNameBytes); + ResultScanner scanner = table.getScanner(scan); + int index = 0; + for (Result result : scanner) { + String vals[] = data[index].toString().split(","); + for (int i = 1; i < vals.length; i++) { + String pair[] = vals[i].split(":"); + assertTrue(result.containsColumn(familyNameBytes, Bytes.toBytes(pair[0]))); + assertEquals(pair[1], Bytes.toString(result.getValue(familyNameBytes, Bytes.toBytes(pair[0])))); + } + index++; + } + //test if load count is the same + assertEquals(data.length, index); + //test if scratch directory was erased + assertFalse(FileSystem.get(job).exists(interPath)); + } + + @Test + public void importSequenceFileTest() throws IOException, ClassNotFoundException, InterruptedException { + String testName = "importSequenceFileTest"; + Path methodTestDir = new Path(getTestDir(), testName); + LOG.info("starting: " + testName); + + String tableName = newTableName(testName).toLowerCase(); + String familyName = "my_family"; + byte[] familyNameBytes = Bytes.toBytes(familyName); + + //include hbase config in conf file + Configuration conf = new Configuration(allConf); + + //create table + createTable(tableName, new String[]{familyName}); + + String data[] = {"1,english:one,spanish:uno", + "2,english:two,spanish:dos", + "3,english:three,spanish:tres"}; + + + // input/output settings + Path inputPath = new Path(methodTestDir, "mr_input"); + getFileSystem().mkdirs(inputPath); + FSDataOutputStream os = getFileSystem().create(new Path(inputPath, "inputFile.txt")); + for (String line : data) + os.write(Bytes.toBytes(line + "\n")); + os.close(); + Path interPath = new Path(methodTestDir, "inter"); + Path scratchPath = new Path(methodTestDir, "scratch"); + + + //create job + Job job = new Job(conf, testName); + job.setWorkingDirectory(new Path(methodTestDir, "mr_work")); + job.setJarByClass(this.getClass()); + job.setMapperClass(MapWrite.class); + + job.setInputFormatClass(TextInputFormat.class); + TextInputFormat.setInputPaths(job, inputPath); + + job.setOutputFormatClass(SequenceFileOutputFormat.class); + SequenceFileOutputFormat.setOutputPath(job, interPath); + + job.setMapOutputKeyClass(ImmutableBytesWritable.class); + job.setMapOutputValueClass(Put.class); + + job.setOutputKeyClass(ImmutableBytesWritable.class); + job.setOutputValueClass(Put.class); + + job.setNumReduceTasks(0); + assertTrue(job.waitForCompletion(true)); + + job = new Job(new Configuration(allConf), testName + "_importer"); + assertTrue(ImportSequenceFile.runJob(job, tableName, interPath, scratchPath)); + + //verify + HTable table = new HTable(conf, tableName); + Scan scan = new Scan(); + scan.addFamily(familyNameBytes); + ResultScanner scanner = table.getScanner(scan); + int index = 0; + for (Result result : scanner) { + String vals[] = data[index].toString().split(","); + for (int i = 1; i < vals.length; i++) { + String pair[] = vals[i].split(":"); + assertTrue(result.containsColumn(familyNameBytes, Bytes.toBytes(pair[0]))); + assertEquals(pair[1], Bytes.toString(result.getValue(familyNameBytes, Bytes.toBytes(pair[0])))); + } + index++; + } + //test if load count is the same + assertEquals(data.length, index); + //test if scratch directory was erased + assertFalse(FileSystem.get(job.getConfiguration()).exists(scratchPath)); + } + + @Test + public void bulkModeHCatOutputFormatTest() throws Exception { + String testName = "bulkModeHCatOutputFormatTest"; + Path methodTestDir = new Path(getTestDir(), testName); + LOG.info("starting: " + testName); + + String databaseName = testName.toLowerCase(); + String dbDir = new Path(methodTestDir, "DB_" + testName).toString(); + String tableName = newTableName(testName).toLowerCase(); + String familyName = "my_family"; + byte[] familyNameBytes = Bytes.toBytes(familyName); + + + //include hbase config in conf file + Configuration conf = new Configuration(allConf); + conf.set(HCatConstants.HCAT_KEY_HIVE_CONF, HCatUtil.serialize(allConf.getAllProperties())); + + + String dbquery = "CREATE DATABASE IF NOT EXISTS " + databaseName + " LOCATION '" + dbDir + "'"; + String tableQuery = "CREATE TABLE " + databaseName + "." + tableName + + "(key int, english string, spanish string) STORED BY " + + "'org.apache.hcatalog.hbase.HBaseHCatStorageHandler'" + + "TBLPROPERTIES ('" + HBaseConstants.PROPERTY_BULK_OUTPUT_MODE_KEY + "'='true'," + + "'hbase.columns.mapping'=':key," + familyName + ":english," + familyName + ":spanish')"; + + assertEquals(0, hcatDriver.run(dbquery).getResponseCode()); + assertEquals(0, hcatDriver.run(tableQuery).getResponseCode()); + + String data[] = {"1,english:ONE,spanish:UNO", + "2,english:TWO,spanish:DOS", + "3,english:THREE,spanish:TRES"}; + + // input/output settings + Path inputPath = new Path(methodTestDir, "mr_input"); + getFileSystem().mkdirs(inputPath); + //create multiple files so we can test with multiple mappers + for (int i = 0; i < data.length; i++) { + FSDataOutputStream os = getFileSystem().create(new Path(inputPath, "inputFile" + i + ".txt")); + os.write(Bytes.toBytes(data[i] + "\n")); + os.close(); + } + + //create job + Job job = new Job(conf, testName); + job.setWorkingDirectory(new Path(methodTestDir, "mr_work")); + job.setJarByClass(this.getClass()); + job.setMapperClass(MapHCatWrite.class); + + job.setInputFormatClass(TextInputFormat.class); + TextInputFormat.setInputPaths(job, inputPath); + + + job.setOutputFormatClass(HCatOutputFormat.class); + OutputJobInfo outputJobInfo = OutputJobInfo.create(databaseName, tableName, null); + HCatOutputFormat.setOutput(job, outputJobInfo); + + job.setMapOutputKeyClass(BytesWritable.class); + job.setMapOutputValueClass(HCatRecord.class); + + job.setOutputKeyClass(BytesWritable.class); + job.setOutputValueClass(HCatRecord.class); + + job.setNumReduceTasks(0); + + assertTrue(job.waitForCompletion(true)); + RevisionManager rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(conf); + try { + TableSnapshot snapshot = rm.createSnapshot(databaseName + "." + tableName); + for (String el : snapshot.getColumnFamilies()) { + assertEquals(1, snapshot.getRevision(el)); + } + } finally { + rm.close(); + } + + //verify + HTable table = new HTable(conf, databaseName + "." + tableName); + Scan scan = new Scan(); + scan.addFamily(familyNameBytes); + ResultScanner scanner = table.getScanner(scan); + int index = 0; + for (Result result : scanner) { + String vals[] = data[index].toString().split(","); + for (int i = 1; i < vals.length; i++) { + String pair[] = vals[i].split(":"); + assertTrue(result.containsColumn(familyNameBytes, Bytes.toBytes(pair[0]))); + assertEquals(pair[1], Bytes.toString(result.getValue(familyNameBytes, Bytes.toBytes(pair[0])))); + assertEquals(1l, result.getColumn(familyNameBytes, Bytes.toBytes(pair[0])).get(0).getTimestamp()); + } + index++; + } + //test if load count is the same + assertEquals(data.length, index); + } + + @Test + public void bulkModeHCatOutputFormatTestWithDefaultDB() throws Exception { + String testName = "bulkModeHCatOutputFormatTestWithDefaultDB"; + Path methodTestDir = new Path(getTestDir(), testName); + + String databaseName = "default"; + String dbDir = new Path(methodTestDir, "DB_" + testName).toString(); + String tableName = newTableName(testName).toLowerCase(); + String familyName = "my_family"; + byte[] familyNameBytes = Bytes.toBytes(familyName); + + + //include hbase config in conf file + Configuration conf = new Configuration(allConf); + conf.set(HCatConstants.HCAT_KEY_HIVE_CONF, HCatUtil.serialize(allConf.getAllProperties())); + + + String dbquery = "CREATE DATABASE IF NOT EXISTS " + databaseName + " LOCATION '" + dbDir + "'"; + String tableQuery = "CREATE TABLE " + databaseName + "." + tableName + + "(key int, english string, spanish string) STORED BY " + + "'org.apache.hcatalog.hbase.HBaseHCatStorageHandler'" + + "TBLPROPERTIES ('" + HBaseConstants.PROPERTY_BULK_OUTPUT_MODE_KEY + "'='true'," + + "'hbase.columns.mapping'=':key," + familyName + ":english," + familyName + ":spanish')"; + + assertEquals(0, hcatDriver.run(dbquery).getResponseCode()); + assertEquals(0, hcatDriver.run(tableQuery).getResponseCode()); + + String data[] = {"1,english:ONE,spanish:UNO", + "2,english:TWO,spanish:DOS", + "3,english:THREE,spanish:TRES"}; + + // input/output settings + Path inputPath = new Path(methodTestDir, "mr_input"); + getFileSystem().mkdirs(inputPath); + FSDataOutputStream os = getFileSystem().create(new Path(inputPath, "inputFile.txt")); + for (String line : data) + os.write(Bytes.toBytes(line + "\n")); + os.close(); + + //create job + Job job = new Job(conf, testName); + job.setWorkingDirectory(new Path(methodTestDir, "mr_work")); + job.setJarByClass(this.getClass()); + job.setMapperClass(MapHCatWrite.class); + + job.setInputFormatClass(TextInputFormat.class); + TextInputFormat.setInputPaths(job, inputPath); + + + job.setOutputFormatClass(HCatOutputFormat.class); + OutputJobInfo outputJobInfo = OutputJobInfo.create(databaseName, tableName, null); + HCatOutputFormat.setOutput(job, outputJobInfo); + + job.setMapOutputKeyClass(BytesWritable.class); + job.setMapOutputValueClass(HCatRecord.class); + + job.setOutputKeyClass(BytesWritable.class); + job.setOutputValueClass(HCatRecord.class); + + job.setNumReduceTasks(0); + + assertTrue(job.waitForCompletion(true)); + + //verify + HTable table = new HTable(conf, tableName); + Scan scan = new Scan(); + scan.addFamily(familyNameBytes); + ResultScanner scanner = table.getScanner(scan); + int index = 0; + for (Result result : scanner) { + String vals[] = data[index].toString().split(","); + for (int i = 1; i < vals.length; i++) { + String pair[] = vals[i].split(":"); + assertTrue(result.containsColumn(familyNameBytes, Bytes.toBytes(pair[0]))); + assertEquals(pair[1], Bytes.toString(result.getValue(familyNameBytes, Bytes.toBytes(pair[0])))); + } + index++; + } + //test if load count is the same + assertEquals(data.length, index); + } + + @Test + public void bulkModeAbortTest() throws Exception { + String testName = "bulkModeAbortTest"; + Path methodTestDir = new Path(getTestDir(), testName); + String databaseName = testName.toLowerCase(); + String dbDir = new Path(methodTestDir, "DB_" + testName).toString(); + String tableName = newTableName(testName).toLowerCase(); + String familyName = "my_family"; + + // include hbase config in conf file + Configuration conf = new Configuration(allConf); + conf.set(HCatConstants.HCAT_KEY_HIVE_CONF, HCatUtil.serialize(allConf.getAllProperties())); + + String dbquery = "CREATE DATABASE IF NOT EXISTS " + databaseName + " LOCATION '" + dbDir + + "'"; + String tableQuery = "CREATE TABLE " + databaseName + "." + tableName + + "(key int, english string, spanish string) STORED BY " + + "'org.apache.hcatalog.hbase.HBaseHCatStorageHandler'" + + "TBLPROPERTIES ('" + HBaseConstants.PROPERTY_BULK_OUTPUT_MODE_KEY + "'='true'," + + "'hbase.columns.mapping'=':key," + familyName + ":english," + familyName + + ":spanish')"; + + assertEquals(0, hcatDriver.run(dbquery).getResponseCode()); + assertEquals(0, hcatDriver.run(tableQuery).getResponseCode()); + + String data[] = {"1,english:ONE,spanish:UNO", + "2,english:TWO,spanish:DOS", + "3,english:THREE,spanish:TRES"}; + + Path inputPath = new Path(methodTestDir, "mr_input"); + getFileSystem().mkdirs(inputPath); + // create multiple files so we can test with multiple mappers + for (int i = 0; i < data.length; i++) { + FSDataOutputStream os = getFileSystem().create( + new Path(inputPath, "inputFile" + i + ".txt")); + os.write(Bytes.toBytes(data[i] + "\n")); + os.close(); + } + + Path workingDir = new Path(methodTestDir, "mr_abort"); + OutputJobInfo outputJobInfo = OutputJobInfo.create(databaseName, + tableName, null); + Job job = configureJob(testName, + conf, workingDir, MapWriteAbortTransaction.class, + outputJobInfo, inputPath); + assertFalse(job.waitForCompletion(true)); + + // verify that revision manager has it as aborted transaction + RevisionManager rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(conf); + try { + TableSnapshot snapshot = rm.createSnapshot(databaseName + "." + tableName); + for (String family : snapshot.getColumnFamilies()) { + assertEquals(1, snapshot.getRevision(family)); + List abortedWriteTransactions = rm.getAbortedWriteTransactions( + databaseName + "." + tableName, family); + assertEquals(1, abortedWriteTransactions.size()); + assertEquals(1, abortedWriteTransactions.get(0).getRevision()); + } + } finally { + rm.close(); + } + + //verify that hbase does not have any of the records. + //Since records are only written during commitJob, + //hbase should not have any records. + HTable table = new HTable(conf, databaseName + "." + tableName); + Scan scan = new Scan(); + scan.addFamily(Bytes.toBytes(familyName)); + ResultScanner scanner = table.getScanner(scan); + assertFalse(scanner.iterator().hasNext()); + + // verify that the storage handler input format returns empty results. + Path outputDir = new Path(getTestDir(), + "mapred/testHBaseTableBulkIgnoreAbortedTransactions"); + FileSystem fs = getFileSystem(); + if (fs.exists(outputDir)) { + fs.delete(outputDir, true); + } + job = new Job(conf, "hbase-bulk-aborted-transaction"); + job.setJarByClass(this.getClass()); + job.setMapperClass(MapReadAbortedTransaction.class); + job.setInputFormatClass(HCatInputFormat.class); + HCatInputFormat.setInput(job, databaseName, tableName); + job.setOutputFormatClass(TextOutputFormat.class); + TextOutputFormat.setOutputPath(job, outputDir); + job.setMapOutputKeyClass(BytesWritable.class); + job.setMapOutputValueClass(Text.class); + job.setOutputKeyClass(BytesWritable.class); + job.setOutputValueClass(Text.class); + job.setNumReduceTasks(0); + assertTrue(job.waitForCompletion(true)); + } + + private Job configureJob(String jobName, Configuration conf, + Path workingDir, Class mapperClass, + OutputJobInfo outputJobInfo, Path inputPath) throws IOException { + Job job = new Job(conf, jobName); + job.setWorkingDirectory(workingDir); + job.setJarByClass(this.getClass()); + job.setMapperClass(mapperClass); + + job.setInputFormatClass(TextInputFormat.class); + TextInputFormat.setInputPaths(job, inputPath); + job.setOutputFormatClass(HCatOutputFormat.class); + HCatOutputFormat.setOutput(job, outputJobInfo); + + job.setMapOutputKeyClass(BytesWritable.class); + job.setMapOutputValueClass(HCatRecord.class); + job.setOutputKeyClass(BytesWritable.class); + job.setOutputValueClass(HCatRecord.class); + + job.setNumReduceTasks(0); + return job; + } + +} + diff --git hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseDirectOutputFormat.java hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseDirectOutputFormat.java new file mode 100644 index 0000000..ccb4715 --- /dev/null +++ hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseDirectOutputFormat.java @@ -0,0 +1,501 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hcatalog.hbase; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.mapred.TableOutputFormat; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hive.cli.CliSessionState; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.session.SessionState; +import org.apache.hadoop.io.BytesWritable; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.WritableComparable; +import org.apache.hadoop.mapred.JobClient; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.OutputCollector; +import org.apache.hadoop.mapred.Reporter; +import org.apache.hadoop.mapred.RunningJob; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.Mapper; +import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; +import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; +import org.apache.hive.hcatalog.cli.HCatDriver; +import org.apache.hive.hcatalog.cli.SemanticAnalysis.HCatSemanticAnalyzer; +import org.apache.hive.hcatalog.common.HCatConstants; +import org.apache.hive.hcatalog.common.HCatUtil; +import org.apache.hive.hcatalog.data.DefaultHCatRecord; +import org.apache.hive.hcatalog.data.HCatRecord; +import org.apache.hive.hcatalog.data.schema.HCatSchema; +import org.apache.hcatalog.hbase.snapshot.FamilyRevision; +import org.apache.hcatalog.hbase.snapshot.RevisionManager; +import org.apache.hcatalog.hbase.snapshot.RevisionManagerConfiguration; +import org.apache.hcatalog.hbase.snapshot.TableSnapshot; +import org.apache.hcatalog.hbase.snapshot.Transaction; +import org.apache.hive.hcatalog.mapreduce.HCatInputFormat; +import org.apache.hive.hcatalog.mapreduce.HCatOutputFormat; +import org.apache.hive.hcatalog.mapreduce.OutputJobInfo; +import org.junit.Test; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertTrue; + +/** + * Test HBaseDirectOUtputFormat and HBaseHCatStorageHandler using a MiniCluster + */ +public class TestHBaseDirectOutputFormat extends SkeletonHBaseTest { + + private final HiveConf allConf; + private final HCatDriver hcatDriver; + + public TestHBaseDirectOutputFormat() { + allConf = getHiveConf(); + allConf.set(HiveConf.ConfVars.SEMANTIC_ANALYZER_HOOK.varname, + HCatSemanticAnalyzer.class.getName()); + allConf.set(HiveConf.ConfVars.HADOOPFS.varname, getFileSystem().getUri().toString()); + allConf.set(HiveConf.ConfVars.METASTOREWAREHOUSE.varname, new Path(getTestDir(), "warehouse").toString()); + + //Add hbase properties + for (Map.Entry el : getHbaseConf()) + allConf.set(el.getKey(), el.getValue()); + for (Map.Entry el : getJobConf()) + allConf.set(el.getKey(), el.getValue()); + HBaseConfiguration.merge( + allConf, + RevisionManagerConfiguration.create()); + SessionState.start(new CliSessionState(allConf)); + hcatDriver = new HCatDriver(); + } + + @Test + public void directOutputFormatTest() throws IOException, ClassNotFoundException, InterruptedException { + String testName = "directOutputFormatTest"; + Path methodTestDir = new Path(getTestDir(), testName); + + String tableName = newTableName(testName).toLowerCase(); + String familyName = "my_family"; + byte[] familyNameBytes = Bytes.toBytes(familyName); + + //include hbase config in conf file + Configuration conf = new Configuration(allConf); + conf.set(HCatConstants.HCAT_KEY_HIVE_CONF, HCatUtil.serialize(allConf.getAllProperties())); + + //create table + createTable(tableName, new String[]{familyName}); + + String data[] = {"1,english:ONE,spanish:UNO", + "2,english:ONE,spanish:DOS", + "3,english:ONE,spanish:TRES"}; + + + // input/output settings + Path inputPath = new Path(methodTestDir, "mr_input"); + getFileSystem().mkdirs(inputPath); + FSDataOutputStream os = getFileSystem().create(new Path(inputPath, "inputFile.txt")); + for (String line : data) + os.write(Bytes.toBytes(line + "\n")); + os.close(); + + //create job + JobConf job = new JobConf(conf); + job.setJobName(testName); + job.setWorkingDirectory(new Path(methodTestDir, "mr_work")); + job.setJarByClass(this.getClass()); + job.setMapperClass(MapWrite.class); + + job.setInputFormat(org.apache.hadoop.mapred.TextInputFormat.class); + org.apache.hadoop.mapred.TextInputFormat.setInputPaths(job, inputPath); + + job.setOutputFormat(HBaseDirectOutputFormat.class); + job.set(TableOutputFormat.OUTPUT_TABLE, tableName); + job.set(HBaseConstants.PROPERTY_OUTPUT_TABLE_NAME_KEY, tableName); + + //manually create transaction + RevisionManager rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(conf); + try { + OutputJobInfo outputJobInfo = OutputJobInfo.create("default", tableName, null); + Transaction txn = rm.beginWriteTransaction(tableName, Arrays.asList(familyName)); + outputJobInfo.getProperties().setProperty(HBaseConstants.PROPERTY_WRITE_TXN_KEY, + HCatUtil.serialize(txn)); + job.set(HCatConstants.HCAT_KEY_OUTPUT_INFO, + HCatUtil.serialize(outputJobInfo)); + } finally { + rm.close(); + } + + job.setMapOutputKeyClass(BytesWritable.class); + job.setMapOutputValueClass(HCatRecord.class); + job.setOutputKeyClass(BytesWritable.class); + job.setOutputValueClass(HCatRecord.class); + job.setNumReduceTasks(0); + + RunningJob runJob = JobClient.runJob(job); + runJob.waitForCompletion(); + assertTrue(runJob.isSuccessful()); + + //verify + HTable table = new HTable(conf, tableName); + Scan scan = new Scan(); + scan.addFamily(familyNameBytes); + ResultScanner scanner = table.getScanner(scan); + int index = 0; + for (Result result : scanner) { + String vals[] = data[index].toString().split(","); + for (int i = 1; i < vals.length; i++) { + String pair[] = vals[i].split(":"); + assertTrue(result.containsColumn(familyNameBytes, Bytes.toBytes(pair[0]))); + assertEquals(pair[1], Bytes.toString(result.getValue(familyNameBytes, Bytes.toBytes(pair[0])))); + } + index++; + } + assertEquals(data.length, index); + } + + @Test + public void directHCatOutputFormatTest() throws Exception { + String testName = "directHCatOutputFormatTest"; + Path methodTestDir = new Path(getTestDir(), testName); + + String databaseName = testName; + String dbDir = new Path(methodTestDir, "DB_" + testName).toString(); + String tableName = newTableName(testName); + String familyName = "my_family"; + byte[] familyNameBytes = Bytes.toBytes(familyName); + //Table name will be lower case unless specified by hbase.table.name property + String hbaseTableName = (databaseName + "." + tableName).toLowerCase(); + + //include hbase config in conf file + Configuration conf = new Configuration(allConf); + conf.set(HCatConstants.HCAT_KEY_HIVE_CONF, HCatUtil.serialize(allConf.getAllProperties())); + + + String dbquery = "CREATE DATABASE IF NOT EXISTS " + databaseName + " LOCATION '" + dbDir + "'"; + String tableQuery = "CREATE TABLE " + databaseName + "." + tableName + + "(key int, english string, spanish string) STORED BY " + + "'org.apache.hcatalog.hbase.HBaseHCatStorageHandler'" + + "TBLPROPERTIES (" + + "'hbase.columns.mapping'=':key," + familyName + ":english," + familyName + ":spanish')"; + + assertEquals(0, hcatDriver.run(dbquery).getResponseCode()); + assertEquals(0, hcatDriver.run(tableQuery).getResponseCode()); + + String data[] = {"1,english:ONE,spanish:UNO", + "2,english:ONE,spanish:DOS", + "3,english:ONE,spanish:TRES"}; + + // input/output settings + Path inputPath = new Path(methodTestDir, "mr_input"); + getFileSystem().mkdirs(inputPath); + //create multiple files so we can test with multiple mappers + for (int i = 0; i < data.length; i++) { + FSDataOutputStream os = getFileSystem().create(new Path(inputPath, "inputFile" + i + ".txt")); + os.write(Bytes.toBytes(data[i] + "\n")); + os.close(); + } + + //create job + Path workingDir = new Path(methodTestDir, "mr_work"); + OutputJobInfo outputJobInfo = OutputJobInfo.create(databaseName, + tableName, null); + Job job = configureJob(testName, conf, workingDir, MapHCatWrite.class, + outputJobInfo, inputPath); + assertTrue(job.waitForCompletion(true)); + + RevisionManager rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(conf); + try { + TableSnapshot snapshot = rm.createSnapshot(hbaseTableName); + for (String el : snapshot.getColumnFamilies()) { + assertEquals(1, snapshot.getRevision(el)); + } + } finally { + rm.close(); + } + + //verify + HTable table = new HTable(conf, hbaseTableName); + Scan scan = new Scan(); + scan.addFamily(familyNameBytes); + ResultScanner scanner = table.getScanner(scan); + int index = 0; + for (Result result : scanner) { + String vals[] = data[index].toString().split(","); + for (int i = 1; i < vals.length; i++) { + String pair[] = vals[i].split(":"); + assertTrue(result.containsColumn(familyNameBytes, Bytes.toBytes(pair[0]))); + assertEquals(pair[1], Bytes.toString(result.getValue(familyNameBytes, Bytes.toBytes(pair[0])))); + assertEquals(1l, result.getColumn(familyNameBytes, Bytes.toBytes(pair[0])).get(0).getTimestamp()); + } + index++; + } + assertEquals(data.length, index); + } + + @Test + public void directModeAbortTest() throws Exception { + String testName = "directModeAbortTest"; + Path methodTestDir = new Path(getTestDir(), testName); + String databaseName = testName; + String dbDir = new Path(methodTestDir, "DB_" + testName).toString(); + String tableName = newTableName(testName); + String familyName = "my_family"; + byte[] familyNameBytes = Bytes.toBytes(familyName); + //Table name as specified by hbase.table.name property + String hbaseTableName = tableName; + + // include hbase config in conf file + Configuration conf = new Configuration(allConf); + conf.set(HCatConstants.HCAT_KEY_HIVE_CONF, HCatUtil.serialize(allConf.getAllProperties())); + + String dbquery = "CREATE DATABASE IF NOT EXISTS " + databaseName + " LOCATION '" + dbDir + + "'"; + String tableQuery = "CREATE TABLE " + databaseName + "." + tableName + + "(key int, english string, spanish string) STORED BY " + + "'org.apache.hcatalog.hbase.HBaseHCatStorageHandler'" + + "TBLPROPERTIES (" + + "'hbase.columns.mapping'=':key," + familyName + ":english," + familyName + + ":spanish','hbase.table.name'='" + hbaseTableName + "')"; + + assertEquals(0, hcatDriver.run(dbquery).getResponseCode()); + assertEquals(0, hcatDriver.run(tableQuery).getResponseCode()); + + String data[] = {"1,english:ONE,spanish:UNO", + "2,english:TWO,spanish:DOS", + "3,english:THREE,spanish:TRES"}; + + Path inputPath = new Path(methodTestDir, "mr_input"); + getFileSystem().mkdirs(inputPath); + // create multiple files so we can test with multiple mappers + for (int i = 0; i < data.length; i++) { + FSDataOutputStream os = getFileSystem().create( + new Path(inputPath, "inputFile" + i + ".txt")); + os.write(Bytes.toBytes(data[i] + "\n")); + os.close(); + } + + Path workingDir = new Path(methodTestDir, "mr_abort"); + OutputJobInfo outputJobInfo = OutputJobInfo.create(databaseName, + tableName, null); + Job job = configureJob(testName, conf, workingDir, MapWriteAbortTransaction.class, + outputJobInfo, inputPath); + assertFalse(job.waitForCompletion(true)); + + // verify that revision manager has it as aborted transaction + RevisionManager rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(conf); + try { + TableSnapshot snapshot = rm.createSnapshot(hbaseTableName); + for (String family : snapshot.getColumnFamilies()) { + assertEquals(1, snapshot.getRevision(family)); + List abortedWriteTransactions = rm.getAbortedWriteTransactions( + hbaseTableName, family); + assertEquals(1, abortedWriteTransactions.size()); + assertEquals(1, abortedWriteTransactions.get(0).getRevision()); + } + } finally { + rm.close(); + } + + // verify that hbase has the records of the successful maps. + HTable table = new HTable(conf, hbaseTableName); + Scan scan = new Scan(); + scan.addFamily(familyNameBytes); + ResultScanner scanner = table.getScanner(scan); + int count = 0; + for (Result result : scanner) { + String key = Bytes.toString(result.getRow()); + assertNotSame(MapWriteAbortTransaction.failedKey, key); + int index = Integer.parseInt(key) - 1; + String vals[] = data[index].toString().split(","); + for (int i = 1; i < vals.length; i++) { + String pair[] = vals[i].split(":"); + assertTrue(result.containsColumn(familyNameBytes, Bytes.toBytes(pair[0]))); + assertEquals(pair[1], + Bytes.toString(result.getValue(familyNameBytes, Bytes.toBytes(pair[0])))); + assertEquals(1l, result.getColumn(familyNameBytes, Bytes.toBytes(pair[0])).get(0) + .getTimestamp()); + } + count++; + } + assertEquals(data.length - 1, count); + + // verify that the inputformat returns empty results. + Path outputDir = new Path(getTestDir(), + "mapred/testHBaseTableIgnoreAbortedTransactions"); + FileSystem fs = getFileSystem(); + if (fs.exists(outputDir)) { + fs.delete(outputDir, true); + } + job = new Job(conf, "hbase-aborted-transaction"); + job.setJarByClass(this.getClass()); + job.setMapperClass(MapReadAbortedTransaction.class); + job.setInputFormatClass(HCatInputFormat.class); + HCatInputFormat.setInput(job, databaseName, tableName); + job.setOutputFormatClass(TextOutputFormat.class); + TextOutputFormat.setOutputPath(job, outputDir); + job.setMapOutputKeyClass(BytesWritable.class); + job.setMapOutputValueClass(Text.class); + job.setOutputKeyClass(BytesWritable.class); + job.setOutputValueClass(Text.class); + job.setNumReduceTasks(0); + assertTrue(job.waitForCompletion(true)); + } + + private Job configureJob(String jobName, Configuration conf, + Path workingDir, Class mapperClass, + OutputJobInfo outputJobInfo, Path inputPath) throws IOException { + Job job = new Job(conf, jobName); + job.setWorkingDirectory(workingDir); + job.setJarByClass(this.getClass()); + job.setMapperClass(mapperClass); + + job.setInputFormatClass(TextInputFormat.class); + TextInputFormat.setInputPaths(job, inputPath); + job.setOutputFormatClass(HCatOutputFormat.class); + HCatOutputFormat.setOutput(job, outputJobInfo); + String txnString = job.getConfiguration().get(HBaseConstants.PROPERTY_WRITE_TXN_KEY); + //Test passing in same OutputJobInfo multiple times and verify 1 transaction is created + String jobString = job.getConfiguration().get(HCatConstants.HCAT_KEY_OUTPUT_INFO); + outputJobInfo = (OutputJobInfo) HCatUtil.deserialize(jobString); + Job job2 = new Job(conf); + HCatOutputFormat.setOutput(job2, outputJobInfo); + assertEquals(txnString, job2.getConfiguration().get(HBaseConstants.PROPERTY_WRITE_TXN_KEY)); + job.setMapOutputKeyClass(BytesWritable.class); + job.setMapOutputValueClass(HCatRecord.class); + job.setOutputKeyClass(BytesWritable.class); + job.setOutputValueClass(HCatRecord.class); + + job.setNumReduceTasks(0); + return job; + } + + public static class MapHCatWrite extends Mapper { + + @Override + public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { + OutputJobInfo jobInfo = (OutputJobInfo) HCatUtil.deserialize(context.getConfiguration().get(HCatConstants.HCAT_KEY_OUTPUT_INFO)); + HCatRecord record = new DefaultHCatRecord(3); + HCatSchema schema = jobInfo.getOutputSchema(); + String vals[] = value.toString().split(","); + record.setInteger("key", schema, Integer.parseInt(vals[0])); + for (int i = 1; i < vals.length; i++) { + String pair[] = vals[i].split(":"); + record.set(pair[0], schema, pair[1]); + } + context.write(null, record); + } + } + + public static class MapWrite implements org.apache.hadoop.mapred.Mapper { + + @Override + public void configure(JobConf job) { + } + + @Override + public void close() throws IOException { + } + + @Override + public void map(LongWritable key, Text value, + OutputCollector output, Reporter reporter) + throws IOException { + String vals[] = value.toString().split(","); + Put put = new Put(Bytes.toBytes(vals[0])); + for (int i = 1; i < vals.length; i++) { + String pair[] = vals[i].split(":"); + put.add(Bytes.toBytes("my_family"), + Bytes.toBytes(pair[0]), + Bytes.toBytes(pair[1])); + } + output.collect(null, put); + } + } + + static class MapWriteAbortTransaction extends Mapper { + public static String failedKey; + private static int count = 0; + + @Override + public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { + OutputJobInfo jobInfo = (OutputJobInfo) HCatUtil.deserialize(context.getConfiguration().get(HCatConstants.HCAT_KEY_OUTPUT_INFO)); + HCatRecord record = new DefaultHCatRecord(3); + HCatSchema schema = jobInfo.getOutputSchema(); + String vals[] = value.toString().split(","); + record.setInteger("key", schema, Integer.parseInt(vals[0])); + synchronized (MapWriteAbortTransaction.class) { + if (count == 2) { + failedKey = vals[0]; + throw new IOException("Failing map to test abort"); + } + for (int i = 1; i < vals.length; i++) { + String pair[] = vals[i].split(":"); + record.set(pair[0], schema, pair[1]); + } + context.write(null, record); + count++; + } + + } + + } + + static class MapReadAbortedTransaction + extends + Mapper, Text> { + + @Override + public void run(Context context) throws IOException, + InterruptedException { + setup(context); + if (context.nextKeyValue()) { + map(context.getCurrentKey(), context.getCurrentValue(), context); + while (context.nextKeyValue()) { + map(context.getCurrentKey(), context.getCurrentValue(), + context); + } + throw new IOException("There should have been no records"); + } + cleanup(context); + } + + @Override + public void map(ImmutableBytesWritable key, HCatRecord value, + Context context) throws IOException, InterruptedException { + System.out.println("HCat record value" + value.toString()); + } + } +} diff --git hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseHCatStorageHandler.java hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseHCatStorageHandler.java new file mode 100644 index 0000000..d377061 --- /dev/null +++ hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseHCatStorageHandler.java @@ -0,0 +1,241 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hcatalog.hbase; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.net.URI; +import java.util.Map; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hive.cli.CliSessionState; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.metastore.Warehouse; +import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; +import org.apache.hadoop.hive.ql.session.SessionState; +import org.apache.hive.hcatalog.cli.HCatDriver; +import org.apache.hive.hcatalog.cli.SemanticAnalysis.HCatSemanticAnalyzer; +import org.apache.hcatalog.hbase.snapshot.RevisionManager; +import org.apache.hcatalog.hbase.snapshot.RevisionManagerConfiguration; +import org.apache.zookeeper.KeeperException.NoNodeException; +import org.junit.Test; + +public class TestHBaseHCatStorageHandler extends SkeletonHBaseTest { + + private static HiveConf hcatConf; + private static HCatDriver hcatDriver; + private static Warehouse wh; + + public void Initialize() throws Exception { + + hcatConf = getHiveConf(); + hcatConf.set(ConfVars.SEMANTIC_ANALYZER_HOOK.varname, + HCatSemanticAnalyzer.class.getName()); + URI fsuri = getFileSystem().getUri(); + Path whPath = new Path(fsuri.getScheme(), fsuri.getAuthority(), + getTestDir()); + hcatConf.set(HiveConf.ConfVars.HADOOPFS.varname, fsuri.toString()); + hcatConf.set(ConfVars.METASTOREWAREHOUSE.varname, whPath.toString()); + + //Add hbase properties + for (Map.Entry el : getHbaseConf()) { + if (el.getKey().startsWith("hbase.")) { + hcatConf.set(el.getKey(), el.getValue()); + } + } + HBaseConfiguration.merge( + hcatConf, + RevisionManagerConfiguration.create()); + + SessionState.start(new CliSessionState(hcatConf)); + hcatDriver = new HCatDriver(); + + } + + @Test + public void testTableCreateDrop() throws Exception { + Initialize(); + + hcatDriver.run("drop table test_table"); + CommandProcessorResponse response = hcatDriver + .run("create table test_table(key int, value string) STORED BY " + + "'org.apache.hcatalog.hbase.HBaseHCatStorageHandler'" + + "TBLPROPERTIES ('hbase.columns.mapping'=':key,cf1:val')"); + + assertEquals(0, response.getResponseCode()); + + HBaseAdmin hAdmin = new HBaseAdmin(getHbaseConf()); + boolean doesTableExist = hAdmin.tableExists("test_table"); + + assertTrue(doesTableExist); + + RevisionManager rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(hcatConf); + rm.open(); + //Should be able to successfully query revision manager + rm.getAbortedWriteTransactions("test_table", "cf1"); + + hcatDriver.run("drop table test_table"); + doesTableExist = hAdmin.tableExists("test_table"); + assertTrue(doesTableExist == false); + + try { + rm.getAbortedWriteTransactions("test_table", "cf1"); + } catch (Exception e) { + assertTrue(e.getCause() instanceof NoNodeException); + } + rm.close(); + + } + + @Test + public void testTableCreateDropDifferentCase() throws Exception { + Initialize(); + + hcatDriver.run("drop table test_Table"); + CommandProcessorResponse response = hcatDriver + .run("create table test_Table(key int, value string) STORED BY " + + "'org.apache.hcatalog.hbase.HBaseHCatStorageHandler'" + + "TBLPROPERTIES ('hbase.columns.mapping'=':key,cf1:val')"); + + assertEquals(0, response.getResponseCode()); + + //HBase table gets created with lower case unless specified as a table property. + HBaseAdmin hAdmin = new HBaseAdmin(getHbaseConf()); + boolean doesTableExist = hAdmin.tableExists("test_table"); + + assertTrue(doesTableExist); + + RevisionManager rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(hcatConf); + rm.open(); + //Should be able to successfully query revision manager + rm.getAbortedWriteTransactions("test_table", "cf1"); + + hcatDriver.run("drop table test_table"); + doesTableExist = hAdmin.tableExists("test_table"); + assertTrue(doesTableExist == false); + + try { + rm.getAbortedWriteTransactions("test_table", "cf1"); + } catch (Exception e) { + assertTrue(e.getCause() instanceof NoNodeException); + } + rm.close(); + + } + + @Test + public void testTableCreateDropCaseSensitive() throws Exception { + Initialize(); + + hcatDriver.run("drop table test_Table"); + CommandProcessorResponse response = hcatDriver + .run("create table test_Table(key int, value string) STORED BY " + + "'org.apache.hcatalog.hbase.HBaseHCatStorageHandler'" + + "TBLPROPERTIES ('hbase.columns.mapping'=':key,cf1:val'," + + " 'hbase.table.name'='CaseSensitiveTable')"); + + assertEquals(0, response.getResponseCode()); + + HBaseAdmin hAdmin = new HBaseAdmin(getHbaseConf()); + boolean doesTableExist = hAdmin.tableExists("CaseSensitiveTable"); + + assertTrue(doesTableExist); + + RevisionManager rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(hcatConf); + rm.open(); + //Should be able to successfully query revision manager + rm.getAbortedWriteTransactions("CaseSensitiveTable", "cf1"); + + hcatDriver.run("drop table test_table"); + doesTableExist = hAdmin.tableExists("CaseSensitiveTable"); + assertTrue(doesTableExist == false); + + try { + rm.getAbortedWriteTransactions("CaseSensitiveTable", "cf1"); + } catch (Exception e) { + assertTrue(e.getCause() instanceof NoNodeException); + } + rm.close(); + + } + + @Test + public void testTableDropNonExistent() throws Exception { + Initialize(); + + hcatDriver.run("drop table mytable"); + CommandProcessorResponse response = hcatDriver + .run("create table mytable(key int, value string) STORED BY " + + "'org.apache.hcatalog.hbase.HBaseHCatStorageHandler'" + + "TBLPROPERTIES ('hbase.columns.mapping'=':key,cf1:val')"); + + assertEquals(0, response.getResponseCode()); + + HBaseAdmin hAdmin = new HBaseAdmin(getHbaseConf()); + boolean doesTableExist = hAdmin.tableExists("mytable"); + assertTrue(doesTableExist); + + //Now delete the table from hbase + if (hAdmin.isTableEnabled("mytable")) { + hAdmin.disableTable("mytable"); + } + hAdmin.deleteTable("mytable"); + doesTableExist = hAdmin.tableExists("mytable"); + assertTrue(doesTableExist == false); + + CommandProcessorResponse responseTwo = hcatDriver.run("drop table mytable"); + assertTrue(responseTwo.getResponseCode() == 0); + + } + + @Test + public void testTableCreateExternal() throws Exception { + + String tableName = "testTable"; + HBaseAdmin hAdmin = new HBaseAdmin(getHbaseConf()); + + HTableDescriptor tableDesc = new HTableDescriptor(tableName); + tableDesc.addFamily(new HColumnDescriptor(Bytes.toBytes("key"))); + tableDesc.addFamily(new HColumnDescriptor(Bytes.toBytes("familyone"))); + tableDesc.addFamily(new HColumnDescriptor(Bytes.toBytes("familytwo"))); + + hAdmin.createTable(tableDesc); + boolean doesTableExist = hAdmin.tableExists(tableName); + assertTrue(doesTableExist); + + hcatDriver.run("drop table mytabletwo"); + CommandProcessorResponse response = hcatDriver + .run("create external table mytabletwo(key int, valueone string, valuetwo string) STORED BY " + + "'org.apache.hcatalog.hbase.HBaseHCatStorageHandler'" + + "TBLPROPERTIES ('hbase.columns.mapping'=':key,familyone:val,familytwo:val'," + + "'hbase.table.name'='testTable')"); + + assertEquals(0, response.getResponseCode()); + + } + + +} diff --git hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseInputFormat.java hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseInputFormat.java new file mode 100644 index 0000000..28d1135 --- /dev/null +++ hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseInputFormat.java @@ -0,0 +1,609 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hcatalog.hbase; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.net.URI; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.mapreduce.TableInputFormat; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hive.cli.CliSessionState; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.metastore.MetaStoreUtils; +import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; +import org.apache.hadoop.hive.ql.session.SessionState; +import org.apache.hadoop.io.BytesWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.WritableComparable; +import org.apache.hadoop.mapred.JobClient; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.OutputCollector; +import org.apache.hadoop.mapred.Reporter; +import org.apache.hadoop.mapred.RunningJob; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.Mapper; +import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; +import org.apache.hive.hcatalog.cli.HCatDriver; +import org.apache.hive.hcatalog.cli.SemanticAnalysis.HCatSemanticAnalyzer; +import org.apache.hive.hcatalog.common.HCatConstants; +import org.apache.hive.hcatalog.common.HCatException; +import org.apache.hive.hcatalog.common.HCatUtil; +import org.apache.hive.hcatalog.data.HCatRecord; +import org.apache.hive.hcatalog.data.schema.HCatFieldSchema; +import org.apache.hive.hcatalog.data.schema.HCatSchema; +import org.apache.hcatalog.hbase.snapshot.RevisionManager; +import org.apache.hcatalog.hbase.snapshot.RevisionManagerConfiguration; +import org.apache.hcatalog.hbase.snapshot.Transaction; +import org.apache.hive.hcatalog.mapreduce.HCatInputFormat; +import org.apache.hive.hcatalog.mapreduce.InputJobInfo; +import org.apache.hive.hcatalog.mapreduce.PartInfo; +import org.junit.Test; + +public class TestHBaseInputFormat extends SkeletonHBaseTest { + + private static HiveConf hcatConf; + private static HCatDriver hcatDriver; + private final byte[] FAMILY = Bytes.toBytes("testFamily"); + private final byte[] QUALIFIER1 = Bytes.toBytes("testQualifier1"); + private final byte[] QUALIFIER2 = Bytes.toBytes("testQualifier2"); + + public TestHBaseInputFormat() throws Exception { + hcatConf = getHiveConf(); + hcatConf.set(ConfVars.SEMANTIC_ANALYZER_HOOK.varname, + HCatSemanticAnalyzer.class.getName()); + URI fsuri = getFileSystem().getUri(); + Path whPath = new Path(fsuri.getScheme(), fsuri.getAuthority(), + getTestDir()); + hcatConf.set(HiveConf.ConfVars.HADOOPFS.varname, fsuri.toString()); + hcatConf.set(ConfVars.METASTOREWAREHOUSE.varname, whPath.toString()); + + //Add hbase properties + + for (Map.Entry el : getHbaseConf()) { + if (el.getKey().startsWith("hbase.")) { + hcatConf.set(el.getKey(), el.getValue()); + } + } + HBaseConfiguration.merge(hcatConf, + RevisionManagerConfiguration.create()); + + + SessionState.start(new CliSessionState(hcatConf)); + hcatDriver = new HCatDriver(); + + } + + private List generatePuts(int num, String tableName) throws IOException { + + List columnFamilies = Arrays.asList("testFamily"); + RevisionManager rm = null; + List myPuts; + try { + rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(hcatConf); + rm.open(); + myPuts = new ArrayList(); + for (int i = 1; i <= num; i++) { + Put put = new Put(Bytes.toBytes("testRow")); + put.add(FAMILY, QUALIFIER1, i, Bytes.toBytes("textValue-" + i)); + put.add(FAMILY, QUALIFIER2, i, Bytes.toBytes("textValue-" + i)); + myPuts.add(put); + Transaction tsx = rm.beginWriteTransaction(tableName, + columnFamilies); + rm.commitWriteTransaction(tsx); + } + } finally { + if (rm != null) + rm.close(); + } + + return myPuts; + } + + private void populateHBaseTable(String tName, int revisions) throws IOException { + List myPuts = generatePuts(revisions, tName); + HTable table = new HTable(getHbaseConf(), Bytes.toBytes(tName)); + table.put(myPuts); + } + + private long populateHBaseTableQualifier1(String tName, int value, Boolean commit) + throws IOException { + List columnFamilies = Arrays.asList("testFamily"); + RevisionManager rm = null; + List myPuts = new ArrayList(); + long revision; + try { + rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(hcatConf); + rm.open(); + Transaction tsx = rm.beginWriteTransaction(tName, columnFamilies); + + Put put = new Put(Bytes.toBytes("testRow")); + revision = tsx.getRevisionNumber(); + put.add(FAMILY, QUALIFIER1, revision, + Bytes.toBytes("textValue-" + value)); + myPuts.add(put); + + // If commit is null it is left as a running transaction + if (commit != null) { + if (commit) { + rm.commitWriteTransaction(tsx); + } else { + rm.abortWriteTransaction(tsx); + } + } + } finally { + if (rm != null) + rm.close(); + } + HTable table = new HTable(getHbaseConf(), Bytes.toBytes(tName)); + table.put(myPuts); + return revision; + } + + @Test + public void TestHBaseTableReadMR() throws Exception { + String tableName = newTableName("MyTable"); + String databaseName = newTableName("MyDatabase"); + //Table name will be lower case unless specified by hbase.table.name property + String hbaseTableName = (databaseName + "." + tableName).toLowerCase(); + String db_dir = new Path(getTestDir(), "hbasedb").toString(); + + String dbquery = "CREATE DATABASE IF NOT EXISTS " + databaseName + " LOCATION '" + + db_dir + "'"; + String tableQuery = "CREATE TABLE " + databaseName + "." + tableName + + "(key string, testqualifier1 string, testqualifier2 string) STORED BY " + + "'org.apache.hcatalog.hbase.HBaseHCatStorageHandler'" + + "TBLPROPERTIES ('hbase.columns.mapping'=':key,testFamily:testQualifier1,testFamily:testQualifier2')"; + + CommandProcessorResponse responseOne = hcatDriver.run(dbquery); + assertEquals(0, responseOne.getResponseCode()); + CommandProcessorResponse responseTwo = hcatDriver.run(tableQuery); + assertEquals(0, responseTwo.getResponseCode()); + + HBaseAdmin hAdmin = new HBaseAdmin(getHbaseConf()); + boolean doesTableExist = hAdmin.tableExists(hbaseTableName); + assertTrue(doesTableExist); + + populateHBaseTable(hbaseTableName, 5); + Configuration conf = new Configuration(hcatConf); + conf.set(HCatConstants.HCAT_KEY_HIVE_CONF, + HCatUtil.serialize(getHiveConf().getAllProperties())); + + // output settings + Path outputDir = new Path(getTestDir(), "mapred/testHbaseTableMRRead"); + FileSystem fs = getFileSystem(); + if (fs.exists(outputDir)) { + fs.delete(outputDir, true); + } + // create job + Job job = new Job(conf, "hbase-mr-read-test"); + job.setJarByClass(this.getClass()); + job.setMapperClass(MapReadHTable.class); + MapReadHTable.resetCounters(); + + job.setInputFormatClass(HCatInputFormat.class); + HCatInputFormat.setInput(job.getConfiguration(), databaseName, tableName); + job.setOutputFormatClass(TextOutputFormat.class); + TextOutputFormat.setOutputPath(job, outputDir); + job.setMapOutputKeyClass(BytesWritable.class); + job.setMapOutputValueClass(Text.class); + job.setOutputKeyClass(BytesWritable.class); + job.setOutputValueClass(Text.class); + job.setNumReduceTasks(0); + assertTrue(job.waitForCompletion(true)); + // Note: These asserts only works in case of LocalJobRunner as they run in same jvm. + // If using MiniMRCluster, the tests will have to be modified. + assertFalse(MapReadHTable.error); + assertEquals(MapReadHTable.count, 1); + + String dropTableQuery = "DROP TABLE " + hbaseTableName; + CommandProcessorResponse responseThree = hcatDriver.run(dropTableQuery); + assertEquals(0, responseThree.getResponseCode()); + + boolean isHbaseTableThere = hAdmin.tableExists(hbaseTableName); + assertFalse(isHbaseTableThere); + + String dropDB = "DROP DATABASE " + databaseName; + CommandProcessorResponse responseFour = hcatDriver.run(dropDB); + assertEquals(0, responseFour.getResponseCode()); + } + + @Test + public void TestHBaseTableProjectionReadMR() throws Exception { + + String tableName = newTableName("MyTable"); + //Table name as specified by hbase.table.name property + String hbaseTableName = "MyDB_" + tableName; + String tableQuery = "CREATE TABLE " + tableName + + "(key string, testqualifier1 string, testqualifier2 string) STORED BY " + + "'org.apache.hcatalog.hbase.HBaseHCatStorageHandler'" + + "TBLPROPERTIES ('hbase.columns.mapping'=" + + "':key,testFamily:testQualifier1,testFamily:testQualifier2'," + + "'hbase.table.name'='" + hbaseTableName + "')"; + + CommandProcessorResponse responseTwo = hcatDriver.run(tableQuery); + assertEquals(0, responseTwo.getResponseCode()); + + HBaseAdmin hAdmin = new HBaseAdmin(getHbaseConf()); + boolean doesTableExist = hAdmin.tableExists(hbaseTableName); + assertTrue(doesTableExist); + + populateHBaseTable(hbaseTableName, 5); + + Configuration conf = new Configuration(hcatConf); + conf.set(HCatConstants.HCAT_KEY_HIVE_CONF, + HCatUtil.serialize(getHiveConf().getAllProperties())); + + // output settings + Path outputDir = new Path(getTestDir(), "mapred/testHBaseTableProjectionReadMR"); + FileSystem fs = getFileSystem(); + if (fs.exists(outputDir)) { + fs.delete(outputDir, true); + } + // create job + Job job = new Job(conf, "hbase-column-projection"); + job.setJarByClass(this.getClass()); + job.setMapperClass(MapReadProjHTable.class); + job.setInputFormatClass(HCatInputFormat.class); + HCatInputFormat.setOutputSchema(job, getProjectionSchema()); + HCatInputFormat.setInput(job, MetaStoreUtils.DEFAULT_DATABASE_NAME, tableName); + job.setOutputFormatClass(TextOutputFormat.class); + TextOutputFormat.setOutputPath(job, outputDir); + job.setMapOutputKeyClass(BytesWritable.class); + job.setMapOutputValueClass(Text.class); + job.setOutputKeyClass(BytesWritable.class); + job.setOutputValueClass(Text.class); + job.setNumReduceTasks(0); + assertTrue(job.waitForCompletion(true)); + assertFalse(MapReadProjHTable.error); + assertEquals(MapReadProjHTable.count, 1); + + String dropTableQuery = "DROP TABLE " + tableName; + CommandProcessorResponse responseThree = hcatDriver.run(dropTableQuery); + assertEquals(0, responseThree.getResponseCode()); + + boolean isHbaseTableThere = hAdmin.tableExists(hbaseTableName); + assertFalse(isHbaseTableThere); + } + + @Test + public void TestHBaseInputFormatProjectionReadMR() throws Exception { + + String tableName = newTableName("mytable"); + String tableQuery = "CREATE TABLE " + tableName + + "(key string, testqualifier1 string, testqualifier2 string) STORED BY " + + "'org.apache.hcatalog.hbase.HBaseHCatStorageHandler'" + + "TBLPROPERTIES ('hbase.columns.mapping'=':key," + + "testFamily:testQualifier1,testFamily:testQualifier2')"; + + CommandProcessorResponse responseTwo = hcatDriver.run(tableQuery); + assertEquals(0, responseTwo.getResponseCode()); + + HBaseAdmin hAdmin = new HBaseAdmin(getHbaseConf()); + boolean doesTableExist = hAdmin.tableExists(tableName); + assertTrue(doesTableExist); + + populateHBaseTable(tableName, 5); + + Configuration conf = new Configuration(hcatConf); + conf.set(HCatConstants.HCAT_KEY_HIVE_CONF, + HCatUtil.serialize(getHiveConf().getAllProperties())); + + // output settings + Path outputDir = new Path(getTestDir(), "mapred/testHBaseTableProjectionReadMR"); + FileSystem fs = getFileSystem(); + if (fs.exists(outputDir)) { + fs.delete(outputDir, true); + } + // create job + JobConf job = new JobConf(conf); + job.setJobName("hbase-scan-column"); + job.setJarByClass(this.getClass()); + job.setMapperClass(MapReadProjectionHTable.class); + job.setInputFormat(HBaseInputFormat.class); + + //Configure projection schema + job.set(HCatConstants.HCAT_KEY_OUTPUT_SCHEMA, HCatUtil.serialize(getProjectionSchema())); + Job newJob = new Job(job); + HCatInputFormat.setInput(newJob, MetaStoreUtils.DEFAULT_DATABASE_NAME, tableName); + String inputJobString = newJob.getConfiguration().get(HCatConstants.HCAT_KEY_JOB_INFO); + InputJobInfo info = (InputJobInfo) HCatUtil.deserialize(inputJobString); + job.set(HCatConstants.HCAT_KEY_JOB_INFO, inputJobString); + for (PartInfo partinfo : info.getPartitions()) { + for (Entry entry : partinfo.getJobProperties().entrySet()) + job.set(entry.getKey(), entry.getValue()); + } + assertEquals("testFamily:testQualifier1", job.get(TableInputFormat.SCAN_COLUMNS)); + + job.setOutputFormat(org.apache.hadoop.mapred.TextOutputFormat.class); + org.apache.hadoop.mapred.TextOutputFormat.setOutputPath(job, outputDir); + job.setMapOutputKeyClass(BytesWritable.class); + job.setMapOutputValueClass(Text.class); + job.setOutputKeyClass(BytesWritable.class); + job.setOutputValueClass(Text.class); + job.setNumReduceTasks(0); + + RunningJob runJob = JobClient.runJob(job); + runJob.waitForCompletion(); + assertTrue(runJob.isSuccessful()); + assertFalse(MapReadProjHTable.error); + assertEquals(MapReadProjHTable.count, 1); + + String dropTableQuery = "DROP TABLE " + tableName; + CommandProcessorResponse responseThree = hcatDriver.run(dropTableQuery); + assertEquals(0, responseThree.getResponseCode()); + + boolean isHbaseTableThere = hAdmin.tableExists(tableName); + assertFalse(isHbaseTableThere); + } + + @Test + public void TestHBaseTableIgnoreAbortedTransactions() throws Exception { + String tableName = newTableName("mytable"); + String tableQuery = "CREATE TABLE " + tableName + + "(key string, testqualifier1 string, testqualifier2 string) STORED BY " + + "'org.apache.hcatalog.hbase.HBaseHCatStorageHandler'" + + "TBLPROPERTIES ('hbase.columns.mapping'=':key," + + "testFamily:testQualifier1,testFamily:testQualifier2')"; + + CommandProcessorResponse responseTwo = hcatDriver.run(tableQuery); + assertEquals(0, responseTwo.getResponseCode()); + + HBaseAdmin hAdmin = new HBaseAdmin(getHbaseConf()); + boolean doesTableExist = hAdmin.tableExists(tableName); + assertTrue(doesTableExist); + + populateHBaseTable(tableName, 5); + populateHBaseTableQualifier1(tableName, 6, false); + populateHBaseTableQualifier1(tableName, 7, false); + + Configuration conf = new Configuration(hcatConf); + conf.set(HCatConstants.HCAT_KEY_HIVE_CONF, + HCatUtil.serialize(getHiveConf().getAllProperties())); + + Path outputDir = new Path(getTestDir(), "mapred/testHBaseTableIgnoreAbortedTransactions"); + FileSystem fs = getFileSystem(); + if (fs.exists(outputDir)) { + fs.delete(outputDir, true); + } + Job job = new Job(conf, "hbase-aborted-transaction"); + job.setJarByClass(this.getClass()); + job.setMapperClass(MapReadHTable.class); + MapReadHTable.resetCounters(); + job.setInputFormatClass(HCatInputFormat.class); + HCatInputFormat.setInput(job, MetaStoreUtils.DEFAULT_DATABASE_NAME, tableName); + job.setOutputFormatClass(TextOutputFormat.class); + TextOutputFormat.setOutputPath(job, outputDir); + job.setMapOutputKeyClass(BytesWritable.class); + job.setMapOutputValueClass(Text.class); + job.setOutputKeyClass(BytesWritable.class); + job.setOutputValueClass(Text.class); + job.setNumReduceTasks(0); + assertTrue(job.waitForCompletion(true)); + // Verify that the records do not contain aborted transaction + // revisions 6 and 7 for testFamily:testQualifier1 and + // fetches revision 5 for both testQualifier1 and testQualifier2 + assertFalse(MapReadHTable.error); + assertEquals(1, MapReadHTable.count); + + String dropTableQuery = "DROP TABLE " + tableName; + CommandProcessorResponse responseThree = hcatDriver.run(dropTableQuery); + assertEquals(0, responseThree.getResponseCode()); + + boolean isHbaseTableThere = hAdmin.tableExists(tableName); + assertFalse(isHbaseTableThere); + } + + @Test + public void TestHBaseTableIgnoreAbortedAndRunningTransactions() throws Exception { + String tableName = newTableName("mytable"); + String tableQuery = "CREATE TABLE " + tableName + + "(key string, testqualifier1 string, testqualifier2 string) STORED BY " + + "'org.apache.hcatalog.hbase.HBaseHCatStorageHandler'" + + "TBLPROPERTIES ('hbase.columns.mapping'=':key," + + "testFamily:testQualifier1,testFamily:testQualifier2')"; + + CommandProcessorResponse responseTwo = hcatDriver.run(tableQuery); + assertEquals(0, responseTwo.getResponseCode()); + + HBaseAdmin hAdmin = new HBaseAdmin(getHbaseConf()); + boolean doesTableExist = hAdmin.tableExists(tableName); + assertTrue(doesTableExist); + + populateHBaseTable(tableName, 2); + populateHBaseTableQualifier1(tableName, 3, Boolean.TRUE); //Committed transaction + populateHBaseTableQualifier1(tableName, 4, null); //Running transaction + populateHBaseTableQualifier1(tableName, 5, Boolean.FALSE); //Aborted transaction + populateHBaseTableQualifier1(tableName, 6, Boolean.TRUE); //Committed transaction + populateHBaseTableQualifier1(tableName, 7, null); //Running Transaction + populateHBaseTableQualifier1(tableName, 8, Boolean.FALSE); //Aborted Transaction + + Configuration conf = new Configuration(hcatConf); + conf.set(HCatConstants.HCAT_KEY_HIVE_CONF, + HCatUtil.serialize(getHiveConf().getAllProperties())); + + Path outputDir = new Path(getTestDir(), "mapred/testHBaseTableIgnoreAbortedTransactions"); + FileSystem fs = getFileSystem(); + if (fs.exists(outputDir)) { + fs.delete(outputDir, true); + } + Job job = new Job(conf, "hbase-running-aborted-transaction"); + job.setJarByClass(this.getClass()); + job.setMapperClass(MapReadHTableRunningAbort.class); + job.setInputFormatClass(HCatInputFormat.class); + HCatInputFormat.setInput(job, MetaStoreUtils.DEFAULT_DATABASE_NAME, tableName); + job.setOutputFormatClass(TextOutputFormat.class); + TextOutputFormat.setOutputPath(job, outputDir); + job.setMapOutputKeyClass(BytesWritable.class); + job.setMapOutputValueClass(Text.class); + job.setOutputKeyClass(BytesWritable.class); + job.setOutputValueClass(Text.class); + job.setNumReduceTasks(0); + assertTrue(job.waitForCompletion(true)); + // Verify that the records do not contain running and aborted transaction + // and it fetches revision 2 for testQualifier1 and testQualifier2 + assertFalse(MapReadHTableRunningAbort.error); + assertEquals(1, MapReadHTableRunningAbort.count); + + String dropTableQuery = "DROP TABLE " + tableName; + CommandProcessorResponse responseThree = hcatDriver.run(dropTableQuery); + assertEquals(0, responseThree.getResponseCode()); + + boolean isHbaseTableThere = hAdmin.tableExists(tableName); + assertFalse(isHbaseTableThere); + } + + + static class MapReadHTable + extends + Mapper, Text> { + + static boolean error = false; + static int count = 0; + + @Override + public void map(ImmutableBytesWritable key, HCatRecord value, + Context context) throws IOException, InterruptedException { + System.out.println("HCat record value" + value.toString()); + boolean correctValues = (value.size() == 3) + && (value.get(0).toString()).equalsIgnoreCase("testRow") + && (value.get(1).toString()).equalsIgnoreCase("textValue-5") + && (value.get(2).toString()).equalsIgnoreCase("textValue-5"); + + if (correctValues == false) { + error = true; + } + count++; + } + + public static void resetCounters() { + error = false; + count = 0; + } + } + + static class MapReadProjHTable + extends + Mapper, Text> { + + static boolean error = false; + static int count = 0; + + @Override + public void map(ImmutableBytesWritable key, HCatRecord value, + Context context) throws IOException, InterruptedException { + System.out.println("HCat record value" + value.toString()); + boolean correctValues = (value.size() == 2) + && (value.get(0).toString()).equalsIgnoreCase("testRow") + && (value.get(1).toString()).equalsIgnoreCase("textValue-5"); + + if (correctValues == false) { + error = true; + } + count++; + } + } + + static class MapReadProjectionHTable + implements org.apache.hadoop.mapred.Mapper, Text> { + + static boolean error = false; + static int count = 0; + + @Override + public void configure(JobConf job) { + } + + @Override + public void close() throws IOException { + } + + @Override + public void map(ImmutableBytesWritable key, Result result, + OutputCollector, Text> output, Reporter reporter) + throws IOException { + System.out.println("Result " + result.toString()); + List list = result.list(); + boolean correctValues = (list.size() == 1) + && (Bytes.toString(list.get(0).getRow())).equalsIgnoreCase("testRow") + && (Bytes.toString(list.get(0).getValue())).equalsIgnoreCase("textValue-5") + && (Bytes.toString(list.get(0).getFamily())).equalsIgnoreCase("testFamily") + && (Bytes.toString(list.get(0).getQualifier())).equalsIgnoreCase("testQualifier1"); + + if (correctValues == false) { + error = true; + } + count++; + } + } + + static class MapReadHTableRunningAbort + extends + Mapper, Text> { + + static boolean error = false; + static int count = 0; + + @Override + public void map(ImmutableBytesWritable key, HCatRecord value, + Context context) throws IOException, InterruptedException { + System.out.println("HCat record value" + value.toString()); + boolean correctValues = (value.size() == 3) + && (value.get(0).toString()).equalsIgnoreCase("testRow") + && (value.get(1).toString()).equalsIgnoreCase("textValue-3") + && (value.get(2).toString()).equalsIgnoreCase("textValue-2"); + + if (correctValues == false) { + error = true; + } + count++; + } + } + + private HCatSchema getProjectionSchema() throws HCatException { + + HCatSchema schema = new HCatSchema(new ArrayList()); + schema.append(new HCatFieldSchema("key", HCatFieldSchema.Type.STRING, + "")); + schema.append(new HCatFieldSchema("testqualifier1", + HCatFieldSchema.Type.STRING, "")); + return schema; + } + + +} diff --git hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestSnapshots.java hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestSnapshots.java new file mode 100644 index 0000000..0fc8c9d --- /dev/null +++ hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestSnapshots.java @@ -0,0 +1,141 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hcatalog.hbase; + +import static org.junit.Assert.assertEquals; + +import java.net.URI; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.cli.CliSessionState; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; +import org.apache.hadoop.hive.ql.session.SessionState; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hive.hcatalog.cli.HCatDriver; +import org.apache.hive.hcatalog.cli.SemanticAnalysis.HCatSemanticAnalyzer; +import org.apache.hive.hcatalog.common.HCatConstants; +import org.apache.hive.hcatalog.common.HCatUtil; +import org.apache.hcatalog.hbase.snapshot.TableSnapshot; +import org.apache.hive.hcatalog.mapreduce.HCatInputFormat; +import org.apache.hive.hcatalog.mapreduce.InputJobInfo; +import org.junit.Test; + +public class TestSnapshots extends SkeletonHBaseTest { + private static HiveConf hcatConf; + private static HCatDriver hcatDriver; + + public void Initialize() throws Exception { + hcatConf = getHiveConf(); + hcatConf.set(ConfVars.SEMANTIC_ANALYZER_HOOK.varname, + HCatSemanticAnalyzer.class.getName()); + URI fsuri = getFileSystem().getUri(); + Path whPath = new Path(fsuri.getScheme(), fsuri.getAuthority(), + getTestDir()); + hcatConf.set(HiveConf.ConfVars.HADOOPFS.varname, fsuri.toString()); + hcatConf.set(ConfVars.METASTOREWAREHOUSE.varname, whPath.toString()); + + //Add hbase properties + + for (Map.Entry el : getHbaseConf()) { + if (el.getKey().startsWith("hbase.")) { + hcatConf.set(el.getKey(), el.getValue()); + } + } + + SessionState.start(new CliSessionState(hcatConf)); + hcatDriver = new HCatDriver(); + + } + + @Test + public void TestSnapshotConversion() throws Exception { + Initialize(); + String tableName = newTableName("mytableOne"); + String databaseName = newTableName("mydatabase"); + String fullyQualTableName = databaseName + "." + tableName; + String db_dir = new Path(getTestDir(), "hbasedb").toString(); + String dbquery = "CREATE DATABASE IF NOT EXISTS " + databaseName + " LOCATION '" + + db_dir + "'"; + String tableQuery = "CREATE TABLE " + fullyQualTableName + + "(key string, value1 string, value2 string) STORED BY " + + "'org.apache.hcatalog.hbase.HBaseHCatStorageHandler'" + + "TBLPROPERTIES ('hbase.columns.mapping'=':key,cf1:q1,cf2:q2')"; + + CommandProcessorResponse cmdResponse = hcatDriver.run(dbquery); + assertEquals(0, cmdResponse.getResponseCode()); + cmdResponse = hcatDriver.run(tableQuery); + assertEquals(0, cmdResponse.getResponseCode()); + + Configuration conf = new Configuration(hcatConf); + conf.set(HCatConstants.HCAT_KEY_HIVE_CONF, + HCatUtil.serialize(getHiveConf().getAllProperties())); + Job job = new Job(conf); + Properties properties = new Properties(); + properties.setProperty(HBaseConstants.PROPERTY_TABLE_SNAPSHOT_KEY, "dummysnapshot"); + HCatInputFormat.setInput(job, databaseName, tableName).setProperties(properties); + String modifiedInputInfo = job.getConfiguration().get(HCatConstants.HCAT_KEY_JOB_INFO); + InputJobInfo inputInfo = (InputJobInfo) HCatUtil.deserialize(modifiedInputInfo); + + Map revMap = new HashMap(); + revMap.put("cf1", 3L); + revMap.put("cf2", 5L); + TableSnapshot hbaseSnapshot = new TableSnapshot(fullyQualTableName, revMap, -1); + HCatTableSnapshot hcatSnapshot = HBaseRevisionManagerUtil.convertSnapshot(hbaseSnapshot, inputInfo.getTableInfo()); + + assertEquals(hcatSnapshot.getRevision("value1"), 3); + assertEquals(hcatSnapshot.getRevision("value2"), 5); + + String dropTable = "DROP TABLE " + fullyQualTableName; + cmdResponse = hcatDriver.run(dropTable); + assertEquals(0, cmdResponse.getResponseCode()); + + tableName = newTableName("mytableTwo"); + fullyQualTableName = databaseName + "." + tableName; + tableQuery = "CREATE TABLE " + fullyQualTableName + + "(key string, value1 string, value2 string) STORED BY " + + "'org.apache.hcatalog.hbase.HBaseHCatStorageHandler'" + + "TBLPROPERTIES ('hbase.columns.mapping'=':key,cf1:q1,cf1:q2')"; + cmdResponse = hcatDriver.run(tableQuery); + assertEquals(0, cmdResponse.getResponseCode()); + revMap.clear(); + revMap.put("cf1", 3L); + hbaseSnapshot = new TableSnapshot(fullyQualTableName, revMap, -1); + HCatInputFormat.setInput(job, databaseName, tableName).setProperties(properties); + modifiedInputInfo = job.getConfiguration().get(HCatConstants.HCAT_KEY_JOB_INFO); + inputInfo = (InputJobInfo) HCatUtil.deserialize(modifiedInputInfo); + hcatSnapshot = HBaseRevisionManagerUtil.convertSnapshot(hbaseSnapshot, inputInfo.getTableInfo()); + assertEquals(hcatSnapshot.getRevision("value1"), 3); + assertEquals(hcatSnapshot.getRevision("value2"), 3); + + dropTable = "DROP TABLE " + fullyQualTableName; + cmdResponse = hcatDriver.run(dropTable); + assertEquals(0, cmdResponse.getResponseCode()); + + String dropDatabase = "DROP DATABASE IF EXISTS " + databaseName + "CASCADE"; + cmdResponse = hcatDriver.run(dropDatabase); + assertEquals(0, cmdResponse.getResponseCode()); + } +} diff --git hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/IDGenClient.java hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/IDGenClient.java new file mode 100644 index 0000000..353402b --- /dev/null +++ hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/IDGenClient.java @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hcatalog.hbase.snapshot; + +import java.util.HashMap; +import java.util.Map; +import java.util.Random; + +public class IDGenClient extends Thread { + + String connectionStr; + String base_dir; + ZKUtil zkutil; + Random sleepTime = new Random(); + int runtime; + HashMap idMap; + String tableName; + + IDGenClient(String connectionStr, String base_dir, int time, String tableName) { + super(); + this.connectionStr = connectionStr; + this.base_dir = base_dir; + this.zkutil = new ZKUtil(connectionStr, base_dir); + this.runtime = time; + idMap = new HashMap(); + this.tableName = tableName; + } + + /* + * @see java.lang.Runnable#run() + */ + @Override + public void run() { + long startTime = System.currentTimeMillis(); + int timeElapsed = 0; + while( timeElapsed <= runtime){ + try { + long id = zkutil.nextId(tableName); + idMap.put(System.currentTimeMillis(), id); + + int sTime = sleepTime.nextInt(2); + Thread.sleep(sTime * 100); + } catch (Exception e) { + e.printStackTrace(); + } + + timeElapsed = (int) Math.ceil((System.currentTimeMillis() - startTime)/(double)1000); + } + + } + + Map getIdMap(){ + return idMap; + } + +} diff --git hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestIDGenerator.java hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestIDGenerator.java new file mode 100644 index 0000000..9b0cd01 --- /dev/null +++ hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestIDGenerator.java @@ -0,0 +1,99 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hcatalog.hbase.snapshot; + +import static org.junit.Assert.assertTrue; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; + +import org.apache.hcatalog.hbase.SkeletonHBaseTest; +import org.junit.Assert; +import org.junit.Test; + +public class TestIDGenerator extends SkeletonHBaseTest { + + @Test + public void testIDGeneration() throws Exception { + + int port = getHbaseConf().getInt("hbase.zookeeper.property.clientPort", 2181); + String servers = getHbaseConf().get("hbase.zookeeper.quorum"); + String[] splits = servers.split(","); + StringBuffer sb = new StringBuffer(); + for (String split : splits) { + sb.append(split); + sb.append(':'); + sb.append(port); + } + ZKUtil zkutil = new ZKUtil(sb.toString(), "/rm_base"); + + String tableName = "myTable"; + long initId = zkutil.nextId(tableName); + for (int i = 0; i < 10; i++) { + long id = zkutil.nextId(tableName); + Assert.assertEquals(initId + (i + 1), id); + } + } + + @Test + public void testMultipleClients() throws InterruptedException { + + int port = getHbaseConf().getInt("hbase.zookeeper.property.clientPort", 2181); + String servers = getHbaseConf().get("hbase.zookeeper.quorum"); + String[] splits = servers.split(","); + StringBuffer sb = new StringBuffer(); + for (String split : splits) { + sb.append(split); + sb.append(':'); + sb.append(port); + } + + ArrayList clients = new ArrayList(); + + for (int i = 0; i < 5; i++) { + IDGenClient idClient = new IDGenClient(sb.toString(), "/rm_base", 10, "testTable"); + clients.add(idClient); + } + + for (IDGenClient idClient : clients) { + idClient.run(); + } + + for (IDGenClient idClient : clients) { + idClient.join(); + } + + HashMap idMap = new HashMap(); + for (IDGenClient idClient : clients) { + idMap.putAll(idClient.getIdMap()); + } + + ArrayList keys = new ArrayList(idMap.keySet()); + Collections.sort(keys); + int startId = 1; + for (Long key : keys) { + Long id = idMap.get(key); + System.out.println("Key: " + key + " Value " + id); + assertTrue(id == startId); + startId++; + + } + } +} diff --git hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManager.java hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManager.java new file mode 100644 index 0000000..114895a --- /dev/null +++ hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManager.java @@ -0,0 +1,260 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hcatalog.hbase.snapshot; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hcatalog.hbase.SkeletonHBaseTest; +import org.apache.hcatalog.hbase.snapshot.transaction.thrift.StoreFamilyRevision; +import org.apache.hcatalog.hbase.snapshot.transaction.thrift.StoreFamilyRevisionList; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.ZooKeeper; +import org.apache.zookeeper.data.Stat; +import org.junit.Test; + +public class TestRevisionManager extends SkeletonHBaseTest { + + @Test + public void testBasicZNodeCreation() throws IOException, KeeperException, InterruptedException { + + int port = getHbaseConf().getInt("hbase.zookeeper.property.clientPort", 2181); + String servers = getHbaseConf().get("hbase.zookeeper.quorum"); + String[] splits = servers.split(","); + StringBuffer sb = new StringBuffer(); + for (String split : splits) { + sb.append(split); + sb.append(':'); + sb.append(port); + } + + ZKUtil zkutil = new ZKUtil(sb.toString(), "/rm_base"); + String tableName = newTableName("testTable"); + List columnFamilies = Arrays.asList("cf001", "cf002", "cf003"); + + zkutil.createRootZNodes(); + ZooKeeper zk = zkutil.getSession(); + Stat tempTwo = zk.exists("/rm_base" + PathUtil.DATA_DIR, false); + assertTrue(tempTwo != null); + Stat tempThree = zk.exists("/rm_base" + PathUtil.CLOCK_NODE, false); + assertTrue(tempThree != null); + + zkutil.setUpZnodesForTable(tableName, columnFamilies); + String transactionDataTablePath = "/rm_base" + PathUtil.DATA_DIR + "/" + tableName; + Stat result = zk.exists(transactionDataTablePath, false); + assertTrue(result != null); + + for (String colFamiliy : columnFamilies) { + String cfPath = transactionDataTablePath + "/" + colFamiliy; + Stat resultTwo = zk.exists(cfPath, false); + assertTrue(resultTwo != null); + } + + } + + @Test + public void testCommitTransaction() throws IOException { + + int port = getHbaseConf().getInt("hbase.zookeeper.property.clientPort", 2181); + String servers = getHbaseConf().get("hbase.zookeeper.quorum"); + String[] splits = servers.split(","); + StringBuffer sb = new StringBuffer(); + for (String split : splits) { + sb.append(split); + sb.append(':'); + sb.append(port); + } + + Configuration conf = RevisionManagerConfiguration.create(getHbaseConf()); + conf.set(RMConstants.ZOOKEEPER_DATADIR, "/rm_base"); + ZKBasedRevisionManager manager = new ZKBasedRevisionManager(); + manager.initialize(conf); + manager.open(); + ZKUtil zkutil = new ZKUtil(sb.toString(), "/rm_base"); + + String tableName = newTableName("testTable"); + List columnFamilies = Arrays.asList("cf1", "cf2", "cf3"); + Transaction txn = manager.beginWriteTransaction(tableName, + columnFamilies); + + List cfs = zkutil.getColumnFamiliesOfTable(tableName); + assertTrue(cfs.size() == columnFamilies.size()); + for (String cf : cfs) { + assertTrue(columnFamilies.contains(cf)); + } + + for (String colFamily : columnFamilies) { + String path = PathUtil.getRunningTxnInfoPath("/rm_base", tableName, colFamily); + byte[] data = zkutil.getRawData(path, null); + StoreFamilyRevisionList list = new StoreFamilyRevisionList(); + ZKUtil.deserialize(list, data); + assertEquals(list.getRevisionListSize(), 1); + StoreFamilyRevision lightTxn = list.getRevisionList().get(0); + assertEquals(lightTxn.timestamp, txn.getTransactionExpireTimeStamp()); + assertEquals(lightTxn.revision, txn.getRevisionNumber()); + + } + manager.commitWriteTransaction(txn); + for (String colFamiliy : columnFamilies) { + String path = PathUtil.getRunningTxnInfoPath("/rm_base", tableName, colFamiliy); + byte[] data = zkutil.getRawData(path, null); + StoreFamilyRevisionList list = new StoreFamilyRevisionList(); + ZKUtil.deserialize(list, data); + assertEquals(list.getRevisionListSize(), 0); + + } + + manager.close(); + } + + @Test + public void testAbortTransaction() throws IOException { + + int port = getHbaseConf().getInt("hbase.zookeeper.property.clientPort", 2181); + String host = getHbaseConf().get("hbase.zookeeper.quorum"); + Configuration conf = RevisionManagerConfiguration.create(getHbaseConf()); + conf.set(RMConstants.ZOOKEEPER_DATADIR, "/rm_base"); + ZKBasedRevisionManager manager = new ZKBasedRevisionManager(); + manager.initialize(conf); + manager.open(); + ZKUtil zkutil = new ZKUtil(host + ':' + port, "/rm_base"); + + String tableName = newTableName("testTable"); + List columnFamilies = Arrays.asList("cf1", "cf2", "cf3"); + Transaction txn = manager.beginWriteTransaction(tableName, columnFamilies); + List cfs = zkutil.getColumnFamiliesOfTable(tableName); + + assertTrue(cfs.size() == columnFamilies.size()); + for (String cf : cfs) { + assertTrue(columnFamilies.contains(cf)); + } + + for (String colFamiliy : columnFamilies) { + String path = PathUtil.getRunningTxnInfoPath("/rm_base", tableName, colFamiliy); + byte[] data = zkutil.getRawData(path, null); + StoreFamilyRevisionList list = new StoreFamilyRevisionList(); + ZKUtil.deserialize(list, data); + assertEquals(list.getRevisionListSize(), 1); + StoreFamilyRevision lightTxn = list.getRevisionList().get(0); + assertEquals(lightTxn.timestamp, txn.getTransactionExpireTimeStamp()); + assertEquals(lightTxn.revision, txn.getRevisionNumber()); + + } + manager.abortWriteTransaction(txn); + for (String colFamiliy : columnFamilies) { + String path = PathUtil.getRunningTxnInfoPath("/rm_base", tableName, colFamiliy); + byte[] data = zkutil.getRawData(path, null); + StoreFamilyRevisionList list = new StoreFamilyRevisionList(); + ZKUtil.deserialize(list, data); + assertEquals(list.getRevisionListSize(), 0); + + } + + for (String colFamiliy : columnFamilies) { + String path = PathUtil.getAbortInformationPath("/rm_base", tableName, colFamiliy); + byte[] data = zkutil.getRawData(path, null); + StoreFamilyRevisionList list = new StoreFamilyRevisionList(); + ZKUtil.deserialize(list, data); + assertEquals(list.getRevisionListSize(), 1); + StoreFamilyRevision abortedTxn = list.getRevisionList().get(0); + assertEquals(abortedTxn.getRevision(), txn.getRevisionNumber()); + } + manager.close(); + } + + @Test + public void testKeepAliveTransaction() throws InterruptedException, IOException { + + int port = getHbaseConf().getInt("hbase.zookeeper.property.clientPort", 2181); + String servers = getHbaseConf().get("hbase.zookeeper.quorum"); + String[] splits = servers.split(","); + StringBuffer sb = new StringBuffer(); + for (String split : splits) { + sb.append(split); + sb.append(':'); + sb.append(port); + } + + Configuration conf = RevisionManagerConfiguration.create(getHbaseConf()); + conf.set(RMConstants.ZOOKEEPER_DATADIR, "/rm_base"); + ZKBasedRevisionManager manager = new ZKBasedRevisionManager(); + manager.initialize(conf); + manager.open(); + String tableName = newTableName("testTable"); + List columnFamilies = Arrays.asList("cf1", "cf2"); + Transaction txn = manager.beginWriteTransaction(tableName, + columnFamilies, 40); + Thread.sleep(100); + try { + manager.commitWriteTransaction(txn); + } catch (Exception e) { + assertTrue(e instanceof IOException); + assertEquals(e.getMessage(), + "The transaction to be removed not found in the data."); + } + + } + + @Test + public void testCreateSnapshot() throws IOException { + int port = getHbaseConf().getInt("hbase.zookeeper.property.clientPort", 2181); + String host = getHbaseConf().get("hbase.zookeeper.quorum"); + Configuration conf = RevisionManagerConfiguration.create(getHbaseConf()); + conf.set(RMConstants.ZOOKEEPER_DATADIR, "/rm_base"); + ZKBasedRevisionManager manager = new ZKBasedRevisionManager(); + manager.initialize(conf); + manager.open(); + String tableName = newTableName("testTable"); + List cfOne = Arrays.asList("cf1", "cf2"); + List cfTwo = Arrays.asList("cf2", "cf3"); + Transaction tsx1 = manager.beginWriteTransaction(tableName, cfOne); + Transaction tsx2 = manager.beginWriteTransaction(tableName, cfTwo); + TableSnapshot snapshotOne = manager.createSnapshot(tableName); + assertEquals(snapshotOne.getRevision("cf1"), 0); + assertEquals(snapshotOne.getRevision("cf2"), 0); + assertEquals(snapshotOne.getRevision("cf3"), 1); + + List cfThree = Arrays.asList("cf1", "cf3"); + Transaction tsx3 = manager.beginWriteTransaction(tableName, cfThree); + manager.commitWriteTransaction(tsx1); + TableSnapshot snapshotTwo = manager.createSnapshot(tableName); + assertEquals(snapshotTwo.getRevision("cf1"), 2); + assertEquals(snapshotTwo.getRevision("cf2"), 1); + assertEquals(snapshotTwo.getRevision("cf3"), 1); + + manager.commitWriteTransaction(tsx2); + TableSnapshot snapshotThree = manager.createSnapshot(tableName); + assertEquals(snapshotThree.getRevision("cf1"), 2); + assertEquals(snapshotThree.getRevision("cf2"), 3); + assertEquals(snapshotThree.getRevision("cf3"), 2); + manager.commitWriteTransaction(tsx3); + TableSnapshot snapshotFour = manager.createSnapshot(tableName); + assertEquals(snapshotFour.getRevision("cf1"), 3); + assertEquals(snapshotFour.getRevision("cf2"), 3); + assertEquals(snapshotFour.getRevision("cf3"), 3); + + } + + +} diff --git hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManagerConfiguration.java hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManagerConfiguration.java new file mode 100644 index 0000000..301bf92 --- /dev/null +++ hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManagerConfiguration.java @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hcatalog.hbase.snapshot; + +import org.apache.hadoop.conf.Configuration; +import org.junit.Assert; +import org.junit.Test; + +public class TestRevisionManagerConfiguration { + + @Test + public void testDefault() { + Configuration conf = RevisionManagerConfiguration.create(); + Assert.assertEquals("org.apache.hcatalog.hbase.snapshot.ZKBasedRevisionManager", + conf.get(RevisionManagerFactory.REVISION_MGR_IMPL_CLASS)); + } +} diff --git hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManagerEndpoint.java hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManagerEndpoint.java new file mode 100644 index 0000000..fe9ca40 --- /dev/null +++ hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManagerEndpoint.java @@ -0,0 +1,206 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hcatalog.hbase.snapshot; + +import java.io.IOException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import org.apache.commons.lang.builder.ToStringBuilder; +import org.apache.commons.lang.builder.ToStringStyle; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.apache.hcatalog.hbase.SkeletonHBaseTest; +import org.junit.Assert; +import org.junit.Test; + +public class TestRevisionManagerEndpoint extends SkeletonHBaseTest { + + static { + // test case specific mini cluster settings + testConf = new Configuration(false); + testConf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, + "org.apache.hcatalog.hbase.snapshot.RevisionManagerEndpoint", + "org.apache.hadoop.hbase.coprocessor.GenericEndpoint"); + testConf.set(RMConstants.REVISION_MGR_ENDPOINT_IMPL_CLASS, MockRM.class.getName()); + } + + /** + * Mock implementation to test the protocol/serialization + */ + public static class MockRM implements RevisionManager { + + private static class Invocation { + Invocation(String methodName, Object ret, Object... args) { + this.methodName = methodName; + this.args = args; + this.ret = ret; + } + + String methodName; + Object[] args; + Object ret; + + private static boolean equals(Object obj1, Object obj2) { + if (obj1 == obj2) return true; + if (obj1 == null || obj2 == null) return false; + if (obj1 instanceof Transaction || obj1 instanceof TableSnapshot) { + return obj1.toString().equals(obj2.toString()); + } + return obj1.equals(obj2); + } + + @Override + public boolean equals(Object obj) { + Invocation other = (Invocation) obj; + if (this == other) return true; + if (other == null) return false; + if (this.args != other.args) { + if (this.args == null || other.args == null) return false; + if (this.args.length != other.args.length) return false; + for (int i = 0; i < args.length; i++) { + if (!equals(this.args[i], other.args[i])) return false; + } + } + return equals(this.ret, other.ret); + } + + @Override + public String toString() { + return new ToStringBuilder(this, ToStringStyle.SHORT_PREFIX_STYLE). + append("method", this.methodName). + append("args", this.args). + append("returns", this.ret). + toString(); + } + } + + final static String DEFAULT_INSTANCE = "default"; + final static Map INSTANCES = new ConcurrentHashMap(); + Invocation lastCall; + boolean isOpen = false; + + private T recordCall(T result, Object... args) { + StackTraceElement[] stackTrace = Thread.currentThread().getStackTrace(); + lastCall = new Invocation(stackTrace[2].getMethodName(), result, args); + return result; + } + + @Override + public void initialize(Configuration conf) { + if (!INSTANCES.containsKey(DEFAULT_INSTANCE)) + INSTANCES.put(DEFAULT_INSTANCE, this); + } + + @Override + public void open() throws IOException { + isOpen = true; + } + + @Override + public void close() throws IOException { + isOpen = false; + } + + @Override + public void createTable(String table, List columnFamilies) throws IOException { + } + + @Override + public void dropTable(String table) throws IOException { + } + + @Override + public Transaction beginWriteTransaction(String table, + List families) throws IOException { + return recordCall(null, table, families); + } + + @Override + public Transaction beginWriteTransaction(String table, + List families, long keepAlive) throws IOException { + return recordCall(null, table, families, keepAlive); + } + + @Override + public void commitWriteTransaction(Transaction transaction) + throws IOException { + } + + @Override + public void abortWriteTransaction(Transaction transaction) + throws IOException { + } + + @Override + public List getAbortedWriteTransactions(String table, + String columnFamily) throws IOException { + return null; + } + + @Override + public TableSnapshot createSnapshot(String tableName) + throws IOException { + return null; + } + + @Override + public TableSnapshot createSnapshot(String tableName, long revision) + throws IOException { + TableSnapshot ret = new TableSnapshot(tableName, new HashMap(), revision); + return recordCall(ret, tableName, revision); + } + + @Override + public void keepAlive(Transaction transaction) throws IOException { + recordCall(null, transaction); + } + } + + @Test + public void testRevisionManagerProtocol() throws Throwable { + + Configuration conf = getHbaseConf(); + RevisionManager rm = RevisionManagerFactory.getOpenedRevisionManager( + RevisionManagerEndpointClient.class.getName(), conf); + + MockRM mockImpl = MockRM.INSTANCES.get(MockRM.DEFAULT_INSTANCE); + Assert.assertNotNull(mockImpl); + Assert.assertTrue(mockImpl.isOpen); + + Transaction t = new Transaction("t1", Arrays.asList("f1", "f2"), 0, 0); + MockRM.Invocation call = new MockRM.Invocation("keepAlive", null, t); + rm.keepAlive(t); + Assert.assertEquals(call.methodName, call, mockImpl.lastCall); + + t = new Transaction("t2", Arrays.asList("f21", "f22"), 0, 0); + call = new MockRM.Invocation("beginWriteTransaction", null, t.getTableName(), t.getColumnFamilies()); + call.ret = rm.beginWriteTransaction(t.getTableName(), t.getColumnFamilies()); + Assert.assertEquals(call.methodName, call, mockImpl.lastCall); + + call = new MockRM.Invocation("createSnapshot", null, "t3", 1L); + call.ret = rm.createSnapshot("t3", 1); + Assert.assertEquals(call.methodName, call, mockImpl.lastCall); + + } + +} diff --git hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestThriftSerialization.java hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestThriftSerialization.java new file mode 100644 index 0000000..e423f65 --- /dev/null +++ hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestThriftSerialization.java @@ -0,0 +1,85 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hcatalog.hbase.snapshot; + +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +import org.apache.hcatalog.hbase.snapshot.transaction.thrift.StoreFamilyRevision; +import org.apache.hcatalog.hbase.snapshot.transaction.thrift.StoreFamilyRevisionList; +import org.junit.Test; + +public class TestThriftSerialization { + + @Test + public void testLightWeightTransaction() { + StoreFamilyRevision trxn = new StoreFamilyRevision(0, 1000); + try { + + byte[] data = ZKUtil.serialize(trxn); + StoreFamilyRevision newWtx = new StoreFamilyRevision(); + ZKUtil.deserialize(newWtx, data); + + assertTrue(newWtx.getRevision() == trxn.getRevision()); + assertTrue(newWtx.getTimestamp() == trxn.getTimestamp()); + + } catch (IOException e) { + e.printStackTrace(); + } + } + + @Test + public void testWriteTransactionList() { + List txnList = new ArrayList(); + long version; + long timestamp; + for (int i = 0; i < 10; i++) { + version = i; + timestamp = 1000 + i; + StoreFamilyRevision wtx = new StoreFamilyRevision(version, timestamp); + txnList.add(wtx); + } + + StoreFamilyRevisionList wList = new StoreFamilyRevisionList(txnList); + + try { + byte[] data = ZKUtil.serialize(wList); + StoreFamilyRevisionList newList = new StoreFamilyRevisionList(); + ZKUtil.deserialize(newList, data); + assertTrue(newList.getRevisionListSize() == wList.getRevisionListSize()); + + Iterator itr = newList.getRevisionListIterator(); + int i = 0; + while (itr.hasNext()) { + StoreFamilyRevision txn = itr.next(); + assertTrue(txn.getRevision() == i); + assertTrue(txn.getTimestamp() == (i + 1000)); + i++; + } + + } catch (IOException e) { + e.printStackTrace(); + } + } + +} diff --git hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestZNodeSetUp.java hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestZNodeSetUp.java new file mode 100644 index 0000000..7a1a3c2 --- /dev/null +++ hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestZNodeSetUp.java @@ -0,0 +1,120 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hcatalog.hbase.snapshot; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.net.URI; +import java.util.Map; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hive.cli.CliSessionState; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; +import org.apache.hadoop.hive.ql.session.SessionState; +import org.apache.hive.hcatalog.cli.HCatDriver; +import org.apache.hive.hcatalog.cli.SemanticAnalysis.HCatSemanticAnalyzer; +import org.apache.hcatalog.hbase.SkeletonHBaseTest; +import org.apache.zookeeper.ZooKeeper; +import org.apache.zookeeper.data.Stat; +import org.junit.Test; + + +public class TestZNodeSetUp extends SkeletonHBaseTest { + + private static HiveConf hcatConf; + private static HCatDriver hcatDriver; + + public void Initialize() throws Exception { + + hcatConf = getHiveConf(); + hcatConf.set(ConfVars.SEMANTIC_ANALYZER_HOOK.varname, + HCatSemanticAnalyzer.class.getName()); + URI fsuri = getFileSystem().getUri(); + Path whPath = new Path(fsuri.getScheme(), fsuri.getAuthority(), + getTestDir()); + hcatConf.set(HiveConf.ConfVars.HADOOPFS.varname, fsuri.toString()); + hcatConf.set(ConfVars.METASTOREWAREHOUSE.varname, whPath.toString()); + + //Add hbase properties + + for (Map.Entry el : getHbaseConf()) { + if (el.getKey().startsWith("hbase.")) { + hcatConf.set(el.getKey(), el.getValue()); + } + } + HBaseConfiguration.merge(hcatConf, + RevisionManagerConfiguration.create()); + hcatConf.set(RMConstants.ZOOKEEPER_DATADIR, "/rm_base"); + SessionState.start(new CliSessionState(hcatConf)); + hcatDriver = new HCatDriver(); + + } + + @Test + public void testBasicZNodeCreation() throws Exception { + + Initialize(); + int port = getHbaseConf().getInt("hbase.zookeeper.property.clientPort", 2181); + String servers = getHbaseConf().get("hbase.zookeeper.quorum"); + String[] splits = servers.split(","); + StringBuffer sb = new StringBuffer(); + for (String split : splits) { + sb.append(split); + sb.append(':'); + sb.append(port); + } + + hcatDriver.run("drop table test_table"); + CommandProcessorResponse response = hcatDriver + .run("create table test_table(key int, value string) STORED BY " + + "'org.apache.hcatalog.hbase.HBaseHCatStorageHandler'" + + "TBLPROPERTIES ('hbase.columns.mapping'=':key,cf1:val')"); + + assertEquals(0, response.getResponseCode()); + + HBaseAdmin hAdmin = new HBaseAdmin(getHbaseConf()); + boolean doesTableExist = hAdmin.tableExists("test_table"); + assertTrue(doesTableExist); + + + ZKUtil zkutil = new ZKUtil(sb.toString(), "/rm_base"); + ZooKeeper zk = zkutil.getSession(); + String tablePath = PathUtil.getTxnDataPath("/rm_base", "test_table"); + Stat tempTwo = zk.exists(tablePath, false); + assertTrue(tempTwo != null); + + String cfPath = PathUtil.getTxnDataPath("/rm_base", "test_table") + "/cf1"; + Stat tempThree = zk.exists(cfPath, false); + assertTrue(tempThree != null); + + hcatDriver.run("drop table test_table"); + + System.out.println("Table path : " + tablePath); + Stat tempFour = zk.exists(tablePath, false); + assertTrue(tempFour == null); + + } + +} diff --git hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/lock/TestWriteLock.java hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/lock/TestWriteLock.java new file mode 100644 index 0000000..c03a00b --- /dev/null +++ hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/lock/TestWriteLock.java @@ -0,0 +1,161 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hcatalog.hbase.snapshot.lock; + +import org.apache.zookeeper.ZooKeeper; +import org.apache.zookeeper.test.ClientBase; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Test; + +/** + * test for writelock + * This class is taken from the zookeeper 3.4.0 as-is to test the zookeeper lock + * Recipe with a change in the package name. + */ +public class TestWriteLock extends ClientBase { + protected int sessionTimeout = 10 * 1000; + protected String dir = "/" + getClass().getName(); + protected WriteLock[] nodes; + protected CountDownLatch latch = new CountDownLatch(1); + private boolean restartServer = true; + private boolean workAroundClosingLastZNodeFails = true; + private boolean killLeader = true; + + @Test + public void testRun() throws Exception { + runTest(3); + } + + class LockCallback implements LockListener { + public void lockAcquired() { + latch.countDown(); + } + + public void lockReleased() { + + } + + } + + protected void runTest(int count) throws Exception { + nodes = new WriteLock[count]; + for (int i = 0; i < count; i++) { + ZooKeeper keeper = createClient(); + WriteLock leader = new WriteLock(keeper, dir, null); + leader.setLockListener(new LockCallback()); + nodes[i] = leader; + + leader.lock(); + } + + // lets wait for any previous leaders to die and one of our new + // nodes to become the new leader + latch.await(30, TimeUnit.SECONDS); + + WriteLock first = nodes[0]; + dumpNodes(count); + + // lets assert that the first election is the leader + Assert.assertTrue("The first znode should be the leader " + first.getId(), first.isOwner()); + + for (int i = 1; i < count; i++) { + WriteLock node = nodes[i]; + Assert.assertFalse("Node should not be the leader " + node.getId(), node.isOwner()); + } + + if (count > 1) { + if (killLeader) { + System.out.println("Now killing the leader"); + // now lets kill the leader + latch = new CountDownLatch(1); + first.unlock(); + latch.await(30, TimeUnit.SECONDS); + //Thread.sleep(10000); + WriteLock second = nodes[1]; + dumpNodes(count); + // lets assert that the first election is the leader + Assert.assertTrue("The second znode should be the leader " + second.getId(), second.isOwner()); + + for (int i = 2; i < count; i++) { + WriteLock node = nodes[i]; + Assert.assertFalse("Node should not be the leader " + node.getId(), node.isOwner()); + } + } + + + if (restartServer) { + // now lets stop the server + System.out.println("Now stopping the server"); + stopServer(); + Thread.sleep(10000); + + // TODO lets assert that we are no longer the leader + dumpNodes(count); + + System.out.println("Starting the server"); + startServer(); + Thread.sleep(10000); + + for (int i = 0; i < count - 1; i++) { + System.out.println("Calling acquire for node: " + i); + nodes[i].lock(); + } + dumpNodes(count); + System.out.println("Now closing down..."); + } + } + } + + protected void dumpNodes(int count) { + for (int i = 0; i < count; i++) { + WriteLock node = nodes[i]; + System.out.println("node: " + i + " id: " + + node.getId() + " is leader: " + node.isOwner()); + } + } + + @After + public void tearDown() throws Exception { + if (nodes != null) { + for (int i = 0; i < nodes.length; i++) { + WriteLock node = nodes[i]; + if (node != null) { + System.out.println("Closing node: " + i); + node.close(); + if (workAroundClosingLastZNodeFails && i == nodes.length - 1) { + System.out.println("Not closing zookeeper: " + i + " due to bug!"); + } else { + System.out.println("Closing zookeeper: " + i); + node.getZookeeper().close(); + System.out.println("Closed zookeeper: " + i); + } + } + } + } + System.out.println("Now lets stop the server"); + super.tearDown(); + + } +} diff --git hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/lock/TestZNodeName.java hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/lock/TestZNodeName.java new file mode 100644 index 0000000..a39c691 --- /dev/null +++ hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/lock/TestZNodeName.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hcatalog.hbase.snapshot.lock; + +import junit.framework.TestCase; + +import java.util.SortedSet; +import java.util.TreeSet; + +import org.junit.Test; + +/** + * test for znodenames. This class is taken as-is from zookeeper lock recipe test. + * The package name has been changed. + */ +public class TestZNodeName extends TestCase { + @Test + public void testOrderWithSamePrefix() throws Exception { + String[] names = { "x-3", "x-5", "x-11", "x-1" }; + String[] expected = { "x-1", "x-3", "x-5", "x-11" }; + assertOrderedNodeNames(names, expected); + } + @Test + public void testOrderWithDifferentPrefixes() throws Exception { + String[] names = { "r-3", "r-2", "r-1", "w-2", "w-1" }; + String[] expected = { "r-1", "r-2", "r-3", "w-1", "w-2" }; + assertOrderedNodeNames(names, expected); + } + + protected void assertOrderedNodeNames(String[] names, String[] expected) { + int size = names.length; + assertEquals("The two arrays should be the same size!", names.length, expected.length); + SortedSet nodeNames = new TreeSet(); + for (String name : names) { + nodeNames.add(new ZNodeName(name)); + } + + int index = 0; + for (ZNodeName nodeName : nodeNames) { + String name = nodeName.getName(); + assertEquals("Node " + index, expected[index++], name); + } + } + +} diff --git hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/ManyMiniCluster.java hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/ManyMiniCluster.java deleted file mode 100644 index d89ec25..0000000 --- hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/ManyMiniCluster.java +++ /dev/null @@ -1,370 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hive.hcatalog.hbase; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.FileUtil; -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.MiniHBaseCluster; -import org.apache.hadoop.hbase.client.HConnectionManager; -import org.apache.hadoop.hbase.client.HTable; -import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster; -import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.MiniMRCluster; - -import java.io.File; -import java.io.IOException; -import java.net.ServerSocket; - -/** - * MiniCluster class composed of a number of Hadoop Minicluster implementations - * and other necessary daemons needed for testing (HBase, Hive MetaStore, Zookeeper, MiniMRCluster) - */ -public class ManyMiniCluster { - - //MR stuff - private boolean miniMRClusterEnabled; - private MiniMRCluster mrCluster; - private int numTaskTrackers; - private JobConf jobConf; - - //HBase stuff - private boolean miniHBaseClusterEnabled; - private MiniHBaseCluster hbaseCluster; - private String hbaseRoot; - private Configuration hbaseConf; - private String hbaseDir; - - //ZK Stuff - private boolean miniZookeeperClusterEnabled; - private MiniZooKeeperCluster zookeeperCluster; - private int zookeeperPort; - private String zookeeperDir; - - //DFS Stuff - private MiniDFSCluster dfsCluster; - - //Hive Stuff - private boolean miniHiveMetastoreEnabled; - private HiveConf hiveConf; - private HiveMetaStoreClient hiveMetaStoreClient; - - private final File workDir; - private boolean started = false; - - - /** - * create a cluster instance using a builder which will expose configurable options - * @param workDir working directory ManyMiniCluster will use for all of it's *Minicluster instances - * @return a Builder instance - */ - public static Builder create(File workDir) { - return new Builder(workDir); - } - - private ManyMiniCluster(Builder b) { - workDir = b.workDir; - numTaskTrackers = b.numTaskTrackers; - hiveConf = b.hiveConf; - jobConf = b.jobConf; - hbaseConf = b.hbaseConf; - miniMRClusterEnabled = b.miniMRClusterEnabled; - miniHBaseClusterEnabled = b.miniHBaseClusterEnabled; - miniHiveMetastoreEnabled = b.miniHiveMetastoreEnabled; - miniZookeeperClusterEnabled = b.miniZookeeperClusterEnabled; - } - - protected synchronized void start() { - try { - if (!started) { - FileUtil.fullyDelete(workDir); - if (miniMRClusterEnabled) { - setupMRCluster(); - } - if (miniZookeeperClusterEnabled || miniHBaseClusterEnabled) { - miniZookeeperClusterEnabled = true; - setupZookeeper(); - } - if (miniHBaseClusterEnabled) { - setupHBaseCluster(); - } - if (miniHiveMetastoreEnabled) { - setUpMetastore(); - } - } - } catch (Exception e) { - throw new IllegalStateException("Failed to setup cluster", e); - } - } - - protected synchronized void stop() { - if (hbaseCluster != null) { - HConnectionManager.deleteAllConnections(true); - try { - hbaseCluster.shutdown(); - } catch (Exception e) { - e.printStackTrace(); - } - hbaseCluster = null; - } - if (zookeeperCluster != null) { - try { - zookeeperCluster.shutdown(); - } catch (Exception e) { - e.printStackTrace(); - } - zookeeperCluster = null; - } - if (mrCluster != null) { - try { - mrCluster.shutdown(); - } catch (Exception e) { - e.printStackTrace(); - } - mrCluster = null; - } - if (dfsCluster != null) { - try { - dfsCluster.getFileSystem().close(); - dfsCluster.shutdown(); - } catch (Exception e) { - e.printStackTrace(); - } - dfsCluster = null; - } - try { - FileSystem.closeAll(); - } catch (IOException e) { - e.printStackTrace(); - } - started = false; - } - - /** - * @return Configuration of mini HBase cluster - */ - public Configuration getHBaseConf() { - return HBaseConfiguration.create(hbaseConf); - } - - /** - * @return Configuration of mini MR cluster - */ - public Configuration getJobConf() { - return new Configuration(jobConf); - } - - /** - * @return Configuration of Hive Metastore, this is a standalone not a daemon - */ - public HiveConf getHiveConf() { - return new HiveConf(hiveConf); - } - - /** - * @return Filesystem used by MiniMRCluster and MiniHBaseCluster - */ - public FileSystem getFileSystem() { - try { - return FileSystem.get(jobConf); - } catch (IOException e) { - throw new IllegalStateException("Failed to get FileSystem", e); - } - } - - /** - * @return Metastore client instance - */ - public HiveMetaStoreClient getHiveMetaStoreClient() { - return hiveMetaStoreClient; - } - - private void setupMRCluster() { - try { - final int jobTrackerPort = findFreePort(); - final int taskTrackerPort = findFreePort(); - - if (jobConf == null) - jobConf = new JobConf(); - - jobConf.setInt("mapred.submit.replication", 1); - jobConf.set("yarn.scheduler.capacity.root.queues", "default"); - jobConf.set("yarn.scheduler.capacity.root.default.capacity", "100"); - //conf.set("hadoop.job.history.location",new File(workDir).getAbsolutePath()+"/history"); - System.setProperty("hadoop.log.dir", new File(workDir, "/logs").getAbsolutePath()); - - mrCluster = new MiniMRCluster(jobTrackerPort, - taskTrackerPort, - numTaskTrackers, - getFileSystem().getUri().toString(), - numTaskTrackers, - null, - null, - null, - jobConf); - - jobConf = mrCluster.createJobConf(); - } catch (IOException e) { - throw new IllegalStateException("Failed to Setup MR Cluster", e); - } - } - - private void setupZookeeper() { - try { - zookeeperDir = new File(workDir, "zk").getAbsolutePath(); - zookeeperPort = findFreePort(); - zookeeperCluster = new MiniZooKeeperCluster(); - zookeeperCluster.setDefaultClientPort(zookeeperPort); - zookeeperCluster.startup(new File(zookeeperDir)); - } catch (Exception e) { - throw new IllegalStateException("Failed to Setup Zookeeper Cluster", e); - } - } - - private void setupHBaseCluster() { - final int numRegionServers = 1; - - try { - hbaseDir = new File(workDir, "hbase").toString(); - hbaseDir = hbaseDir.replaceAll("\\\\", "/"); - hbaseRoot = "file://" + hbaseDir; - - if (hbaseConf == null) - hbaseConf = HBaseConfiguration.create(); - - hbaseConf.set("hbase.rootdir", hbaseRoot); - hbaseConf.set("hbase.master", "local"); - hbaseConf.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, zookeeperPort); - hbaseConf.set(HConstants.ZOOKEEPER_QUORUM, "127.0.0.1"); - hbaseConf.setInt("hbase.master.port", findFreePort()); - hbaseConf.setInt("hbase.master.info.port", -1); - hbaseConf.setInt("hbase.regionserver.port", findFreePort()); - hbaseConf.setInt("hbase.regionserver.info.port", -1); - - hbaseCluster = new MiniHBaseCluster(hbaseConf, numRegionServers); - hbaseConf.set("hbase.master", hbaseCluster.getMaster().getServerName().getHostAndPort()); - //opening the META table ensures that cluster is running - new HTable(hbaseConf, HConstants.META_TABLE_NAME); - } catch (Exception e) { - throw new IllegalStateException("Failed to setup HBase Cluster", e); - } - } - - private void setUpMetastore() throws Exception { - if (hiveConf == null) - hiveConf = new HiveConf(this.getClass()); - - //The default org.apache.hadoop.hive.ql.hooks.PreExecutePrinter hook - //is present only in the ql/test directory - hiveConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, ""); - hiveConf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, ""); - hiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false"); - hiveConf.set(HiveConf.ConfVars.METASTORECONNECTURLKEY.varname, - "jdbc:derby:" + new File(workDir + "/metastore_db") + ";create=true"); - hiveConf.set(HiveConf.ConfVars.METASTOREWAREHOUSE.toString(), - new File(workDir, "warehouse").toString()); - //set where derby logs - File derbyLogFile = new File(workDir + "/derby.log"); - derbyLogFile.createNewFile(); - System.setProperty("derby.stream.error.file", derbyLogFile.getPath()); - - -// Driver driver = new Driver(hiveConf); -// SessionState.start(new CliSessionState(hiveConf)); - - hiveMetaStoreClient = new HiveMetaStoreClient(hiveConf); - } - - private static int findFreePort() throws IOException { - ServerSocket server = new ServerSocket(0); - int port = server.getLocalPort(); - server.close(); - return port; - } - - public static class Builder { - private File workDir; - private int numTaskTrackers = 1; - private JobConf jobConf; - private Configuration hbaseConf; - private HiveConf hiveConf; - - private boolean miniMRClusterEnabled = true; - private boolean miniHBaseClusterEnabled = true; - private boolean miniHiveMetastoreEnabled = true; - private boolean miniZookeeperClusterEnabled = true; - - - private Builder(File workDir) { - this.workDir = workDir; - } - - public Builder numTaskTrackers(int num) { - numTaskTrackers = num; - return this; - } - - public Builder jobConf(JobConf jobConf) { - this.jobConf = jobConf; - return this; - } - - public Builder hbaseConf(Configuration hbaseConf) { - this.hbaseConf = hbaseConf; - return this; - } - - public Builder hiveConf(HiveConf hiveConf) { - this.hiveConf = hiveConf; - return this; - } - - public Builder miniMRClusterEnabled(boolean enabled) { - this.miniMRClusterEnabled = enabled; - return this; - } - - public Builder miniHBaseClusterEnabled(boolean enabled) { - this.miniHBaseClusterEnabled = enabled; - return this; - } - - public Builder miniZookeeperClusterEnabled(boolean enabled) { - this.miniZookeeperClusterEnabled = enabled; - return this; - } - - public Builder miniHiveMetastoreEnabled(boolean enabled) { - this.miniHiveMetastoreEnabled = enabled; - return this; - } - - - public ManyMiniCluster build() { - return new ManyMiniCluster(this); - } - - } -} diff --git hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/SkeletonHBaseTest.java hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/SkeletonHBaseTest.java deleted file mode 100644 index 9495549..0000000 --- hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/SkeletonHBaseTest.java +++ /dev/null @@ -1,237 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hive.hcatalog.hbase; - -import java.io.File; -import java.io.IOException; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Random; -import java.util.Set; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.hive.conf.HiveConf; -import org.junit.AfterClass; -import org.junit.BeforeClass; - -/** - * Base class for HBase Tests which need a mini cluster instance - */ -public abstract class SkeletonHBaseTest { - - protected static String TEST_DIR = "/tmp/build/test/data/"; - - protected final static String DEFAULT_CONTEXT_HANDLE = "default"; - - protected static Map contextMap = new HashMap(); - protected static Set tableNames = new HashSet(); - - /** - * Allow tests to alter the default MiniCluster configuration. - * (requires static initializer block as all setup here is static) - */ - protected static Configuration testConf = null; - - protected void createTable(String tableName, String[] families) { - try { - HBaseAdmin admin = new HBaseAdmin(getHbaseConf()); - HTableDescriptor tableDesc = new HTableDescriptor(tableName); - for (String family : families) { - HColumnDescriptor columnDescriptor = new HColumnDescriptor(family); - tableDesc.addFamily(columnDescriptor); - } - admin.createTable(tableDesc); - } catch (Exception e) { - e.printStackTrace(); - throw new IllegalStateException(e); - } - - } - - protected String newTableName(String prefix) { - String name = null; - int tries = 100; - do { - name = prefix + "_" + Math.abs(new Random().nextLong()); - } while (tableNames.contains(name) && --tries > 0); - if (tableNames.contains(name)) - throw new IllegalStateException("Couldn't find a unique table name, tableNames size: " + tableNames.size()); - tableNames.add(name); - return name; - } - - - /** - * startup an hbase cluster instance before a test suite runs - */ - @BeforeClass - public static void setup() { - if (!contextMap.containsKey(getContextHandle())) - contextMap.put(getContextHandle(), new Context(getContextHandle())); - - contextMap.get(getContextHandle()).start(); - } - - /** - * shutdown an hbase cluster instance ant the end of the test suite - */ - @AfterClass - public static void tearDown() { - contextMap.get(getContextHandle()).stop(); - } - - /** - * override this with a different context handle if tests suites are run simultaneously - * and ManyMiniCluster instances shouldn't be shared - * @return - */ - public static String getContextHandle() { - return DEFAULT_CONTEXT_HANDLE; - } - - /** - * @return working directory for a given test context, which normally is a test suite - */ - public String getTestDir() { - return contextMap.get(getContextHandle()).getTestDir(); - } - - /** - * @return ManyMiniCluster instance - */ - public ManyMiniCluster getCluster() { - return contextMap.get(getContextHandle()).getCluster(); - } - - /** - * @return configuration of MiniHBaseCluster - */ - public Configuration getHbaseConf() { - return contextMap.get(getContextHandle()).getHbaseConf(); - } - - /** - * @return configuration of MiniMRCluster - */ - public Configuration getJobConf() { - return contextMap.get(getContextHandle()).getJobConf(); - } - - /** - * @return configuration of Hive Metastore - */ - public HiveConf getHiveConf() { - return contextMap.get(getContextHandle()).getHiveConf(); - } - - /** - * @return filesystem used by ManyMiniCluster daemons - */ - public FileSystem getFileSystem() { - return contextMap.get(getContextHandle()).getFileSystem(); - } - - /** - * class used to encapsulate a context which is normally used by - * a single TestSuite or across TestSuites when multi-threaded testing is turned on - */ - public static class Context { - protected String testDir; - protected ManyMiniCluster cluster; - - protected Configuration hbaseConf; - protected Configuration jobConf; - protected HiveConf hiveConf; - - protected FileSystem fileSystem; - - protected int usageCount = 0; - - public Context(String handle) { - testDir = new File(TEST_DIR + "/test_" + handle + "_" + Math.abs(new Random().nextLong()) + "/").getPath(); - System.out.println("Cluster work directory: " + testDir); - } - - public void start() { - if (usageCount++ == 0) { - ManyMiniCluster.Builder b = ManyMiniCluster.create(new File(testDir)); - if (testConf != null) { - b.hbaseConf(HBaseConfiguration.create(testConf)); - } - cluster = b.build(); - cluster.start(); - this.hbaseConf = cluster.getHBaseConf(); - jobConf = cluster.getJobConf(); - fileSystem = cluster.getFileSystem(); - hiveConf = cluster.getHiveConf(); - } - } - - public void stop() { - if (--usageCount == 0) { - try { - cluster.stop(); - cluster = null; - } finally { - System.out.println("Trying to cleanup: " + testDir); - try { - FileSystem fs = FileSystem.get(jobConf); - fs.delete(new Path(testDir), true); - } catch (IOException e) { - throw new IllegalStateException("Failed to cleanup test dir", e); - } - - } - } - } - - public String getTestDir() { - return testDir; - } - - public ManyMiniCluster getCluster() { - return cluster; - } - - public Configuration getHbaseConf() { - return hbaseConf; - } - - public Configuration getJobConf() { - return jobConf; - } - - public HiveConf getHiveConf() { - return hiveConf; - } - - public FileSystem getFileSystem() { - return fileSystem; - } - } - -} diff --git hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/TestHBaseBulkOutputFormat.java hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/TestHBaseBulkOutputFormat.java deleted file mode 100644 index df328ef..0000000 --- hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/TestHBaseBulkOutputFormat.java +++ /dev/null @@ -1,631 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hive.hcatalog.hbase; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.client.HTable; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.ResultScanner; -import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hive.cli.CliSessionState; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.ql.session.SessionState; -import org.apache.hadoop.io.BytesWritable; -import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.mapred.JobClient; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.OutputCollector; -import org.apache.hadoop.mapred.Reporter; -import org.apache.hadoop.mapred.RunningJob; -import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.mapreduce.Mapper; -import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; -import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat; -import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; -import org.apache.hive.hcatalog.cli.HCatDriver; -import org.apache.hive.hcatalog.cli.SemanticAnalysis.HCatSemanticAnalyzer; -import org.apache.hive.hcatalog.common.HCatConstants; -import org.apache.hive.hcatalog.common.HCatUtil; -import org.apache.hive.hcatalog.data.DefaultHCatRecord; -import org.apache.hive.hcatalog.data.HCatRecord; -import org.apache.hive.hcatalog.data.schema.HCatSchema; -import org.apache.hive.hcatalog.hbase.HBaseBulkOutputFormat.HBaseBulkOutputCommitter; -import org.apache.hive.hcatalog.hbase.TestHBaseDirectOutputFormat.MapReadAbortedTransaction; -import org.apache.hive.hcatalog.hbase.TestHBaseDirectOutputFormat.MapWriteAbortTransaction; -import org.apache.hive.hcatalog.hbase.snapshot.FamilyRevision; -import org.apache.hive.hcatalog.hbase.snapshot.RevisionManager; -import org.apache.hive.hcatalog.hbase.snapshot.RevisionManagerConfiguration; -import org.apache.hive.hcatalog.hbase.snapshot.TableSnapshot; -import org.apache.hive.hcatalog.hbase.snapshot.Transaction; -import org.apache.hive.hcatalog.mapreduce.HCatInputFormat; -import org.apache.hive.hcatalog.mapreduce.HCatOutputFormat; -import org.apache.hive.hcatalog.mapreduce.OutputJobInfo; - -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.Arrays; -import java.util.List; -import java.util.Map; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -/** - * Tests components of HBaseHCatStorageHandler using ManyMiniCluster. - * Including ImprtSequenceFile and HBaseBulkOutputFormat - */ -public class TestHBaseBulkOutputFormat extends SkeletonHBaseTest { - private final static Logger LOG = LoggerFactory.getLogger(TestHBaseBulkOutputFormat.class); - - private final HiveConf allConf; - private final HCatDriver hcatDriver; - - public TestHBaseBulkOutputFormat() { - allConf = getHiveConf(); - allConf.set(HiveConf.ConfVars.SEMANTIC_ANALYZER_HOOK.varname, - HCatSemanticAnalyzer.class.getName()); - allConf.set(HiveConf.ConfVars.HADOOPFS.varname, getFileSystem().getUri().toString()); - allConf.set(HiveConf.ConfVars.METASTOREWAREHOUSE.varname, new Path(getTestDir(), "warehouse").toString()); - - //Add hbase properties - for (Map.Entry el : getHbaseConf()) - allConf.set(el.getKey(), el.getValue()); - for (Map.Entry el : getJobConf()) - allConf.set(el.getKey(), el.getValue()); - - HBaseConfiguration.merge( - allConf, - RevisionManagerConfiguration.create()); - SessionState.start(new CliSessionState(allConf)); - hcatDriver = new HCatDriver(); - } - - public static class MapWriteOldMapper implements org.apache.hadoop.mapred.Mapper { - - @Override - public void close() throws IOException { - } - - @Override - public void configure(JobConf job) { - } - - @Override - public void map(LongWritable key, Text value, - OutputCollector output, - Reporter reporter) throws IOException { - String vals[] = value.toString().split(","); - Put put = new Put(Bytes.toBytes(vals[0])); - for (int i = 1; i < vals.length; i++) { - String pair[] = vals[i].split(":"); - put.add(Bytes.toBytes("my_family"), - Bytes.toBytes(pair[0]), - Bytes.toBytes(pair[1])); - } - output.collect(new ImmutableBytesWritable(Bytes.toBytes(vals[0])), put); - } - - } - - public static class MapWrite extends Mapper { - - @Override - public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { - String vals[] = value.toString().split(","); - Put put = new Put(Bytes.toBytes(vals[0])); - for (int i = 1; i < vals.length; i++) { - String pair[] = vals[i].split(":"); - put.add(Bytes.toBytes("my_family"), - Bytes.toBytes(pair[0]), - Bytes.toBytes(pair[1])); - } - context.write(new ImmutableBytesWritable(Bytes.toBytes(vals[0])), put); - } - } - - public static class MapHCatWrite extends Mapper { - @Override - public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { - OutputJobInfo jobInfo = (OutputJobInfo) HCatUtil.deserialize(context.getConfiguration().get(HCatConstants.HCAT_KEY_OUTPUT_INFO)); - HCatRecord record = new DefaultHCatRecord(3); - HCatSchema schema = jobInfo.getOutputSchema(); - String vals[] = value.toString().split(","); - record.setInteger("key", schema, Integer.parseInt(vals[0])); - for (int i = 1; i < vals.length; i++) { - String pair[] = vals[i].split(":"); - record.set(pair[0], schema, pair[1]); - } - context.write(null, record); - } - } - - @Test - public void hbaseBulkOutputFormatTest() throws IOException, ClassNotFoundException, InterruptedException { - String testName = "hbaseBulkOutputFormatTest"; - Path methodTestDir = new Path(getTestDir(), testName); - LOG.info("starting: " + testName); - - String tableName = newTableName(testName).toLowerCase(); - String familyName = "my_family"; - byte[] familyNameBytes = Bytes.toBytes(familyName); - - //include hbase config in conf file - Configuration conf = new Configuration(allConf); - - //create table - conf.set(HBaseConstants.PROPERTY_OUTPUT_TABLE_NAME_KEY, tableName); - conf.set("yarn.scheduler.capacity.root.queues", "default"); - conf.set("yarn.scheduler.capacity.root.default.capacity", "100"); - createTable(tableName, new String[]{familyName}); - - String data[] = {"1,english:one,spanish:uno", - "2,english:two,spanish:dos", - "3,english:three,spanish:tres"}; - - - // input/output settings - Path inputPath = new Path(methodTestDir, "mr_input"); - FSDataOutputStream os = getFileSystem().create(new Path(inputPath, "inputFile.txt")); - for (String line : data) - os.write(Bytes.toBytes(line + "\n")); - os.close(); - Path interPath = new Path(methodTestDir, "inter"); - //create job - JobConf job = new JobConf(conf); - job.setWorkingDirectory(new Path(methodTestDir, "mr_work")); - job.setJarByClass(this.getClass()); - job.setMapperClass(MapWriteOldMapper.class); - - job.setInputFormat(org.apache.hadoop.mapred.TextInputFormat.class); - org.apache.hadoop.mapred.TextInputFormat.setInputPaths(job, inputPath); - - job.setOutputFormat(HBaseBulkOutputFormat.class); - org.apache.hadoop.mapred.SequenceFileOutputFormat.setOutputPath(job, interPath); - job.setOutputCommitter(HBaseBulkOutputCommitter.class); - - //manually create transaction - RevisionManager rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(conf); - try { - OutputJobInfo outputJobInfo = OutputJobInfo.create("default", tableName, null); - Transaction txn = rm.beginWriteTransaction(tableName, Arrays.asList(familyName)); - outputJobInfo.getProperties().setProperty(HBaseConstants.PROPERTY_WRITE_TXN_KEY, - HCatUtil.serialize(txn)); - job.set(HCatConstants.HCAT_KEY_OUTPUT_INFO, - HCatUtil.serialize(outputJobInfo)); - } finally { - rm.close(); - } - - job.setMapOutputKeyClass(ImmutableBytesWritable.class); - job.setMapOutputValueClass(HCatRecord.class); - - job.setOutputKeyClass(ImmutableBytesWritable.class); - job.setOutputValueClass(HCatRecord.class); - - job.setNumReduceTasks(0); - - RunningJob runJob = JobClient.runJob(job); - runJob.waitForCompletion(); - assertTrue(runJob.isSuccessful()); - - //verify - HTable table = new HTable(conf, tableName); - Scan scan = new Scan(); - scan.addFamily(familyNameBytes); - ResultScanner scanner = table.getScanner(scan); - int index = 0; - for (Result result : scanner) { - String vals[] = data[index].toString().split(","); - for (int i = 1; i < vals.length; i++) { - String pair[] = vals[i].split(":"); - assertTrue(result.containsColumn(familyNameBytes, Bytes.toBytes(pair[0]))); - assertEquals(pair[1], Bytes.toString(result.getValue(familyNameBytes, Bytes.toBytes(pair[0])))); - } - index++; - } - //test if load count is the same - assertEquals(data.length, index); - //test if scratch directory was erased - assertFalse(FileSystem.get(job).exists(interPath)); - } - - @Test - public void importSequenceFileTest() throws IOException, ClassNotFoundException, InterruptedException { - String testName = "importSequenceFileTest"; - Path methodTestDir = new Path(getTestDir(), testName); - LOG.info("starting: " + testName); - - String tableName = newTableName(testName).toLowerCase(); - String familyName = "my_family"; - byte[] familyNameBytes = Bytes.toBytes(familyName); - - //include hbase config in conf file - Configuration conf = new Configuration(allConf); - - //create table - createTable(tableName, new String[]{familyName}); - - String data[] = {"1,english:one,spanish:uno", - "2,english:two,spanish:dos", - "3,english:three,spanish:tres"}; - - - // input/output settings - Path inputPath = new Path(methodTestDir, "mr_input"); - getFileSystem().mkdirs(inputPath); - FSDataOutputStream os = getFileSystem().create(new Path(inputPath, "inputFile.txt")); - for (String line : data) - os.write(Bytes.toBytes(line + "\n")); - os.close(); - Path interPath = new Path(methodTestDir, "inter"); - Path scratchPath = new Path(methodTestDir, "scratch"); - - - //create job - Job job = new Job(conf, testName); - job.setWorkingDirectory(new Path(methodTestDir, "mr_work")); - job.setJarByClass(this.getClass()); - job.setMapperClass(MapWrite.class); - - job.setInputFormatClass(TextInputFormat.class); - TextInputFormat.setInputPaths(job, inputPath); - - job.setOutputFormatClass(SequenceFileOutputFormat.class); - SequenceFileOutputFormat.setOutputPath(job, interPath); - - job.setMapOutputKeyClass(ImmutableBytesWritable.class); - job.setMapOutputValueClass(Put.class); - - job.setOutputKeyClass(ImmutableBytesWritable.class); - job.setOutputValueClass(Put.class); - - job.setNumReduceTasks(0); - assertTrue(job.waitForCompletion(true)); - - job = new Job(new Configuration(allConf), testName + "_importer"); - assertTrue(ImportSequenceFile.runJob(job, tableName, interPath, scratchPath)); - - //verify - HTable table = new HTable(conf, tableName); - Scan scan = new Scan(); - scan.addFamily(familyNameBytes); - ResultScanner scanner = table.getScanner(scan); - int index = 0; - for (Result result : scanner) { - String vals[] = data[index].toString().split(","); - for (int i = 1; i < vals.length; i++) { - String pair[] = vals[i].split(":"); - assertTrue(result.containsColumn(familyNameBytes, Bytes.toBytes(pair[0]))); - assertEquals(pair[1], Bytes.toString(result.getValue(familyNameBytes, Bytes.toBytes(pair[0])))); - } - index++; - } - //test if load count is the same - assertEquals(data.length, index); - //test if scratch directory was erased - assertFalse(FileSystem.get(job.getConfiguration()).exists(scratchPath)); - } - - @Test - public void bulkModeHCatOutputFormatTest() throws Exception { - String testName = "bulkModeHCatOutputFormatTest"; - Path methodTestDir = new Path(getTestDir(), testName); - LOG.info("starting: " + testName); - - String databaseName = testName.toLowerCase(); - String dbDir = new Path(methodTestDir, "DB_" + testName).toString(); - String tableName = newTableName(testName).toLowerCase(); - String familyName = "my_family"; - byte[] familyNameBytes = Bytes.toBytes(familyName); - - - //include hbase config in conf file - Configuration conf = new Configuration(allConf); - conf.set(HCatConstants.HCAT_KEY_HIVE_CONF, HCatUtil.serialize(allConf.getAllProperties())); - - - String dbquery = "CREATE DATABASE IF NOT EXISTS " + databaseName + " LOCATION '" + dbDir + "'"; - String tableQuery = "CREATE TABLE " + databaseName + "." + tableName + - "(key int, english string, spanish string) STORED BY " + - "'org.apache.hive.hcatalog.hbase.HBaseHCatStorageHandler'" + - "TBLPROPERTIES ('" + HBaseConstants.PROPERTY_BULK_OUTPUT_MODE_KEY + "'='true'," + - "'hbase.columns.mapping'=':key," + familyName + ":english," + familyName + ":spanish')"; - - assertEquals(0, hcatDriver.run(dbquery).getResponseCode()); - assertEquals(0, hcatDriver.run(tableQuery).getResponseCode()); - - String data[] = {"1,english:ONE,spanish:UNO", - "2,english:TWO,spanish:DOS", - "3,english:THREE,spanish:TRES"}; - - // input/output settings - Path inputPath = new Path(methodTestDir, "mr_input"); - getFileSystem().mkdirs(inputPath); - //create multiple files so we can test with multiple mappers - for (int i = 0; i < data.length; i++) { - FSDataOutputStream os = getFileSystem().create(new Path(inputPath, "inputFile" + i + ".txt")); - os.write(Bytes.toBytes(data[i] + "\n")); - os.close(); - } - - //create job - Job job = new Job(conf, testName); - job.setWorkingDirectory(new Path(methodTestDir, "mr_work")); - job.setJarByClass(this.getClass()); - job.setMapperClass(MapHCatWrite.class); - - job.setInputFormatClass(TextInputFormat.class); - TextInputFormat.setInputPaths(job, inputPath); - - - job.setOutputFormatClass(HCatOutputFormat.class); - OutputJobInfo outputJobInfo = OutputJobInfo.create(databaseName, tableName, null); - HCatOutputFormat.setOutput(job, outputJobInfo); - - job.setMapOutputKeyClass(BytesWritable.class); - job.setMapOutputValueClass(HCatRecord.class); - - job.setOutputKeyClass(BytesWritable.class); - job.setOutputValueClass(HCatRecord.class); - - job.setNumReduceTasks(0); - - assertTrue(job.waitForCompletion(true)); - RevisionManager rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(conf); - try { - TableSnapshot snapshot = rm.createSnapshot(databaseName + "." + tableName); - for (String el : snapshot.getColumnFamilies()) { - assertEquals(1, snapshot.getRevision(el)); - } - } finally { - rm.close(); - } - - //verify - HTable table = new HTable(conf, databaseName + "." + tableName); - Scan scan = new Scan(); - scan.addFamily(familyNameBytes); - ResultScanner scanner = table.getScanner(scan); - int index = 0; - for (Result result : scanner) { - String vals[] = data[index].toString().split(","); - for (int i = 1; i < vals.length; i++) { - String pair[] = vals[i].split(":"); - assertTrue(result.containsColumn(familyNameBytes, Bytes.toBytes(pair[0]))); - assertEquals(pair[1], Bytes.toString(result.getValue(familyNameBytes, Bytes.toBytes(pair[0])))); - assertEquals(1l, result.getColumn(familyNameBytes, Bytes.toBytes(pair[0])).get(0).getTimestamp()); - } - index++; - } - //test if load count is the same - assertEquals(data.length, index); - } - - @Test - public void bulkModeHCatOutputFormatTestWithDefaultDB() throws Exception { - String testName = "bulkModeHCatOutputFormatTestWithDefaultDB"; - Path methodTestDir = new Path(getTestDir(), testName); - - String databaseName = "default"; - String dbDir = new Path(methodTestDir, "DB_" + testName).toString(); - String tableName = newTableName(testName).toLowerCase(); - String familyName = "my_family"; - byte[] familyNameBytes = Bytes.toBytes(familyName); - - - //include hbase config in conf file - Configuration conf = new Configuration(allConf); - conf.set(HCatConstants.HCAT_KEY_HIVE_CONF, HCatUtil.serialize(allConf.getAllProperties())); - - - String dbquery = "CREATE DATABASE IF NOT EXISTS " + databaseName + " LOCATION '" + dbDir + "'"; - String tableQuery = "CREATE TABLE " + databaseName + "." + tableName + - "(key int, english string, spanish string) STORED BY " + - "'org.apache.hive.hcatalog.hbase.HBaseHCatStorageHandler'" + - "TBLPROPERTIES ('" + HBaseConstants.PROPERTY_BULK_OUTPUT_MODE_KEY + "'='true'," + - "'hbase.columns.mapping'=':key," + familyName + ":english," + familyName + ":spanish')"; - - assertEquals(0, hcatDriver.run(dbquery).getResponseCode()); - assertEquals(0, hcatDriver.run(tableQuery).getResponseCode()); - - String data[] = {"1,english:ONE,spanish:UNO", - "2,english:TWO,spanish:DOS", - "3,english:THREE,spanish:TRES"}; - - // input/output settings - Path inputPath = new Path(methodTestDir, "mr_input"); - getFileSystem().mkdirs(inputPath); - FSDataOutputStream os = getFileSystem().create(new Path(inputPath, "inputFile.txt")); - for (String line : data) - os.write(Bytes.toBytes(line + "\n")); - os.close(); - - //create job - Job job = new Job(conf, testName); - job.setWorkingDirectory(new Path(methodTestDir, "mr_work")); - job.setJarByClass(this.getClass()); - job.setMapperClass(MapHCatWrite.class); - - job.setInputFormatClass(TextInputFormat.class); - TextInputFormat.setInputPaths(job, inputPath); - - - job.setOutputFormatClass(HCatOutputFormat.class); - OutputJobInfo outputJobInfo = OutputJobInfo.create(databaseName, tableName, null); - HCatOutputFormat.setOutput(job, outputJobInfo); - - job.setMapOutputKeyClass(BytesWritable.class); - job.setMapOutputValueClass(HCatRecord.class); - - job.setOutputKeyClass(BytesWritable.class); - job.setOutputValueClass(HCatRecord.class); - - job.setNumReduceTasks(0); - - assertTrue(job.waitForCompletion(true)); - - //verify - HTable table = new HTable(conf, tableName); - Scan scan = new Scan(); - scan.addFamily(familyNameBytes); - ResultScanner scanner = table.getScanner(scan); - int index = 0; - for (Result result : scanner) { - String vals[] = data[index].toString().split(","); - for (int i = 1; i < vals.length; i++) { - String pair[] = vals[i].split(":"); - assertTrue(result.containsColumn(familyNameBytes, Bytes.toBytes(pair[0]))); - assertEquals(pair[1], Bytes.toString(result.getValue(familyNameBytes, Bytes.toBytes(pair[0])))); - } - index++; - } - //test if load count is the same - assertEquals(data.length, index); - } - - @Test - public void bulkModeAbortTest() throws Exception { - String testName = "bulkModeAbortTest"; - Path methodTestDir = new Path(getTestDir(), testName); - String databaseName = testName.toLowerCase(); - String dbDir = new Path(methodTestDir, "DB_" + testName).toString(); - String tableName = newTableName(testName).toLowerCase(); - String familyName = "my_family"; - - // include hbase config in conf file - Configuration conf = new Configuration(allConf); - conf.set(HCatConstants.HCAT_KEY_HIVE_CONF, HCatUtil.serialize(allConf.getAllProperties())); - - String dbquery = "CREATE DATABASE IF NOT EXISTS " + databaseName + " LOCATION '" + dbDir - + "'"; - String tableQuery = "CREATE TABLE " + databaseName + "." + tableName + - "(key int, english string, spanish string) STORED BY " + - "'org.apache.hive.hcatalog.hbase.HBaseHCatStorageHandler'" + - "TBLPROPERTIES ('" + HBaseConstants.PROPERTY_BULK_OUTPUT_MODE_KEY + "'='true'," + - "'hbase.columns.mapping'=':key," + familyName + ":english," + familyName - + ":spanish')"; - - assertEquals(0, hcatDriver.run(dbquery).getResponseCode()); - assertEquals(0, hcatDriver.run(tableQuery).getResponseCode()); - - String data[] = {"1,english:ONE,spanish:UNO", - "2,english:TWO,spanish:DOS", - "3,english:THREE,spanish:TRES"}; - - Path inputPath = new Path(methodTestDir, "mr_input"); - getFileSystem().mkdirs(inputPath); - // create multiple files so we can test with multiple mappers - for (int i = 0; i < data.length; i++) { - FSDataOutputStream os = getFileSystem().create( - new Path(inputPath, "inputFile" + i + ".txt")); - os.write(Bytes.toBytes(data[i] + "\n")); - os.close(); - } - - Path workingDir = new Path(methodTestDir, "mr_abort"); - OutputJobInfo outputJobInfo = OutputJobInfo.create(databaseName, - tableName, null); - Job job = configureJob(testName, - conf, workingDir, MapWriteAbortTransaction.class, - outputJobInfo, inputPath); - assertFalse(job.waitForCompletion(true)); - - // verify that revision manager has it as aborted transaction - RevisionManager rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(conf); - try { - TableSnapshot snapshot = rm.createSnapshot(databaseName + "." + tableName); - for (String family : snapshot.getColumnFamilies()) { - assertEquals(1, snapshot.getRevision(family)); - List abortedWriteTransactions = rm.getAbortedWriteTransactions( - databaseName + "." + tableName, family); - assertEquals(1, abortedWriteTransactions.size()); - assertEquals(1, abortedWriteTransactions.get(0).getRevision()); - } - } finally { - rm.close(); - } - - //verify that hbase does not have any of the records. - //Since records are only written during commitJob, - //hbase should not have any records. - HTable table = new HTable(conf, databaseName + "." + tableName); - Scan scan = new Scan(); - scan.addFamily(Bytes.toBytes(familyName)); - ResultScanner scanner = table.getScanner(scan); - assertFalse(scanner.iterator().hasNext()); - - // verify that the storage handler input format returns empty results. - Path outputDir = new Path(getTestDir(), - "mapred/testHBaseTableBulkIgnoreAbortedTransactions"); - FileSystem fs = getFileSystem(); - if (fs.exists(outputDir)) { - fs.delete(outputDir, true); - } - job = new Job(conf, "hbase-bulk-aborted-transaction"); - job.setJarByClass(this.getClass()); - job.setMapperClass(MapReadAbortedTransaction.class); - job.setInputFormatClass(HCatInputFormat.class); - HCatInputFormat.setInput(job, databaseName, tableName); - job.setOutputFormatClass(TextOutputFormat.class); - TextOutputFormat.setOutputPath(job, outputDir); - job.setMapOutputKeyClass(BytesWritable.class); - job.setMapOutputValueClass(Text.class); - job.setOutputKeyClass(BytesWritable.class); - job.setOutputValueClass(Text.class); - job.setNumReduceTasks(0); - assertTrue(job.waitForCompletion(true)); - } - - private Job configureJob(String jobName, Configuration conf, - Path workingDir, Class mapperClass, - OutputJobInfo outputJobInfo, Path inputPath) throws IOException { - Job job = new Job(conf, jobName); - job.setWorkingDirectory(workingDir); - job.setJarByClass(this.getClass()); - job.setMapperClass(mapperClass); - - job.setInputFormatClass(TextInputFormat.class); - TextInputFormat.setInputPaths(job, inputPath); - job.setOutputFormatClass(HCatOutputFormat.class); - HCatOutputFormat.setOutput(job, outputJobInfo); - - job.setMapOutputKeyClass(BytesWritable.class); - job.setMapOutputValueClass(HCatRecord.class); - job.setOutputKeyClass(BytesWritable.class); - job.setOutputValueClass(HCatRecord.class); - - job.setNumReduceTasks(0); - return job; - } - -} - diff --git hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/TestHBaseDirectOutputFormat.java hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/TestHBaseDirectOutputFormat.java deleted file mode 100644 index aadf472..0000000 --- hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/TestHBaseDirectOutputFormat.java +++ /dev/null @@ -1,501 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hive.hcatalog.hbase; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.client.HTable; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.ResultScanner; -import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.hadoop.hbase.mapred.TableOutputFormat; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hive.cli.CliSessionState; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.ql.session.SessionState; -import org.apache.hadoop.io.BytesWritable; -import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.io.WritableComparable; -import org.apache.hadoop.mapred.JobClient; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.OutputCollector; -import org.apache.hadoop.mapred.Reporter; -import org.apache.hadoop.mapred.RunningJob; -import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.mapreduce.Mapper; -import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; -import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; -import org.apache.hive.hcatalog.cli.HCatDriver; -import org.apache.hive.hcatalog.cli.SemanticAnalysis.HCatSemanticAnalyzer; -import org.apache.hive.hcatalog.common.HCatConstants; -import org.apache.hive.hcatalog.common.HCatUtil; -import org.apache.hive.hcatalog.data.DefaultHCatRecord; -import org.apache.hive.hcatalog.data.HCatRecord; -import org.apache.hive.hcatalog.data.schema.HCatSchema; -import org.apache.hive.hcatalog.hbase.snapshot.FamilyRevision; -import org.apache.hive.hcatalog.hbase.snapshot.RevisionManager; -import org.apache.hive.hcatalog.hbase.snapshot.RevisionManagerConfiguration; -import org.apache.hive.hcatalog.hbase.snapshot.TableSnapshot; -import org.apache.hive.hcatalog.hbase.snapshot.Transaction; -import org.apache.hive.hcatalog.mapreduce.HCatInputFormat; -import org.apache.hive.hcatalog.mapreduce.HCatOutputFormat; -import org.apache.hive.hcatalog.mapreduce.OutputJobInfo; -import org.junit.Test; - -import java.io.IOException; -import java.util.Arrays; -import java.util.List; -import java.util.Map; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotSame; -import static org.junit.Assert.assertTrue; - -/** - * Test HBaseDirectOUtputFormat and HBaseHCatStorageHandler using a MiniCluster - */ -public class TestHBaseDirectOutputFormat extends SkeletonHBaseTest { - - private final HiveConf allConf; - private final HCatDriver hcatDriver; - - public TestHBaseDirectOutputFormat() { - allConf = getHiveConf(); - allConf.set(HiveConf.ConfVars.SEMANTIC_ANALYZER_HOOK.varname, - HCatSemanticAnalyzer.class.getName()); - allConf.set(HiveConf.ConfVars.HADOOPFS.varname, getFileSystem().getUri().toString()); - allConf.set(HiveConf.ConfVars.METASTOREWAREHOUSE.varname, new Path(getTestDir(), "warehouse").toString()); - - //Add hbase properties - for (Map.Entry el : getHbaseConf()) - allConf.set(el.getKey(), el.getValue()); - for (Map.Entry el : getJobConf()) - allConf.set(el.getKey(), el.getValue()); - HBaseConfiguration.merge( - allConf, - RevisionManagerConfiguration.create()); - SessionState.start(new CliSessionState(allConf)); - hcatDriver = new HCatDriver(); - } - - @Test - public void directOutputFormatTest() throws IOException, ClassNotFoundException, InterruptedException { - String testName = "directOutputFormatTest"; - Path methodTestDir = new Path(getTestDir(), testName); - - String tableName = newTableName(testName).toLowerCase(); - String familyName = "my_family"; - byte[] familyNameBytes = Bytes.toBytes(familyName); - - //include hbase config in conf file - Configuration conf = new Configuration(allConf); - conf.set(HCatConstants.HCAT_KEY_HIVE_CONF, HCatUtil.serialize(allConf.getAllProperties())); - - //create table - createTable(tableName, new String[]{familyName}); - - String data[] = {"1,english:ONE,spanish:UNO", - "2,english:ONE,spanish:DOS", - "3,english:ONE,spanish:TRES"}; - - - // input/output settings - Path inputPath = new Path(methodTestDir, "mr_input"); - getFileSystem().mkdirs(inputPath); - FSDataOutputStream os = getFileSystem().create(new Path(inputPath, "inputFile.txt")); - for (String line : data) - os.write(Bytes.toBytes(line + "\n")); - os.close(); - - //create job - JobConf job = new JobConf(conf); - job.setJobName(testName); - job.setWorkingDirectory(new Path(methodTestDir, "mr_work")); - job.setJarByClass(this.getClass()); - job.setMapperClass(MapWrite.class); - - job.setInputFormat(org.apache.hadoop.mapred.TextInputFormat.class); - org.apache.hadoop.mapred.TextInputFormat.setInputPaths(job, inputPath); - - job.setOutputFormat(HBaseDirectOutputFormat.class); - job.set(TableOutputFormat.OUTPUT_TABLE, tableName); - job.set(HBaseConstants.PROPERTY_OUTPUT_TABLE_NAME_KEY, tableName); - - //manually create transaction - RevisionManager rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(conf); - try { - OutputJobInfo outputJobInfo = OutputJobInfo.create("default", tableName, null); - Transaction txn = rm.beginWriteTransaction(tableName, Arrays.asList(familyName)); - outputJobInfo.getProperties().setProperty(HBaseConstants.PROPERTY_WRITE_TXN_KEY, - HCatUtil.serialize(txn)); - job.set(HCatConstants.HCAT_KEY_OUTPUT_INFO, - HCatUtil.serialize(outputJobInfo)); - } finally { - rm.close(); - } - - job.setMapOutputKeyClass(BytesWritable.class); - job.setMapOutputValueClass(HCatRecord.class); - job.setOutputKeyClass(BytesWritable.class); - job.setOutputValueClass(HCatRecord.class); - job.setNumReduceTasks(0); - - RunningJob runJob = JobClient.runJob(job); - runJob.waitForCompletion(); - assertTrue(runJob.isSuccessful()); - - //verify - HTable table = new HTable(conf, tableName); - Scan scan = new Scan(); - scan.addFamily(familyNameBytes); - ResultScanner scanner = table.getScanner(scan); - int index = 0; - for (Result result : scanner) { - String vals[] = data[index].toString().split(","); - for (int i = 1; i < vals.length; i++) { - String pair[] = vals[i].split(":"); - assertTrue(result.containsColumn(familyNameBytes, Bytes.toBytes(pair[0]))); - assertEquals(pair[1], Bytes.toString(result.getValue(familyNameBytes, Bytes.toBytes(pair[0])))); - } - index++; - } - assertEquals(data.length, index); - } - - @Test - public void directHCatOutputFormatTest() throws Exception { - String testName = "directHCatOutputFormatTest"; - Path methodTestDir = new Path(getTestDir(), testName); - - String databaseName = testName; - String dbDir = new Path(methodTestDir, "DB_" + testName).toString(); - String tableName = newTableName(testName); - String familyName = "my_family"; - byte[] familyNameBytes = Bytes.toBytes(familyName); - //Table name will be lower case unless specified by hbase.table.name property - String hbaseTableName = (databaseName + "." + tableName).toLowerCase(); - - //include hbase config in conf file - Configuration conf = new Configuration(allConf); - conf.set(HCatConstants.HCAT_KEY_HIVE_CONF, HCatUtil.serialize(allConf.getAllProperties())); - - - String dbquery = "CREATE DATABASE IF NOT EXISTS " + databaseName + " LOCATION '" + dbDir + "'"; - String tableQuery = "CREATE TABLE " + databaseName + "." + tableName + - "(key int, english string, spanish string) STORED BY " + - "'org.apache.hive.hcatalog.hbase.HBaseHCatStorageHandler'" + - "TBLPROPERTIES (" + - "'hbase.columns.mapping'=':key," + familyName + ":english," + familyName + ":spanish')"; - - assertEquals(0, hcatDriver.run(dbquery).getResponseCode()); - assertEquals(0, hcatDriver.run(tableQuery).getResponseCode()); - - String data[] = {"1,english:ONE,spanish:UNO", - "2,english:ONE,spanish:DOS", - "3,english:ONE,spanish:TRES"}; - - // input/output settings - Path inputPath = new Path(methodTestDir, "mr_input"); - getFileSystem().mkdirs(inputPath); - //create multiple files so we can test with multiple mappers - for (int i = 0; i < data.length; i++) { - FSDataOutputStream os = getFileSystem().create(new Path(inputPath, "inputFile" + i + ".txt")); - os.write(Bytes.toBytes(data[i] + "\n")); - os.close(); - } - - //create job - Path workingDir = new Path(methodTestDir, "mr_work"); - OutputJobInfo outputJobInfo = OutputJobInfo.create(databaseName, - tableName, null); - Job job = configureJob(testName, conf, workingDir, MapHCatWrite.class, - outputJobInfo, inputPath); - assertTrue(job.waitForCompletion(true)); - - RevisionManager rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(conf); - try { - TableSnapshot snapshot = rm.createSnapshot(hbaseTableName); - for (String el : snapshot.getColumnFamilies()) { - assertEquals(1, snapshot.getRevision(el)); - } - } finally { - rm.close(); - } - - //verify - HTable table = new HTable(conf, hbaseTableName); - Scan scan = new Scan(); - scan.addFamily(familyNameBytes); - ResultScanner scanner = table.getScanner(scan); - int index = 0; - for (Result result : scanner) { - String vals[] = data[index].toString().split(","); - for (int i = 1; i < vals.length; i++) { - String pair[] = vals[i].split(":"); - assertTrue(result.containsColumn(familyNameBytes, Bytes.toBytes(pair[0]))); - assertEquals(pair[1], Bytes.toString(result.getValue(familyNameBytes, Bytes.toBytes(pair[0])))); - assertEquals(1l, result.getColumn(familyNameBytes, Bytes.toBytes(pair[0])).get(0).getTimestamp()); - } - index++; - } - assertEquals(data.length, index); - } - - @Test - public void directModeAbortTest() throws Exception { - String testName = "directModeAbortTest"; - Path methodTestDir = new Path(getTestDir(), testName); - String databaseName = testName; - String dbDir = new Path(methodTestDir, "DB_" + testName).toString(); - String tableName = newTableName(testName); - String familyName = "my_family"; - byte[] familyNameBytes = Bytes.toBytes(familyName); - //Table name as specified by hbase.table.name property - String hbaseTableName = tableName; - - // include hbase config in conf file - Configuration conf = new Configuration(allConf); - conf.set(HCatConstants.HCAT_KEY_HIVE_CONF, HCatUtil.serialize(allConf.getAllProperties())); - - String dbquery = "CREATE DATABASE IF NOT EXISTS " + databaseName + " LOCATION '" + dbDir - + "'"; - String tableQuery = "CREATE TABLE " + databaseName + "." + tableName + - "(key int, english string, spanish string) STORED BY " + - "'org.apache.hive.hcatalog.hbase.HBaseHCatStorageHandler'" + - "TBLPROPERTIES (" + - "'hbase.columns.mapping'=':key," + familyName + ":english," + familyName + - ":spanish','hbase.table.name'='" + hbaseTableName + "')"; - - assertEquals(0, hcatDriver.run(dbquery).getResponseCode()); - assertEquals(0, hcatDriver.run(tableQuery).getResponseCode()); - - String data[] = {"1,english:ONE,spanish:UNO", - "2,english:TWO,spanish:DOS", - "3,english:THREE,spanish:TRES"}; - - Path inputPath = new Path(methodTestDir, "mr_input"); - getFileSystem().mkdirs(inputPath); - // create multiple files so we can test with multiple mappers - for (int i = 0; i < data.length; i++) { - FSDataOutputStream os = getFileSystem().create( - new Path(inputPath, "inputFile" + i + ".txt")); - os.write(Bytes.toBytes(data[i] + "\n")); - os.close(); - } - - Path workingDir = new Path(methodTestDir, "mr_abort"); - OutputJobInfo outputJobInfo = OutputJobInfo.create(databaseName, - tableName, null); - Job job = configureJob(testName, conf, workingDir, MapWriteAbortTransaction.class, - outputJobInfo, inputPath); - assertFalse(job.waitForCompletion(true)); - - // verify that revision manager has it as aborted transaction - RevisionManager rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(conf); - try { - TableSnapshot snapshot = rm.createSnapshot(hbaseTableName); - for (String family : snapshot.getColumnFamilies()) { - assertEquals(1, snapshot.getRevision(family)); - List abortedWriteTransactions = rm.getAbortedWriteTransactions( - hbaseTableName, family); - assertEquals(1, abortedWriteTransactions.size()); - assertEquals(1, abortedWriteTransactions.get(0).getRevision()); - } - } finally { - rm.close(); - } - - // verify that hbase has the records of the successful maps. - HTable table = new HTable(conf, hbaseTableName); - Scan scan = new Scan(); - scan.addFamily(familyNameBytes); - ResultScanner scanner = table.getScanner(scan); - int count = 0; - for (Result result : scanner) { - String key = Bytes.toString(result.getRow()); - assertNotSame(MapWriteAbortTransaction.failedKey, key); - int index = Integer.parseInt(key) - 1; - String vals[] = data[index].toString().split(","); - for (int i = 1; i < vals.length; i++) { - String pair[] = vals[i].split(":"); - assertTrue(result.containsColumn(familyNameBytes, Bytes.toBytes(pair[0]))); - assertEquals(pair[1], - Bytes.toString(result.getValue(familyNameBytes, Bytes.toBytes(pair[0])))); - assertEquals(1l, result.getColumn(familyNameBytes, Bytes.toBytes(pair[0])).get(0) - .getTimestamp()); - } - count++; - } - assertEquals(data.length - 1, count); - - // verify that the inputformat returns empty results. - Path outputDir = new Path(getTestDir(), - "mapred/testHBaseTableIgnoreAbortedTransactions"); - FileSystem fs = getFileSystem(); - if (fs.exists(outputDir)) { - fs.delete(outputDir, true); - } - job = new Job(conf, "hbase-aborted-transaction"); - job.setJarByClass(this.getClass()); - job.setMapperClass(MapReadAbortedTransaction.class); - job.setInputFormatClass(HCatInputFormat.class); - HCatInputFormat.setInput(job, databaseName, tableName); - job.setOutputFormatClass(TextOutputFormat.class); - TextOutputFormat.setOutputPath(job, outputDir); - job.setMapOutputKeyClass(BytesWritable.class); - job.setMapOutputValueClass(Text.class); - job.setOutputKeyClass(BytesWritable.class); - job.setOutputValueClass(Text.class); - job.setNumReduceTasks(0); - assertTrue(job.waitForCompletion(true)); - } - - private Job configureJob(String jobName, Configuration conf, - Path workingDir, Class mapperClass, - OutputJobInfo outputJobInfo, Path inputPath) throws IOException { - Job job = new Job(conf, jobName); - job.setWorkingDirectory(workingDir); - job.setJarByClass(this.getClass()); - job.setMapperClass(mapperClass); - - job.setInputFormatClass(TextInputFormat.class); - TextInputFormat.setInputPaths(job, inputPath); - job.setOutputFormatClass(HCatOutputFormat.class); - HCatOutputFormat.setOutput(job, outputJobInfo); - String txnString = job.getConfiguration().get(HBaseConstants.PROPERTY_WRITE_TXN_KEY); - //Test passing in same OutputJobInfo multiple times and verify 1 transaction is created - String jobString = job.getConfiguration().get(HCatConstants.HCAT_KEY_OUTPUT_INFO); - outputJobInfo = (OutputJobInfo) HCatUtil.deserialize(jobString); - Job job2 = new Job(conf); - HCatOutputFormat.setOutput(job2, outputJobInfo); - assertEquals(txnString, job2.getConfiguration().get(HBaseConstants.PROPERTY_WRITE_TXN_KEY)); - job.setMapOutputKeyClass(BytesWritable.class); - job.setMapOutputValueClass(HCatRecord.class); - job.setOutputKeyClass(BytesWritable.class); - job.setOutputValueClass(HCatRecord.class); - - job.setNumReduceTasks(0); - return job; - } - - public static class MapHCatWrite extends Mapper { - - @Override - public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { - OutputJobInfo jobInfo = (OutputJobInfo) HCatUtil.deserialize(context.getConfiguration().get(HCatConstants.HCAT_KEY_OUTPUT_INFO)); - HCatRecord record = new DefaultHCatRecord(3); - HCatSchema schema = jobInfo.getOutputSchema(); - String vals[] = value.toString().split(","); - record.setInteger("key", schema, Integer.parseInt(vals[0])); - for (int i = 1; i < vals.length; i++) { - String pair[] = vals[i].split(":"); - record.set(pair[0], schema, pair[1]); - } - context.write(null, record); - } - } - - public static class MapWrite implements org.apache.hadoop.mapred.Mapper { - - @Override - public void configure(JobConf job) { - } - - @Override - public void close() throws IOException { - } - - @Override - public void map(LongWritable key, Text value, - OutputCollector output, Reporter reporter) - throws IOException { - String vals[] = value.toString().split(","); - Put put = new Put(Bytes.toBytes(vals[0])); - for (int i = 1; i < vals.length; i++) { - String pair[] = vals[i].split(":"); - put.add(Bytes.toBytes("my_family"), - Bytes.toBytes(pair[0]), - Bytes.toBytes(pair[1])); - } - output.collect(null, put); - } - } - - static class MapWriteAbortTransaction extends Mapper { - public static String failedKey; - private static int count = 0; - - @Override - public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { - OutputJobInfo jobInfo = (OutputJobInfo) HCatUtil.deserialize(context.getConfiguration().get(HCatConstants.HCAT_KEY_OUTPUT_INFO)); - HCatRecord record = new DefaultHCatRecord(3); - HCatSchema schema = jobInfo.getOutputSchema(); - String vals[] = value.toString().split(","); - record.setInteger("key", schema, Integer.parseInt(vals[0])); - synchronized (MapWriteAbortTransaction.class) { - if (count == 2) { - failedKey = vals[0]; - throw new IOException("Failing map to test abort"); - } - for (int i = 1; i < vals.length; i++) { - String pair[] = vals[i].split(":"); - record.set(pair[0], schema, pair[1]); - } - context.write(null, record); - count++; - } - - } - - } - - static class MapReadAbortedTransaction - extends - Mapper, Text> { - - @Override - public void run(Context context) throws IOException, - InterruptedException { - setup(context); - if (context.nextKeyValue()) { - map(context.getCurrentKey(), context.getCurrentValue(), context); - while (context.nextKeyValue()) { - map(context.getCurrentKey(), context.getCurrentValue(), - context); - } - throw new IOException("There should have been no records"); - } - cleanup(context); - } - - @Override - public void map(ImmutableBytesWritable key, HCatRecord value, - Context context) throws IOException, InterruptedException { - System.out.println("HCat record value" + value.toString()); - } - } -} diff --git hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/TestHBaseHCatStorageHandler.java hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/TestHBaseHCatStorageHandler.java deleted file mode 100644 index f1fd54a..0000000 --- hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/TestHBaseHCatStorageHandler.java +++ /dev/null @@ -1,241 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.hive.hcatalog.hbase; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.net.URI; -import java.util.Map; - -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hive.cli.CliSessionState; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.conf.HiveConf.ConfVars; -import org.apache.hadoop.hive.metastore.Warehouse; -import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; -import org.apache.hadoop.hive.ql.session.SessionState; -import org.apache.hive.hcatalog.cli.HCatDriver; -import org.apache.hive.hcatalog.cli.SemanticAnalysis.HCatSemanticAnalyzer; -import org.apache.hive.hcatalog.hbase.snapshot.RevisionManager; -import org.apache.hive.hcatalog.hbase.snapshot.RevisionManagerConfiguration; -import org.apache.zookeeper.KeeperException.NoNodeException; -import org.junit.Test; - -public class TestHBaseHCatStorageHandler extends SkeletonHBaseTest { - - private static HiveConf hcatConf; - private static HCatDriver hcatDriver; - private static Warehouse wh; - - public void Initialize() throws Exception { - - hcatConf = getHiveConf(); - hcatConf.set(ConfVars.SEMANTIC_ANALYZER_HOOK.varname, - HCatSemanticAnalyzer.class.getName()); - URI fsuri = getFileSystem().getUri(); - Path whPath = new Path(fsuri.getScheme(), fsuri.getAuthority(), - getTestDir()); - hcatConf.set(HiveConf.ConfVars.HADOOPFS.varname, fsuri.toString()); - hcatConf.set(ConfVars.METASTOREWAREHOUSE.varname, whPath.toString()); - - //Add hbase properties - for (Map.Entry el : getHbaseConf()) { - if (el.getKey().startsWith("hbase.")) { - hcatConf.set(el.getKey(), el.getValue()); - } - } - HBaseConfiguration.merge( - hcatConf, - RevisionManagerConfiguration.create()); - - SessionState.start(new CliSessionState(hcatConf)); - hcatDriver = new HCatDriver(); - - } - - @Test - public void testTableCreateDrop() throws Exception { - Initialize(); - - hcatDriver.run("drop table test_table"); - CommandProcessorResponse response = hcatDriver - .run("create table test_table(key int, value string) STORED BY " + - "'org.apache.hive.hcatalog.hbase.HBaseHCatStorageHandler'" - + "TBLPROPERTIES ('hbase.columns.mapping'=':key,cf1:val')"); - - assertEquals(0, response.getResponseCode()); - - HBaseAdmin hAdmin = new HBaseAdmin(getHbaseConf()); - boolean doesTableExist = hAdmin.tableExists("test_table"); - - assertTrue(doesTableExist); - - RevisionManager rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(hcatConf); - rm.open(); - //Should be able to successfully query revision manager - rm.getAbortedWriteTransactions("test_table", "cf1"); - - hcatDriver.run("drop table test_table"); - doesTableExist = hAdmin.tableExists("test_table"); - assertTrue(doesTableExist == false); - - try { - rm.getAbortedWriteTransactions("test_table", "cf1"); - } catch (Exception e) { - assertTrue(e.getCause() instanceof NoNodeException); - } - rm.close(); - - } - - @Test - public void testTableCreateDropDifferentCase() throws Exception { - Initialize(); - - hcatDriver.run("drop table test_Table"); - CommandProcessorResponse response = hcatDriver - .run("create table test_Table(key int, value string) STORED BY " + - "'org.apache.hive.hcatalog.hbase.HBaseHCatStorageHandler'" - + "TBLPROPERTIES ('hbase.columns.mapping'=':key,cf1:val')"); - - assertEquals(0, response.getResponseCode()); - - //HBase table gets created with lower case unless specified as a table property. - HBaseAdmin hAdmin = new HBaseAdmin(getHbaseConf()); - boolean doesTableExist = hAdmin.tableExists("test_table"); - - assertTrue(doesTableExist); - - RevisionManager rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(hcatConf); - rm.open(); - //Should be able to successfully query revision manager - rm.getAbortedWriteTransactions("test_table", "cf1"); - - hcatDriver.run("drop table test_table"); - doesTableExist = hAdmin.tableExists("test_table"); - assertTrue(doesTableExist == false); - - try { - rm.getAbortedWriteTransactions("test_table", "cf1"); - } catch (Exception e) { - assertTrue(e.getCause() instanceof NoNodeException); - } - rm.close(); - - } - - @Test - public void testTableCreateDropCaseSensitive() throws Exception { - Initialize(); - - hcatDriver.run("drop table test_Table"); - CommandProcessorResponse response = hcatDriver - .run("create table test_Table(key int, value string) STORED BY " + - "'org.apache.hive.hcatalog.hbase.HBaseHCatStorageHandler'" - + "TBLPROPERTIES ('hbase.columns.mapping'=':key,cf1:val'," + - " 'hbase.table.name'='CaseSensitiveTable')"); - - assertEquals(0, response.getResponseCode()); - - HBaseAdmin hAdmin = new HBaseAdmin(getHbaseConf()); - boolean doesTableExist = hAdmin.tableExists("CaseSensitiveTable"); - - assertTrue(doesTableExist); - - RevisionManager rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(hcatConf); - rm.open(); - //Should be able to successfully query revision manager - rm.getAbortedWriteTransactions("CaseSensitiveTable", "cf1"); - - hcatDriver.run("drop table test_table"); - doesTableExist = hAdmin.tableExists("CaseSensitiveTable"); - assertTrue(doesTableExist == false); - - try { - rm.getAbortedWriteTransactions("CaseSensitiveTable", "cf1"); - } catch (Exception e) { - assertTrue(e.getCause() instanceof NoNodeException); - } - rm.close(); - - } - - @Test - public void testTableDropNonExistent() throws Exception { - Initialize(); - - hcatDriver.run("drop table mytable"); - CommandProcessorResponse response = hcatDriver - .run("create table mytable(key int, value string) STORED BY " + - "'org.apache.hive.hcatalog.hbase.HBaseHCatStorageHandler'" - + "TBLPROPERTIES ('hbase.columns.mapping'=':key,cf1:val')"); - - assertEquals(0, response.getResponseCode()); - - HBaseAdmin hAdmin = new HBaseAdmin(getHbaseConf()); - boolean doesTableExist = hAdmin.tableExists("mytable"); - assertTrue(doesTableExist); - - //Now delete the table from hbase - if (hAdmin.isTableEnabled("mytable")) { - hAdmin.disableTable("mytable"); - } - hAdmin.deleteTable("mytable"); - doesTableExist = hAdmin.tableExists("mytable"); - assertTrue(doesTableExist == false); - - CommandProcessorResponse responseTwo = hcatDriver.run("drop table mytable"); - assertTrue(responseTwo.getResponseCode() == 0); - - } - - @Test - public void testTableCreateExternal() throws Exception { - - String tableName = "testTable"; - HBaseAdmin hAdmin = new HBaseAdmin(getHbaseConf()); - - HTableDescriptor tableDesc = new HTableDescriptor(tableName); - tableDesc.addFamily(new HColumnDescriptor(Bytes.toBytes("key"))); - tableDesc.addFamily(new HColumnDescriptor(Bytes.toBytes("familyone"))); - tableDesc.addFamily(new HColumnDescriptor(Bytes.toBytes("familytwo"))); - - hAdmin.createTable(tableDesc); - boolean doesTableExist = hAdmin.tableExists(tableName); - assertTrue(doesTableExist); - - hcatDriver.run("drop table mytabletwo"); - CommandProcessorResponse response = hcatDriver - .run("create external table mytabletwo(key int, valueone string, valuetwo string) STORED BY " + - "'org.apache.hive.hcatalog.hbase.HBaseHCatStorageHandler'" - + "TBLPROPERTIES ('hbase.columns.mapping'=':key,familyone:val,familytwo:val'," + - "'hbase.table.name'='testTable')"); - - assertEquals(0, response.getResponseCode()); - - } - - -} diff --git hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/TestHBaseInputFormat.java hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/TestHBaseInputFormat.java deleted file mode 100644 index 92430d1..0000000 --- hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/TestHBaseInputFormat.java +++ /dev/null @@ -1,609 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.hive.hcatalog.hbase; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -import java.io.IOException; -import java.net.URI; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.hbase.client.HTable; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.hadoop.hbase.mapreduce.TableInputFormat; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hive.cli.CliSessionState; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.conf.HiveConf.ConfVars; -import org.apache.hadoop.hive.metastore.MetaStoreUtils; -import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; -import org.apache.hadoop.hive.ql.session.SessionState; -import org.apache.hadoop.io.BytesWritable; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.io.WritableComparable; -import org.apache.hadoop.mapred.JobClient; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.OutputCollector; -import org.apache.hadoop.mapred.Reporter; -import org.apache.hadoop.mapred.RunningJob; -import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.mapreduce.Mapper; -import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; -import org.apache.hive.hcatalog.cli.HCatDriver; -import org.apache.hive.hcatalog.cli.SemanticAnalysis.HCatSemanticAnalyzer; -import org.apache.hive.hcatalog.common.HCatConstants; -import org.apache.hive.hcatalog.common.HCatException; -import org.apache.hive.hcatalog.common.HCatUtil; -import org.apache.hive.hcatalog.data.HCatRecord; -import org.apache.hive.hcatalog.data.schema.HCatFieldSchema; -import org.apache.hive.hcatalog.data.schema.HCatSchema; -import org.apache.hive.hcatalog.hbase.snapshot.RevisionManager; -import org.apache.hive.hcatalog.hbase.snapshot.RevisionManagerConfiguration; -import org.apache.hive.hcatalog.hbase.snapshot.Transaction; -import org.apache.hive.hcatalog.mapreduce.HCatInputFormat; -import org.apache.hive.hcatalog.mapreduce.InputJobInfo; -import org.apache.hive.hcatalog.mapreduce.PartInfo; -import org.junit.Test; - -public class TestHBaseInputFormat extends SkeletonHBaseTest { - - private static HiveConf hcatConf; - private static HCatDriver hcatDriver; - private final byte[] FAMILY = Bytes.toBytes("testFamily"); - private final byte[] QUALIFIER1 = Bytes.toBytes("testQualifier1"); - private final byte[] QUALIFIER2 = Bytes.toBytes("testQualifier2"); - - public TestHBaseInputFormat() throws Exception { - hcatConf = getHiveConf(); - hcatConf.set(ConfVars.SEMANTIC_ANALYZER_HOOK.varname, - HCatSemanticAnalyzer.class.getName()); - URI fsuri = getFileSystem().getUri(); - Path whPath = new Path(fsuri.getScheme(), fsuri.getAuthority(), - getTestDir()); - hcatConf.set(HiveConf.ConfVars.HADOOPFS.varname, fsuri.toString()); - hcatConf.set(ConfVars.METASTOREWAREHOUSE.varname, whPath.toString()); - - //Add hbase properties - - for (Map.Entry el : getHbaseConf()) { - if (el.getKey().startsWith("hbase.")) { - hcatConf.set(el.getKey(), el.getValue()); - } - } - HBaseConfiguration.merge(hcatConf, - RevisionManagerConfiguration.create()); - - - SessionState.start(new CliSessionState(hcatConf)); - hcatDriver = new HCatDriver(); - - } - - private List generatePuts(int num, String tableName) throws IOException { - - List columnFamilies = Arrays.asList("testFamily"); - RevisionManager rm = null; - List myPuts; - try { - rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(hcatConf); - rm.open(); - myPuts = new ArrayList(); - for (int i = 1; i <= num; i++) { - Put put = new Put(Bytes.toBytes("testRow")); - put.add(FAMILY, QUALIFIER1, i, Bytes.toBytes("textValue-" + i)); - put.add(FAMILY, QUALIFIER2, i, Bytes.toBytes("textValue-" + i)); - myPuts.add(put); - Transaction tsx = rm.beginWriteTransaction(tableName, - columnFamilies); - rm.commitWriteTransaction(tsx); - } - } finally { - if (rm != null) - rm.close(); - } - - return myPuts; - } - - private void populateHBaseTable(String tName, int revisions) throws IOException { - List myPuts = generatePuts(revisions, tName); - HTable table = new HTable(getHbaseConf(), Bytes.toBytes(tName)); - table.put(myPuts); - } - - private long populateHBaseTableQualifier1(String tName, int value, Boolean commit) - throws IOException { - List columnFamilies = Arrays.asList("testFamily"); - RevisionManager rm = null; - List myPuts = new ArrayList(); - long revision; - try { - rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(hcatConf); - rm.open(); - Transaction tsx = rm.beginWriteTransaction(tName, columnFamilies); - - Put put = new Put(Bytes.toBytes("testRow")); - revision = tsx.getRevisionNumber(); - put.add(FAMILY, QUALIFIER1, revision, - Bytes.toBytes("textValue-" + value)); - myPuts.add(put); - - // If commit is null it is left as a running transaction - if (commit != null) { - if (commit) { - rm.commitWriteTransaction(tsx); - } else { - rm.abortWriteTransaction(tsx); - } - } - } finally { - if (rm != null) - rm.close(); - } - HTable table = new HTable(getHbaseConf(), Bytes.toBytes(tName)); - table.put(myPuts); - return revision; - } - - @Test - public void TestHBaseTableReadMR() throws Exception { - String tableName = newTableName("MyTable"); - String databaseName = newTableName("MyDatabase"); - //Table name will be lower case unless specified by hbase.table.name property - String hbaseTableName = (databaseName + "." + tableName).toLowerCase(); - String db_dir = new Path(getTestDir(), "hbasedb").toString(); - - String dbquery = "CREATE DATABASE IF NOT EXISTS " + databaseName + " LOCATION '" - + db_dir + "'"; - String tableQuery = "CREATE TABLE " + databaseName + "." + tableName - + "(key string, testqualifier1 string, testqualifier2 string) STORED BY " + - "'org.apache.hive.hcatalog.hbase.HBaseHCatStorageHandler'" - + "TBLPROPERTIES ('hbase.columns.mapping'=':key,testFamily:testQualifier1,testFamily:testQualifier2')"; - - CommandProcessorResponse responseOne = hcatDriver.run(dbquery); - assertEquals(0, responseOne.getResponseCode()); - CommandProcessorResponse responseTwo = hcatDriver.run(tableQuery); - assertEquals(0, responseTwo.getResponseCode()); - - HBaseAdmin hAdmin = new HBaseAdmin(getHbaseConf()); - boolean doesTableExist = hAdmin.tableExists(hbaseTableName); - assertTrue(doesTableExist); - - populateHBaseTable(hbaseTableName, 5); - Configuration conf = new Configuration(hcatConf); - conf.set(HCatConstants.HCAT_KEY_HIVE_CONF, - HCatUtil.serialize(getHiveConf().getAllProperties())); - - // output settings - Path outputDir = new Path(getTestDir(), "mapred/testHbaseTableMRRead"); - FileSystem fs = getFileSystem(); - if (fs.exists(outputDir)) { - fs.delete(outputDir, true); - } - // create job - Job job = new Job(conf, "hbase-mr-read-test"); - job.setJarByClass(this.getClass()); - job.setMapperClass(MapReadHTable.class); - MapReadHTable.resetCounters(); - - job.setInputFormatClass(HCatInputFormat.class); - HCatInputFormat.setInput(job.getConfiguration(), databaseName, tableName); - job.setOutputFormatClass(TextOutputFormat.class); - TextOutputFormat.setOutputPath(job, outputDir); - job.setMapOutputKeyClass(BytesWritable.class); - job.setMapOutputValueClass(Text.class); - job.setOutputKeyClass(BytesWritable.class); - job.setOutputValueClass(Text.class); - job.setNumReduceTasks(0); - assertTrue(job.waitForCompletion(true)); - // Note: These asserts only works in case of LocalJobRunner as they run in same jvm. - // If using MiniMRCluster, the tests will have to be modified. - assertFalse(MapReadHTable.error); - assertEquals(MapReadHTable.count, 1); - - String dropTableQuery = "DROP TABLE " + hbaseTableName; - CommandProcessorResponse responseThree = hcatDriver.run(dropTableQuery); - assertEquals(0, responseThree.getResponseCode()); - - boolean isHbaseTableThere = hAdmin.tableExists(hbaseTableName); - assertFalse(isHbaseTableThere); - - String dropDB = "DROP DATABASE " + databaseName; - CommandProcessorResponse responseFour = hcatDriver.run(dropDB); - assertEquals(0, responseFour.getResponseCode()); - } - - @Test - public void TestHBaseTableProjectionReadMR() throws Exception { - - String tableName = newTableName("MyTable"); - //Table name as specified by hbase.table.name property - String hbaseTableName = "MyDB_" + tableName; - String tableQuery = "CREATE TABLE " + tableName - + "(key string, testqualifier1 string, testqualifier2 string) STORED BY " - + "'org.apache.hive.hcatalog.hbase.HBaseHCatStorageHandler'" - + "TBLPROPERTIES ('hbase.columns.mapping'=" - + "':key,testFamily:testQualifier1,testFamily:testQualifier2'," - + "'hbase.table.name'='" + hbaseTableName + "')"; - - CommandProcessorResponse responseTwo = hcatDriver.run(tableQuery); - assertEquals(0, responseTwo.getResponseCode()); - - HBaseAdmin hAdmin = new HBaseAdmin(getHbaseConf()); - boolean doesTableExist = hAdmin.tableExists(hbaseTableName); - assertTrue(doesTableExist); - - populateHBaseTable(hbaseTableName, 5); - - Configuration conf = new Configuration(hcatConf); - conf.set(HCatConstants.HCAT_KEY_HIVE_CONF, - HCatUtil.serialize(getHiveConf().getAllProperties())); - - // output settings - Path outputDir = new Path(getTestDir(), "mapred/testHBaseTableProjectionReadMR"); - FileSystem fs = getFileSystem(); - if (fs.exists(outputDir)) { - fs.delete(outputDir, true); - } - // create job - Job job = new Job(conf, "hbase-column-projection"); - job.setJarByClass(this.getClass()); - job.setMapperClass(MapReadProjHTable.class); - job.setInputFormatClass(HCatInputFormat.class); - HCatInputFormat.setOutputSchema(job, getProjectionSchema()); - HCatInputFormat.setInput(job, MetaStoreUtils.DEFAULT_DATABASE_NAME, tableName); - job.setOutputFormatClass(TextOutputFormat.class); - TextOutputFormat.setOutputPath(job, outputDir); - job.setMapOutputKeyClass(BytesWritable.class); - job.setMapOutputValueClass(Text.class); - job.setOutputKeyClass(BytesWritable.class); - job.setOutputValueClass(Text.class); - job.setNumReduceTasks(0); - assertTrue(job.waitForCompletion(true)); - assertFalse(MapReadProjHTable.error); - assertEquals(MapReadProjHTable.count, 1); - - String dropTableQuery = "DROP TABLE " + tableName; - CommandProcessorResponse responseThree = hcatDriver.run(dropTableQuery); - assertEquals(0, responseThree.getResponseCode()); - - boolean isHbaseTableThere = hAdmin.tableExists(hbaseTableName); - assertFalse(isHbaseTableThere); - } - - @Test - public void TestHBaseInputFormatProjectionReadMR() throws Exception { - - String tableName = newTableName("mytable"); - String tableQuery = "CREATE TABLE " + tableName - + "(key string, testqualifier1 string, testqualifier2 string) STORED BY " + - "'org.apache.hive.hcatalog.hbase.HBaseHCatStorageHandler'" - + "TBLPROPERTIES ('hbase.columns.mapping'=':key," + - "testFamily:testQualifier1,testFamily:testQualifier2')"; - - CommandProcessorResponse responseTwo = hcatDriver.run(tableQuery); - assertEquals(0, responseTwo.getResponseCode()); - - HBaseAdmin hAdmin = new HBaseAdmin(getHbaseConf()); - boolean doesTableExist = hAdmin.tableExists(tableName); - assertTrue(doesTableExist); - - populateHBaseTable(tableName, 5); - - Configuration conf = new Configuration(hcatConf); - conf.set(HCatConstants.HCAT_KEY_HIVE_CONF, - HCatUtil.serialize(getHiveConf().getAllProperties())); - - // output settings - Path outputDir = new Path(getTestDir(), "mapred/testHBaseTableProjectionReadMR"); - FileSystem fs = getFileSystem(); - if (fs.exists(outputDir)) { - fs.delete(outputDir, true); - } - // create job - JobConf job = new JobConf(conf); - job.setJobName("hbase-scan-column"); - job.setJarByClass(this.getClass()); - job.setMapperClass(MapReadProjectionHTable.class); - job.setInputFormat(HBaseInputFormat.class); - - //Configure projection schema - job.set(HCatConstants.HCAT_KEY_OUTPUT_SCHEMA, HCatUtil.serialize(getProjectionSchema())); - Job newJob = new Job(job); - HCatInputFormat.setInput(newJob, MetaStoreUtils.DEFAULT_DATABASE_NAME, tableName); - String inputJobString = newJob.getConfiguration().get(HCatConstants.HCAT_KEY_JOB_INFO); - InputJobInfo info = (InputJobInfo) HCatUtil.deserialize(inputJobString); - job.set(HCatConstants.HCAT_KEY_JOB_INFO, inputJobString); - for (PartInfo partinfo : info.getPartitions()) { - for (Entry entry : partinfo.getJobProperties().entrySet()) - job.set(entry.getKey(), entry.getValue()); - } - assertEquals("testFamily:testQualifier1", job.get(TableInputFormat.SCAN_COLUMNS)); - - job.setOutputFormat(org.apache.hadoop.mapred.TextOutputFormat.class); - org.apache.hadoop.mapred.TextOutputFormat.setOutputPath(job, outputDir); - job.setMapOutputKeyClass(BytesWritable.class); - job.setMapOutputValueClass(Text.class); - job.setOutputKeyClass(BytesWritable.class); - job.setOutputValueClass(Text.class); - job.setNumReduceTasks(0); - - RunningJob runJob = JobClient.runJob(job); - runJob.waitForCompletion(); - assertTrue(runJob.isSuccessful()); - assertFalse(MapReadProjHTable.error); - assertEquals(MapReadProjHTable.count, 1); - - String dropTableQuery = "DROP TABLE " + tableName; - CommandProcessorResponse responseThree = hcatDriver.run(dropTableQuery); - assertEquals(0, responseThree.getResponseCode()); - - boolean isHbaseTableThere = hAdmin.tableExists(tableName); - assertFalse(isHbaseTableThere); - } - - @Test - public void TestHBaseTableIgnoreAbortedTransactions() throws Exception { - String tableName = newTableName("mytable"); - String tableQuery = "CREATE TABLE " + tableName - + "(key string, testqualifier1 string, testqualifier2 string) STORED BY " + - "'org.apache.hive.hcatalog.hbase.HBaseHCatStorageHandler'" - + "TBLPROPERTIES ('hbase.columns.mapping'=':key," + - "testFamily:testQualifier1,testFamily:testQualifier2')"; - - CommandProcessorResponse responseTwo = hcatDriver.run(tableQuery); - assertEquals(0, responseTwo.getResponseCode()); - - HBaseAdmin hAdmin = new HBaseAdmin(getHbaseConf()); - boolean doesTableExist = hAdmin.tableExists(tableName); - assertTrue(doesTableExist); - - populateHBaseTable(tableName, 5); - populateHBaseTableQualifier1(tableName, 6, false); - populateHBaseTableQualifier1(tableName, 7, false); - - Configuration conf = new Configuration(hcatConf); - conf.set(HCatConstants.HCAT_KEY_HIVE_CONF, - HCatUtil.serialize(getHiveConf().getAllProperties())); - - Path outputDir = new Path(getTestDir(), "mapred/testHBaseTableIgnoreAbortedTransactions"); - FileSystem fs = getFileSystem(); - if (fs.exists(outputDir)) { - fs.delete(outputDir, true); - } - Job job = new Job(conf, "hbase-aborted-transaction"); - job.setJarByClass(this.getClass()); - job.setMapperClass(MapReadHTable.class); - MapReadHTable.resetCounters(); - job.setInputFormatClass(HCatInputFormat.class); - HCatInputFormat.setInput(job, MetaStoreUtils.DEFAULT_DATABASE_NAME, tableName); - job.setOutputFormatClass(TextOutputFormat.class); - TextOutputFormat.setOutputPath(job, outputDir); - job.setMapOutputKeyClass(BytesWritable.class); - job.setMapOutputValueClass(Text.class); - job.setOutputKeyClass(BytesWritable.class); - job.setOutputValueClass(Text.class); - job.setNumReduceTasks(0); - assertTrue(job.waitForCompletion(true)); - // Verify that the records do not contain aborted transaction - // revisions 6 and 7 for testFamily:testQualifier1 and - // fetches revision 5 for both testQualifier1 and testQualifier2 - assertFalse(MapReadHTable.error); - assertEquals(1, MapReadHTable.count); - - String dropTableQuery = "DROP TABLE " + tableName; - CommandProcessorResponse responseThree = hcatDriver.run(dropTableQuery); - assertEquals(0, responseThree.getResponseCode()); - - boolean isHbaseTableThere = hAdmin.tableExists(tableName); - assertFalse(isHbaseTableThere); - } - - @Test - public void TestHBaseTableIgnoreAbortedAndRunningTransactions() throws Exception { - String tableName = newTableName("mytable"); - String tableQuery = "CREATE TABLE " + tableName - + "(key string, testqualifier1 string, testqualifier2 string) STORED BY " + - "'org.apache.hive.hcatalog.hbase.HBaseHCatStorageHandler'" - + "TBLPROPERTIES ('hbase.columns.mapping'=':key," + - "testFamily:testQualifier1,testFamily:testQualifier2')"; - - CommandProcessorResponse responseTwo = hcatDriver.run(tableQuery); - assertEquals(0, responseTwo.getResponseCode()); - - HBaseAdmin hAdmin = new HBaseAdmin(getHbaseConf()); - boolean doesTableExist = hAdmin.tableExists(tableName); - assertTrue(doesTableExist); - - populateHBaseTable(tableName, 2); - populateHBaseTableQualifier1(tableName, 3, Boolean.TRUE); //Committed transaction - populateHBaseTableQualifier1(tableName, 4, null); //Running transaction - populateHBaseTableQualifier1(tableName, 5, Boolean.FALSE); //Aborted transaction - populateHBaseTableQualifier1(tableName, 6, Boolean.TRUE); //Committed transaction - populateHBaseTableQualifier1(tableName, 7, null); //Running Transaction - populateHBaseTableQualifier1(tableName, 8, Boolean.FALSE); //Aborted Transaction - - Configuration conf = new Configuration(hcatConf); - conf.set(HCatConstants.HCAT_KEY_HIVE_CONF, - HCatUtil.serialize(getHiveConf().getAllProperties())); - - Path outputDir = new Path(getTestDir(), "mapred/testHBaseTableIgnoreAbortedTransactions"); - FileSystem fs = getFileSystem(); - if (fs.exists(outputDir)) { - fs.delete(outputDir, true); - } - Job job = new Job(conf, "hbase-running-aborted-transaction"); - job.setJarByClass(this.getClass()); - job.setMapperClass(MapReadHTableRunningAbort.class); - job.setInputFormatClass(HCatInputFormat.class); - HCatInputFormat.setInput(job, MetaStoreUtils.DEFAULT_DATABASE_NAME, tableName); - job.setOutputFormatClass(TextOutputFormat.class); - TextOutputFormat.setOutputPath(job, outputDir); - job.setMapOutputKeyClass(BytesWritable.class); - job.setMapOutputValueClass(Text.class); - job.setOutputKeyClass(BytesWritable.class); - job.setOutputValueClass(Text.class); - job.setNumReduceTasks(0); - assertTrue(job.waitForCompletion(true)); - // Verify that the records do not contain running and aborted transaction - // and it fetches revision 2 for testQualifier1 and testQualifier2 - assertFalse(MapReadHTableRunningAbort.error); - assertEquals(1, MapReadHTableRunningAbort.count); - - String dropTableQuery = "DROP TABLE " + tableName; - CommandProcessorResponse responseThree = hcatDriver.run(dropTableQuery); - assertEquals(0, responseThree.getResponseCode()); - - boolean isHbaseTableThere = hAdmin.tableExists(tableName); - assertFalse(isHbaseTableThere); - } - - - static class MapReadHTable - extends - Mapper, Text> { - - static boolean error = false; - static int count = 0; - - @Override - public void map(ImmutableBytesWritable key, HCatRecord value, - Context context) throws IOException, InterruptedException { - System.out.println("HCat record value" + value.toString()); - boolean correctValues = (value.size() == 3) - && (value.get(0).toString()).equalsIgnoreCase("testRow") - && (value.get(1).toString()).equalsIgnoreCase("textValue-5") - && (value.get(2).toString()).equalsIgnoreCase("textValue-5"); - - if (correctValues == false) { - error = true; - } - count++; - } - - public static void resetCounters() { - error = false; - count = 0; - } - } - - static class MapReadProjHTable - extends - Mapper, Text> { - - static boolean error = false; - static int count = 0; - - @Override - public void map(ImmutableBytesWritable key, HCatRecord value, - Context context) throws IOException, InterruptedException { - System.out.println("HCat record value" + value.toString()); - boolean correctValues = (value.size() == 2) - && (value.get(0).toString()).equalsIgnoreCase("testRow") - && (value.get(1).toString()).equalsIgnoreCase("textValue-5"); - - if (correctValues == false) { - error = true; - } - count++; - } - } - - static class MapReadProjectionHTable - implements org.apache.hadoop.mapred.Mapper, Text> { - - static boolean error = false; - static int count = 0; - - @Override - public void configure(JobConf job) { - } - - @Override - public void close() throws IOException { - } - - @Override - public void map(ImmutableBytesWritable key, Result result, - OutputCollector, Text> output, Reporter reporter) - throws IOException { - System.out.println("Result " + result.toString()); - List list = result.list(); - boolean correctValues = (list.size() == 1) - && (Bytes.toString(list.get(0).getRow())).equalsIgnoreCase("testRow") - && (Bytes.toString(list.get(0).getValue())).equalsIgnoreCase("textValue-5") - && (Bytes.toString(list.get(0).getFamily())).equalsIgnoreCase("testFamily") - && (Bytes.toString(list.get(0).getQualifier())).equalsIgnoreCase("testQualifier1"); - - if (correctValues == false) { - error = true; - } - count++; - } - } - - static class MapReadHTableRunningAbort - extends - Mapper, Text> { - - static boolean error = false; - static int count = 0; - - @Override - public void map(ImmutableBytesWritable key, HCatRecord value, - Context context) throws IOException, InterruptedException { - System.out.println("HCat record value" + value.toString()); - boolean correctValues = (value.size() == 3) - && (value.get(0).toString()).equalsIgnoreCase("testRow") - && (value.get(1).toString()).equalsIgnoreCase("textValue-3") - && (value.get(2).toString()).equalsIgnoreCase("textValue-2"); - - if (correctValues == false) { - error = true; - } - count++; - } - } - - private HCatSchema getProjectionSchema() throws HCatException { - - HCatSchema schema = new HCatSchema(new ArrayList()); - schema.append(new HCatFieldSchema("key", HCatFieldSchema.Type.STRING, - "")); - schema.append(new HCatFieldSchema("testqualifier1", - HCatFieldSchema.Type.STRING, "")); - return schema; - } - - -} diff --git hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/TestSnapshots.java hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/TestSnapshots.java deleted file mode 100644 index cfe2e9d..0000000 --- hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/TestSnapshots.java +++ /dev/null @@ -1,141 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hive.hcatalog.hbase; - -import static org.junit.Assert.assertEquals; - -import java.net.URI; -import java.util.HashMap; -import java.util.Map; -import java.util.Properties; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.cli.CliSessionState; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.conf.HiveConf.ConfVars; -import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; -import org.apache.hadoop.hive.ql.session.SessionState; -import org.apache.hadoop.mapreduce.Job; -import org.apache.hive.hcatalog.cli.HCatDriver; -import org.apache.hive.hcatalog.cli.SemanticAnalysis.HCatSemanticAnalyzer; -import org.apache.hive.hcatalog.common.HCatConstants; -import org.apache.hive.hcatalog.common.HCatUtil; -import org.apache.hive.hcatalog.hbase.snapshot.TableSnapshot; -import org.apache.hive.hcatalog.mapreduce.HCatInputFormat; -import org.apache.hive.hcatalog.mapreduce.InputJobInfo; -import org.junit.Test; - -public class TestSnapshots extends SkeletonHBaseTest { - private static HiveConf hcatConf; - private static HCatDriver hcatDriver; - - public void Initialize() throws Exception { - hcatConf = getHiveConf(); - hcatConf.set(ConfVars.SEMANTIC_ANALYZER_HOOK.varname, - HCatSemanticAnalyzer.class.getName()); - URI fsuri = getFileSystem().getUri(); - Path whPath = new Path(fsuri.getScheme(), fsuri.getAuthority(), - getTestDir()); - hcatConf.set(HiveConf.ConfVars.HADOOPFS.varname, fsuri.toString()); - hcatConf.set(ConfVars.METASTOREWAREHOUSE.varname, whPath.toString()); - - //Add hbase properties - - for (Map.Entry el : getHbaseConf()) { - if (el.getKey().startsWith("hbase.")) { - hcatConf.set(el.getKey(), el.getValue()); - } - } - - SessionState.start(new CliSessionState(hcatConf)); - hcatDriver = new HCatDriver(); - - } - - @Test - public void TestSnapshotConversion() throws Exception { - Initialize(); - String tableName = newTableName("mytableOne"); - String databaseName = newTableName("mydatabase"); - String fullyQualTableName = databaseName + "." + tableName; - String db_dir = new Path(getTestDir(), "hbasedb").toString(); - String dbquery = "CREATE DATABASE IF NOT EXISTS " + databaseName + " LOCATION '" - + db_dir + "'"; - String tableQuery = "CREATE TABLE " + fullyQualTableName - + "(key string, value1 string, value2 string) STORED BY " + - "'org.apache.hive.hcatalog.hbase.HBaseHCatStorageHandler'" - + "TBLPROPERTIES ('hbase.columns.mapping'=':key,cf1:q1,cf2:q2')"; - - CommandProcessorResponse cmdResponse = hcatDriver.run(dbquery); - assertEquals(0, cmdResponse.getResponseCode()); - cmdResponse = hcatDriver.run(tableQuery); - assertEquals(0, cmdResponse.getResponseCode()); - - Configuration conf = new Configuration(hcatConf); - conf.set(HCatConstants.HCAT_KEY_HIVE_CONF, - HCatUtil.serialize(getHiveConf().getAllProperties())); - Job job = new Job(conf); - Properties properties = new Properties(); - properties.setProperty(HBaseConstants.PROPERTY_TABLE_SNAPSHOT_KEY, "dummysnapshot"); - HCatInputFormat.setInput(job, databaseName, tableName).setProperties(properties); - String modifiedInputInfo = job.getConfiguration().get(HCatConstants.HCAT_KEY_JOB_INFO); - InputJobInfo inputInfo = (InputJobInfo) HCatUtil.deserialize(modifiedInputInfo); - - Map revMap = new HashMap(); - revMap.put("cf1", 3L); - revMap.put("cf2", 5L); - TableSnapshot hbaseSnapshot = new TableSnapshot(fullyQualTableName, revMap, -1); - HCatTableSnapshot hcatSnapshot = HBaseRevisionManagerUtil.convertSnapshot(hbaseSnapshot, inputInfo.getTableInfo()); - - assertEquals(hcatSnapshot.getRevision("value1"), 3); - assertEquals(hcatSnapshot.getRevision("value2"), 5); - - String dropTable = "DROP TABLE " + fullyQualTableName; - cmdResponse = hcatDriver.run(dropTable); - assertEquals(0, cmdResponse.getResponseCode()); - - tableName = newTableName("mytableTwo"); - fullyQualTableName = databaseName + "." + tableName; - tableQuery = "CREATE TABLE " + fullyQualTableName - + "(key string, value1 string, value2 string) STORED BY " + - "'org.apache.hive.hcatalog.hbase.HBaseHCatStorageHandler'" - + "TBLPROPERTIES ('hbase.columns.mapping'=':key,cf1:q1,cf1:q2')"; - cmdResponse = hcatDriver.run(tableQuery); - assertEquals(0, cmdResponse.getResponseCode()); - revMap.clear(); - revMap.put("cf1", 3L); - hbaseSnapshot = new TableSnapshot(fullyQualTableName, revMap, -1); - HCatInputFormat.setInput(job, databaseName, tableName).setProperties(properties); - modifiedInputInfo = job.getConfiguration().get(HCatConstants.HCAT_KEY_JOB_INFO); - inputInfo = (InputJobInfo) HCatUtil.deserialize(modifiedInputInfo); - hcatSnapshot = HBaseRevisionManagerUtil.convertSnapshot(hbaseSnapshot, inputInfo.getTableInfo()); - assertEquals(hcatSnapshot.getRevision("value1"), 3); - assertEquals(hcatSnapshot.getRevision("value2"), 3); - - dropTable = "DROP TABLE " + fullyQualTableName; - cmdResponse = hcatDriver.run(dropTable); - assertEquals(0, cmdResponse.getResponseCode()); - - String dropDatabase = "DROP DATABASE IF EXISTS " + databaseName + "CASCADE"; - cmdResponse = hcatDriver.run(dropDatabase); - assertEquals(0, cmdResponse.getResponseCode()); - } -} diff --git hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/snapshot/IDGenClient.java hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/snapshot/IDGenClient.java deleted file mode 100644 index 42234f4..0000000 --- hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/snapshot/IDGenClient.java +++ /dev/null @@ -1,72 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.hive.hcatalog.hbase.snapshot; - -import java.util.HashMap; -import java.util.Map; -import java.util.Random; - -public class IDGenClient extends Thread { - - String connectionStr; - String base_dir; - ZKUtil zkutil; - Random sleepTime = new Random(); - int runtime; - HashMap idMap; - String tableName; - - IDGenClient(String connectionStr, String base_dir, int time, String tableName) { - super(); - this.connectionStr = connectionStr; - this.base_dir = base_dir; - this.zkutil = new ZKUtil(connectionStr, base_dir); - this.runtime = time; - idMap = new HashMap(); - this.tableName = tableName; - } - - /* - * @see java.lang.Runnable#run() - */ - @Override - public void run() { - long startTime = System.currentTimeMillis(); - int timeElapsed = 0; - while( timeElapsed <= runtime){ - try { - long id = zkutil.nextId(tableName); - idMap.put(System.currentTimeMillis(), id); - - int sTime = sleepTime.nextInt(2); - Thread.sleep(sTime * 100); - } catch (Exception e) { - e.printStackTrace(); - } - - timeElapsed = (int) Math.ceil((System.currentTimeMillis() - startTime)/(double)1000); - } - - } - - Map getIdMap(){ - return idMap; - } - -} diff --git hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/snapshot/TestIDGenerator.java hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/snapshot/TestIDGenerator.java deleted file mode 100644 index fcc4017..0000000 --- hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/snapshot/TestIDGenerator.java +++ /dev/null @@ -1,99 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.hive.hcatalog.hbase.snapshot; - -import static org.junit.Assert.assertTrue; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; - -import org.apache.hive.hcatalog.hbase.SkeletonHBaseTest; -import org.junit.Assert; -import org.junit.Test; - -public class TestIDGenerator extends SkeletonHBaseTest { - - @Test - public void testIDGeneration() throws Exception { - - int port = getHbaseConf().getInt("hbase.zookeeper.property.clientPort", 2181); - String servers = getHbaseConf().get("hbase.zookeeper.quorum"); - String[] splits = servers.split(","); - StringBuffer sb = new StringBuffer(); - for (String split : splits) { - sb.append(split); - sb.append(':'); - sb.append(port); - } - ZKUtil zkutil = new ZKUtil(sb.toString(), "/rm_base"); - - String tableName = "myTable"; - long initId = zkutil.nextId(tableName); - for (int i = 0; i < 10; i++) { - long id = zkutil.nextId(tableName); - Assert.assertEquals(initId + (i + 1), id); - } - } - - @Test - public void testMultipleClients() throws InterruptedException { - - int port = getHbaseConf().getInt("hbase.zookeeper.property.clientPort", 2181); - String servers = getHbaseConf().get("hbase.zookeeper.quorum"); - String[] splits = servers.split(","); - StringBuffer sb = new StringBuffer(); - for (String split : splits) { - sb.append(split); - sb.append(':'); - sb.append(port); - } - - ArrayList clients = new ArrayList(); - - for (int i = 0; i < 5; i++) { - IDGenClient idClient = new IDGenClient(sb.toString(), "/rm_base", 10, "testTable"); - clients.add(idClient); - } - - for (IDGenClient idClient : clients) { - idClient.run(); - } - - for (IDGenClient idClient : clients) { - idClient.join(); - } - - HashMap idMap = new HashMap(); - for (IDGenClient idClient : clients) { - idMap.putAll(idClient.getIdMap()); - } - - ArrayList keys = new ArrayList(idMap.keySet()); - Collections.sort(keys); - int startId = 1; - for (Long key : keys) { - Long id = idMap.get(key); - System.out.println("Key: " + key + " Value " + id); - assertTrue(id == startId); - startId++; - - } - } -} diff --git hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/snapshot/TestRevisionManager.java hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/snapshot/TestRevisionManager.java deleted file mode 100644 index 989b22f..0000000 --- hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/snapshot/TestRevisionManager.java +++ /dev/null @@ -1,260 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.hive.hcatalog.hbase.snapshot; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.io.IOException; -import java.util.Arrays; -import java.util.List; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hive.hcatalog.hbase.SkeletonHBaseTest; -import org.apache.hive.hcatalog.hbase.snapshot.transaction.thrift.StoreFamilyRevision; -import org.apache.hive.hcatalog.hbase.snapshot.transaction.thrift.StoreFamilyRevisionList; -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.ZooKeeper; -import org.apache.zookeeper.data.Stat; -import org.junit.Test; - -public class TestRevisionManager extends SkeletonHBaseTest { - - @Test - public void testBasicZNodeCreation() throws IOException, KeeperException, InterruptedException { - - int port = getHbaseConf().getInt("hbase.zookeeper.property.clientPort", 2181); - String servers = getHbaseConf().get("hbase.zookeeper.quorum"); - String[] splits = servers.split(","); - StringBuffer sb = new StringBuffer(); - for (String split : splits) { - sb.append(split); - sb.append(':'); - sb.append(port); - } - - ZKUtil zkutil = new ZKUtil(sb.toString(), "/rm_base"); - String tableName = newTableName("testTable"); - List columnFamilies = Arrays.asList("cf001", "cf002", "cf003"); - - zkutil.createRootZNodes(); - ZooKeeper zk = zkutil.getSession(); - Stat tempTwo = zk.exists("/rm_base" + PathUtil.DATA_DIR, false); - assertTrue(tempTwo != null); - Stat tempThree = zk.exists("/rm_base" + PathUtil.CLOCK_NODE, false); - assertTrue(tempThree != null); - - zkutil.setUpZnodesForTable(tableName, columnFamilies); - String transactionDataTablePath = "/rm_base" + PathUtil.DATA_DIR + "/" + tableName; - Stat result = zk.exists(transactionDataTablePath, false); - assertTrue(result != null); - - for (String colFamiliy : columnFamilies) { - String cfPath = transactionDataTablePath + "/" + colFamiliy; - Stat resultTwo = zk.exists(cfPath, false); - assertTrue(resultTwo != null); - } - - } - - @Test - public void testCommitTransaction() throws IOException { - - int port = getHbaseConf().getInt("hbase.zookeeper.property.clientPort", 2181); - String servers = getHbaseConf().get("hbase.zookeeper.quorum"); - String[] splits = servers.split(","); - StringBuffer sb = new StringBuffer(); - for (String split : splits) { - sb.append(split); - sb.append(':'); - sb.append(port); - } - - Configuration conf = RevisionManagerConfiguration.create(getHbaseConf()); - conf.set(RMConstants.ZOOKEEPER_DATADIR, "/rm_base"); - ZKBasedRevisionManager manager = new ZKBasedRevisionManager(); - manager.initialize(conf); - manager.open(); - ZKUtil zkutil = new ZKUtil(sb.toString(), "/rm_base"); - - String tableName = newTableName("testTable"); - List columnFamilies = Arrays.asList("cf1", "cf2", "cf3"); - Transaction txn = manager.beginWriteTransaction(tableName, - columnFamilies); - - List cfs = zkutil.getColumnFamiliesOfTable(tableName); - assertTrue(cfs.size() == columnFamilies.size()); - for (String cf : cfs) { - assertTrue(columnFamilies.contains(cf)); - } - - for (String colFamily : columnFamilies) { - String path = PathUtil.getRunningTxnInfoPath("/rm_base", tableName, colFamily); - byte[] data = zkutil.getRawData(path, null); - StoreFamilyRevisionList list = new StoreFamilyRevisionList(); - ZKUtil.deserialize(list, data); - assertEquals(list.getRevisionListSize(), 1); - StoreFamilyRevision lightTxn = list.getRevisionList().get(0); - assertEquals(lightTxn.timestamp, txn.getTransactionExpireTimeStamp()); - assertEquals(lightTxn.revision, txn.getRevisionNumber()); - - } - manager.commitWriteTransaction(txn); - for (String colFamiliy : columnFamilies) { - String path = PathUtil.getRunningTxnInfoPath("/rm_base", tableName, colFamiliy); - byte[] data = zkutil.getRawData(path, null); - StoreFamilyRevisionList list = new StoreFamilyRevisionList(); - ZKUtil.deserialize(list, data); - assertEquals(list.getRevisionListSize(), 0); - - } - - manager.close(); - } - - @Test - public void testAbortTransaction() throws IOException { - - int port = getHbaseConf().getInt("hbase.zookeeper.property.clientPort", 2181); - String host = getHbaseConf().get("hbase.zookeeper.quorum"); - Configuration conf = RevisionManagerConfiguration.create(getHbaseConf()); - conf.set(RMConstants.ZOOKEEPER_DATADIR, "/rm_base"); - ZKBasedRevisionManager manager = new ZKBasedRevisionManager(); - manager.initialize(conf); - manager.open(); - ZKUtil zkutil = new ZKUtil(host + ':' + port, "/rm_base"); - - String tableName = newTableName("testTable"); - List columnFamilies = Arrays.asList("cf1", "cf2", "cf3"); - Transaction txn = manager.beginWriteTransaction(tableName, columnFamilies); - List cfs = zkutil.getColumnFamiliesOfTable(tableName); - - assertTrue(cfs.size() == columnFamilies.size()); - for (String cf : cfs) { - assertTrue(columnFamilies.contains(cf)); - } - - for (String colFamiliy : columnFamilies) { - String path = PathUtil.getRunningTxnInfoPath("/rm_base", tableName, colFamiliy); - byte[] data = zkutil.getRawData(path, null); - StoreFamilyRevisionList list = new StoreFamilyRevisionList(); - ZKUtil.deserialize(list, data); - assertEquals(list.getRevisionListSize(), 1); - StoreFamilyRevision lightTxn = list.getRevisionList().get(0); - assertEquals(lightTxn.timestamp, txn.getTransactionExpireTimeStamp()); - assertEquals(lightTxn.revision, txn.getRevisionNumber()); - - } - manager.abortWriteTransaction(txn); - for (String colFamiliy : columnFamilies) { - String path = PathUtil.getRunningTxnInfoPath("/rm_base", tableName, colFamiliy); - byte[] data = zkutil.getRawData(path, null); - StoreFamilyRevisionList list = new StoreFamilyRevisionList(); - ZKUtil.deserialize(list, data); - assertEquals(list.getRevisionListSize(), 0); - - } - - for (String colFamiliy : columnFamilies) { - String path = PathUtil.getAbortInformationPath("/rm_base", tableName, colFamiliy); - byte[] data = zkutil.getRawData(path, null); - StoreFamilyRevisionList list = new StoreFamilyRevisionList(); - ZKUtil.deserialize(list, data); - assertEquals(list.getRevisionListSize(), 1); - StoreFamilyRevision abortedTxn = list.getRevisionList().get(0); - assertEquals(abortedTxn.getRevision(), txn.getRevisionNumber()); - } - manager.close(); - } - - @Test - public void testKeepAliveTransaction() throws InterruptedException, IOException { - - int port = getHbaseConf().getInt("hbase.zookeeper.property.clientPort", 2181); - String servers = getHbaseConf().get("hbase.zookeeper.quorum"); - String[] splits = servers.split(","); - StringBuffer sb = new StringBuffer(); - for (String split : splits) { - sb.append(split); - sb.append(':'); - sb.append(port); - } - - Configuration conf = RevisionManagerConfiguration.create(getHbaseConf()); - conf.set(RMConstants.ZOOKEEPER_DATADIR, "/rm_base"); - ZKBasedRevisionManager manager = new ZKBasedRevisionManager(); - manager.initialize(conf); - manager.open(); - String tableName = newTableName("testTable"); - List columnFamilies = Arrays.asList("cf1", "cf2"); - Transaction txn = manager.beginWriteTransaction(tableName, - columnFamilies, 40); - Thread.sleep(100); - try { - manager.commitWriteTransaction(txn); - } catch (Exception e) { - assertTrue(e instanceof IOException); - assertEquals(e.getMessage(), - "The transaction to be removed not found in the data."); - } - - } - - @Test - public void testCreateSnapshot() throws IOException { - int port = getHbaseConf().getInt("hbase.zookeeper.property.clientPort", 2181); - String host = getHbaseConf().get("hbase.zookeeper.quorum"); - Configuration conf = RevisionManagerConfiguration.create(getHbaseConf()); - conf.set(RMConstants.ZOOKEEPER_DATADIR, "/rm_base"); - ZKBasedRevisionManager manager = new ZKBasedRevisionManager(); - manager.initialize(conf); - manager.open(); - String tableName = newTableName("testTable"); - List cfOne = Arrays.asList("cf1", "cf2"); - List cfTwo = Arrays.asList("cf2", "cf3"); - Transaction tsx1 = manager.beginWriteTransaction(tableName, cfOne); - Transaction tsx2 = manager.beginWriteTransaction(tableName, cfTwo); - TableSnapshot snapshotOne = manager.createSnapshot(tableName); - assertEquals(snapshotOne.getRevision("cf1"), 0); - assertEquals(snapshotOne.getRevision("cf2"), 0); - assertEquals(snapshotOne.getRevision("cf3"), 1); - - List cfThree = Arrays.asList("cf1", "cf3"); - Transaction tsx3 = manager.beginWriteTransaction(tableName, cfThree); - manager.commitWriteTransaction(tsx1); - TableSnapshot snapshotTwo = manager.createSnapshot(tableName); - assertEquals(snapshotTwo.getRevision("cf1"), 2); - assertEquals(snapshotTwo.getRevision("cf2"), 1); - assertEquals(snapshotTwo.getRevision("cf3"), 1); - - manager.commitWriteTransaction(tsx2); - TableSnapshot snapshotThree = manager.createSnapshot(tableName); - assertEquals(snapshotThree.getRevision("cf1"), 2); - assertEquals(snapshotThree.getRevision("cf2"), 3); - assertEquals(snapshotThree.getRevision("cf3"), 2); - manager.commitWriteTransaction(tsx3); - TableSnapshot snapshotFour = manager.createSnapshot(tableName); - assertEquals(snapshotFour.getRevision("cf1"), 3); - assertEquals(snapshotFour.getRevision("cf2"), 3); - assertEquals(snapshotFour.getRevision("cf3"), 3); - - } - - -} diff --git hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/snapshot/TestRevisionManagerConfiguration.java hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/snapshot/TestRevisionManagerConfiguration.java deleted file mode 100644 index 38d5f42..0000000 --- hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/snapshot/TestRevisionManagerConfiguration.java +++ /dev/null @@ -1,34 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hive.hcatalog.hbase.snapshot; - -import org.apache.hadoop.conf.Configuration; -import org.junit.Assert; -import org.junit.Test; - -public class TestRevisionManagerConfiguration { - - @Test - public void testDefault() { - Configuration conf = RevisionManagerConfiguration.create(); - Assert.assertEquals("org.apache.hive.hcatalog.hbase.snapshot.ZKBasedRevisionManager", - conf.get(RevisionManagerFactory.REVISION_MGR_IMPL_CLASS)); - } -} diff --git hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/snapshot/TestRevisionManagerEndpoint.java hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/snapshot/TestRevisionManagerEndpoint.java deleted file mode 100644 index 2afd463..0000000 --- hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/snapshot/TestRevisionManagerEndpoint.java +++ /dev/null @@ -1,206 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.hive.hcatalog.hbase.snapshot; - -import java.io.IOException; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -import org.apache.commons.lang.builder.ToStringBuilder; -import org.apache.commons.lang.builder.ToStringStyle; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; -import org.apache.hive.hcatalog.hbase.SkeletonHBaseTest; -import org.junit.Assert; -import org.junit.Test; - -public class TestRevisionManagerEndpoint extends SkeletonHBaseTest { - - static { - // test case specific mini cluster settings - testConf = new Configuration(false); - testConf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, - "org.apache.hive.hcatalog.hbase.snapshot.RevisionManagerEndpoint", - "org.apache.hadoop.hbase.coprocessor.GenericEndpoint"); - testConf.set(RMConstants.REVISION_MGR_ENDPOINT_IMPL_CLASS, MockRM.class.getName()); - } - - /** - * Mock implementation to test the protocol/serialization - */ - public static class MockRM implements RevisionManager { - - private static class Invocation { - Invocation(String methodName, Object ret, Object... args) { - this.methodName = methodName; - this.args = args; - this.ret = ret; - } - - String methodName; - Object[] args; - Object ret; - - private static boolean equals(Object obj1, Object obj2) { - if (obj1 == obj2) return true; - if (obj1 == null || obj2 == null) return false; - if (obj1 instanceof Transaction || obj1 instanceof TableSnapshot) { - return obj1.toString().equals(obj2.toString()); - } - return obj1.equals(obj2); - } - - @Override - public boolean equals(Object obj) { - Invocation other = (Invocation) obj; - if (this == other) return true; - if (other == null) return false; - if (this.args != other.args) { - if (this.args == null || other.args == null) return false; - if (this.args.length != other.args.length) return false; - for (int i = 0; i < args.length; i++) { - if (!equals(this.args[i], other.args[i])) return false; - } - } - return equals(this.ret, other.ret); - } - - @Override - public String toString() { - return new ToStringBuilder(this, ToStringStyle.SHORT_PREFIX_STYLE). - append("method", this.methodName). - append("args", this.args). - append("returns", this.ret). - toString(); - } - } - - final static String DEFAULT_INSTANCE = "default"; - final static Map INSTANCES = new ConcurrentHashMap(); - Invocation lastCall; - boolean isOpen = false; - - private T recordCall(T result, Object... args) { - StackTraceElement[] stackTrace = Thread.currentThread().getStackTrace(); - lastCall = new Invocation(stackTrace[2].getMethodName(), result, args); - return result; - } - - @Override - public void initialize(Configuration conf) { - if (!INSTANCES.containsKey(DEFAULT_INSTANCE)) - INSTANCES.put(DEFAULT_INSTANCE, this); - } - - @Override - public void open() throws IOException { - isOpen = true; - } - - @Override - public void close() throws IOException { - isOpen = false; - } - - @Override - public void createTable(String table, List columnFamilies) throws IOException { - } - - @Override - public void dropTable(String table) throws IOException { - } - - @Override - public Transaction beginWriteTransaction(String table, - List families) throws IOException { - return recordCall(null, table, families); - } - - @Override - public Transaction beginWriteTransaction(String table, - List families, long keepAlive) throws IOException { - return recordCall(null, table, families, keepAlive); - } - - @Override - public void commitWriteTransaction(Transaction transaction) - throws IOException { - } - - @Override - public void abortWriteTransaction(Transaction transaction) - throws IOException { - } - - @Override - public List getAbortedWriteTransactions(String table, - String columnFamily) throws IOException { - return null; - } - - @Override - public TableSnapshot createSnapshot(String tableName) - throws IOException { - return null; - } - - @Override - public TableSnapshot createSnapshot(String tableName, long revision) - throws IOException { - TableSnapshot ret = new TableSnapshot(tableName, new HashMap(), revision); - return recordCall(ret, tableName, revision); - } - - @Override - public void keepAlive(Transaction transaction) throws IOException { - recordCall(null, transaction); - } - } - - @Test - public void testRevisionManagerProtocol() throws Throwable { - - Configuration conf = getHbaseConf(); - RevisionManager rm = RevisionManagerFactory.getOpenedRevisionManager( - RevisionManagerEndpointClient.class.getName(), conf); - - MockRM mockImpl = MockRM.INSTANCES.get(MockRM.DEFAULT_INSTANCE); - Assert.assertNotNull(mockImpl); - Assert.assertTrue(mockImpl.isOpen); - - Transaction t = new Transaction("t1", Arrays.asList("f1", "f2"), 0, 0); - MockRM.Invocation call = new MockRM.Invocation("keepAlive", null, t); - rm.keepAlive(t); - Assert.assertEquals(call.methodName, call, mockImpl.lastCall); - - t = new Transaction("t2", Arrays.asList("f21", "f22"), 0, 0); - call = new MockRM.Invocation("beginWriteTransaction", null, t.getTableName(), t.getColumnFamilies()); - call.ret = rm.beginWriteTransaction(t.getTableName(), t.getColumnFamilies()); - Assert.assertEquals(call.methodName, call, mockImpl.lastCall); - - call = new MockRM.Invocation("createSnapshot", null, "t3", 1L); - call.ret = rm.createSnapshot("t3", 1); - Assert.assertEquals(call.methodName, call, mockImpl.lastCall); - - } - -} diff --git hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/snapshot/TestThriftSerialization.java hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/snapshot/TestThriftSerialization.java deleted file mode 100644 index 8cd4e84..0000000 --- hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/snapshot/TestThriftSerialization.java +++ /dev/null @@ -1,85 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.hive.hcatalog.hbase.snapshot; - -import static org.junit.Assert.assertTrue; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; - -import org.apache.hive.hcatalog.hbase.snapshot.transaction.thrift.StoreFamilyRevision; -import org.apache.hive.hcatalog.hbase.snapshot.transaction.thrift.StoreFamilyRevisionList; -import org.junit.Test; - -public class TestThriftSerialization { - - @Test - public void testLightWeightTransaction() { - StoreFamilyRevision trxn = new StoreFamilyRevision(0, 1000); - try { - - byte[] data = ZKUtil.serialize(trxn); - StoreFamilyRevision newWtx = new StoreFamilyRevision(); - ZKUtil.deserialize(newWtx, data); - - assertTrue(newWtx.getRevision() == trxn.getRevision()); - assertTrue(newWtx.getTimestamp() == trxn.getTimestamp()); - - } catch (IOException e) { - e.printStackTrace(); - } - } - - @Test - public void testWriteTransactionList() { - List txnList = new ArrayList(); - long version; - long timestamp; - for (int i = 0; i < 10; i++) { - version = i; - timestamp = 1000 + i; - StoreFamilyRevision wtx = new StoreFamilyRevision(version, timestamp); - txnList.add(wtx); - } - - StoreFamilyRevisionList wList = new StoreFamilyRevisionList(txnList); - - try { - byte[] data = ZKUtil.serialize(wList); - StoreFamilyRevisionList newList = new StoreFamilyRevisionList(); - ZKUtil.deserialize(newList, data); - assertTrue(newList.getRevisionListSize() == wList.getRevisionListSize()); - - Iterator itr = newList.getRevisionListIterator(); - int i = 0; - while (itr.hasNext()) { - StoreFamilyRevision txn = itr.next(); - assertTrue(txn.getRevision() == i); - assertTrue(txn.getTimestamp() == (i + 1000)); - i++; - } - - } catch (IOException e) { - e.printStackTrace(); - } - } - -} diff --git hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/snapshot/TestZNodeSetUp.java hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/snapshot/TestZNodeSetUp.java deleted file mode 100644 index 7d051a2..0000000 --- hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/snapshot/TestZNodeSetUp.java +++ /dev/null @@ -1,120 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hive.hcatalog.hbase.snapshot; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.net.URI; -import java.util.Map; - -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.hive.cli.CliSessionState; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.conf.HiveConf.ConfVars; -import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; -import org.apache.hadoop.hive.ql.session.SessionState; -import org.apache.hive.hcatalog.cli.HCatDriver; -import org.apache.hive.hcatalog.cli.SemanticAnalysis.HCatSemanticAnalyzer; -import org.apache.hive.hcatalog.hbase.SkeletonHBaseTest; -import org.apache.zookeeper.ZooKeeper; -import org.apache.zookeeper.data.Stat; -import org.junit.Test; - - -public class TestZNodeSetUp extends SkeletonHBaseTest { - - private static HiveConf hcatConf; - private static HCatDriver hcatDriver; - - public void Initialize() throws Exception { - - hcatConf = getHiveConf(); - hcatConf.set(ConfVars.SEMANTIC_ANALYZER_HOOK.varname, - HCatSemanticAnalyzer.class.getName()); - URI fsuri = getFileSystem().getUri(); - Path whPath = new Path(fsuri.getScheme(), fsuri.getAuthority(), - getTestDir()); - hcatConf.set(HiveConf.ConfVars.HADOOPFS.varname, fsuri.toString()); - hcatConf.set(ConfVars.METASTOREWAREHOUSE.varname, whPath.toString()); - - //Add hbase properties - - for (Map.Entry el : getHbaseConf()) { - if (el.getKey().startsWith("hbase.")) { - hcatConf.set(el.getKey(), el.getValue()); - } - } - HBaseConfiguration.merge(hcatConf, - RevisionManagerConfiguration.create()); - hcatConf.set(RMConstants.ZOOKEEPER_DATADIR, "/rm_base"); - SessionState.start(new CliSessionState(hcatConf)); - hcatDriver = new HCatDriver(); - - } - - @Test - public void testBasicZNodeCreation() throws Exception { - - Initialize(); - int port = getHbaseConf().getInt("hbase.zookeeper.property.clientPort", 2181); - String servers = getHbaseConf().get("hbase.zookeeper.quorum"); - String[] splits = servers.split(","); - StringBuffer sb = new StringBuffer(); - for (String split : splits) { - sb.append(split); - sb.append(':'); - sb.append(port); - } - - hcatDriver.run("drop table test_table"); - CommandProcessorResponse response = hcatDriver - .run("create table test_table(key int, value string) STORED BY " + - "'org.apache.hive.hcatalog.hbase.HBaseHCatStorageHandler'" - + "TBLPROPERTIES ('hbase.columns.mapping'=':key,cf1:val')"); - - assertEquals(0, response.getResponseCode()); - - HBaseAdmin hAdmin = new HBaseAdmin(getHbaseConf()); - boolean doesTableExist = hAdmin.tableExists("test_table"); - assertTrue(doesTableExist); - - - ZKUtil zkutil = new ZKUtil(sb.toString(), "/rm_base"); - ZooKeeper zk = zkutil.getSession(); - String tablePath = PathUtil.getTxnDataPath("/rm_base", "test_table"); - Stat tempTwo = zk.exists(tablePath, false); - assertTrue(tempTwo != null); - - String cfPath = PathUtil.getTxnDataPath("/rm_base", "test_table") + "/cf1"; - Stat tempThree = zk.exists(cfPath, false); - assertTrue(tempThree != null); - - hcatDriver.run("drop table test_table"); - - System.out.println("Table path : " + tablePath); - Stat tempFour = zk.exists(tablePath, false); - assertTrue(tempFour == null); - - } - -} diff --git hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/snapshot/lock/TestWriteLock.java hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/snapshot/lock/TestWriteLock.java deleted file mode 100644 index 4cb8478..0000000 --- hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/snapshot/lock/TestWriteLock.java +++ /dev/null @@ -1,161 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hive.hcatalog.hbase.snapshot.lock; - -import org.apache.zookeeper.ZooKeeper; -import org.apache.zookeeper.test.ClientBase; - -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -import org.junit.After; -import org.junit.Assert; -import org.junit.Test; - -/** - * test for writelock - * This class is taken from the zookeeper 3.4.0 as-is to test the zookeeper lock - * Recipe with a change in the package name. - */ -public class TestWriteLock extends ClientBase { - protected int sessionTimeout = 10 * 1000; - protected String dir = "/" + getClass().getName(); - protected WriteLock[] nodes; - protected CountDownLatch latch = new CountDownLatch(1); - private boolean restartServer = true; - private boolean workAroundClosingLastZNodeFails = true; - private boolean killLeader = true; - - @Test - public void testRun() throws Exception { - runTest(3); - } - - class LockCallback implements LockListener { - public void lockAcquired() { - latch.countDown(); - } - - public void lockReleased() { - - } - - } - - protected void runTest(int count) throws Exception { - nodes = new WriteLock[count]; - for (int i = 0; i < count; i++) { - ZooKeeper keeper = createClient(); - WriteLock leader = new WriteLock(keeper, dir, null); - leader.setLockListener(new LockCallback()); - nodes[i] = leader; - - leader.lock(); - } - - // lets wait for any previous leaders to die and one of our new - // nodes to become the new leader - latch.await(30, TimeUnit.SECONDS); - - WriteLock first = nodes[0]; - dumpNodes(count); - - // lets assert that the first election is the leader - Assert.assertTrue("The first znode should be the leader " + first.getId(), first.isOwner()); - - for (int i = 1; i < count; i++) { - WriteLock node = nodes[i]; - Assert.assertFalse("Node should not be the leader " + node.getId(), node.isOwner()); - } - - if (count > 1) { - if (killLeader) { - System.out.println("Now killing the leader"); - // now lets kill the leader - latch = new CountDownLatch(1); - first.unlock(); - latch.await(30, TimeUnit.SECONDS); - //Thread.sleep(10000); - WriteLock second = nodes[1]; - dumpNodes(count); - // lets assert that the first election is the leader - Assert.assertTrue("The second znode should be the leader " + second.getId(), second.isOwner()); - - for (int i = 2; i < count; i++) { - WriteLock node = nodes[i]; - Assert.assertFalse("Node should not be the leader " + node.getId(), node.isOwner()); - } - } - - - if (restartServer) { - // now lets stop the server - System.out.println("Now stopping the server"); - stopServer(); - Thread.sleep(10000); - - // TODO lets assert that we are no longer the leader - dumpNodes(count); - - System.out.println("Starting the server"); - startServer(); - Thread.sleep(10000); - - for (int i = 0; i < count - 1; i++) { - System.out.println("Calling acquire for node: " + i); - nodes[i].lock(); - } - dumpNodes(count); - System.out.println("Now closing down..."); - } - } - } - - protected void dumpNodes(int count) { - for (int i = 0; i < count; i++) { - WriteLock node = nodes[i]; - System.out.println("node: " + i + " id: " + - node.getId() + " is leader: " + node.isOwner()); - } - } - - @After - public void tearDown() throws Exception { - if (nodes != null) { - for (int i = 0; i < nodes.length; i++) { - WriteLock node = nodes[i]; - if (node != null) { - System.out.println("Closing node: " + i); - node.close(); - if (workAroundClosingLastZNodeFails && i == nodes.length - 1) { - System.out.println("Not closing zookeeper: " + i + " due to bug!"); - } else { - System.out.println("Closing zookeeper: " + i); - node.getZookeeper().close(); - System.out.println("Closed zookeeper: " + i); - } - } - } - } - System.out.println("Now lets stop the server"); - super.tearDown(); - - } -} diff --git hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/snapshot/lock/TestZNodeName.java hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/snapshot/lock/TestZNodeName.java deleted file mode 100644 index 3780e23..0000000 --- hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/snapshot/lock/TestZNodeName.java +++ /dev/null @@ -1,62 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hive.hcatalog.hbase.snapshot.lock; - -import junit.framework.TestCase; - -import java.util.SortedSet; -import java.util.TreeSet; - -import org.junit.Test; - -/** - * test for znodenames. This class is taken as-is from zookeeper lock recipe test. - * The package name has been changed. - */ -public class TestZNodeName extends TestCase { - @Test - public void testOrderWithSamePrefix() throws Exception { - String[] names = { "x-3", "x-5", "x-11", "x-1" }; - String[] expected = { "x-1", "x-3", "x-5", "x-11" }; - assertOrderedNodeNames(names, expected); - } - @Test - public void testOrderWithDifferentPrefixes() throws Exception { - String[] names = { "r-3", "r-2", "r-1", "w-2", "w-1" }; - String[] expected = { "r-1", "r-2", "r-3", "w-1", "w-2" }; - assertOrderedNodeNames(names, expected); - } - - protected void assertOrderedNodeNames(String[] names, String[] expected) { - int size = names.length; - assertEquals("The two arrays should be the same size!", names.length, expected.length); - SortedSet nodeNames = new TreeSet(); - for (String name : names) { - nodeNames.add(new ZNodeName(name)); - } - - int index = 0; - for (ZNodeName nodeName : nodeNames) { - String name = nodeName.getName(); - assertEquals("Node " + index, expected[index++], name); - } - } - -}