Index: metastore/src/gen-py/hive_metastore/ttypes.py =================================================================== --- metastore/src/gen-py/hive_metastore/ttypes.py (revision 959407) +++ metastore/src/gen-py/hive_metastore/ttypes.py (working copy) @@ -1055,26 +1055,23 @@ - tableName - dbName - colNames - - partName """ thrift_spec = ( None, # 0 (1, TType.STRING, 'indexName', None, None, ), # 1 - (2, TType.I32, 'indexType', None, None, ), # 2 + (2, TType.STRING, 'indexType', None, None, ), # 2 (3, TType.STRING, 'tableName', None, None, ), # 3 (4, TType.STRING, 'dbName', None, None, ), # 4 (5, TType.LIST, 'colNames', (TType.STRING,None), None, ), # 5 - (6, TType.STRING, 'partName', None, None, ), # 6 ) - def __init__(self, indexName=None, indexType=None, tableName=None, dbName=None, colNames=None, partName=None,): + def __init__(self, indexName=None, indexType=None, tableName=None, dbName=None, colNames=None,): self.indexName = indexName self.indexType = indexType self.tableName = tableName self.dbName = dbName self.colNames = colNames - self.partName = partName def read(self, iprot): if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: @@ -1091,8 +1088,8 @@ else: iprot.skip(ftype) elif fid == 2: - if ftype == TType.I32: - self.indexType = iprot.readI32(); + if ftype == TType.STRING: + self.indexType = iprot.readString(); else: iprot.skip(ftype) elif fid == 3: @@ -1115,11 +1112,6 @@ iprot.readListEnd() else: iprot.skip(ftype) - elif fid == 6: - if ftype == TType.STRING: - self.partName = iprot.readString(); - else: - iprot.skip(ftype) else: iprot.skip(ftype) iprot.readFieldEnd() @@ -1135,8 +1127,8 @@ oprot.writeString(self.indexName) oprot.writeFieldEnd() if self.indexType != None: - oprot.writeFieldBegin('indexType', TType.I32, 2) - oprot.writeI32(self.indexType) + oprot.writeFieldBegin('indexType', TType.STRING, 2) + oprot.writeString(self.indexType) oprot.writeFieldEnd() if self.tableName != None: oprot.writeFieldBegin('tableName', TType.STRING, 3) @@ -1153,10 +1145,6 @@ oprot.writeString(iter84) oprot.writeListEnd() oprot.writeFieldEnd() - if self.partName != None: - oprot.writeFieldBegin('partName', TType.STRING, 6) - oprot.writeString(self.partName) - oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() Index: metastore/src/gen-py/hive_metastore/ThriftHiveMetastore.py =================================================================== --- metastore/src/gen-py/hive_metastore/ThriftHiveMetastore.py (revision 959407) +++ metastore/src/gen-py/hive_metastore/ThriftHiveMetastore.py (working copy) @@ -262,7 +262,14 @@ """ pass + def create_index(self, index): + """ + Parameters: + - index + """ + pass + class Client(fb303.FacebookService.Client, Iface): """ This interface is live. @@ -1339,7 +1346,43 @@ raise result.o1 raise TApplicationException(TApplicationException.MISSING_RESULT, "partition_name_to_spec failed: unknown result"); + def create_index(self, index): + """ + Parameters: + - index + """ + self.send_create_index(index) + self.recv_create_index() + def send_create_index(self, index): + self._oprot.writeMessageBegin('create_index', TMessageType.CALL, self._seqid) + args = create_index_args() + args.index = index + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_create_index(self, ): + (fname, mtype, rseqid) = self._iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(self._iprot) + self._iprot.readMessageEnd() + raise x + result = create_index_result() + result.read(self._iprot) + self._iprot.readMessageEnd() + if result.o1 != None: + raise result.o1 + if result.o2 != None: + raise result.o2 + if result.o3 != None: + raise result.o3 + if result.o4 != None: + raise result.o4 + return + + class Processor(fb303.FacebookService.Processor, Iface, TProcessor): def __init__(self, handler): fb303.FacebookService.Processor.__init__(self, handler) @@ -1373,6 +1416,7 @@ self._processMap["get_config_value"] = Processor.process_get_config_value self._processMap["partition_name_to_vals"] = Processor.process_partition_name_to_vals self._processMap["partition_name_to_spec"] = Processor.process_partition_name_to_spec + self._processMap["create_index"] = Processor.process_create_index def process(self, iprot, oprot): (name, type, seqid) = iprot.readMessageBegin() @@ -1859,7 +1903,27 @@ oprot.writeMessageEnd() oprot.trans.flush() + def process_create_index(self, seqid, iprot, oprot): + args = create_index_args() + args.read(iprot) + iprot.readMessageEnd() + result = create_index_result() + try: + self._handler.create_index(args.index) + except AlreadyExistsException, o1: + result.o1 = o1 + except InvalidObjectException, o2: + result.o2 = o2 + except MetaException, o3: + result.o3 = o3 + except NoSuchObjectException, o4: + result.o4 = o4 + oprot.writeMessageBegin("create_index", TMessageType.REPLY, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + # HELPER FUNCTIONS AND STRUCTURES class create_database_args: @@ -6428,4 +6492,157 @@ def __ne__(self, other): return not (self == other) +class create_index_args: + """ + Attributes: + - index + """ + thrift_spec = ( + None, # 0 + (1, TType.STRUCT, 'index', (Index, Index.thrift_spec), None, ), # 1 + ) + + def __init__(self, index=None,): + self.index = index + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.index = Index() + self.index.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('create_index_args') + if self.index != None: + oprot.writeFieldBegin('index', TType.STRUCT, 1) + self.index.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class create_index_result: + """ + Attributes: + - o1 + - o2 + - o3 + - o4 + """ + + thrift_spec = ( + None, # 0 + (1, TType.STRUCT, 'o1', (AlreadyExistsException, AlreadyExistsException.thrift_spec), None, ), # 1 + (2, TType.STRUCT, 'o2', (InvalidObjectException, InvalidObjectException.thrift_spec), None, ), # 2 + (3, TType.STRUCT, 'o3', (MetaException, MetaException.thrift_spec), None, ), # 3 + (4, TType.STRUCT, 'o4', (NoSuchObjectException, NoSuchObjectException.thrift_spec), None, ), # 4 + ) + + def __init__(self, o1=None, o2=None, o3=None, o4=None,): + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + self.o4 = o4 + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = AlreadyExistsException() + self.o1.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = InvalidObjectException() + self.o2.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = MetaException() + self.o3.read(iprot) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRUCT: + self.o4 = NoSuchObjectException() + self.o4.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('create_index_result') + if self.o1 != None: + oprot.writeFieldBegin('o1', TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 != None: + oprot.writeFieldBegin('o2', TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 != None: + oprot.writeFieldBegin('o3', TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + if self.o4 != None: + oprot.writeFieldBegin('o4', TType.STRUCT, 4) + self.o4.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + Index: metastore/src/gen-py/hive_metastore/ThriftHiveMetastore-remote =================================================================== --- metastore/src/gen-py/hive_metastore/ThriftHiveMetastore-remote (revision 959407) +++ metastore/src/gen-py/hive_metastore/ThriftHiveMetastore-remote (working copy) @@ -51,6 +51,7 @@ print ' string get_config_value(string name, string defaultValue)' print ' partition_name_to_vals(string part_name)' print ' partition_name_to_spec(string part_name)' + print ' void create_index(Index index)' print '' sys.exit(0) @@ -279,4 +280,10 @@ sys.exit(1) pp.pprint(client.partition_name_to_spec(args[0],)) +elif cmd == 'create_index': + if len(args) != 1: + print 'create_index requires 1 args' + sys.exit(1) + pp.pprint(client.create_index(eval(args[0]),)) + transport.close() Index: metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java (revision 959407) +++ metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java (working copy) @@ -328,4 +328,13 @@ } return FileUtils.makePartName(colNames, vals); } + + public static List getPartValuesFromPartName(String partName) + throws MetaException { + LinkedHashMap partSpec = Warehouse.makeSpecFromName(partName); + List values = new ArrayList(); + values.addAll(partSpec.values()); + return values; + } + } Index: metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java (revision 959407) +++ metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java (working copy) @@ -24,6 +24,8 @@ import org.apache.hadoop.hive.metastore.api.AlreadyExistsException; import org.apache.hadoop.hive.metastore.api.ConfigValSecurityException; import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.Index; +import org.apache.hadoop.hive.metastore.api.IndexAlreadyExistsException; import org.apache.hadoop.hive.metastore.api.InvalidObjectException; import org.apache.hadoop.hive.metastore.api.InvalidOperationException; import org.apache.hadoop.hive.metastore.api.MetaException; @@ -329,4 +331,16 @@ */ public Map partitionNameToSpec(String name) throws MetaException, TException; + + /** + * create an index + * @param index the index object + * @throws InvalidObjectException + * @throws MetaException + * @throws NoSuchObjectException + * @throws TException + * @throws AlreadyExistsException + */ + public void createIndex(Index index) throws InvalidObjectException, + MetaException, NoSuchObjectException, TException, AlreadyExistsException; } Index: metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java (revision 959407) +++ metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java (working copy) @@ -32,6 +32,8 @@ import org.apache.hadoop.hive.metastore.api.ConfigValSecurityException; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.Index; +import org.apache.hadoop.hive.metastore.api.IndexAlreadyExistsException; import org.apache.hadoop.hive.metastore.api.InvalidObjectException; import org.apache.hadoop.hive.metastore.api.InvalidOperationException; import org.apache.hadoop.hive.metastore.api.MetaException; @@ -604,6 +606,11 @@ return deepCopyFieldSchemas(client.get_fields(db, tableName)); } + @Override + public void createIndex(Index index) throws AlreadyExistsException, InvalidObjectException, MetaException, NoSuchObjectException, TException { + client.create_index(index); + } + /** * @param db * @param tableName @@ -654,7 +661,7 @@ } @Override - public Map partitionNameToSpec(String name) throws MetaException, TException { + public Map partitionNameToSpec(String name) throws MetaException, TException{ return client.partition_name_to_spec(name); } Index: metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (revision 959407) +++ metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (working copy) @@ -603,6 +603,7 @@ assert(e instanceof RuntimeException); throw (RuntimeException)e; } + return ret; } @@ -651,24 +652,14 @@ try { ms.openTransaction(); if (!TableType.VIRTUAL_VIEW.toString().equals(tbl.getTableType())) { - if (tbl.getSd().getLocation() == null - || tbl.getSd().getLocation().isEmpty()) { - tblPath = wh.getDefaultTablePath( - tbl.getDbName(), tbl.getTableName()); - } else { - if (!isExternal(tbl) && !MetaStoreUtils.isNonNativeTable(tbl)) { - LOG.warn("Location: " + tbl.getSd().getLocation() - + " specified for non-external table:" + tbl.getTableName()); - } - tblPath = wh.getDnsPath(new Path(tbl.getSd().getLocation())); - } + tblPath = getTableLocation(tbl); tbl.getSd().setLocation(tblPath.toString()); } // get_table checks whether database exists, it should be moved here if (is_table_exists(tbl.getDbName(), tbl.getTableName())) { - throw new AlreadyExistsException("Table " + tbl.getTableName() - + " already exists"); + throw new AlreadyExistsException( + "The name is already used by an existing table or index, please choose another one"); } if (tblPath != null) { @@ -699,6 +690,21 @@ } } + private Path getTableLocation(Table tbl) throws MetaException { + Path tblPath; + if (tbl.getSd().getLocation() == null + || tbl.getSd().getLocation().isEmpty()) { + tblPath = wh.getDefaultTablePath(tbl.getDbName(), tbl.getTableName()); + } else { + if (!isExternal(tbl)) { + LOG.warn("Location: " + tbl.getSd().getLocation() + + "specified for non-external table:" + tbl.getTableName()); + } + tblPath = wh.getDnsPath(new Path(tbl.getSd().getLocation())); + } + return tblPath; + } + public void create_table(final Table tbl) throws AlreadyExistsException, MetaException, InvalidObjectException { incrementCounter("create_table"); @@ -1260,13 +1266,125 @@ return; } - public boolean create_index(Index index_def) - throws IndexAlreadyExistsException, MetaException { - incrementCounter("create_index"); - // TODO Auto-generated method stub - throw new MetaException("Not yet implemented"); + + public void create_index(Index index_def) throws AlreadyExistsException, + MetaException, NoSuchObjectException, InvalidObjectException, AlreadyExistsException { + this.incrementCounter("create_index"); + validateCreateIndex(index_def); + createIndexTableEntry(index_def); + + List partitions = new ArrayList(); + if (index_def.getPartName() != null) { + Partition part = get_partition(index_def.getDbName(), index_def + .getTableName(), Warehouse.getPartValuesFromPartName(index_def + .getPartName())); + partitions.add(part); + }else{ + partitions = get_partitions(index_def.getDbName(), index_def.getTableName(), Short.MAX_VALUE); + } + + if(partitions.size()>0){ + for(Partition part : partitions){ + part.setTableName(index_def.getIndexName()); + part.setCreateTime(getCurrentTime()); + part.setLastAccessTime(getCurrentTime()); + part.getSd().setBucketCols(null); + part.getSd().setLocation(null); + part.getSd().setNumBuckets(1); + add_partition(part); + } + } } + private void createIndexTableEntry(Index index_def) throws MetaException, + NoSuchObjectException, InvalidObjectException { + if (!is_table_exists(index_def.getDbName(), index_def.getIndexName())) { + // we need to first create one table entry for the index table + Table tbl = get_table(index_def.getDbName(), index_def.getTableName()).clone(); + tbl.setCreateTime(getCurrentTime()); + tbl.setParameters(null); + MetaStoreUtils.setIndexTable(tbl); + MetaStoreUtils.setBaseTableOfIndexTable(tbl, index_def.getTableName()); + MetaStoreUtils.setIndexType(tbl, index_def.getIndexType()); + tbl.setTableName(index_def.getIndexName()); + tbl.setLastAccessTime(getCurrentTime()); + tbl.getSd().setBucketCols(null); + List indexedCols = new ArrayList(); + List indexColumns = index_def.getColNames(); + int k =0; + for (int i = 0; i < tbl.getSd().getCols().size(); i++) { + FieldSchema col = tbl.getSd().getCols().get(i); + if (indexColumns.contains(col.getName())) { + indexedCols.add(col); + k++; + } + } + if(k!=indexColumns.size()) + throw new RuntimeException( + "Check the index columns, they should appear in the table being indexed."); + tbl.getSd().setCols(indexedCols); + tbl.getSd().setLocation(null); + Path tblPath = getTableLocation(tbl); + tbl.getSd().setLocation(tblPath.toString()); + tbl.getSd().setNumBuckets(1); + tbl.getSd().setInputFormat("org.apache.hadoop.mapred.TextInputFormat"); + tbl.getSd().setOutputFormat("org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"); + getMS(false).createTable(tbl); + } + } + + private int getCurrentTime() { + return (int) (System.currentTimeMillis() / 1000); + } + + private void validateCreateIndex(Index index_def) throws MetaException, AlreadyExistsException { + Index index = null; + try { + index = get_index(index_def.getDbName(), index_def.getTableName(), index_def.getIndexName(), index_def.getPartName()); + } catch (NoSuchObjectException e) { + } + + /* + * will throw exception either when there already is a table with the same + * name as index_def's index name, or when index is already there but no + * partition specified in the index_def + */ + if ((index == null && is_table_exists(index_def.getDbName(), index_def + .getIndexName())) + || (index != null && index_def.getPartName() == null)) { + throw new AlreadyExistsException( + "The name is already used by an existing table or index, please choose another one"); + } + + } + + public Index get_index(String dbName, String tableName, String indexName, + String partName) throws MetaException, NoSuchObjectException { + this.incrementCounter("get_index"); + Table t = get_table(dbName, indexName); + boolean isIndexTable = MetaStoreUtils.isIndexTable(t); + if (isIndexTable + && MetaStoreUtils.getBaseTableNameOfIndexTable(t).equals(tableName)) { + if(partName != null) { + Partition part = get_partition(dbName, tableName, Warehouse.getPartValuesFromPartName(partName)); + if(part == null) + throw new NoSuchObjectException("index not found"); + } + + String indexType = MetaStoreUtils.getIndexType(t); + List cols = new ArrayList(); + List fieldSchemas = t.getSd().getCols(); + for (int i = 0; i < fieldSchemas.size(); i++) { + cols.add(fieldSchemas.get(i).getName()); + } + + Index index = new Index(indexName, indexType, tableName, dbName, cols, + partName); + return index; + } + throw new NoSuchObjectException("index not found"); + } + public String getVersion() throws TException { incrementCounter("getVersion"); logStartFunction("getVersion"); Index: metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java (revision 959407) +++ metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java (working copy) @@ -60,7 +60,7 @@ protected static final Log LOG = LogFactory.getLog("hive.log"); public static final String DEFAULT_DATABASE_NAME = "default"; - + /** * printStackTrace * @@ -883,4 +883,42 @@ } return true; } + + //used in base table to record index tables + public static final String INDEX_TABLES = "hive.index.tables"; + + //used in index tables to record index types, the base table. + public static String INDEX_TABLE_PROPERTY="INDEX_TABLE"; + public static String INDEX_BASE_TABLE_PROPERTY="INDEX_ORIGIN_TABLE"; + public static String INDEX_TYPE_PROPERTY="INDEX_TYPE"; + + public static void appendIndexTable(Table baseTbl, String indexTblName) { + String listOfIndexTbls = baseTbl.getParameters().get(INDEX_TABLES); + if (listOfIndexTbls == null || listOfIndexTbls.trim().equals("")) { + listOfIndexTbls = indexTblName; + } else { + listOfIndexTbls = listOfIndexTbls + "," + indexTblName; + } + baseTbl.getParameters().put(INDEX_TABLES, listOfIndexTbls); + } + + public static String getBaseTableNameOfIndexTable(Table indextbl) { + return indextbl.getParameters().get(INDEX_BASE_TABLE_PROPERTY); + } + public static boolean isIndexTable(Table t) { + return t.getParameters().get(INDEX_TABLE_PROPERTY).equalsIgnoreCase("true"); + } + public static String getIndexType(Table t) { + return t.getParameters().get(INDEX_TYPE_PROPERTY); + } + public static void setIndexTable(Table tbl) { + tbl.putToParameters(INDEX_TABLE_PROPERTY, "true"); + } + public static void setBaseTableOfIndexTable(Table tbl, String tableName) { + tbl.putToParameters(INDEX_BASE_TABLE_PROPERTY, tableName); + } + public static void setIndexType(Table tbl, String indexType) { + tbl.putToParameters(INDEX_TYPE_PROPERTY, indexType); + } + } Index: metastore/src/gen-cpp/ThriftHiveMetastore.cpp =================================================================== --- metastore/src/gen-cpp/ThriftHiveMetastore.cpp (revision 959407) +++ metastore/src/gen-cpp/ThriftHiveMetastore.cpp (working copy) @@ -6763,6 +6763,224 @@ return xfer; } +uint32_t ThriftHiveMetastore_create_index_args::read(apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 1: + if (ftype == apache::thrift::protocol::T_STRUCT) { + xfer += this->index.read(iprot); + this->__isset.index = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t ThriftHiveMetastore_create_index_args::write(apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + xfer += oprot->writeStructBegin("ThriftHiveMetastore_create_index_args"); + xfer += oprot->writeFieldBegin("index", apache::thrift::protocol::T_STRUCT, 1); + xfer += this->index.write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +uint32_t ThriftHiveMetastore_create_index_pargs::write(apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + xfer += oprot->writeStructBegin("ThriftHiveMetastore_create_index_pargs"); + xfer += oprot->writeFieldBegin("index", apache::thrift::protocol::T_STRUCT, 1); + xfer += (*(this->index)).write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +uint32_t ThriftHiveMetastore_create_index_result::read(apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 1: + if (ftype == apache::thrift::protocol::T_STRUCT) { + xfer += this->o1.read(iprot); + this->__isset.o1 = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 2: + if (ftype == apache::thrift::protocol::T_STRUCT) { + xfer += this->o2.read(iprot); + this->__isset.o2 = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 3: + if (ftype == apache::thrift::protocol::T_STRUCT) { + xfer += this->o3.read(iprot); + this->__isset.o3 = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 4: + if (ftype == apache::thrift::protocol::T_STRUCT) { + xfer += this->o4.read(iprot); + this->__isset.o4 = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t ThriftHiveMetastore_create_index_result::write(apache::thrift::protocol::TProtocol* oprot) const { + + uint32_t xfer = 0; + + xfer += oprot->writeStructBegin("ThriftHiveMetastore_create_index_result"); + + if (this->__isset.o1) { + xfer += oprot->writeFieldBegin("o1", apache::thrift::protocol::T_STRUCT, 1); + xfer += this->o1.write(oprot); + xfer += oprot->writeFieldEnd(); + } else if (this->__isset.o2) { + xfer += oprot->writeFieldBegin("o2", apache::thrift::protocol::T_STRUCT, 2); + xfer += this->o2.write(oprot); + xfer += oprot->writeFieldEnd(); + } else if (this->__isset.o3) { + xfer += oprot->writeFieldBegin("o3", apache::thrift::protocol::T_STRUCT, 3); + xfer += this->o3.write(oprot); + xfer += oprot->writeFieldEnd(); + } else if (this->__isset.o4) { + xfer += oprot->writeFieldBegin("o4", apache::thrift::protocol::T_STRUCT, 4); + xfer += this->o4.write(oprot); + xfer += oprot->writeFieldEnd(); + } + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +uint32_t ThriftHiveMetastore_create_index_presult::read(apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 1: + if (ftype == apache::thrift::protocol::T_STRUCT) { + xfer += this->o1.read(iprot); + this->__isset.o1 = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 2: + if (ftype == apache::thrift::protocol::T_STRUCT) { + xfer += this->o2.read(iprot); + this->__isset.o2 = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 3: + if (ftype == apache::thrift::protocol::T_STRUCT) { + xfer += this->o3.read(iprot); + this->__isset.o3 = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 4: + if (ftype == apache::thrift::protocol::T_STRUCT) { + xfer += this->o4.read(iprot); + this->__isset.o4 = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + bool ThriftHiveMetastoreClient::create_database(const std::string& name, const std::string& description) { send_create_database(name, description); @@ -8743,6 +8961,73 @@ throw apache::thrift::TApplicationException(apache::thrift::TApplicationException::MISSING_RESULT, "partition_name_to_spec failed: unknown result"); } +void ThriftHiveMetastoreClient::create_index(const Index& index) +{ + send_create_index(index); + recv_create_index(); +} + +void ThriftHiveMetastoreClient::send_create_index(const Index& index) +{ + int32_t cseqid = 0; + oprot_->writeMessageBegin("create_index", apache::thrift::protocol::T_CALL, cseqid); + + ThriftHiveMetastore_create_index_pargs args; + args.index = &index; + args.write(oprot_); + + oprot_->writeMessageEnd(); + oprot_->getTransport()->flush(); + oprot_->getTransport()->writeEnd(); +} + +void ThriftHiveMetastoreClient::recv_create_index() +{ + + int32_t rseqid = 0; + std::string fname; + apache::thrift::protocol::TMessageType mtype; + + iprot_->readMessageBegin(fname, mtype, rseqid); + if (mtype == apache::thrift::protocol::T_EXCEPTION) { + apache::thrift::TApplicationException x; + x.read(iprot_); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + throw x; + } + if (mtype != apache::thrift::protocol::T_REPLY) { + iprot_->skip(apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + throw apache::thrift::TApplicationException(apache::thrift::TApplicationException::INVALID_MESSAGE_TYPE); + } + if (fname.compare("create_index") != 0) { + iprot_->skip(apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + throw apache::thrift::TApplicationException(apache::thrift::TApplicationException::WRONG_METHOD_NAME); + } + ThriftHiveMetastore_create_index_presult result; + result.read(iprot_); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + + if (result.__isset.o1) { + throw result.o1; + } + if (result.__isset.o2) { + throw result.o2; + } + if (result.__isset.o3) { + throw result.o3; + } + if (result.__isset.o4) { + throw result.o4; + } + return; +} + bool ThriftHiveMetastoreProcessor::process(boost::shared_ptr piprot, boost::shared_ptr poprot) { apache::thrift::protocol::TProtocol* iprot = piprot.get(); @@ -9780,5 +10065,44 @@ oprot->getTransport()->writeEnd(); } +void ThriftHiveMetastoreProcessor::process_create_index(int32_t seqid, apache::thrift::protocol::TProtocol* iprot, apache::thrift::protocol::TProtocol* oprot) +{ + ThriftHiveMetastore_create_index_args args; + args.read(iprot); + iprot->readMessageEnd(); + iprot->getTransport()->readEnd(); + + ThriftHiveMetastore_create_index_result result; + try { + iface_->create_index(args.index); + } catch (AlreadyExistsException &o1) { + result.o1 = o1; + result.__isset.o1 = true; + } catch (InvalidObjectException &o2) { + result.o2 = o2; + result.__isset.o2 = true; + } catch (MetaException &o3) { + result.o3 = o3; + result.__isset.o3 = true; + } catch (NoSuchObjectException &o4) { + result.o4 = o4; + result.__isset.o4 = true; + } catch (const std::exception& e) { + apache::thrift::TApplicationException x(e.what()); + oprot->writeMessageBegin("create_index", apache::thrift::protocol::T_EXCEPTION, seqid); + x.write(oprot); + oprot->writeMessageEnd(); + oprot->getTransport()->flush(); + oprot->getTransport()->writeEnd(); + return; + } + + oprot->writeMessageBegin("create_index", apache::thrift::protocol::T_REPLY, seqid); + result.write(oprot); + oprot->writeMessageEnd(); + oprot->getTransport()->flush(); + oprot->getTransport()->writeEnd(); +} + }}} // namespace Index: metastore/src/gen-cpp/hive_metastore_types.cpp =================================================================== --- metastore/src/gen-cpp/hive_metastore_types.cpp (revision 959407) +++ metastore/src/gen-cpp/hive_metastore_types.cpp (working copy) @@ -1111,8 +1111,8 @@ return xfer; } -const char* Index::ascii_fingerprint = "3163EDEDA2214D868610157908B1AB7A"; -const uint8_t Index::binary_fingerprint[16] = {0x31,0x63,0xED,0xED,0xA2,0x21,0x4D,0x86,0x86,0x10,0x15,0x79,0x08,0xB1,0xAB,0x7A}; +const char* Index::ascii_fingerprint = "E372B7F5FC9673FD276367227B6C378D"; +const uint8_t Index::binary_fingerprint[16] = {0xE3,0x72,0xB7,0xF5,0xFC,0x96,0x73,0xFD,0x27,0x63,0x67,0x22,0x7B,0x6C,0x37,0x8D}; uint32_t Index::read(apache::thrift::protocol::TProtocol* iprot) { @@ -1143,8 +1143,8 @@ } break; case 2: - if (ftype == apache::thrift::protocol::T_I32) { - xfer += iprot->readI32(this->indexType); + if (ftype == apache::thrift::protocol::T_STRING) { + xfer += iprot->readString(this->indexType); this->__isset.indexType = true; } else { xfer += iprot->skip(ftype); @@ -1186,14 +1186,6 @@ xfer += iprot->skip(ftype); } break; - case 6: - if (ftype == apache::thrift::protocol::T_STRING) { - xfer += iprot->readString(this->partName); - this->__isset.partName = true; - } else { - xfer += iprot->skip(ftype); - } - break; default: xfer += iprot->skip(ftype); break; @@ -1212,8 +1204,8 @@ xfer += oprot->writeFieldBegin("indexName", apache::thrift::protocol::T_STRING, 1); xfer += oprot->writeString(this->indexName); xfer += oprot->writeFieldEnd(); - xfer += oprot->writeFieldBegin("indexType", apache::thrift::protocol::T_I32, 2); - xfer += oprot->writeI32(this->indexType); + xfer += oprot->writeFieldBegin("indexType", apache::thrift::protocol::T_STRING, 2); + xfer += oprot->writeString(this->indexType); xfer += oprot->writeFieldEnd(); xfer += oprot->writeFieldBegin("tableName", apache::thrift::protocol::T_STRING, 3); xfer += oprot->writeString(this->tableName); @@ -1232,9 +1224,6 @@ xfer += oprot->writeListEnd(); } xfer += oprot->writeFieldEnd(); - xfer += oprot->writeFieldBegin("partName", apache::thrift::protocol::T_STRING, 6); - xfer += oprot->writeString(this->partName); - xfer += oprot->writeFieldEnd(); xfer += oprot->writeFieldStop(); xfer += oprot->writeStructEnd(); return xfer; Index: metastore/src/gen-cpp/ThriftHiveMetastore.h =================================================================== --- metastore/src/gen-cpp/ThriftHiveMetastore.h (revision 959407) +++ metastore/src/gen-cpp/ThriftHiveMetastore.h (working copy) @@ -45,6 +45,7 @@ virtual void get_config_value(std::string& _return, const std::string& name, const std::string& defaultValue) = 0; virtual void partition_name_to_vals(std::vector & _return, const std::string& part_name) = 0; virtual void partition_name_to_spec(std::map & _return, const std::string& part_name) = 0; + virtual void create_index(const Index& index) = 0; }; class ThriftHiveMetastoreNull : virtual public ThriftHiveMetastoreIf , virtual public facebook::fb303::FacebookServiceNull { @@ -146,6 +147,9 @@ void partition_name_to_spec(std::map & /* _return */, const std::string& /* part_name */) { return; } + void create_index(const Index& /* index */) { + return; + } }; class ThriftHiveMetastore_create_database_args { @@ -3415,6 +3419,117 @@ }; +class ThriftHiveMetastore_create_index_args { + public: + + ThriftHiveMetastore_create_index_args() { + } + + virtual ~ThriftHiveMetastore_create_index_args() throw() {} + + Index index; + + struct __isset { + __isset() : index(false) {} + bool index; + } __isset; + + bool operator == (const ThriftHiveMetastore_create_index_args & rhs) const + { + if (!(index == rhs.index)) + return false; + return true; + } + bool operator != (const ThriftHiveMetastore_create_index_args &rhs) const { + return !(*this == rhs); + } + + bool operator < (const ThriftHiveMetastore_create_index_args & ) const; + + uint32_t read(apache::thrift::protocol::TProtocol* iprot); + uint32_t write(apache::thrift::protocol::TProtocol* oprot) const; + +}; + +class ThriftHiveMetastore_create_index_pargs { + public: + + + virtual ~ThriftHiveMetastore_create_index_pargs() throw() {} + + const Index* index; + + uint32_t write(apache::thrift::protocol::TProtocol* oprot) const; + +}; + +class ThriftHiveMetastore_create_index_result { + public: + + ThriftHiveMetastore_create_index_result() { + } + + virtual ~ThriftHiveMetastore_create_index_result() throw() {} + + AlreadyExistsException o1; + InvalidObjectException o2; + MetaException o3; + NoSuchObjectException o4; + + struct __isset { + __isset() : o1(false), o2(false), o3(false), o4(false) {} + bool o1; + bool o2; + bool o3; + bool o4; + } __isset; + + bool operator == (const ThriftHiveMetastore_create_index_result & rhs) const + { + if (!(o1 == rhs.o1)) + return false; + if (!(o2 == rhs.o2)) + return false; + if (!(o3 == rhs.o3)) + return false; + if (!(o4 == rhs.o4)) + return false; + return true; + } + bool operator != (const ThriftHiveMetastore_create_index_result &rhs) const { + return !(*this == rhs); + } + + bool operator < (const ThriftHiveMetastore_create_index_result & ) const; + + uint32_t read(apache::thrift::protocol::TProtocol* iprot); + uint32_t write(apache::thrift::protocol::TProtocol* oprot) const; + +}; + +class ThriftHiveMetastore_create_index_presult { + public: + + + virtual ~ThriftHiveMetastore_create_index_presult() throw() {} + + AlreadyExistsException o1; + InvalidObjectException o2; + MetaException o3; + NoSuchObjectException o4; + + struct __isset { + __isset() : o1(false), o2(false), o3(false), o4(false) {} + bool o1; + bool o2; + bool o3; + bool o4; + } __isset; + + uint32_t read(apache::thrift::protocol::TProtocol* iprot); + +}; + class ThriftHiveMetastoreClient : virtual public ThriftHiveMetastoreIf, public facebook::fb303::FacebookServiceClient { public: ThriftHiveMetastoreClient(boost::shared_ptr prot) : @@ -3517,6 +3632,9 @@ void partition_name_to_spec(std::map & _return, const std::string& part_name); void send_partition_name_to_spec(const std::string& part_name); void recv_partition_name_to_spec(std::map & _return); + void create_index(const Index& index); + void send_create_index(const Index& index); + void recv_create_index(); }; class ThriftHiveMetastoreProcessor : virtual public apache::thrift::TProcessor, public facebook::fb303::FacebookServiceProcessor { @@ -3555,6 +3673,7 @@ void process_get_config_value(int32_t seqid, apache::thrift::protocol::TProtocol* iprot, apache::thrift::protocol::TProtocol* oprot); void process_partition_name_to_vals(int32_t seqid, apache::thrift::protocol::TProtocol* iprot, apache::thrift::protocol::TProtocol* oprot); void process_partition_name_to_spec(int32_t seqid, apache::thrift::protocol::TProtocol* iprot, apache::thrift::protocol::TProtocol* oprot); + void process_create_index(int32_t seqid, apache::thrift::protocol::TProtocol* iprot, apache::thrift::protocol::TProtocol* oprot); public: ThriftHiveMetastoreProcessor(boost::shared_ptr iface) : facebook::fb303::FacebookServiceProcessor(iface), @@ -3589,6 +3708,7 @@ processMap_["get_config_value"] = &ThriftHiveMetastoreProcessor::process_get_config_value; processMap_["partition_name_to_vals"] = &ThriftHiveMetastoreProcessor::process_partition_name_to_vals; processMap_["partition_name_to_spec"] = &ThriftHiveMetastoreProcessor::process_partition_name_to_spec; + processMap_["create_index"] = &ThriftHiveMetastoreProcessor::process_create_index; } virtual bool process(boost::shared_ptr piprot, boost::shared_ptr poprot); @@ -3946,6 +4066,13 @@ } } + void create_index(const Index& index) { + uint32_t sz = ifaces_.size(); + for (uint32_t i = 0; i < sz; ++i) { + ifaces_[i]->create_index(index); + } + } + }; }}} // namespace Index: metastore/src/gen-cpp/hive_metastore_types.h =================================================================== --- metastore/src/gen-cpp/hive_metastore_types.h (revision 959407) +++ metastore/src/gen-cpp/hive_metastore_types.h (working copy) @@ -483,29 +483,27 @@ class Index { public: - static const char* ascii_fingerprint; // = "3163EDEDA2214D868610157908B1AB7A"; - static const uint8_t binary_fingerprint[16]; // = {0x31,0x63,0xED,0xED,0xA2,0x21,0x4D,0x86,0x86,0x10,0x15,0x79,0x08,0xB1,0xAB,0x7A}; + static const char* ascii_fingerprint; // = "E372B7F5FC9673FD276367227B6C378D"; + static const uint8_t binary_fingerprint[16]; // = {0xE3,0x72,0xB7,0xF5,0xFC,0x96,0x73,0xFD,0x27,0x63,0x67,0x22,0x7B,0x6C,0x37,0x8D}; - Index() : indexName(""), indexType(0), tableName(""), dbName(""), partName("") { + Index() : indexName(""), indexType(""), tableName(""), dbName("") { } virtual ~Index() throw() {} std::string indexName; - int32_t indexType; + std::string indexType; std::string tableName; std::string dbName; std::vector colNames; - std::string partName; struct __isset { - __isset() : indexName(false), indexType(false), tableName(false), dbName(false), colNames(false), partName(false) {} + __isset() : indexName(false), indexType(false), tableName(false), dbName(false), colNames(false) {} bool indexName; bool indexType; bool tableName; bool dbName; bool colNames; - bool partName; } __isset; bool operator == (const Index & rhs) const @@ -520,8 +518,6 @@ return false; if (!(colNames == rhs.colNames)) return false; - if (!(partName == rhs.partName)) - return false; return true; } bool operator != (const Index &rhs) const { Index: metastore/src/gen-cpp/ThriftHiveMetastore_server.skeleton.cpp =================================================================== --- metastore/src/gen-cpp/ThriftHiveMetastore_server.skeleton.cpp (revision 959407) +++ metastore/src/gen-cpp/ThriftHiveMetastore_server.skeleton.cpp (working copy) @@ -172,6 +172,11 @@ printf("partition_name_to_spec\n"); } + void create_index(const Index& index) { + // Your implementation goes here + printf("create_index\n"); + } + }; int main(int argc, char **argv) { Index: metastore/src/gen-javabean/org/apache/hadoop/hive/metastore/api/ThriftHiveMetastore.java =================================================================== --- metastore/src/gen-javabean/org/apache/hadoop/hive/metastore/api/ThriftHiveMetastore.java (revision 959407) +++ metastore/src/gen-javabean/org/apache/hadoop/hive/metastore/api/ThriftHiveMetastore.java (working copy) @@ -85,6 +85,8 @@ public Map partition_name_to_spec(String part_name) throws MetaException, TException; + public void create_index(Index index) throws AlreadyExistsException, InvalidObjectException, MetaException, NoSuchObjectException, TException; + } public static class Client extends com.facebook.fb303.FacebookService.Client implements Iface { @@ -1276,6 +1278,48 @@ throw new TApplicationException(TApplicationException.MISSING_RESULT, "partition_name_to_spec failed: unknown result"); } + public void create_index(Index index) throws AlreadyExistsException, InvalidObjectException, MetaException, NoSuchObjectException, TException + { + send_create_index(index); + recv_create_index(); + } + + public void send_create_index(Index index) throws TException + { + oprot_.writeMessageBegin(new TMessage("create_index", TMessageType.CALL, seqid_)); + create_index_args args = new create_index_args(); + args.index = index; + args.write(oprot_); + oprot_.writeMessageEnd(); + oprot_.getTransport().flush(); + } + + public void recv_create_index() throws AlreadyExistsException, InvalidObjectException, MetaException, NoSuchObjectException, TException + { + TMessage msg = iprot_.readMessageBegin(); + if (msg.type == TMessageType.EXCEPTION) { + TApplicationException x = TApplicationException.read(iprot_); + iprot_.readMessageEnd(); + throw x; + } + create_index_result result = new create_index_result(); + result.read(iprot_); + iprot_.readMessageEnd(); + if (result.o1 != null) { + throw result.o1; + } + if (result.o2 != null) { + throw result.o2; + } + if (result.o3 != null) { + throw result.o3; + } + if (result.o4 != null) { + throw result.o4; + } + return; + } + } public static class Processor extends com.facebook.fb303.FacebookService.Processor implements TProcessor { private static final Logger LOGGER = Logger.getLogger(Processor.class.getName()); @@ -1313,6 +1357,7 @@ processMap_.put("get_config_value", new get_config_value()); processMap_.put("partition_name_to_vals", new partition_name_to_vals()); processMap_.put("partition_name_to_spec", new partition_name_to_spec()); + processMap_.put("create_index", new create_index()); } private Iface iface_; @@ -2231,6 +2276,40 @@ } + private class create_index implements ProcessFunction { + public void process(int seqid, TProtocol iprot, TProtocol oprot) throws TException + { + create_index_args args = new create_index_args(); + args.read(iprot); + iprot.readMessageEnd(); + create_index_result result = new create_index_result(); + try { + iface_.create_index(args.index); + } catch (AlreadyExistsException o1) { + result.o1 = o1; + } catch (InvalidObjectException o2) { + result.o2 = o2; + } catch (MetaException o3) { + result.o3 = o3; + } catch (NoSuchObjectException o4) { + result.o4 = o4; + } catch (Throwable th) { + LOGGER.error("Internal error processing create_index", th); + TApplicationException x = new TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error processing create_index"); + oprot.writeMessageBegin(new TMessage("create_index", TMessageType.EXCEPTION, seqid)); + x.write(oprot); + oprot.writeMessageEnd(); + oprot.getTransport().flush(); + return; + } + oprot.writeMessageBegin(new TMessage("create_index", TMessageType.REPLY, seqid)); + result.write(oprot); + oprot.writeMessageEnd(); + oprot.getTransport().flush(); + } + + } + } public static class create_database_args implements TBase, java.io.Serializable, Cloneable { @@ -20183,4 +20262,594 @@ } + public static class create_index_args implements TBase, java.io.Serializable, Cloneable { + private static final TStruct STRUCT_DESC = new TStruct("create_index_args"); + private static final TField INDEX_FIELD_DESC = new TField("index", TType.STRUCT, (short)1); + + private Index index; + public static final int INDEX = 1; + + private final Isset __isset = new Isset(); + private static final class Isset implements java.io.Serializable { + } + + public static final Map metaDataMap = Collections.unmodifiableMap(new HashMap() {{ + put(INDEX, new FieldMetaData("index", TFieldRequirementType.DEFAULT, + new StructMetaData(TType.STRUCT, Index.class))); + }}); + + static { + FieldMetaData.addStructMetaDataMap(create_index_args.class, metaDataMap); + } + + public create_index_args() { + } + + public create_index_args( + Index index) + { + this(); + this.index = index; + } + + /** + * Performs a deep copy on other. + */ + public create_index_args(create_index_args other) { + if (other.isSetIndex()) { + this.index = new Index(other.index); + } + } + + @Override + public create_index_args clone() { + return new create_index_args(this); + } + + public Index getIndex() { + return this.index; + } + + public void setIndex(Index index) { + this.index = index; + } + + public void unsetIndex() { + this.index = null; + } + + // Returns true if field index is set (has been asigned a value) and false otherwise + public boolean isSetIndex() { + return this.index != null; + } + + public void setFieldValue(int fieldID, Object value) { + switch (fieldID) { + case INDEX: + if (value == null) { + unsetIndex(); + } else { + setIndex((Index)value); + } + break; + + default: + throw new IllegalArgumentException("Field " + fieldID + " doesn't exist!"); + } + } + + public Object getFieldValue(int fieldID) { + switch (fieldID) { + case INDEX: + return getIndex(); + + default: + throw new IllegalArgumentException("Field " + fieldID + " doesn't exist!"); + } + } + + // Returns true if field corresponding to fieldID is set (has been asigned a value) and false otherwise + public boolean isSet(int fieldID) { + switch (fieldID) { + case INDEX: + return isSetIndex(); + default: + throw new IllegalArgumentException("Field " + fieldID + " doesn't exist!"); + } + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof create_index_args) + return this.equals((create_index_args)that); + return false; + } + + public boolean equals(create_index_args that) { + if (that == null) + return false; + + boolean this_present_index = true && this.isSetIndex(); + boolean that_present_index = true && that.isSetIndex(); + if (this_present_index || that_present_index) { + if (!(this_present_index && that_present_index)) + return false; + if (!this.index.equals(that.index)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public void read(TProtocol iprot) throws TException { + TField field; + iprot.readStructBegin(); + while (true) + { + field = iprot.readFieldBegin(); + if (field.type == TType.STOP) { + break; + } + switch (field.id) + { + case INDEX: + if (field.type == TType.STRUCT) { + this.index = new Index(); + this.index.read(iprot); + } else { + TProtocolUtil.skip(iprot, field.type); + } + break; + default: + TProtocolUtil.skip(iprot, field.type); + break; + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + validate(); + } + + public void write(TProtocol oprot) throws TException { + validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (this.index != null) { + oprot.writeFieldBegin(INDEX_FIELD_DESC); + this.index.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("create_index_args("); + boolean first = true; + + sb.append("index:"); + if (this.index == null) { + sb.append("null"); + } else { + sb.append(this.index); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws TException { + // check for required fields + // check that fields of type enum have valid values + } + + } + + public static class create_index_result implements TBase, java.io.Serializable, Cloneable { + private static final TStruct STRUCT_DESC = new TStruct("create_index_result"); + private static final TField O1_FIELD_DESC = new TField("o1", TType.STRUCT, (short)1); + private static final TField O2_FIELD_DESC = new TField("o2", TType.STRUCT, (short)2); + private static final TField O3_FIELD_DESC = new TField("o3", TType.STRUCT, (short)3); + private static final TField O4_FIELD_DESC = new TField("o4", TType.STRUCT, (short)4); + + private AlreadyExistsException o1; + public static final int O1 = 1; + private InvalidObjectException o2; + public static final int O2 = 2; + private MetaException o3; + public static final int O3 = 3; + private NoSuchObjectException o4; + public static final int O4 = 4; + + private final Isset __isset = new Isset(); + private static final class Isset implements java.io.Serializable { + } + + public static final Map metaDataMap = Collections.unmodifiableMap(new HashMap() {{ + put(O1, new FieldMetaData("o1", TFieldRequirementType.DEFAULT, + new FieldValueMetaData(TType.STRUCT))); + put(O2, new FieldMetaData("o2", TFieldRequirementType.DEFAULT, + new FieldValueMetaData(TType.STRUCT))); + put(O3, new FieldMetaData("o3", TFieldRequirementType.DEFAULT, + new FieldValueMetaData(TType.STRUCT))); + put(O4, new FieldMetaData("o4", TFieldRequirementType.DEFAULT, + new FieldValueMetaData(TType.STRUCT))); + }}); + + static { + FieldMetaData.addStructMetaDataMap(create_index_result.class, metaDataMap); + } + + public create_index_result() { + } + + public create_index_result( + AlreadyExistsException o1, + InvalidObjectException o2, + MetaException o3, + NoSuchObjectException o4) + { + this(); + this.o1 = o1; + this.o2 = o2; + this.o3 = o3; + this.o4 = o4; + } + + /** + * Performs a deep copy on other. + */ + public create_index_result(create_index_result other) { + if (other.isSetO1()) { + this.o1 = new AlreadyExistsException(other.o1); + } + if (other.isSetO2()) { + this.o2 = new InvalidObjectException(other.o2); + } + if (other.isSetO3()) { + this.o3 = new MetaException(other.o3); + } + if (other.isSetO4()) { + this.o4 = new NoSuchObjectException(other.o4); + } + } + + @Override + public create_index_result clone() { + return new create_index_result(this); + } + + public AlreadyExistsException getO1() { + return this.o1; + } + + public void setO1(AlreadyExistsException o1) { + this.o1 = o1; + } + + public void unsetO1() { + this.o1 = null; + } + + // Returns true if field o1 is set (has been asigned a value) and false otherwise + public boolean isSetO1() { + return this.o1 != null; + } + + public InvalidObjectException getO2() { + return this.o2; + } + + public void setO2(InvalidObjectException o2) { + this.o2 = o2; + } + + public void unsetO2() { + this.o2 = null; + } + + // Returns true if field o2 is set (has been asigned a value) and false otherwise + public boolean isSetO2() { + return this.o2 != null; + } + + public MetaException getO3() { + return this.o3; + } + + public void setO3(MetaException o3) { + this.o3 = o3; + } + + public void unsetO3() { + this.o3 = null; + } + + // Returns true if field o3 is set (has been asigned a value) and false otherwise + public boolean isSetO3() { + return this.o3 != null; + } + + public NoSuchObjectException getO4() { + return this.o4; + } + + public void setO4(NoSuchObjectException o4) { + this.o4 = o4; + } + + public void unsetO4() { + this.o4 = null; + } + + // Returns true if field o4 is set (has been asigned a value) and false otherwise + public boolean isSetO4() { + return this.o4 != null; + } + + public void setFieldValue(int fieldID, Object value) { + switch (fieldID) { + case O1: + if (value == null) { + unsetO1(); + } else { + setO1((AlreadyExistsException)value); + } + break; + + case O2: + if (value == null) { + unsetO2(); + } else { + setO2((InvalidObjectException)value); + } + break; + + case O3: + if (value == null) { + unsetO3(); + } else { + setO3((MetaException)value); + } + break; + + case O4: + if (value == null) { + unsetO4(); + } else { + setO4((NoSuchObjectException)value); + } + break; + + default: + throw new IllegalArgumentException("Field " + fieldID + " doesn't exist!"); + } + } + + public Object getFieldValue(int fieldID) { + switch (fieldID) { + case O1: + return getO1(); + + case O2: + return getO2(); + + case O3: + return getO3(); + + case O4: + return getO4(); + + default: + throw new IllegalArgumentException("Field " + fieldID + " doesn't exist!"); + } + } + + // Returns true if field corresponding to fieldID is set (has been asigned a value) and false otherwise + public boolean isSet(int fieldID) { + switch (fieldID) { + case O1: + return isSetO1(); + case O2: + return isSetO2(); + case O3: + return isSetO3(); + case O4: + return isSetO4(); + default: + throw new IllegalArgumentException("Field " + fieldID + " doesn't exist!"); + } + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof create_index_result) + return this.equals((create_index_result)that); + return false; + } + + public boolean equals(create_index_result that) { + if (that == null) + return false; + + boolean this_present_o1 = true && this.isSetO1(); + boolean that_present_o1 = true && that.isSetO1(); + if (this_present_o1 || that_present_o1) { + if (!(this_present_o1 && that_present_o1)) + return false; + if (!this.o1.equals(that.o1)) + return false; + } + + boolean this_present_o2 = true && this.isSetO2(); + boolean that_present_o2 = true && that.isSetO2(); + if (this_present_o2 || that_present_o2) { + if (!(this_present_o2 && that_present_o2)) + return false; + if (!this.o2.equals(that.o2)) + return false; + } + + boolean this_present_o3 = true && this.isSetO3(); + boolean that_present_o3 = true && that.isSetO3(); + if (this_present_o3 || that_present_o3) { + if (!(this_present_o3 && that_present_o3)) + return false; + if (!this.o3.equals(that.o3)) + return false; + } + + boolean this_present_o4 = true && this.isSetO4(); + boolean that_present_o4 = true && that.isSetO4(); + if (this_present_o4 || that_present_o4) { + if (!(this_present_o4 && that_present_o4)) + return false; + if (!this.o4.equals(that.o4)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public void read(TProtocol iprot) throws TException { + TField field; + iprot.readStructBegin(); + while (true) + { + field = iprot.readFieldBegin(); + if (field.type == TType.STOP) { + break; + } + switch (field.id) + { + case O1: + if (field.type == TType.STRUCT) { + this.o1 = new AlreadyExistsException(); + this.o1.read(iprot); + } else { + TProtocolUtil.skip(iprot, field.type); + } + break; + case O2: + if (field.type == TType.STRUCT) { + this.o2 = new InvalidObjectException(); + this.o2.read(iprot); + } else { + TProtocolUtil.skip(iprot, field.type); + } + break; + case O3: + if (field.type == TType.STRUCT) { + this.o3 = new MetaException(); + this.o3.read(iprot); + } else { + TProtocolUtil.skip(iprot, field.type); + } + break; + case O4: + if (field.type == TType.STRUCT) { + this.o4 = new NoSuchObjectException(); + this.o4.read(iprot); + } else { + TProtocolUtil.skip(iprot, field.type); + } + break; + default: + TProtocolUtil.skip(iprot, field.type); + break; + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + validate(); + } + + public void write(TProtocol oprot) throws TException { + oprot.writeStructBegin(STRUCT_DESC); + + if (this.isSetO1()) { + oprot.writeFieldBegin(O1_FIELD_DESC); + this.o1.write(oprot); + oprot.writeFieldEnd(); + } else if (this.isSetO2()) { + oprot.writeFieldBegin(O2_FIELD_DESC); + this.o2.write(oprot); + oprot.writeFieldEnd(); + } else if (this.isSetO3()) { + oprot.writeFieldBegin(O3_FIELD_DESC); + this.o3.write(oprot); + oprot.writeFieldEnd(); + } else if (this.isSetO4()) { + oprot.writeFieldBegin(O4_FIELD_DESC); + this.o4.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("create_index_result("); + boolean first = true; + + sb.append("o1:"); + if (this.o1 == null) { + sb.append("null"); + } else { + sb.append(this.o1); + } + first = false; + if (!first) sb.append(", "); + sb.append("o2:"); + if (this.o2 == null) { + sb.append("null"); + } else { + sb.append(this.o2); + } + first = false; + if (!first) sb.append(", "); + sb.append("o3:"); + if (this.o3 == null) { + sb.append("null"); + } else { + sb.append(this.o3); + } + first = false; + if (!first) sb.append(", "); + sb.append("o4:"); + if (this.o4 == null) { + sb.append("null"); + } else { + sb.append(this.o4); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws TException { + // check for required fields + // check that fields of type enum have valid values + } + + } + } Index: metastore/src/gen-javabean/org/apache/hadoop/hive/metastore/api/Index.java =================================================================== --- metastore/src/gen-javabean/org/apache/hadoop/hive/metastore/api/Index.java (revision 959407) +++ metastore/src/gen-javabean/org/apache/hadoop/hive/metastore/api/Index.java (working copy) @@ -21,15 +21,14 @@ public class Index implements TBase, java.io.Serializable, Cloneable { private static final TStruct STRUCT_DESC = new TStruct("Index"); private static final TField INDEX_NAME_FIELD_DESC = new TField("indexName", TType.STRING, (short)1); - private static final TField INDEX_TYPE_FIELD_DESC = new TField("indexType", TType.I32, (short)2); + private static final TField INDEX_TYPE_FIELD_DESC = new TField("indexType", TType.STRING, (short)2); private static final TField TABLE_NAME_FIELD_DESC = new TField("tableName", TType.STRING, (short)3); private static final TField DB_NAME_FIELD_DESC = new TField("dbName", TType.STRING, (short)4); private static final TField COL_NAMES_FIELD_DESC = new TField("colNames", TType.LIST, (short)5); - private static final TField PART_NAME_FIELD_DESC = new TField("partName", TType.STRING, (short)6); private String indexName; public static final int INDEXNAME = 1; - private int indexType; + private String indexType; public static final int INDEXTYPE = 2; private String tableName; public static final int TABLENAME = 3; @@ -37,19 +36,16 @@ public static final int DBNAME = 4; private List colNames; public static final int COLNAMES = 5; - private String partName; - public static final int PARTNAME = 6; private final Isset __isset = new Isset(); private static final class Isset implements java.io.Serializable { - public boolean indexType = false; } public static final Map metaDataMap = Collections.unmodifiableMap(new HashMap() {{ put(INDEXNAME, new FieldMetaData("indexName", TFieldRequirementType.DEFAULT, new FieldValueMetaData(TType.STRING))); put(INDEXTYPE, new FieldMetaData("indexType", TFieldRequirementType.DEFAULT, - new FieldValueMetaData(TType.I32))); + new FieldValueMetaData(TType.STRING))); put(TABLENAME, new FieldMetaData("tableName", TFieldRequirementType.DEFAULT, new FieldValueMetaData(TType.STRING))); put(DBNAME, new FieldMetaData("dbName", TFieldRequirementType.DEFAULT, @@ -57,8 +53,6 @@ put(COLNAMES, new FieldMetaData("colNames", TFieldRequirementType.DEFAULT, new ListMetaData(TType.LIST, new FieldValueMetaData(TType.STRING)))); - put(PARTNAME, new FieldMetaData("partName", TFieldRequirementType.DEFAULT, - new FieldValueMetaData(TType.STRING))); }}); static { @@ -70,20 +64,17 @@ public Index( String indexName, - int indexType, + String indexType, String tableName, String dbName, - List colNames, - String partName) + List colNames) { this(); this.indexName = indexName; this.indexType = indexType; - this.__isset.indexType = true; this.tableName = tableName; this.dbName = dbName; this.colNames = colNames; - this.partName = partName; } /** @@ -93,8 +84,9 @@ if (other.isSetIndexName()) { this.indexName = other.indexName; } - __isset.indexType = other.__isset.indexType; - this.indexType = other.indexType; + if (other.isSetIndexType()) { + this.indexType = other.indexType; + } if (other.isSetTableName()) { this.tableName = other.tableName; } @@ -108,9 +100,6 @@ } this.colNames = __this__colNames; } - if (other.isSetPartName()) { - this.partName = other.partName; - } } @Override @@ -135,22 +124,21 @@ return this.indexName != null; } - public int getIndexType() { + public String getIndexType() { return this.indexType; } - public void setIndexType(int indexType) { + public void setIndexType(String indexType) { this.indexType = indexType; - this.__isset.indexType = true; } public void unsetIndexType() { - this.__isset.indexType = false; + this.indexType = null; } // Returns true if field indexType is set (has been asigned a value) and false otherwise public boolean isSetIndexType() { - return this.__isset.indexType; + return this.indexType != null; } public String getTableName() { @@ -219,23 +207,6 @@ return this.colNames != null; } - public String getPartName() { - return this.partName; - } - - public void setPartName(String partName) { - this.partName = partName; - } - - public void unsetPartName() { - this.partName = null; - } - - // Returns true if field partName is set (has been asigned a value) and false otherwise - public boolean isSetPartName() { - return this.partName != null; - } - public void setFieldValue(int fieldID, Object value) { switch (fieldID) { case INDEXNAME: @@ -250,7 +221,7 @@ if (value == null) { unsetIndexType(); } else { - setIndexType((Integer)value); + setIndexType((String)value); } break; @@ -278,14 +249,6 @@ } break; - case PARTNAME: - if (value == null) { - unsetPartName(); - } else { - setPartName((String)value); - } - break; - default: throw new IllegalArgumentException("Field " + fieldID + " doesn't exist!"); } @@ -297,7 +260,7 @@ return getIndexName(); case INDEXTYPE: - return new Integer(getIndexType()); + return getIndexType(); case TABLENAME: return getTableName(); @@ -308,9 +271,6 @@ case COLNAMES: return getColNames(); - case PARTNAME: - return getPartName(); - default: throw new IllegalArgumentException("Field " + fieldID + " doesn't exist!"); } @@ -329,8 +289,6 @@ return isSetDbName(); case COLNAMES: return isSetColNames(); - case PARTNAME: - return isSetPartName(); default: throw new IllegalArgumentException("Field " + fieldID + " doesn't exist!"); } @@ -358,12 +316,12 @@ return false; } - boolean this_present_indexType = true; - boolean that_present_indexType = true; + boolean this_present_indexType = true && this.isSetIndexType(); + boolean that_present_indexType = true && that.isSetIndexType(); if (this_present_indexType || that_present_indexType) { if (!(this_present_indexType && that_present_indexType)) return false; - if (this.indexType != that.indexType) + if (!this.indexType.equals(that.indexType)) return false; } @@ -394,15 +352,6 @@ return false; } - boolean this_present_partName = true && this.isSetPartName(); - boolean that_present_partName = true && that.isSetPartName(); - if (this_present_partName || that_present_partName) { - if (!(this_present_partName && that_present_partName)) - return false; - if (!this.partName.equals(that.partName)) - return false; - } - return true; } @@ -430,9 +379,8 @@ } break; case INDEXTYPE: - if (field.type == TType.I32) { - this.indexType = iprot.readI32(); - this.__isset.indexType = true; + if (field.type == TType.STRING) { + this.indexType = iprot.readString(); } else { TProtocolUtil.skip(iprot, field.type); } @@ -468,13 +416,6 @@ TProtocolUtil.skip(iprot, field.type); } break; - case PARTNAME: - if (field.type == TType.STRING) { - this.partName = iprot.readString(); - } else { - TProtocolUtil.skip(iprot, field.type); - } - break; default: TProtocolUtil.skip(iprot, field.type); break; @@ -495,9 +436,11 @@ oprot.writeString(this.indexName); oprot.writeFieldEnd(); } - oprot.writeFieldBegin(INDEX_TYPE_FIELD_DESC); - oprot.writeI32(this.indexType); - oprot.writeFieldEnd(); + if (this.indexType != null) { + oprot.writeFieldBegin(INDEX_TYPE_FIELD_DESC); + oprot.writeString(this.indexType); + oprot.writeFieldEnd(); + } if (this.tableName != null) { oprot.writeFieldBegin(TABLE_NAME_FIELD_DESC); oprot.writeString(this.tableName); @@ -519,11 +462,6 @@ } oprot.writeFieldEnd(); } - if (this.partName != null) { - oprot.writeFieldBegin(PART_NAME_FIELD_DESC); - oprot.writeString(this.partName); - oprot.writeFieldEnd(); - } oprot.writeFieldStop(); oprot.writeStructEnd(); } @@ -542,7 +480,11 @@ first = false; if (!first) sb.append(", "); sb.append("indexType:"); - sb.append(this.indexType); + if (this.indexType == null) { + sb.append("null"); + } else { + sb.append(this.indexType); + } first = false; if (!first) sb.append(", "); sb.append("tableName:"); @@ -568,14 +510,6 @@ sb.append(this.colNames); } first = false; - if (!first) sb.append(", "); - sb.append("partName:"); - if (this.partName == null) { - sb.append("null"); - } else { - sb.append(this.partName); - } - first = false; sb.append(")"); return sb.toString(); } Index: metastore/src/gen-php/ThriftHiveMetastore.php =================================================================== --- metastore/src/gen-php/ThriftHiveMetastore.php (revision 959407) +++ metastore/src/gen-php/ThriftHiveMetastore.php (working copy) @@ -40,6 +40,7 @@ public function get_config_value($name, $defaultValue); public function partition_name_to_vals($part_name); public function partition_name_to_spec($part_name); + public function create_index($index); } class ThriftHiveMetastoreClient extends FacebookServiceClient implements ThriftHiveMetastoreIf { @@ -1765,6 +1766,66 @@ throw new Exception("partition_name_to_spec failed: unknown result"); } + public function create_index($index) + { + $this->send_create_index($index); + $this->recv_create_index(); + } + + public function send_create_index($index) + { + $args = new metastore_ThriftHiveMetastore_create_index_args(); + $args->index = $index; + $bin_accel = ($this->output_ instanceof TProtocol::$TBINARYPROTOCOLACCELERATED) && function_exists('thrift_protocol_write_binary'); + if ($bin_accel) + { + thrift_protocol_write_binary($this->output_, 'create_index', TMessageType::CALL, $args, $this->seqid_, $this->output_->isStrictWrite()); + } + else + { + $this->output_->writeMessageBegin('create_index', TMessageType::CALL, $this->seqid_); + $args->write($this->output_); + $this->output_->writeMessageEnd(); + $this->output_->getTransport()->flush(); + } + } + + public function recv_create_index() + { + $bin_accel = ($this->input_ instanceof TProtocol::$TBINARYPROTOCOLACCELERATED) && function_exists('thrift_protocol_read_binary'); + if ($bin_accel) $result = thrift_protocol_read_binary($this->input_, 'metastore_ThriftHiveMetastore_create_index_result', $this->input_->isStrictRead()); + else + { + $rseqid = 0; + $fname = null; + $mtype = 0; + + $this->input_->readMessageBegin($fname, $mtype, $rseqid); + if ($mtype == TMessageType::EXCEPTION) { + $x = new TApplicationException(); + $x->read($this->input_); + $this->input_->readMessageEnd(); + throw $x; + } + $result = new metastore_ThriftHiveMetastore_create_index_result(); + $result->read($this->input_); + $this->input_->readMessageEnd(); + } + if ($result->o1 !== null) { + throw $result->o1; + } + if ($result->o2 !== null) { + throw $result->o2; + } + if ($result->o3 !== null) { + throw $result->o3; + } + if ($result->o4 !== null) { + throw $result->o4; + } + return; + } + } // HELPER FUNCTIONS AND STRUCTURES @@ -8424,4 +8485,221 @@ } +class metastore_ThriftHiveMetastore_create_index_args { + static $_TSPEC; + + public $index = null; + + public function __construct($vals=null) { + if (!isset(self::$_TSPEC)) { + self::$_TSPEC = array( + 1 => array( + 'var' => 'index', + 'type' => TType::STRUCT, + 'class' => 'metastore_Index', + ), + ); + } + if (is_array($vals)) { + if (isset($vals['index'])) { + $this->index = $vals['index']; + } + } + } + + public function getName() { + return 'ThriftHiveMetastore_create_index_args'; + } + + public function read($input) + { + $xfer = 0; + $fname = null; + $ftype = 0; + $fid = 0; + $xfer += $input->readStructBegin($fname); + while (true) + { + $xfer += $input->readFieldBegin($fname, $ftype, $fid); + if ($ftype == TType::STOP) { + break; + } + switch ($fid) + { + case 1: + if ($ftype == TType::STRUCT) { + $this->index = new metastore_Index(); + $xfer += $this->index->read($input); + } else { + $xfer += $input->skip($ftype); + } + break; + default: + $xfer += $input->skip($ftype); + break; + } + $xfer += $input->readFieldEnd(); + } + $xfer += $input->readStructEnd(); + return $xfer; + } + + public function write($output) { + $xfer = 0; + $xfer += $output->writeStructBegin('ThriftHiveMetastore_create_index_args'); + if ($this->index !== null) { + if (!is_object($this->index)) { + throw new TProtocolException('Bad type in structure.', TProtocolException::INVALID_DATA); + } + $xfer += $output->writeFieldBegin('index', TType::STRUCT, 1); + $xfer += $this->index->write($output); + $xfer += $output->writeFieldEnd(); + } + $xfer += $output->writeFieldStop(); + $xfer += $output->writeStructEnd(); + return $xfer; + } + +} + +class metastore_ThriftHiveMetastore_create_index_result { + static $_TSPEC; + + public $o1 = null; + public $o2 = null; + public $o3 = null; + public $o4 = null; + + public function __construct($vals=null) { + if (!isset(self::$_TSPEC)) { + self::$_TSPEC = array( + 1 => array( + 'var' => 'o1', + 'type' => TType::STRUCT, + 'class' => 'metastore_AlreadyExistsException', + ), + 2 => array( + 'var' => 'o2', + 'type' => TType::STRUCT, + 'class' => 'metastore_InvalidObjectException', + ), + 3 => array( + 'var' => 'o3', + 'type' => TType::STRUCT, + 'class' => 'metastore_MetaException', + ), + 4 => array( + 'var' => 'o4', + 'type' => TType::STRUCT, + 'class' => 'metastore_NoSuchObjectException', + ), + ); + } + if (is_array($vals)) { + if (isset($vals['o1'])) { + $this->o1 = $vals['o1']; + } + if (isset($vals['o2'])) { + $this->o2 = $vals['o2']; + } + if (isset($vals['o3'])) { + $this->o3 = $vals['o3']; + } + if (isset($vals['o4'])) { + $this->o4 = $vals['o4']; + } + } + } + + public function getName() { + return 'ThriftHiveMetastore_create_index_result'; + } + + public function read($input) + { + $xfer = 0; + $fname = null; + $ftype = 0; + $fid = 0; + $xfer += $input->readStructBegin($fname); + while (true) + { + $xfer += $input->readFieldBegin($fname, $ftype, $fid); + if ($ftype == TType::STOP) { + break; + } + switch ($fid) + { + case 1: + if ($ftype == TType::STRUCT) { + $this->o1 = new metastore_AlreadyExistsException(); + $xfer += $this->o1->read($input); + } else { + $xfer += $input->skip($ftype); + } + break; + case 2: + if ($ftype == TType::STRUCT) { + $this->o2 = new metastore_InvalidObjectException(); + $xfer += $this->o2->read($input); + } else { + $xfer += $input->skip($ftype); + } + break; + case 3: + if ($ftype == TType::STRUCT) { + $this->o3 = new metastore_MetaException(); + $xfer += $this->o3->read($input); + } else { + $xfer += $input->skip($ftype); + } + break; + case 4: + if ($ftype == TType::STRUCT) { + $this->o4 = new metastore_NoSuchObjectException(); + $xfer += $this->o4->read($input); + } else { + $xfer += $input->skip($ftype); + } + break; + default: + $xfer += $input->skip($ftype); + break; + } + $xfer += $input->readFieldEnd(); + } + $xfer += $input->readStructEnd(); + return $xfer; + } + + public function write($output) { + $xfer = 0; + $xfer += $output->writeStructBegin('ThriftHiveMetastore_create_index_result'); + if ($this->o1 !== null) { + $xfer += $output->writeFieldBegin('o1', TType::STRUCT, 1); + $xfer += $this->o1->write($output); + $xfer += $output->writeFieldEnd(); + } + if ($this->o2 !== null) { + $xfer += $output->writeFieldBegin('o2', TType::STRUCT, 2); + $xfer += $this->o2->write($output); + $xfer += $output->writeFieldEnd(); + } + if ($this->o3 !== null) { + $xfer += $output->writeFieldBegin('o3', TType::STRUCT, 3); + $xfer += $this->o3->write($output); + $xfer += $output->writeFieldEnd(); + } + if ($this->o4 !== null) { + $xfer += $output->writeFieldBegin('o4', TType::STRUCT, 4); + $xfer += $this->o4->write($output); + $xfer += $output->writeFieldEnd(); + } + $xfer += $output->writeFieldStop(); + $xfer += $output->writeStructEnd(); + return $xfer; + } + +} + ?> Index: metastore/src/gen-php/hive_metastore_types.php =================================================================== --- metastore/src/gen-php/hive_metastore_types.php (revision 959407) +++ metastore/src/gen-php/hive_metastore_types.php (working copy) @@ -1699,7 +1699,6 @@ public $tableName = null; public $dbName = null; public $colNames = null; - public $partName = null; public function __construct($vals=null) { if (!isset(self::$_TSPEC)) { @@ -1710,7 +1709,7 @@ ), 2 => array( 'var' => 'indexType', - 'type' => TType::I32, + 'type' => TType::STRING, ), 3 => array( 'var' => 'tableName', @@ -1728,10 +1727,6 @@ 'type' => TType::STRING, ), ), - 6 => array( - 'var' => 'partName', - 'type' => TType::STRING, - ), ); } if (is_array($vals)) { @@ -1750,9 +1745,6 @@ if (isset($vals['colNames'])) { $this->colNames = $vals['colNames']; } - if (isset($vals['partName'])) { - $this->partName = $vals['partName']; - } } } @@ -1783,8 +1775,8 @@ } break; case 2: - if ($ftype == TType::I32) { - $xfer += $input->readI32($this->indexType); + if ($ftype == TType::STRING) { + $xfer += $input->readString($this->indexType); } else { $xfer += $input->skip($ftype); } @@ -1820,13 +1812,6 @@ $xfer += $input->skip($ftype); } break; - case 6: - if ($ftype == TType::STRING) { - $xfer += $input->readString($this->partName); - } else { - $xfer += $input->skip($ftype); - } - break; default: $xfer += $input->skip($ftype); break; @@ -1846,8 +1831,8 @@ $xfer += $output->writeFieldEnd(); } if ($this->indexType !== null) { - $xfer += $output->writeFieldBegin('indexType', TType::I32, 2); - $xfer += $output->writeI32($this->indexType); + $xfer += $output->writeFieldBegin('indexType', TType::STRING, 2); + $xfer += $output->writeString($this->indexType); $xfer += $output->writeFieldEnd(); } if ($this->tableName !== null) { @@ -1877,11 +1862,6 @@ } $xfer += $output->writeFieldEnd(); } - if ($this->partName !== null) { - $xfer += $output->writeFieldBegin('partName', TType::STRING, 6); - $xfer += $output->writeString($this->partName); - $xfer += $output->writeFieldEnd(); - } $xfer += $output->writeFieldStop(); $xfer += $output->writeStructEnd(); return $xfer; Index: metastore/if/hive_metastore.thrift =================================================================== --- metastore/if/hive_metastore.thrift (revision 959407) +++ metastore/if/hive_metastore.thrift (working copy) @@ -88,15 +88,15 @@ 7: map parameters } -// index on a hive table is also another table whose columns are the subset of the base table columns along with the offset -// this will automatically generate table (table_name_index_name) +// index on a hive table is also another table whose columns are the subset +// of the base table columns along with the offset this will automatically +// generate table (table_name_index_name) struct Index { 1: string indexName, // unique with in the whole database namespace - 2: i32 indexType, // reserved + 2: string indexType, // index type 3: string tableName, 4: string dbName, 5: list colNames, // for now columns will be sorted in the ascending order - 6: string partName // partition name } // schema of the table/query results etc. @@ -243,7 +243,9 @@ // converts a partition name into a partition specification (a mapping from // the partition cols to the values) map partition_name_to_spec(1: string part_name) - throws(1: MetaException o1) + throws(1: MetaException o1) + void create_index(1:Index index) + throws(1:AlreadyExistsException o1, 2:InvalidObjectException o2, 3:MetaException o3, 4:NoSuchObjectException o4) } // For storing info about archived partitions in parameters Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java (revision 959407) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java (working copy) @@ -18,6 +18,9 @@ package org.apache.hadoop.hive.serde2.objectinspector.primitive; +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; import java.util.HashMap; import java.util.Map; @@ -36,6 +39,7 @@ import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.WritableUtils; /** * ObjectInspectorFactory is the primary way to create new ObjectInspector @@ -49,7 +53,7 @@ /** * TypeEntry stores information about a Hive Primitive TypeInfo. */ - public static class PrimitiveTypeEntry { + public static class PrimitiveTypeEntry implements Writable{ /** * The category of the PrimitiveType. @@ -85,6 +89,29 @@ primitiveWritableClass = hiveClass; this.typeName = typeName; } + + @Override + public void readFields(DataInput in) throws IOException { + primitiveCategory = WritableUtils.readEnum(in, + PrimitiveObjectInspector.PrimitiveCategory.class); + typeName = WritableUtils.readString(in); + try { + primitiveJavaType = Class.forName(WritableUtils.readString(in)); + primitiveJavaClass = Class.forName(WritableUtils.readString(in)); + primitiveWritableClass = Class.forName(WritableUtils.readString(in)); + } catch (ClassNotFoundException e) { + throw new IOException(e); + } + } + + @Override + public void write(DataOutput out) throws IOException { + WritableUtils.writeEnum(out, primitiveCategory); + WritableUtils.writeString(out, typeName); + WritableUtils.writeString(out, primitiveJavaType.getName()); + WritableUtils.writeString(out, primitiveJavaClass.getName()); + WritableUtils.writeString(out, primitiveWritableClass.getName()); + } } static final Map primitiveCategoryToTypeEntry = new HashMap(); Index: ql/src/test/results/clientpositive/index_compact.q.out =================================================================== --- ql/src/test/results/clientpositive/index_compact.q.out (revision 0) +++ ql/src/test/results/clientpositive/index_compact.q.out (revision 0) @@ -0,0 +1,70 @@ +PREHOOK: query: EXPLAIN +CREATE INDEX srcpart_index_proj TYPE COMPACT ON TABLE srcpart(key) +PREHOOK: type: null +POSTHOOK: query: EXPLAIN +CREATE INDEX srcpart_index_proj TYPE COMPACT ON TABLE srcpart(key) +POSTHOOK: type: null +ABSTRACT SYNTAX TREE: + (TOK_CREATEINDEX srcpart_index_proj COMPACT srcpart (TOK_TABCOLNAME key)) + +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + + +PREHOOK: query: CREATE INDEX srcpart_index_proj TYPE COMPACT ON TABLE srcpart(key) +PREHOOK: type: null +POSTHOOK: query: CREATE INDEX srcpart_index_proj TYPE COMPACT ON TABLE srcpart(key) +POSTHOOK: type: null +PREHOOK: query: UPDATE INDEX srcpart_index_proj +PREHOOK: type: null +POSTHOOK: query: UPDATE INDEX srcpart_index_proj +POSTHOOK: type: null +PREHOOK: query: SELECT x.* FROM srcpart_index_proj x WHERE x.ds = '2008-04-08' and x.hr = 11 +PREHOOK: type: QUERY +PREHOOK: Output: file:/Users/heyongqiang/Documents/workspace/Hive-Index/build/ql/scratchdir/hive_2010-06-01_18-59-55_156_2360593265979076422/10000 +POSTHOOK: query: SELECT x.* FROM srcpart_index_proj x WHERE x.ds = '2008-04-08' and x.hr = 11 +POSTHOOK: type: QUERY +POSTHOOK: Output: file:/Users/heyongqiang/Documents/workspace/Hive-Index/build/ql/scratchdir/hive_2010-06-01_18-59-55_156_2360593265979076422/10000 +PREHOOK: query: DROP TABLE srcpart_index_proj +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE srcpart_index_proj +POSTHOOK: type: DROPTABLE +POSTHOOK: Output: default@srcpart_index_proj +PREHOOK: query: EXPLAIN +CREATE INDEX srcpart_index_proj TYPE COMPACT ON TABLE srcpart(key) +PREHOOK: type: null +POSTHOOK: query: EXPLAIN +CREATE INDEX srcpart_index_proj TYPE COMPACT ON TABLE srcpart(key) +POSTHOOK: type: null +ABSTRACT SYNTAX TREE: + (TOK_CREATEINDEX srcpart_index_proj COMPACT srcpart (TOK_TABCOLNAME key)) + +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + + +PREHOOK: query: CREATE INDEX srcpart_index_proj TYPE COMPACT ON TABLE srcpart(key) +PREHOOK: type: null +POSTHOOK: query: CREATE INDEX srcpart_index_proj TYPE COMPACT ON TABLE srcpart(key) +POSTHOOK: type: null +PREHOOK: query: UPDATE INDEX srcpart_index_proj +PREHOOK: type: null +POSTHOOK: query: UPDATE INDEX srcpart_index_proj +POSTHOOK: type: null +PREHOOK: query: SELECT x.* FROM srcpart_index_proj x +PREHOOK: type: QUERY +PREHOOK: Output: file:/Users/heyongqiang/Documents/workspace/Hive-Index/build/ql/scratchdir/hive_2010-06-01_18-59-55_681_5703964119870136891/10000 +POSTHOOK: query: SELECT x.* FROM srcpart_index_proj x +POSTHOOK: type: QUERY +POSTHOOK: Output: file:/Users/heyongqiang/Documents/workspace/Hive-Index/build/ql/scratchdir/hive_2010-06-01_18-59-55_681_5703964119870136891/10000 +PREHOOK: query: DROP TABLE srcpart_index_proj +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE srcpart_index_proj +POSTHOOK: type: DROPTABLE +POSTHOOK: Output: default@srcpart_index_proj Index: ql/src/test/queries/clientnegative/bad_indextype.q =================================================================== --- ql/src/test/queries/clientnegative/bad_indextype.q (revision 0) +++ ql/src/test/queries/clientnegative/bad_indextype.q (revision 0) @@ -0,0 +1 @@ +CREATE INDEX srcpart_index_proj TYPE UNKNOWN ON TABLE srcpart(key); \ No newline at end of file Index: ql/src/test/queries/clientnegative/index_partition_not_exist.q =================================================================== --- ql/src/test/queries/clientnegative/index_partition_not_exist.q (revision 0) +++ ql/src/test/queries/clientnegative/index_partition_not_exist.q (revision 0) @@ -0,0 +1,2 @@ +CREATE INDEX srcpart_index_proj TYPE SUMMARY ON TABLE srcpart(key) PARTITION(ds='2008-04-08', hr=11); +UPDATE INDEX srcpart_index_proj PARTITION(ds='2008-04-08', hr=12); \ No newline at end of file Index: ql/src/test/queries/clientnegative/index_name_unavailable.q =================================================================== --- ql/src/test/queries/clientnegative/index_name_unavailable.q (revision 0) +++ ql/src/test/queries/clientnegative/index_name_unavailable.q (revision 0) @@ -0,0 +1 @@ +CREATE INDEX srcpart TYPE PROJECTION ON TABLE src(key); \ No newline at end of file Index: ql/src/test/queries/clientpositive/index_summary.q =================================================================== --- ql/src/test/queries/clientpositive/index_summary.q (revision 0) +++ ql/src/test/queries/clientpositive/index_summary.q (revision 0) @@ -0,0 +1,13 @@ +EXPLAIN +CREATE INDEX srcpart_index_proj TYPE SUMMARY ON TABLE srcpart(key); +CREATE INDEX srcpart_index_proj TYPE SUMMARY ON TABLE srcpart(key); +UPDATE INDEX srcpart_index_proj; +SELECT x.* FROM srcpart_index_proj x WHERE x.ds = '2008-04-08' and x.hr = 11; +DROP TABLE srcpart_index_proj; + +EXPLAIN +CREATE INDEX srcpart_index_proj TYPE SUMMARY ON TABLE srcpart(key) PARTITION(ds='2008-04-08', hr=11); +CREATE INDEX srcpart_index_proj TYPE SUMMARY ON TABLE srcpart(key) PARTITION(ds='2008-04-08', hr=11); +UPDATE INDEX srcpart_index_proj; +SELECT x.* FROM srcpart_index_proj x; +DROP TABLE srcpart_index_proj; \ No newline at end of file Index: ql/src/test/queries/clientpositive/index_projection.q =================================================================== --- ql/src/test/queries/clientpositive/index_projection.q (revision 0) +++ ql/src/test/queries/clientpositive/index_projection.q (revision 0) @@ -0,0 +1,13 @@ +EXPLAIN +CREATE INDEX srcpart_index_proj TYPE PROJECTION ON TABLE srcpart(key); +CREATE INDEX srcpart_index_proj TYPE PROJECTION ON TABLE srcpart(key); +UPDATE INDEX srcpart_index_proj; +SELECT x.* FROM srcpart_index_proj x WHERE x.ds = '2008-04-08' and x.hr = 11; +DROP TABLE srcpart_index_proj; + +EXPLAIN +CREATE INDEX srcpart_index_proj TYPE PROJECTION ON TABLE srcpart(key) PARTITION(ds='2008-04-08', hr=11); +CREATE INDEX srcpart_index_proj TYPE PROJECTION ON TABLE srcpart(key) PARTITION(ds='2008-04-08', hr=11); +UPDATE INDEX srcpart_index_proj; +SELECT x.* FROM srcpart_index_proj x; +DROP TABLE srcpart_index_proj; \ No newline at end of file Index: ql/src/test/queries/clientpositive/index_compact.q =================================================================== --- ql/src/test/queries/clientpositive/index_compact.q (revision 0) +++ ql/src/test/queries/clientpositive/index_compact.q (revision 0) @@ -0,0 +1,13 @@ +EXPLAIN +CREATE INDEX srcpart_index_proj TYPE COMPACT ON TABLE srcpart(key); +CREATE INDEX srcpart_index_proj TYPE COMPACT ON TABLE srcpart(key); +UPDATE INDEX srcpart_index_proj; +SELECT x.* FROM srcpart_index_proj x WHERE x.ds = '2008-04-08' and x.hr = 11; +DROP TABLE srcpart_index_proj; + +EXPLAIN +CREATE INDEX srcpart_index_proj TYPE COMPACT ON TABLE srcpart(key); +CREATE INDEX srcpart_index_proj TYPE COMPACT ON TABLE srcpart(key); +UPDATE INDEX srcpart_index_proj; +SELECT x.* FROM srcpart_index_proj x; +DROP TABLE srcpart_index_proj; \ No newline at end of file Index: ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java (revision 959407) +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java (working copy) @@ -20,6 +20,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -44,7 +45,9 @@ import org.apache.hadoop.hive.metastore.api.InvalidOperationException; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; +import org.apache.hadoop.hive.metastore.api.Order; import org.apache.hadoop.hive.ql.exec.Utilities; +import org.apache.hadoop.hive.ql.io.RCFileInputFormat; import org.apache.hadoop.hive.serde2.Deserializer; import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; @@ -327,6 +330,88 @@ } /** + * Creates the index with the given objects + * + * @param indexName + * the index name + * @param tableName + * the table name that this index is built on + * @param indexType + * the type of the index + * @param indexedCols + * @param inputFormat + * @param outputFormat + * @param serde + * @throws HiveException + */ + public void createIndex(String indexName, String tableName, String indexType, + List indexedCols, String inputFormat, String outputFormat, + String serde) + throws HiveException { + try { + String dbName = MetaStoreUtils.DEFAULT_DATABASE_NAME; + org.apache.hadoop.hive.metastore.api.Table baseTbl = getMSC().getTable(dbName, tableName); + org.apache.hadoop.hive.metastore.api.Table tbl = baseTbl.clone(); + tbl.setParameters(null); + MetaStoreUtils.setIndexTable(tbl); + MetaStoreUtils.setBaseTableOfIndexTable(tbl, tableName); + MetaStoreUtils.setIndexType(tbl, indexType); + tbl.setTableName(indexName); + List indexTblCols = new ArrayList(); + List sortCols = new ArrayList(); + tbl.getSd().setBucketCols(null); + int k = 0; + for (int i = 0; i < tbl.getSd().getCols().size(); i++) { + FieldSchema col = tbl.getSd().getCols().get(i); + if (indexedCols.contains(col.getName())) { + indexTblCols.add(col); + sortCols.add(new Order(col.getName(), 1)); + k++; + } + } + if (k != indexedCols.size()) + throw new RuntimeException( + "Check the index columns, they should appear in the table being indexed."); + + FieldSchema bucketFileName = new FieldSchema("_bucketname", "string", ""); + indexTblCols.add(bucketFileName); + FieldSchema offSets = new FieldSchema("_offsets", "array", ""); + indexTblCols.add(offSets); + tbl.getSd().setCols(indexTblCols); + tbl.getSd().setLocation(null); + tbl.getSd().setNumBuckets(1); + tbl.getSd().setSortCols(sortCols); + if(inputFormat == null) { + inputFormat = "org.apache.hadoop.mapred.TextInputFormat"; + } + tbl.getSd().setInputFormat(inputFormat); + + if(outputFormat == null) { + outputFormat = + "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"; + } + tbl.getSd().setOutputFormat(outputFormat); + + if(serde == null) { + serde = org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class.getName(); + } + + tbl.getSd().getSerdeInfo().setSerializationLib(serde); + getMSC().createTable(tbl); + + MetaStoreUtils.appendIndexTable(baseTbl, indexName); + getMSC().alter_table(MetaStoreUtils.DEFAULT_DATABASE_NAME, baseTbl.getTableName(), baseTbl); + } catch (Exception e) { + throw new HiveException(e); + } + } + + /** + * Drops table along with the data in it. If the table doesn't exist + * then it is a no-op + * @param dbName database where the table lives + * @param tableName table to drop + * @throws HiveException thrown if the drop fails * Drops table along with the data in it. If the table doesn't exist then it * is a no-op * @@ -1112,4 +1197,6 @@ + e.getMessage(), e); } } + + }; Index: ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java (revision 959407) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java (working copy) @@ -153,6 +153,14 @@ return (ret); } + makeChild(ret, tasklist); + + return (ret); + } + + + public static void makeChild(Task ret, + Task... tasklist) { // Add the new task as child of each of the passed in tasks for (Task tsk : tasklist) { List> children = tsk.getChildTasks(); @@ -162,8 +170,6 @@ children.add(ret); tsk.setChildTasks(children); } - - return (ret); } private TaskFactory() { Index: ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java (revision 959407) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java (working copy) @@ -57,6 +57,7 @@ import org.apache.hadoop.hive.ql.exec.errors.ErrorAndSolution; import org.apache.hadoop.hive.ql.exec.errors.TaskLogProcessor; import org.apache.hadoop.hive.ql.history.HiveHistory.Keys; +import org.apache.hadoop.hive.ql.index.IndexBuilderBaseReducer; import org.apache.hadoop.hive.ql.io.HiveKey; import org.apache.hadoop.hive.ql.io.HiveOutputFormat; import org.apache.hadoop.hive.ql.metadata.HiveException; @@ -71,11 +72,14 @@ import org.apache.hadoop.io.Text; import org.apache.hadoop.mapred.Counters; import org.apache.hadoop.mapred.FileInputFormat; +import org.apache.hadoop.mapred.FileOutputFormat; import org.apache.hadoop.mapred.InputFormat; import org.apache.hadoop.mapred.JobClient; import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.JobStatus; +import org.apache.hadoop.mapred.Mapper; +import org.apache.hadoop.mapred.OutputFormat; import org.apache.hadoop.mapred.Partitioner; +import org.apache.hadoop.mapred.Reducer; import org.apache.hadoop.mapred.RunningJob; import org.apache.hadoop.mapred.TaskCompletionEvent; import org.apache.log4j.BasicConfigurator; @@ -434,7 +438,7 @@ // this is a temporary hack to fix things that are not fixed in the compiler Integer numReducersFromWork = work.getNumReduceTasks(); - if (work.getReducer() == null) { + if (work.getReducer() == null && work.getIndexCols() == null) { console .printInfo("Number of reduce tasks is set to 0 since there's no reduce operator"); work.setNumReduceTasks(Integer.valueOf(0)); @@ -558,13 +562,14 @@ throw new RuntimeException("Plan invalid, Reason: " + invalidReason); } - String hiveScratchDir; - if (driverContext.getCtx() != null && driverContext.getCtx().getQueryPath() != null) { - hiveScratchDir = driverContext.getCtx().getQueryPath().toString(); - } else { - hiveScratchDir = HiveConf.getVar(job, HiveConf.ConfVars.SCRATCHDIR); - } + String hiveScratchDir = getScratchDir(driverContext); + + String jobScratchDirStr = hiveScratchDir + File.separator + + Utilities.randGen.nextInt(); + Path jobScratchDir = new Path(jobScratchDirStr); + FileOutputFormat.setOutputPath(job, jobScratchDir); + String emptyScratchDirStr = null; Path emptyScratchDir = null; @@ -587,13 +592,18 @@ } } } + + job.setMapperClass(getMapperClass()); - ShimLoader.getHadoopShims().setNullOutputFormat(job); - job.setMapperClass(ExecMapper.class); + job.setMapOutputKeyClass(getMapOutputKeyClass()); + job.setMapOutputValueClass(getMapOutputValueClass()); + Class output = getOutputFormatCls(); + if (output != null) { + job.setOutputFormat(output); + } else { + ShimLoader.getHadoopShims().setNullOutputFormat(job); + } - job.setMapOutputKeyClass(HiveKey.class); - job.setMapOutputValueClass(BytesWritable.class); - try { job.setPartitionerClass((Class) (Class.forName(HiveConf.getVar(job, HiveConf.ConfVars.HIVEPARTITIONER)))); @@ -609,7 +619,7 @@ work.getMinSplitSize().intValue()); } job.setNumReduceTasks(work.getNumReduceTasks().intValue()); - job.setReducerClass(ExecReducer.class); + job.setReducerClass(getReducerClass()); if (work.getInputformat() != null) { HiveConf.setVar(job, HiveConf.ConfVars.HIVEINPUTFORMAT, work.getInputformat()); @@ -623,24 +633,16 @@ HiveConf.ConfVars.HADOOPSPECULATIVEEXECREDUCERS, useSpeculativeExecReducers); - String inpFormat = HiveConf.getVar(job, HiveConf.ConfVars.HIVEINPUTFORMAT); - if ((inpFormat == null) || (!StringUtils.isNotBlank(inpFormat))) { - inpFormat = ShimLoader.getHadoopShims().getInputFormatClassName(); - } - - LOG.info("Using " + inpFormat); - try { - job.setInputFormat((Class) (Class - .forName(inpFormat))); + job.setInputFormat(getInputFormatCls()); } catch (ClassNotFoundException e) { - throw new RuntimeException(e.getMessage()); + throw new RuntimeException(e); } // No-Op - we don't really write anything here .. - job.setOutputKeyClass(Text.class); - job.setOutputValueClass(Text.class); - + job.setOutputKeyClass(getOutputKeyClass()); + job.setOutputValueClass(getOutputValueClass()); + // Transfer HIVEAUXJARS and HIVEADDEDJARS to "tmpjars" so hadoop understands // it String auxJars = HiveConf.getVar(job, HiveConf.ConfVars.HIVEAUXJARS); @@ -676,7 +678,7 @@ HiveConf.setVar(job, HiveConf.ConfVars.HADOOPJOBNAME, "JOB" + randGen.nextInt()); } - + try { addInputPaths(job, work, emptyScratchDirStr); @@ -774,6 +776,11 @@ if (work.getReducer() != null) { work.getReducer().jobClose(job, success, feedBack); } + + if (IndexBuilderBaseReducer.class.isAssignableFrom(this + .getReducerClass())) { + this.closeIndexBuilder(job, success); + } } } catch (Exception e) { // jobClose needs to execute successfully otherwise fail task @@ -789,7 +796,27 @@ return (returnVal); } + + private void closeIndexBuilder(JobConf job, boolean success) + throws HiveException, IOException { + String outputPath = this.work.getOutputPath(); + if (outputPath == null) { + outputPath = getScratchDir(driverContext) + Utilities.randGen.nextInt(); + } + console.printInfo("Closing Index builder job. Output path is " + outputPath); + IndexBuilderBaseReducer.indexBuilderJobClose(outputPath, success, job, console); + } + private String getScratchDir(DriverContext driverContext) { + String hiveScratchDir; + if (driverContext.getCtx() != null && driverContext.getCtx().getQueryPath() != null) { + hiveScratchDir = driverContext.getCtx().getQueryPath().toString(); + } else { + hiveScratchDir = HiveConf.getVar(job, HiveConf.ConfVars.SCRATCHDIR); + } + return hiveScratchDir; + } + /** * This msg pattern is used to track when a job is started. * @@ -1343,6 +1370,80 @@ return StageType.MAPRED; } + private boolean delOutputIfExists; + + public Class getInputFormatCls() throws ClassNotFoundException { + if(this.getWork() == null || this.getWork().getInputFormatCls() == null) { + String inpFormat = HiveConf.getVar(job, HiveConf.ConfVars.HIVEINPUTFORMAT); + if ((inpFormat == null) || (!StringUtils.isNotBlank(inpFormat))) + inpFormat = ShimLoader.getHadoopShims().getInputFormatClassName(); + if(inpFormat == null) { + return org.apache.hadoop.hive.ql.io.HiveInputFormat.class; + } + return (Class) Class.forName(inpFormat); + } else { + return this.getWork().getInputFormatCls(); + } + + } + + public Class getMapperClass() { + if(this.getWork() != null && this.getWork().getMapperClass() != null) { + return this.getWork().getMapperClass(); + } + return ExecMapper.class; + } + + public Class getReducerClass() { + if(this.getWork() != null && this.getWork().getReducerClass() != null) { + return this.getWork().getReducerClass(); + } + return ExecReducer.class; + } + + public Class getMapOutputKeyClass() { + if(this.getWork()!=null && this.getWork().getMapOutputKeyClass() != null) { + return this.getWork().getMapOutputKeyClass(); + } + return HiveKey.class; + } + + public Class getMapOutputValueClass() { + if(this.getWork()!=null && this.getWork().getMapOutputValueClass() != null) { + return this.getWork().getMapOutputValueClass(); + } + return BytesWritable.class; + } + + public Class getOutputKeyClass() { + if(this.getWork()!=null && this.getWork().getOutputKeyClass() !=null) { + return this.getWork().getOutputKeyClass(); + } + return Text.class; + } + + public Class getOutputValueClass() { + if(this.getWork()!=null && this.getWork().getOutputValueClass() != null) { + return this.getWork().getOutputValueClass(); + } + return Text.class; + } + + public boolean isDelOutputIfExists() { + return delOutputIfExists; + } + + public void setDelOutputIfExists(boolean delOutputIfExists) { + this.delOutputIfExists = delOutputIfExists; + } + + public Class getOutputFormatCls() { + if (this.work != null) { + return this.getWork().getOutputFormatCls(); + } + return null; + } + @Override public String getName() { return "EXEC"; Index: ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java (revision 959407) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java (working copy) @@ -53,6 +53,7 @@ import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.Index; import org.apache.hadoop.hive.metastore.api.InvalidOperationException; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.Order; @@ -84,6 +85,7 @@ import org.apache.hadoop.hive.ql.plan.ShowTableStatusDesc; import org.apache.hadoop.hive.ql.plan.ShowTablesDesc; import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes; +import org.apache.hadoop.hive.ql.plan.createIndexDesc; import org.apache.hadoop.hive.ql.plan.api.StageType; import org.apache.hadoop.hive.serde.Constants; import org.apache.hadoop.hive.serde2.Deserializer; @@ -144,7 +146,13 @@ return createTable(db, crtTbl); } + createIndexDesc crtIndex = work.getCreateIndexDesc(); + if (crtIndex != null) { + return createIndex(db, crtIndex); + } + CreateTableLikeDesc crtTblLike = work.getCreateTblLikeDesc(); + if (crtTblLike != null) { return createTableLike(db, crtTblLike); } @@ -233,6 +241,14 @@ return 0; } + private int createIndex(Hive db, createIndexDesc crtIndex) throws HiveException { + db + .createIndex(crtIndex.getIndexName(), crtIndex.getTableName(), crtIndex + .getIndexType(), crtIndex.getIndexedCols(), crtIndex.getInputFormat(), + crtIndex.getOutputFormat(), crtIndex.getSerde()); + return 0; + } + /** * Add a partition to a table. * Index: ql/src/java/org/apache/hadoop/hive/ql/plan/DDLWork.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/plan/DDLWork.java (revision 959407) +++ ql/src/java/org/apache/hadoop/hive/ql/plan/DDLWork.java (working copy) @@ -30,7 +30,7 @@ */ public class DDLWork implements Serializable { private static final long serialVersionUID = 1L; - + private createIndexDesc createIndexDesc; private CreateTableDesc createTblDesc; private CreateTableLikeDesc createTblLikeDesc; private CreateViewDesc createVwDesc; @@ -63,6 +63,10 @@ this.outputs = outputs; } + public DDLWork(createIndexDesc createIndex) { + this.createIndexDesc = createIndex; + } + /** * @param alterTblDesc * alter table descriptor @@ -222,7 +226,15 @@ public void setCreateTblDesc(CreateTableDesc createTblDesc) { this.createTblDesc = createTblDesc; } + + public createIndexDesc getCreateIndexDesc() { + return createIndexDesc; + } + public void setCreateIndexDesc(createIndexDesc createIndexDesc) { + this.createIndexDesc = createIndexDesc; + } + /** * @return the createTblDesc */ Index: ql/src/java/org/apache/hadoop/hive/ql/plan/createIndexDesc.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/plan/createIndexDesc.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/plan/createIndexDesc.java (revision 0) @@ -0,0 +1,107 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.plan; + +import java.io.Serializable; +import java.util.List; + +public class createIndexDesc extends DDLDesc implements Serializable { + + private static final long serialVersionUID = 1L; + String tableName; + String indexName; + List indexedCols; + String inputFormat; + String outputFormat; + String serde; + String indexType; + + public createIndexDesc() { + super(); + } + + public createIndexDesc(String tableName, String indexName, + List indexedCols,String inputFormat, String outputFormat, String serde, String indexType) { + super(); + this.tableName = tableName; + this.indexName = indexName; + this.indexedCols = indexedCols; + this.indexType = indexType; + this.inputFormat = inputFormat; + this.outputFormat = outputFormat; + this.serde = serde; + } + + public String getTableName() { + return tableName; + } + + public void setTableName(String tableName) { + this.tableName = tableName; + } + + public String getIndexName() { + return indexName; + } + + public void setIndexName(String indexName) { + this.indexName = indexName; + } + + public List getIndexedCols() { + return indexedCols; + } + + public void setIndexedCols(List indexedCols) { + this.indexedCols = indexedCols; + } + + public String getInputFormat() { + return inputFormat; + } + + public void setInputFormat(String inputFormat) { + this.inputFormat = inputFormat; + } + + public String getOutputFormat() { + return outputFormat; + } + + public void setOutputFormat(String outputFormat) { + this.outputFormat = outputFormat; + } + + public String getSerde() { + return serde; + } + + public void setSerde(String serde) { + this.serde = serde; + } + + public String getIndexType() { + return indexType; + } + + public void setIndexType(String indexType) { + this.indexType = indexType; + } + +} \ No newline at end of file Index: ql/src/java/org/apache/hadoop/hive/ql/plan/MapredWork.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/plan/MapredWork.java (revision 959407) +++ ql/src/java/org/apache/hadoop/hive/ql/plan/MapredWork.java (working copy) @@ -25,8 +25,13 @@ import java.util.List; import java.util.Map; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.Utilities; +import org.apache.hadoop.mapred.InputFormat; +import org.apache.hadoop.mapred.Mapper; +import org.apache.hadoop.mapred.OutputFormat; +import org.apache.hadoop.mapred.Reducer; /** * MapredWork. @@ -65,7 +70,26 @@ private MapredLocalWork mapLocalWork; private String inputformat; - + + //theses fields are used for building indexing. + private Class mapperClass; + private Class reducerClass; + private Class mapOutputKeyClass; + private Class mapOutputValueClass; + private Class outputKeyClass; + private Class outputValueClass; + private Class inputFormatCls; + private Class outputFormatCls; + private boolean compressed; + private String compressCodec; + private String outputPath; + private TableDesc indexTableDesc; + + /* + * used for indexing. + */ + private String indexCols; + public MapredWork() { aliasToPartnInfo = new LinkedHashMap(); } @@ -240,7 +264,7 @@ @SuppressWarnings("nls") public String isInvalid() { - if ((getNumReduceTasks() >= 1) && (getReducer() == null)) { + if((getNumReduceTasks() >= 1) && (getReducer() == null) && (getIndexCols() == null)) { return "Reducers > 0 but no reduce operator"; } @@ -321,5 +345,109 @@ public void setInputformat(String inputformat) { this.inputformat = inputformat; } + + public String getIndexCols() { + return indexCols; + } + public void setIndexCols(String indexCols) { + this.indexCols = indexCols; + } + + public Class getMapperClass() { + return mapperClass; + } + + public void setMapperClass(Class mapperClass) { + this.mapperClass = mapperClass; + } + + public Class getReducerClass() { + return reducerClass; + } + + public void setReducerClass(Class reducerClass) { + this.reducerClass = reducerClass; + } + + public Class getMapOutputKeyClass() { + return mapOutputKeyClass; + } + + public void setMapOutputKeyClass(Class mapOutputKeyClass) { + this.mapOutputKeyClass = mapOutputKeyClass; + } + + public Class getMapOutputValueClass() { + return mapOutputValueClass; + } + + public void setMapOutputValueClass(Class mapOutputValueClass) { + this.mapOutputValueClass = mapOutputValueClass; + } + + public Class getOutputKeyClass() { + return outputKeyClass; + } + + public void setOutputKeyClass(Class outputKeyClass) { + this.outputKeyClass = outputKeyClass; + } + + public Class getOutputValueClass() { + return outputValueClass; + } + + public void setOutputValueClass(Class outputValueClass) { + this.outputValueClass = outputValueClass; + } + + public Class getInputFormatCls() { + return inputFormatCls; + } + + public void setInputFormatCls(Class inputFormatCls) { + this.inputFormatCls = inputFormatCls; + } + + public Class getOutputFormatCls() { + return outputFormatCls; + } + + public void setOutputFormatCls(Class outputFormat) { + this.outputFormatCls = outputFormat; + } + + public boolean getCompressed() { + return compressed; + } + + public void setCompressed(boolean isCompressed) { + this.compressed = isCompressed; + } + + public String getCompressCodec() { + return compressCodec; + } + + public void setCompressCodec(String compressCodec) { + this.compressCodec = compressCodec; + } + + public String getOutputPath() { + return outputPath; + } + + public void setOutputPath(String outputPath) { + this.outputPath = outputPath; + } + + public TableDesc getIndexTableDesc() { + return indexTableDesc; + } + + public void setIndexTableDesc(TableDesc indexTableDesc) { + this.indexTableDesc = indexTableDesc; + } + } Index: ql/src/java/org/apache/hadoop/hive/ql/io/RCFile.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/io/RCFile.java (revision 959407) +++ ql/src/java/org/apache/hadoop/hive/ql/io/RCFile.java (working copy) @@ -432,6 +432,7 @@ NonSyncDataOutputBuffer valBuf = loadedColumnsValueBuffer[addIndex]; valBuf.reset(); + valBuf.write(in, vaRowsLen); if (codec != null) { decompressedFlag[addIndex] = false; @@ -1486,6 +1487,10 @@ public String toString() { return file.toString(); } + + public boolean isCompressedRCFile() { + return this.decompress; + } /** Close the reader. */ public void close() { Index: ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java (revision 959407) +++ ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java (working copy) @@ -179,8 +179,9 @@ */ protected static Map> inputFormats; - static InputFormat getInputFormatFromCache( - Class inputFormatClass, JobConf job) throws IOException { + public static InputFormat getInputFormatFromCache( + Class inputFormatClass, JobConf job) throws IOException { + if (inputFormats == null) { inputFormats = new HashMap>(); } @@ -329,7 +330,7 @@ for (String alias : aliases) { Operator op = this.mrwork.getAliasToWork().get( alias); - if (op instanceof TableScanOperator) { + if (op != null && op instanceof TableScanOperator) { TableScanOperator tableScan = (TableScanOperator) op; ArrayList list = tableScan.getNeededColumnIDs(); if (list != null) { Index: ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java (revision 959407) +++ ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java (working copy) @@ -206,22 +206,30 @@ public static RecordWriter getHiveRecordWriter(JobConf jc, TableDesc tableInfo, Class outputClass, FileSinkDesc conf, Path outPath) throws HiveException { + String codecStr = conf.getCompressCodec(); + String type = conf.getCompressType(); + boolean compressed = conf.getCompressed(); + return getHiveRecordWriter(jc, tableInfo, outputClass, compressed, + codecStr, type, outPath); + } + + public static RecordWriter getHiveRecordWriter(JobConf jc, + TableDesc tableInfo, Class outputClass, boolean isCompressed, + String codecStr, String compressionType, Path outPath) throws HiveException { try { HiveOutputFormat hiveOutputFormat = tableInfo .getOutputFileFormatClass().newInstance(); - boolean isCompressed = conf.getCompressed(); + JobConf jc_output = jc; if (isCompressed) { jc_output = new JobConf(jc); - String codecStr = conf.getCompressCodec(); if (codecStr != null && !codecStr.trim().equals("")) { Class codec = (Class) Class .forName(codecStr); FileOutputFormat.setOutputCompressorClass(jc_output, codec); } - String type = conf.getCompressType(); - if (type != null && !type.trim().equals("")) { - CompressionType style = CompressionType.valueOf(type); + if (compressionType != null && !compressionType.trim().equals("")) { + CompressionType style = CompressionType.valueOf(compressionType); SequenceFileOutputFormat.setOutputCompressionType(jc, style); } } Index: ql/src/java/org/apache/hadoop/hive/ql/io/FilterMapper.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/io/FilterMapper.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/io/FilterMapper.java (revision 0) @@ -0,0 +1,109 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.io; + +import java.io.IOException; +import java.util.ArrayList; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.WritableComparable; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.MapReduceBase; +import org.apache.hadoop.mapred.Mapper; +import org.apache.hadoop.mapred.OutputCollector; +import org.apache.hadoop.mapred.Reporter; + +/* + * This is the mapper class for filtering the hive stream output for the given predicates. + * Conf Input: hive.index.pred = comma separated list of value to search for + * hive.index.pred_pos = position of the predicate column in the input list (0 based) + * TODO this should come from execution plan but for now this is input by the client program + * Map Input: output from HiveStreaming's select clause + * Output: input rows for which output matches + */ +public class FilterMapper extends MapReduceBase +implements Mapper { + + private ArrayList predicateValList = null; + private int predPos; + private boolean pr = true; + public static final Log l4j = LogFactory.getLog("FilterMapper"); + + + @Override + public void configure(JobConf job) { + + String predClause = job.get("hive.index.pred"); + String[] predList = predClause.split(","); + if (predList.length < 1) { + throw new RuntimeException("Configure: predicate clause should have ip addresses seperated by a comma"); + } + + predicateValList = new ArrayList (predList.length); + for (String pred : predList) { + predicateValList.add(pred); + } + for (String string : predList) { + l4j.info(string); + } + String predPosStr = job.get("hive.index.pred_pos"); + predPos = Integer.parseInt(predPosStr); + } + + public void map(WritableComparable key, Writable value, + OutputCollector output, Reporter reporter) throws IOException { + // key and value (ip and user should be non-null + if((key == null) || (value == null)) { + return; + } + String [] cols = ((Text)value).toString().split("\t"); + if (cols.length < predPos) { + if (pr) { + pr = false; + // there are not enough columns so just ignore this row + l4j.info("Number of columns: " + cols.length + " Predicate pos: " + predPos); + for (String string : cols) { + l4j.info(string); + } + } + return; + } + if(predPos == 0) { + String indexedKeyCol = ((Text)key).toString(); + if(predicateValList.indexOf(indexedKeyCol) == -1) { + // current key is not in the predicate val list so nothing do to for this row + return; + } + } else { + if(predicateValList.indexOf(cols[predPos-1]) == -1) { + return; + } + } + +// String viewTime = ((Text)key).toString(); +// dt.setTime(Long.parseLong(viewTime) * 1000); +// outKey.set(sdf.format(dt)); + + // if it passes the equality predicate then just output it +// output.collect(outKey, value); + output.collect(key, value); + } +} Index: ql/src/java/org/apache/hadoop/hive/ql/index/IndexBuilderSummaryMapper.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/index/IndexBuilderSummaryMapper.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/index/IndexBuilderSummaryMapper.java (revision 0) @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.index; + +import java.io.IOException; + +import org.apache.hadoop.mapred.OutputCollector; +import org.apache.hadoop.mapred.Reporter; + +public class IndexBuilderSummaryMapper extends IndexBuilderBaseMapper { + + public void map(Object key, Object value, OutputCollector oc, + Reporter reporter) throws IOException { + try { + reporter.progress(); + perpareMapper(key, value); + oc.collect(cachedIndexCols, cachedPos); + } catch (Throwable e) { + throw new RuntimeException("Error in indexer's mapper:", e); + } + } +} Index: ql/src/java/org/apache/hadoop/hive/ql/index/IndexBuilderBaseReducer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/index/IndexBuilderBaseReducer.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/index/IndexBuilderBaseReducer.java (revision 0) @@ -0,0 +1,249 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.index; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.ql.exec.Utilities; +import org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter; +import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils; +import org.apache.hadoop.hive.ql.io.HiveOutputFormat; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.plan.MapredWork; +import org.apache.hadoop.hive.ql.plan.TableDesc; +import org.apache.hadoop.hive.ql.session.SessionState.LogHelper; +import org.apache.hadoop.hive.serde2.Deserializer; +import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.hive.serde2.Serializer; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.StructField; +import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption; +import org.apache.hadoop.hive.shims.ShimLoader; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.WritableComparable; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.MapReduceBase; +import org.apache.hadoop.mapred.OutputCollector; +import org.apache.hadoop.mapred.Reducer; +import org.apache.hadoop.mapred.Reporter; + +public abstract class IndexBuilderBaseReducer extends MapReduceBase implements + Reducer { + + Serializer serializer; + Deserializer deserializer; + Class outputClass; + RecordWriter outWriter; + + Path finalPath; + FileSystem fs; + + String[] indexColumns; + StructObjectInspector rowObjectInspector; + StructObjectInspector keyObjectInspector; + StandardStructObjectInspector indexRowOutputObjInspector; + List structFields = new ArrayList(); + List indexColsInspectors; + + boolean exception = false; + boolean autoDelete = false; + Path outPath; + + Object[] indexOutputObjects; + + public void configure(JobConf job) { + MapredWork conf = Utilities.getMapRedWork(job); + TableDesc table = conf.getIndexTableDesc(); + try { + serializer = (Serializer) table.getDeserializerClass().newInstance(); + serializer.initialize(job, table.getProperties()); + outputClass = serializer.getSerializedClass(); + + indexColumns = conf.getIndexCols().split(","); + indexOutputObjects = new Object[indexColumns.length]; + indexColsInspectors = new ArrayList(); + List indexOutputRowInspectors = new ArrayList(); + List outputColName = new ArrayList(); + List indexColNames = new ArrayList(); + deserializer = (Deserializer) HiveIndex.INDEX_MAPRED_BOUN_SERDE + .newInstance(); + deserializer.initialize(job, table.getProperties()); + rowObjectInspector = (StructObjectInspector) deserializer + .getObjectInspector(); + for (int k = 0; k < indexColumns.length - 2; k++) { + String col = indexColumns[k]; + StructField field = rowObjectInspector.getStructFieldRef(col); + structFields.add(field); + ObjectInspector inspector = field.getFieldObjectInspector(); + if (!inspector.getCategory().equals(Category.PRIMITIVE)) { + throw new RuntimeException("Only primitive columns can be indexed."); + } + outputColName.add(col); + indexColNames.add(col); + indexColsInspectors.add(inspector); + indexOutputRowInspectors.add(ObjectInspectorUtils + .getStandardObjectInspector(inspector, ObjectInspectorCopyOption.JAVA)); + } + + for (int k = indexColumns.length - 2; k < indexColumns.length; k++) { + String col = indexColumns[k]; + StructField field = rowObjectInspector.getStructFieldRef(col); + structFields.add(field); + ObjectInspector inspector = field.getFieldObjectInspector(); + outputColName.add(col); + indexOutputRowInspectors.add(ObjectInspectorUtils + .getStandardObjectInspector(inspector, ObjectInspectorCopyOption.JAVA)); + } + + keyObjectInspector = ObjectInspectorFactory + .getStandardStructObjectInspector(indexColNames, + indexColsInspectors); + indexRowOutputObjInspector = ObjectInspectorFactory + .getStandardStructObjectInspector(outputColName, + indexOutputRowInspectors); + + boolean isCompressed = conf.getCompressed(); + String codecStr = conf.getCompressCodec(); + //the job's final output path + String specPath = conf.getOutputPath(); + Path tmpPath = Utilities.toTempPath(specPath); + String taskId = Utilities.getTaskId(job); + finalPath = new Path(tmpPath, taskId); + outPath = new Path(tmpPath, Utilities.toTempPath(taskId)); + fs = (new Path(specPath)).getFileSystem(job); + outWriter = HiveFileFormatUtils.getHiveRecordWriter(job, table, + outputClass, isCompressed, codecStr, "", outPath); + HiveOutputFormat hiveOutputFormat = table + .getOutputFileFormatClass().newInstance(); + + finalPath = HiveFileFormatUtils.getOutputFormatFinalPath(tmpPath, taskId, + job, hiveOutputFormat, isCompressed, finalPath); + autoDelete = ShimLoader.getHadoopShims().fileSystemDeleteOnExit(fs, outPath); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + public void reduce(WritableComparable key, Iterator values, + OutputCollector output, Reporter reporter) throws IOException { + try { + Object row = deserializer.deserialize((Writable) key); + for (int i = 0; i < indexColumns.length - 2; i++) { + this.indexOutputObjects[i] = ObjectInspectorUtils.copyToStandardObject( + rowObjectInspector.getStructFieldData(row, structFields.get(i)), + indexColsInspectors.get(i), ObjectInspectorCopyOption.JAVA); + } + doReduce(this.indexOutputObjects, values, reporter); + } catch (Throwable e) { + this.exception = true; + close(); + throw new IOException(e); + } + } + + public abstract void doReduce(Object[] keys, Iterator values, + Reporter reporter) throws IOException, SerDeException; + + @Override + public void close() throws IOException { + // close writer + if (outWriter != null) { + outWriter.close(exception); + outWriter = null; + } + + if (!exception) { + FileStatus fss = fs.getFileStatus(outPath); + System.out.println("renamed path " + outPath + " to " + finalPath + + " . File size is " + fss.getLen()); + if (!fs.rename(outPath, finalPath)) { + throw new IOException("Unable to rename output to " + finalPath); + } + } else { + if(!autoDelete) { + fs.delete(outPath, true); + } + } + } + + public static void indexBuilderJobClose(String outputPath, boolean success, + JobConf job, LogHelper console) throws HiveException, IOException { + FileSystem fs = (new Path(outputPath)).getFileSystem(job); + Path tmpPath = Utilities.toTempPath(outputPath); + Path intermediatePath = new Path(tmpPath.getParent(), tmpPath.getName() + ".intemediate"); + Path finalPath = new Path(outputPath); + System.out.println("tmpPath is " + tmpPath); + System.out.println("intemediatePath is " + intermediatePath); + System.out.println("finalPath is " + finalPath); + if (success) { + if (fs.exists(tmpPath)) { + + if(fs.exists(intermediatePath)) { + fs.delete(intermediatePath, true); + } + fs.mkdirs(intermediatePath); + FileStatus[] fss = fs.listStatus(tmpPath); + for (FileStatus f : fss) { + fs.rename(f.getPath(), new Path(intermediatePath, f.getPath() + .getName())); + } + + fss = fs.listStatus(intermediatePath); + long len = 0; + for (FileStatus f : fss) { + len += f.getLen(); + } + console.printInfo("IntermediatePath Path's file number is " + fss.length + ", total size is " + + len); + + Utilities.removeTempOrDuplicateFiles(fs, intermediatePath); + + fss = fs.listStatus(finalPath); + for (FileStatus f : fss) { + fs.delete(f.getPath(), true); + } + Utilities.renameOrMoveFiles(fs, intermediatePath, finalPath); + fs.delete(tmpPath, true); + fs.delete(intermediatePath, true); + + fss = fs.listStatus(finalPath); + len = 0; + for (FileStatus f : fss) { + len += f.getLen(); + } + console.printInfo("Final Path's file number is " + fss.length + ", total size is " + + len); + } + } else { + fs.delete(tmpPath, true); + } + } + +} Index: ql/src/java/org/apache/hadoop/hive/ql/index/HiveIndex.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/index/HiveIndex.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/index/HiveIndex.java (revision 0) @@ -0,0 +1,339 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.index; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.SortedSet; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.Vector; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.ql.io.RCFile; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.serde2.columnar.BytesRefWritable; +import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; +import org.apache.hadoop.io.SequenceFile; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.WritableComparable; +import org.apache.hadoop.mapred.FileSplit; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.RecordReader; +import org.apache.hadoop.mapred.LineRecordReader.LineReader; +import org.apache.hadoop.util.ReflectionUtils; + +/** + * Holds index related constants + * Actual logic is spread in HiveStreamingRecordReder and IndexReducer classes. + * + */ +public class HiveIndex { + + public static final Log l4j = LogFactory.getLog("HiveIndex"); + + public static final Class INDEX_MAPRED_BOUN_SERDE = LazySimpleSerDe.class; + + public static enum IndexType { + COMPACT_SUMMARY_TABLE("compact"), + SUMMARY_TABLE("summary"), + PROJECTION("projection"); + + private IndexType(String indexType) { + indexTypeName = indexType; + } + + private String indexTypeName; + + public String getName() { + return indexTypeName; + } + } + + public static IndexType getIndexType(String name) { + IndexType[] types = IndexType.values(); + for (IndexType type : types) { + if(type.getName().equals(name.toLowerCase())) + return type; + } + throw new IllegalArgumentException(name + " is not a valid index type."); + } + + + // modeled on sequence file record reader + public static class IndexSequenceFileRecordReader + implements RecordReader { + + private SequenceFile.Reader in; + private long start; + private long end; + private boolean more = true; + protected Configuration conf; + + public IndexSequenceFileRecordReader(Configuration conf, FileSplit split) + throws IOException { + Path path = split.getPath(); + FileSystem fs = path.getFileSystem(conf); + this.in = new SequenceFile.Reader(fs, path, conf); + this.end = split.getStart() + split.getLength(); + this.conf = conf; + + if (split.getStart() > in.getPosition()) + in.sync(split.getStart()); // sync to start + + this.start = in.getPosition(); + more = start < end; + } + + public void sync(long offset) throws IOException { + if(offset > end) { + offset = end; + } + in.sync(offset); + this.start = in.getPosition(); + more = start < end; + } + + /** + * The class of key that must be passed to {@link + * #next(WritableComparable,Writable)}.. + */ + public Class getKeyClass() { + return in.getKeyClass(); + } + + /** + * The class of value that must be passed to {@link + * #next(WritableComparable,Writable)}.. + */ + public Class getValueClass() { + return in.getValueClass(); + } + + @SuppressWarnings("unchecked") + public K createKey() { + return (K) ReflectionUtils.newInstance(getKeyClass(), conf); + } + + @SuppressWarnings("unchecked") + public V createValue() { + return (V) ReflectionUtils.newInstance(getValueClass(), conf); + } + + public synchronized boolean next(K key, V value) throws IOException { + if (!more) + return false; + long pos = in.getPosition(); + boolean eof = in.next(key, value); + if (pos >= end && in.syncSeen()) { + more = false; + } else { + more = eof; + } + return more; + } + + public synchronized boolean next(K key) throws IOException { + if (!more) + return false; + long pos = in.getPosition(); + boolean eof = in.next(key); + if (pos >= end && in.syncSeen()) { + more = false; + } else { + more = eof; + } + return more; + } + + public synchronized void getCurrentValue(V value) throws IOException { + in.getCurrentValue(value); + } + + /** + * Return the progress within the input split + * + * @return 0.0 to 1.0 of the input byte range + */ + public float getProgress() throws IOException { + if (end == start) { + return 0.0f; + } else { + return Math.min(1.0f, (in.getPosition() - start) + / (float) (end - start)); + } + } + + public synchronized long getPos() throws IOException { + return in.getPosition(); + } + + public synchronized void seek(long pos) throws IOException { + in.seek(pos); + } + + public synchronized void close() throws IOException { + in.close(); + } + + public boolean syncSeen() { + return in.syncSeen(); + } + } + + //IndexBucket + public static class IBucket { + private String name = null; + private SortedSet offsets = new TreeSet(); + public IBucket(String n) { + name = n; + } + public void add(Long offset) { + offsets.add(offset); + } + public String getName() { + return name; + } + public SortedSet getOffsets() { + return offsets; + } + public boolean equals(Object obj) { + if(obj.getClass() != this.getClass()) { + return false; + } + return (((IBucket)obj).name.compareToIgnoreCase(this.name) == 0); + } + } + + public static class HiveIndexResult { + private HashMap>> indexResult = new HashMap>>(); + JobConf job = null; + + BytesRefWritable[] bytesRef = new BytesRefWritable[2]; + + public HiveIndexResult(String indexFile, JobConf conf) throws IOException, HiveException { + job = conf; + + bytesRef[0] = new BytesRefWritable(); + bytesRef[1] = new BytesRefWritable(); + + if(indexFile != null) { + Path indexFilePath = new Path(indexFile); + FileSystem fs = FileSystem.get(conf); + FileStatus indexStat = fs.getFileStatus(indexFilePath); + List paths = new ArrayList(); + if(indexStat.isDir()) { + FileStatus[] fss = fs.listStatus(indexFilePath); + for (FileStatus f : fss) { + paths.add(f.getPath()); + } + } else { + paths.add(indexFilePath); + } + + for(Path indexFinalPath : paths) { + FSDataInputStream ifile = fs.open(indexFinalPath); + LineReader lr = new LineReader(ifile, conf); + Text line = new Text(); + while( lr.readLine(line) > 0) { + add(line); + } + // this will close the input stream + lr.close(); + } + } + } + + + Map buckets = new HashMap(); + + private void add(Text line) throws HiveException { + String l = line.toString(); + byte[] bytes = l.getBytes(); + int firstEnd = 0; + int i = 0; + for (int index = 0; index < bytes.length; index++) { + if (bytes[index] == LazySimpleSerDe.DefaultSeparators[0]) { + i++; + firstEnd = index; + } + } + if (i > 1) { + throw new HiveException( + "Bad index file row (index file should only contain two columns: bucket_file_name and offset lists.) ." + + line.toString()); + } + String bucketFileName = new String(bytes, 0, firstEnd); + IBucket bucket = buckets.get(bucketFileName); + if(bucket == null) { + bucket = new IBucket(bucketFileName); + buckets.put(bucketFileName, bucket); + } + + int currentStart = firstEnd + 1; + int currentEnd = firstEnd + 1; + for (; currentEnd < bytes.length; currentEnd++) { + if (bytes[currentEnd] == LazySimpleSerDe.DefaultSeparators[1]) { + String one_offset = new String (bytes, currentStart, currentEnd - currentStart); + Long offset = Long.parseLong(one_offset); + bucket.getOffsets().add(offset); + currentStart = currentEnd + 1; + } + } + String one_offset = new String(bytes, currentStart, currentEnd + - currentStart); + bucket.getOffsets().add(Long.parseLong(one_offset)); + } + + + public boolean contains(FileSplit split) throws HiveException { + + if(buckets == null) { + return false; + } + String bucketName = split.getPath().toString(); + IBucket bucket = buckets.get(bucketName); + if(bucket == null) { + bucketName = split.getPath().toUri().getPath(); + bucket = buckets.get(bucketName); + if(bucket == null) { + return false; + } + } + + for (Long offset : bucket.getOffsets()) { + if ( (offset >= split.getStart()) && (offset <= split.getStart() + split.getLength())) { + return true; + } + } + return false; + } + } + +} Index: ql/src/java/org/apache/hadoop/hive/ql/index/IndexBuilderCompactSumReducer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/index/IndexBuilderCompactSumReducer.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/index/IndexBuilderCompactSumReducer.java (revision 0) @@ -0,0 +1,86 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.index; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.SortedSet; +import java.util.TreeSet; + +import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.mapred.Reporter; + +/** + * This class is reducer for CreateIndex Hive command. Input: <, + * +> Output: A Hive table of key, value separated by + * ^A^A>^B>^B> so on + * + */ +public class IndexBuilderCompactSumReducer extends IndexBuilderBaseReducer { + + private Map> bucketOffsetMap = new HashMap>(); + + private Writable outVal; + + //private NullWritable key = NullWritable.get(); + + public IndexBuilderCompactSumReducer() { + } + + public void doReduce(Object[] keys, Iterator values, + Reporter reporter) throws IOException, SerDeException { + reporter.progress(); + bucketOffsetMap.clear(); + while (values.hasNext()) { + IndexEntryValueCell value = (IndexEntryValueCell) values.next(); + String bucketName = value.getBucketName(); + long offset = value.getPosition(); + SortedSet bucketPos = bucketOffsetMap.get(bucketName); + if (bucketPos == null) { + bucketPos = new TreeSet(); + bucketOffsetMap.put(bucketName, bucketPos); + } + if (!bucketPos.contains(offset)) + bucketPos.add(offset); + } + + Iterator it = bucketOffsetMap.keySet().iterator(); + List offsets = new ArrayList(); + while (it.hasNext()) { + String bucketName = it.next(); + this.indexOutputObjects[this.indexColumns.length-2] = bucketName; + SortedSet poses = bucketOffsetMap.get(bucketName); + Iterator posIter = poses.iterator(); + offsets.clear(); + while (posIter.hasNext()) { + offsets.add(posIter.next().toString()); + } + this.indexOutputObjects[this.indexColumns.length-1] = offsets; + outVal = this.serializer.serialize(this.indexOutputObjects, indexRowOutputObjInspector); + this.outWriter.write(outVal); + } + + } + +} Index: ql/src/java/org/apache/hadoop/hive/ql/index/HiveIndexRecordReader.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/index/HiveIndexRecordReader.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/index/HiveIndexRecordReader.java (revision 0) @@ -0,0 +1,112 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.index; + +import java.io.IOException; + +import org.apache.hadoop.hive.ql.io.RCFile; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.mapred.RecordReader; + +public class HiveIndexRecordReader implements IndexRecordReader { + + private RecordReader rawReader; + + private LongWritable cachedKey = new LongWritable(); + + private Object rawKey; + + private boolean blockPointer = false; + private long blockStart = -1; + + private long currentBlockStart = 0; + private long nextBlockStart = -1; + + public HiveIndexRecordReader(RecordReader recordReader) throws IOException { + this.rawReader = recordReader; + rawKey = recordReader.createKey(); + this.currentBlockStart = this.rawReader.getPos(); + } + + public void close() throws IOException { + rawReader.close(); + } + + public LongWritable createKey() { + return cachedKey; + } + + public Object createValue() { + return rawReader.createValue(); + } + + public long getPos() throws IOException { + return rawReader.getPos(); + } + + public float getProgress() throws IOException { + return rawReader.getProgress(); + } + + /** + * the key object from raw record reader is throw away here, and is not passed + * to upper. + */ + public boolean next(Object key, Object value) throws IOException { + boolean result = rawReader.next(rawKey, value); + ((LongWritable) key).set(this.getCurrentValIndexOffset()); + return result; + } + + public void setBlockPointer(boolean b) { + blockPointer = b; + } + + public void setBlockStart(long blockStart) { + this.blockStart = blockStart; + } + + @Override + public long getCurrentValIndexOffset() throws IOException { + if(this.rawReader instanceof IndexRecordReader) { + return ((IndexRecordReader)this.rawReader).getCurrentValIndexOffset(); + } + + long ret = this.currentBlockStart; + long pointerPos = this.rawReader.getPos(); + if(this.nextBlockStart == -1) { + this.nextBlockStart = pointerPos; + return ret; + } + + if (pointerPos != this.nextBlockStart) { + // the reader pointer has moved to the end of next block, or the end of + // next record. (actually, here's next is current, because we already did + // the read action.) + + this.currentBlockStart = this.nextBlockStart; + this.nextBlockStart = pointerPos; + if(blockPointer) { + // we need the beginning of the current block. + ret = this.currentBlockStart; + } + } + + return ret; + } +} Index: ql/src/java/org/apache/hadoop/hive/ql/index/IndexEntryValueCell.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/index/IndexEntryValueCell.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/index/IndexEntryValueCell.java (revision 0) @@ -0,0 +1,70 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.index; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.WritableFactories; +import org.apache.hadoop.io.WritableFactory; +import org.apache.hadoop.io.WritableUtils; + +public class IndexEntryValueCell implements Writable { + + public String getBucketName() { + return bucketName; + } + + public void setBucketName(String bucketName) { + this.bucketName = bucketName; + } + + public long getPosition() { + return position; + } + + public void setPosition(long position) { + this.position = position; + } + + private String bucketName; + private long position; + + @Override + public void readFields(DataInput in) throws IOException { + bucketName = WritableUtils.readString(in); + position = in.readLong(); + } + + @Override + public void write(DataOutput out) throws IOException { + WritableUtils.writeString(out, bucketName); + out.writeLong(position); + } + + static { // register a ctor + WritableFactories.setFactory(IndexEntryValueCell.class, new WritableFactory() { + public Writable newInstance() { + return new IndexEntryValueCell(); + } + }); + } + +} Index: ql/src/java/org/apache/hadoop/hive/ql/index/IndexBuilderSummaryReducer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/index/IndexBuilderSummaryReducer.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/index/IndexBuilderSummaryReducer.java (revision 0) @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.index; + +import java.io.IOException; +import java.util.Iterator; + +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapred.Reporter; + +public class IndexBuilderSummaryReducer extends IndexBuilderBaseReducer { + + private StringBuilder bl = null; + private Text outVal = new Text(); + + private NullWritable key = NullWritable.get(); + + public IndexBuilderSummaryReducer() { + } + + public void doReduce(Object[] keys, Iterator values, Reporter reporter) throws IOException { + throw new RuntimeException("Not supported."); + } +} Index: ql/src/java/org/apache/hadoop/hive/ql/index/IndexBuilderFileFormat.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/index/IndexBuilderFileFormat.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/index/IndexBuilderFileFormat.java (revision 0) @@ -0,0 +1,85 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.index; + +import java.io.IOException; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.ql.io.HiveInputFormat; +import org.apache.hadoop.hive.ql.io.RCFile; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.io.SequenceFile; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.mapred.FileSplit; +import org.apache.hadoop.mapred.InputFormat; +import org.apache.hadoop.mapred.InputSplit; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.RecordReader; +import org.apache.hadoop.mapred.Reporter; + +public class IndexBuilderFileFormat + extends HiveInputFormat { + + public RecordReader getRecordReader(InputSplit split, JobConf job, + Reporter reporter) throws IOException { + + HiveInputSplit hsplit = (HiveInputSplit) split; + + InputSplit inputSplit = hsplit.getInputSplit(); + String inputFormatClassName = null; + Class inputFormatClass = null; + try { + inputFormatClassName = hsplit.inputFormatClassName(); + inputFormatClass = Class.forName(inputFormatClassName); + } catch (Exception e) { + throw new IOException("cannot find class " + inputFormatClassName); + } + + InputFormat inputFormat = getInputFormatFromCache(inputFormatClass, job); + boolean blockPointer = false; + long blockStart = -1; + FileSplit fileSplit = (FileSplit) split; + Path path = fileSplit.getPath(); + FileSystem fs = path.getFileSystem(job); + if (inputFormatClass.getName().contains("SequenceFile")) { + SequenceFile.Reader in = new SequenceFile.Reader(fs, path, job); + blockPointer = in.isBlockCompressed(); + in.sync(fileSplit.getStart()); + blockStart = in.getPosition(); + in.close(); + } else if (inputFormatClass.getName().contains("RCFile")) { + RCFile.Reader in = new RCFile.Reader(fs, path, job); + blockPointer = true; + in.sync(fileSplit.getStart()); + blockStart = in.getPosition(); + in.close(); + } + + HiveIndexRecordReader indexReader = new HiveIndexRecordReader(inputFormat + .getRecordReader(inputSplit, job, reporter)); + + if (blockPointer) { + indexReader.setBlockPointer(true); + indexReader.setBlockStart(blockStart); + } + + return indexReader; + } + +} Index: ql/src/java/org/apache/hadoop/hive/ql/index/io/HiveIndexInputFormat.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/index/io/HiveIndexInputFormat.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/index/io/HiveIndexInputFormat.java (revision 0) @@ -0,0 +1,75 @@ +package org.apache.hadoop.hive.ql.index.io; + +import java.io.IOException; +import java.util.ArrayList; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.ql.index.HiveIndex.HiveIndexResult; +import org.apache.hadoop.hive.ql.io.HiveInputFormat; +import org.apache.hadoop.hive.ql.io.RCFile; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.mapred.FileSplit; +import org.apache.hadoop.mapred.InputSplit; +import org.apache.hadoop.mapred.JobConf; + +public class HiveIndexInputFormat extends HiveInputFormat { + + public static final Log l4j = LogFactory.getLog("HiveIndexInputFormat"); + + public HiveIndexInputFormat() { + super(); + } + + @Override + public InputSplit[] getSplits(JobConf job, int numSplits) throws IOException { + String indexFileStr = job.get("hive.exec.index_file"); + l4j.info("index_file is " + indexFileStr); + if (indexFileStr == null) { + // ok it is not index accessed plan, what can we do? + // return super.getSplits(job, numSplits); + throw new IOException("No index file."); + } + HiveInputSplit[] splits = (HiveInputSplit[]) super + .getSplits(job, numSplits); + + HiveIndexResult hiveIndexResult = null; + try { + hiveIndexResult = new HiveIndexResult(indexFileStr, job); + } catch (HiveException e) { + // there is + l4j.error("Unable to read index so we will go with all the file splits."); + e.printStackTrace(); + } + + ArrayList newSplits = new ArrayList( + numSplits); + for (HiveInputSplit split : splits) { + l4j.info("split start : " + split.getStart()); + l4j.info("split end : " + (split.getStart() + split.getLength())); + + try { + if (hiveIndexResult.contains(split)) { + // we may miss a sync here + HiveInputSplit newSplit = split; + if (split.getStart() > RCFile.SYNC_INTERVAL) { + newSplit = new HiveInputSplit(new FileSplit(split.getPath(), split + .getStart() + - RCFile.SYNC_INTERVAL, split.getLength() + + RCFile.SYNC_INTERVAL, split.getLocations()), split + .inputFormatClassName()); + } + + newSplits.add(newSplit); + } + } catch (HiveException e) { + throw new RuntimeException( + "Unable to get metadata for input table split" + split.getPath()); + } + } + InputSplit retA[] = newSplits.toArray((new FileSplit[newSplits.size()])); + l4j.info("Number of input splits: " + splits.length + " new input splits: " + + retA.length); + return retA; + } +} Index: ql/src/java/org/apache/hadoop/hive/ql/index/IndexRecordReader.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/index/IndexRecordReader.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/index/IndexRecordReader.java (revision 0) @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.index; + +import java.io.IOException; + +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.mapred.RecordReader; + +public interface IndexRecordReader extends RecordReader { + + public long getCurrentValIndexOffset() throws IOException; + +} Index: ql/src/java/org/apache/hadoop/hive/ql/index/IndexBuilderProjectMapper.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/index/IndexBuilderProjectMapper.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/index/IndexBuilderProjectMapper.java (revision 0) @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.index; + +import java.io.IOException; + +import org.apache.hadoop.mapred.OutputCollector; +import org.apache.hadoop.mapred.Reporter; + +public class IndexBuilderProjectMapper extends IndexBuilderBaseMapper { + + public void map(Object key, Object value, OutputCollector oc, + Reporter reporter) throws IOException { + try { + reporter.progress(); + perpareMapper(key, value); + oc.collect(cachedIndexCols, cachedIndexCols); + } catch (Throwable e) { + throw new RuntimeException("Error in indexer's mapper:", e); + } + } +} Index: ql/src/java/org/apache/hadoop/hive/ql/index/IndexBuilderBaseMapper.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/index/IndexBuilderBaseMapper.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/index/IndexBuilderBaseMapper.java (revision 0) @@ -0,0 +1,166 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.index; + +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.ql.exec.Utilities; +import org.apache.hadoop.hive.ql.io.HiveKey; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.plan.MapredWork; +import org.apache.hadoop.hive.ql.plan.TableDesc; +import org.apache.hadoop.hive.serde2.Deserializer; +import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.hive.serde2.Serializer; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.objectinspector.StructField; +import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry; +import org.apache.hadoop.io.BytesWritable; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.WritableFactories; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.MapReduceBase; +import org.apache.hadoop.mapred.Mapper; + +public abstract class IndexBuilderBaseMapper extends MapReduceBase implements Mapper { + + private JobConf jc; + private MapredWork conf; + + private Deserializer deserializer; + private StructObjectInspector rowObjectInspector; + + public final Log LOG = LogFactory.getLog("IndexBuilderBaseMapper"); + + private String bucketName; + String[] indexColumns; + + private List structFields = new ArrayList(); + + protected HiveKey cachedIndexCols = new HiveKey(); + + private Object[] cachedKeys; + + protected IndexEntryValueCell cachedPos; + + transient Serializer keySerializer; + transient StructObjectInspector keyObjectInspector; + transient boolean keyIsText; + + public IndexBuilderBaseMapper() { + } + + public void configure(JobConf job) { + try { + jc = job; + + conf = Utilities.getMapRedWork(job); + bucketName = jc.get("map.input.file"); + TableDesc table = conf.getKeyDesc(); + + Properties p = table.getProperties(); + Class deserializerClass = table + .getDeserializerClass(); + if (deserializerClass == null) { + String className = table.getSerdeClassName(); + if ((className == "") || (className == null)) { + throw new HiveException( + "SerDe class or the SerDe class name is not set for table: " + + table.getProperties().getProperty("name")); + } + deserializerClass = (Class) getClass() + .getClassLoader().loadClass(className); + } + + deserializer = (Deserializer) deserializerClass.newInstance(); + deserializer.initialize(jc, p); + rowObjectInspector = (StructObjectInspector) deserializer + .getObjectInspector(); + indexColumns = conf.getIndexCols().split(","); + + List fieldObjectInspectors = new ArrayList(); + List outputColName = new ArrayList(); + for (int k = 0; k < indexColumns.length -2 ; k++) { + String col = indexColumns[k]; + StructField field = rowObjectInspector.getStructFieldRef(col); + structFields.add(field); + ObjectInspector inspector = field.getFieldObjectInspector(); + if (!inspector.getCategory().equals(Category.PRIMITIVE)) { + throw new RuntimeException("Only primitive columns can be indexed."); + } + outputColName.add(col); + fieldObjectInspectors.add(inspector); + } + + keyObjectInspector = ObjectInspectorFactory + .getStandardStructObjectInspector(outputColName, + fieldObjectInspectors); + + TableDesc keyTableDesc = conf.getIndexTableDesc(); + keySerializer = (Serializer) HiveIndex.INDEX_MAPRED_BOUN_SERDE.newInstance(); + keySerializer.initialize(jc, keyTableDesc.getProperties()); + keyIsText = keySerializer.getSerializedClass().equals(Text.class); + + cachedKeys = new Object[indexColumns.length - 2]; + cachedPos = (IndexEntryValueCell) WritableFactories.newInstance(IndexEntryValueCell.class); + cachedPos.setBucketName(bucketName); + + } catch (Exception e) { + LOG.error("Error in building index: " + e.getMessage(), e); + throw new RuntimeException(e); + } + } + + protected void perpareMapper(Object key, Object value) throws SerDeException { + Object row = deserializer.deserialize((Writable) value); + for (int i = 0; i < indexColumns.length -2; i++) { + cachedKeys[i] = rowObjectInspector.getStructFieldData(row, structFields + .get(i)); + } + + if (keyIsText) { + Text k = (Text) keySerializer.serialize(cachedKeys, keyObjectInspector); + cachedIndexCols.set(k.getBytes(), 0, k.getLength()); + } else { + // Must be BytesWritable + BytesWritable k = (BytesWritable) keySerializer.serialize(cachedKeys, + keyObjectInspector); + cachedIndexCols.set(k.get(), 0, k.getSize()); + } + + int keyHashCode = 0; + for (int i = 0; i < cachedKeys.length; i++) { + Object o = cachedKeys[i]; + keyHashCode = keyHashCode * 31 + + ObjectInspectorUtils.hashCode(o, structFields.get(i).getFieldObjectInspector()); + } + cachedIndexCols.setHashCode(keyHashCode); + + cachedPos.setPosition(((LongWritable) key).get()); + } +} Index: ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g (revision 959407) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g (working copy) @@ -85,6 +85,8 @@ TOK_STRUCT; TOK_MAP; TOK_CREATETABLE; +TOK_CREATEINDEX; +TOK_UPDATEINDEX; TOK_LIKETABLE; TOK_DESCTABLE; TOK_DESCFUNCTION; @@ -218,6 +220,8 @@ | createViewStatement | dropViewStatement | createFunctionStatement + | createIndexStatement + | updateIndexStatement | dropFunctionStatement ; @@ -258,6 +262,20 @@ ) ; +createIndexStatement +@init { msgs.push("create index statement");} +@after {msgs.pop();} + : KW_CREATE KW_INDEX indexName=Identifier KW_TYPE typeName=Identifier KW_ON KW_TABLE tab=Identifier LPAREN indexedCols=columnNameList RPAREN tableFileFormat? + ->^(TOK_CREATEINDEX $indexName $typeName $tab $indexedCols tableFileFormat?) + ; + +updateIndexStatement +@init { msgs.push("update index statement");} +@after {msgs.pop();} + : KW_UPDATE KW_INDEX indexName=Identifier partitionSpec? + ->^(TOK_UPDATEINDEX $indexName partitionSpec?) + ; + dropTableStatement @init { msgs.push("drop statement"); } @after { msgs.pop(); } @@ -1509,6 +1527,8 @@ KW_PARTITIONS : 'PARTITIONS'; KW_TABLE: 'TABLE'; KW_TABLES: 'TABLES'; +KW_INDEX: 'INDEX'; +KW_TYPE: 'TYPE'; KW_FUNCTIONS: 'FUNCTIONS'; KW_SHOW: 'SHOW'; KW_MSCK: 'MSCK'; @@ -1527,6 +1547,7 @@ KW_IS: 'IS'; KW_NULL: 'NULL'; KW_CREATE: 'CREATE'; +KW_UPDATE: 'UPDATE'; KW_EXTERNAL: 'EXTERNAL'; KW_ALTER: 'ALTER'; KW_CHANGE: 'CHANGE'; Index: ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java (revision 959407) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java (working copy) @@ -97,6 +97,8 @@ case HiveParser.TOK_SHOW_TABLESTATUS: case HiveParser.TOK_SHOWFUNCTIONS: case HiveParser.TOK_SHOWPARTITIONS: + case HiveParser.TOK_CREATEINDEX: + case HiveParser.TOK_UPDATEINDEX: case HiveParser.TOK_ALTERTABLE_FILEFORMAT: case HiveParser.TOK_ALTERTABLE_CLUSTER_SORT: case HiveParser.TOK_ALTERTABLE_TOUCH: Index: ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java (revision 959407) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java (working copy) @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.ql.parse; +import java.io.Serializable; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -39,9 +40,24 @@ import org.apache.hadoop.hive.metastore.MetaStoreUtils; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Order; +import org.apache.hadoop.hive.ql.exec.ExecDriver; import org.apache.hadoop.hive.ql.exec.FetchTask; +import org.apache.hadoop.hive.ql.exec.ForwardOperator; +import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.exec.TaskFactory; +import org.apache.hadoop.hive.ql.exec.Utilities; +import org.apache.hadoop.hive.ql.index.HiveIndex; +import org.apache.hadoop.hive.ql.index.IndexEntryValueCell; +import org.apache.hadoop.hive.ql.index.HiveIndex.IndexType; +import org.apache.hadoop.hive.ql.io.HiveKey; import org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat; +import org.apache.hadoop.hive.ql.io.RCFileOutputFormat; +import org.apache.hadoop.hive.ql.lib.Node; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.metadata.Partition; +import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.AddPartitionDesc; import org.apache.hadoop.hive.ql.plan.AlterTableDesc; import org.apache.hadoop.hive.ql.plan.AlterTableSimpleDesc; @@ -50,15 +66,19 @@ import org.apache.hadoop.hive.ql.plan.DescTableDesc; import org.apache.hadoop.hive.ql.plan.DropTableDesc; import org.apache.hadoop.hive.ql.plan.FetchWork; +import org.apache.hadoop.hive.ql.plan.MapredWork; import org.apache.hadoop.hive.ql.plan.MsckDesc; +import org.apache.hadoop.hive.ql.plan.PartitionDesc; import org.apache.hadoop.hive.ql.plan.ShowFunctionsDesc; import org.apache.hadoop.hive.ql.plan.ShowPartitionsDesc; import org.apache.hadoop.hive.ql.plan.ShowTableStatusDesc; import org.apache.hadoop.hive.ql.plan.ShowTablesDesc; import org.apache.hadoop.hive.ql.plan.TableDesc; +import org.apache.hadoop.hive.ql.plan.createIndexDesc; import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes; import org.apache.hadoop.hive.serde.Constants; import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; +import org.apache.hadoop.mapred.OutputFormat; import org.apache.hadoop.mapred.TextInputFormat; /** @@ -107,6 +127,10 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { if (ast.getToken().getType() == HiveParser.TOK_DROPTABLE) { analyzeDropTable(ast, false); + } else if (ast.getToken().getType() == HiveParser.TOK_CREATEINDEX) { + analyzeCreateIndex(ast); + } else if (ast.getToken().getType() == HiveParser.TOK_UPDATEINDEX) { + analyzeUpdateIndex(ast); } else if (ast.getToken().getType() == HiveParser.TOK_DESCTABLE) { ctx.setResFile(new Path(ctx.getLocalTmpFileURI())); analyzeDescribeTable(ast); @@ -172,7 +196,211 @@ rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), dropTblDesc), conf)); } + + private HashMap extractPartitionSpecs(Tree partspec) + throws SemanticException { + HashMap partSpec = new LinkedHashMap(); + for (int i = 0; i < partspec.getChildCount(); ++i) { + CommonTree partspec_val = (CommonTree) partspec.getChild(i); + String val = stripQuotes(partspec_val.getChild(1).getText()); + partSpec.put(partspec_val.getChild(0).getText().toLowerCase(), val); + } + return partSpec; + } + + + private void analyzeCreateIndex(ASTNode ast) throws SemanticException { + String indexName = unescapeIdentifier(ast.getChild(0).getText()); + String typeName = unescapeIdentifier(ast.getChild(1).getText()); + String tableName = unescapeIdentifier(ast.getChild(2).getText()); + List indexedCols = getColumnNames((ASTNode) ast.getChild(3)); + ASTNode fileFormat = (ASTNode) ast.getChild(4); + String serde = null; + String inputFormat = null; + String outputFormat = null; + if (fileFormat != null) { + switch (fileFormat.getToken().getType()) { + case HiveParser.TOK_TBLSEQUENCEFILE: + inputFormat = SEQUENCEFILE_INPUT; + outputFormat = SEQUENCEFILE_OUTPUT; + break; + case HiveParser.TOK_TBLTEXTFILE: + inputFormat = TEXTFILE_INPUT; + outputFormat = TEXTFILE_OUTPUT; + break; + case HiveParser.TOK_TABLEFILEFORMAT: + inputFormat = unescapeSQLString(fileFormat.getChild(0).getText()); + outputFormat = unescapeSQLString(fileFormat.getChild(1).getText()); + break; + case HiveParser.TOK_TBLRCFILE: + default: + inputFormat = RCFILE_INPUT; + outputFormat = RCFILE_OUTPUT; + serde = COLUMNAR_SERDE; + break; + } + } + try { + HiveIndex.getIndexType(typeName); + } catch (Exception e) { + throw new SemanticException(e); + } + createIndexDesc crtIndexDesc = new createIndexDesc(tableName, indexName, + indexedCols,inputFormat, outputFormat, serde, typeName); + Task createIndex = TaskFactory.get(new DDLWork(crtIndexDesc), conf); + rootTasks.add(createIndex); + } + + private void analyzeUpdateIndex(ASTNode ast) throws SemanticException { + String indexName = unescapeIdentifier(ast.getChild(0).getText()); + HashMap partSpec = null; + Tree part = ast.getChild(1); + if (part != null) + partSpec = extractPartitionSpecs(part); + List> indexBuilder = getIndexBuilderMapRed(indexName, partSpec); + rootTasks.addAll(indexBuilder); + } + + private List> getIndexBuilderMapRed(String indexTableName, + HashMap partSpec) throws SemanticException { + try { + Table tbl = db.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, + indexTableName); + String baseTblName = MetaStoreUtils.getBaseTableNameOfIndexTable(tbl + .getTTable()); + Table baseTbl = db.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, + baseTblName); + TableDesc desc = Utilities.getTableDesc(tbl); + IndexType indexType = HiveIndex.getIndexType(MetaStoreUtils + .getIndexType(tbl.getTTable())); + boolean compact = false; + boolean projection = false; + if (indexType.equals(IndexType.COMPACT_SUMMARY_TABLE)) { + compact = true; + } else if (indexType.equals(IndexType.PROJECTION)) { + projection = true; + } + List baseTblPartitions = new ArrayList(); + List indexTblPartitions = new ArrayList(); + List newBaseTblPartitions = new ArrayList(); + if (partSpec != null) { + // if partspec is specified, then only producing index for that + // partition + Partition part = db.getPartition(baseTbl, partSpec, false); + Partition indexPart = db.getPartition(tbl, partSpec, false); + baseTblPartitions.add(part); + indexTblPartitions.add(indexPart); + } else if (baseTbl.isPartitioned()) { + // if no partition is specified, create indexes for all partitions one + // by one. + baseTblPartitions = db.getPartitions(baseTbl); + indexTblPartitions = db.getPartitions(tbl); + } + List> indexBuilderTasks = new ArrayList>(); + String indexCols = MetaStoreUtils.getColumnNamesFromFieldSchema(tbl + .getCols()); + if (!baseTbl.isPartitioned()) { + // the table does not have any partition, then create index for the + // whole table + Task indexBuilder = getIndexBuilderMapRedTask(indexCols, tbl + .getDataLocation().toString(), new PartitionDesc(desc, null), + new PartitionDesc(Utilities.getTableDesc(baseTbl), null), + baseTbl.getDataLocation().toString(), compact, projection, null); + indexBuilderTasks.add(indexBuilder); + } else { + + // check whether the index table partitions are still exists in base + // table + for (int i = 0; i < indexTblPartitions.size(); i++) { + Partition indexPart = indexTblPartitions.get(i); + Partition basePart = null; + for (int j = 0; j < baseTblPartitions.size(); j++) { + if (baseTblPartitions.get(j).getName().equals(indexPart.getName())) { + basePart = baseTblPartitions.get(j); + newBaseTblPartitions.add(baseTblPartitions.get(j)); + break; + } + } + if (basePart == null) + throw new RuntimeException( + "Partitions of base table and index table are inconsistent."); + // for each partition, spawn a map reduce task? + Task indexBuilder = getIndexBuilderMapRedTask(indexCols, + indexTblPartitions.get(i).getDataLocation().toString(), + new PartitionDesc(desc, indexTblPartitions.get(i).getSpec()), + new PartitionDesc(basePart), + newBaseTblPartitions.get(i).getDataLocation().toString(), + compact, projection, null); + indexBuilderTasks.add(indexBuilder); + } + + } + return indexBuilderTasks; + } catch (HiveException e) { + throw new SemanticException(e); + } + } + + private Task getIndexBuilderMapRedTask( + String indexCols, String destOutputPath, PartitionDesc indexPartDesc, + PartitionDesc basePartDesc, String inputPath, boolean compact, + boolean projection, Class outputFormat) { + MapredWork work = new MapredWork(); + work.setIndexTableDesc(indexPartDesc.getTableDesc()); + work.setKeyDesc(basePartDesc.getTableDesc()); + String aliasString = "building-index"; + LinkedHashMap> pathToAliases = new LinkedHashMap>(); + ArrayList alias = new ArrayList(); + alias.add(aliasString); + pathToAliases.put(inputPath, alias); + work.setPathToAliases(pathToAliases); + LinkedHashMap aliasToPartnInfo = new LinkedHashMap(); + aliasToPartnInfo.put(aliasString, basePartDesc); + work.setAliasToPartnInfo(aliasToPartnInfo); + LinkedHashMap> aliasToWork = new LinkedHashMap>(); + aliasToWork.put(aliasString, new ForwardOperator()); + work.setAliasToWork(aliasToWork); + LinkedHashMap pathToPart = new LinkedHashMap(); + pathToPart.put(inputPath, basePartDesc); + work.setPathToPartitionInfo(pathToPart); + work.setIndexCols(indexCols); + work.setNumReduceTasks(-1); + ExecDriver exec = new ExecDriver(); + if (!projection) { + work + .setMapperClass(org.apache.hadoop.hive.ql.index.IndexBuilderSummaryMapper.class); + if (compact) + work + .setReducerClass(org.apache.hadoop.hive.ql.index.IndexBuilderCompactSumReducer.class); + else + work + .setReducerClass(org.apache.hadoop.hive.ql.index.IndexBuilderSummaryReducer.class); + } else { + work + .setMapperClass(org.apache.hadoop.hive.ql.index.IndexBuilderProjectMapper.class); + work.setNumReduceTasks(0); + } + + work.setMapOutputKeyClass(HiveKey.class); + work.setMapOutputValueClass(IndexEntryValueCell.class); + work.setCompressed(this.conf.getBoolVar(HiveConf.ConfVars.COMPRESSRESULT)); + work.setCompressCodec(this.conf + .getVar(HiveConf.ConfVars.COMPRESSINTERMEDIATECODEC)); + work.setOutputPath(destOutputPath); + + exec.setDelOutputIfExists(true); + work + .setInputFormatCls(org.apache.hadoop.hive.ql.index.IndexBuilderFileFormat.class); + if (outputFormat == null) { + outputFormat = RCFileOutputFormat.class; + } + work.setOutputFormatCls(outputFormat); + exec.setWork(work); + exec.setId("Stage-0"); + return exec; + } + private void analyzeAlterTableProps(ASTNode ast, boolean expectView) throws SemanticException {