Index: src/java/org/apache/hcatalog/cli/SemanticAnalysis/CreateTableHook.java =================================================================== --- src/java/org/apache/hcatalog/cli/SemanticAnalysis/CreateTableHook.java (revision 1170359) +++ src/java/org/apache/hcatalog/cli/SemanticAnalysis/CreateTableHook.java (working copy) @@ -22,11 +22,11 @@ import java.util.List; import java.util.Map; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.hive.metastore.Warehouse; -import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.ql.exec.DDLTask; @@ -42,172 +42,249 @@ import org.apache.hadoop.hive.ql.parse.HiveSemanticAnalyzerHookContext; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.CreateTableDesc; +import org.apache.hadoop.hive.ql.security.authorization.HiveAuthorizationProvider; import org.apache.hcatalog.common.AuthUtils; import org.apache.hcatalog.common.HCatConstants; import org.apache.hcatalog.common.HCatException; +import org.apache.hcatalog.common.HCatUtil; import org.apache.hcatalog.rcfile.RCFileInputDriver; import org.apache.hcatalog.rcfile.RCFileOutputDriver; +import org.apache.hcatalog.storagehandler.HCatStorageHandler; -final class CreateTableHook extends AbstractSemanticAnalyzerHook{ +final class CreateTableHook extends AbstractSemanticAnalyzerHook { - private String inStorageDriver, outStorageDriver, tableName; + private String inStorageDriver, outStorageDriver, tableName; - @Override - public ASTNode preAnalyze(HiveSemanticAnalyzerHookContext context, ASTNode ast) - throws SemanticException { + @Override + public ASTNode preAnalyze(HiveSemanticAnalyzerHookContext context, + ASTNode ast) throws SemanticException { - Hive db; - try { - db = context.getHive(); - } catch (HiveException e) { - throw new SemanticException("Couldn't get Hive DB instance in semantic analysis phase.", e); - } + Hive db; + try { + db = context.getHive(); + } catch (HiveException e) { + throw new SemanticException( + "Couldn't get Hive DB instance in semantic analysis phase.", + e); + } - // Analyze and create tbl properties object - int numCh = ast.getChildCount(); + // Analyze and create tbl properties object + int numCh = ast.getChildCount(); - String inputFormat = null, outputFormat = null; - tableName = BaseSemanticAnalyzer.getUnescapedName((ASTNode)ast.getChild(0)); + String inputFormat = null, outputFormat = null; + tableName = BaseSemanticAnalyzer.getUnescapedName((ASTNode) ast + .getChild(0)); - for (int num = 1; num < numCh; num++) { - ASTNode child = (ASTNode) ast.getChild(num); + for (int num = 1; num < numCh; num++) { + ASTNode child = (ASTNode) ast.getChild(num); - switch (child.getToken().getType()) { + switch (child.getToken().getType()) { - case HiveParser.TOK_QUERY: // CTAS - throw new SemanticException("Operation not supported. Create table as Select is not a valid operation."); + case HiveParser.TOK_QUERY: // CTAS + throw new SemanticException( + "Operation not supported. Create table as Select is not a valid operation."); - case HiveParser.TOK_TABLEBUCKETS: - throw new SemanticException("Operation not supported. HCatalog doesn't allow Clustered By in create table."); + case HiveParser.TOK_TABLEBUCKETS: + throw new SemanticException( + "Operation not supported. HCatalog doesn't allow Clustered By in create table."); - case HiveParser.TOK_TBLSEQUENCEFILE: - throw new SemanticException("Operation not supported. HCatalog doesn't support Sequence File by default yet. " + - "You may specify it through INPUT/OUTPUT storage drivers."); + case HiveParser.TOK_TBLSEQUENCEFILE: + throw new SemanticException( + "Operation not supported. HCatalog doesn't support Sequence File by default yet. " + + "You may specify it through INPUT/OUTPUT storage drivers."); - case HiveParser.TOK_TBLTEXTFILE: - throw new SemanticException("Operation not supported. HCatalog doesn't support Text File by default yet. " + - "You may specify it through INPUT/OUTPUT storage drivers."); + case HiveParser.TOK_TBLTEXTFILE: + throw new SemanticException( + "Operation not supported. HCatalog doesn't support Text File by default yet. " + + "You may specify it through INPUT/OUTPUT storage drivers."); - case HiveParser.TOK_LIKETABLE: + case HiveParser.TOK_LIKETABLE: - String likeTableName; - if (child.getChildCount() > 0 && (likeTableName = BaseSemanticAnalyzer.getUnescapedName((ASTNode)ast.getChild(0))) != null) { + String likeTableName; + if (child.getChildCount() > 0 + && (likeTableName = BaseSemanticAnalyzer + .getUnescapedName((ASTNode) ast.getChild(0))) != null) { - throw new SemanticException("Operation not supported. CREATE TABLE LIKE is not supported."); -// Map tblProps; -// try { -// tblProps = db.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, likeTableName).getParameters(); -// } catch (HiveException he) { -// throw new SemanticException(he); -// } -// if(!(tblProps.containsKey(InitializeInput.HOWL_ISD_CLASS) && tblProps.containsKey(InitializeInput.HOWL_OSD_CLASS))){ -// throw new SemanticException("Operation not supported. Table "+likeTableName+" should have been created through HCat. Seems like its not."); -// } -// return ast; - } - break; + throw new SemanticException( + "Operation not supported. CREATE TABLE LIKE is not supported."); + // Map tblProps; + // try { + // tblProps = + // db.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, + // likeTableName).getParameters(); + // } catch (HiveException he) { + // throw new SemanticException(he); + // } + // if(!(tblProps.containsKey(InitializeInput.HOWL_ISD_CLASS) + // && + // tblProps.containsKey(InitializeInput.HOWL_OSD_CLASS))){ + // throw new + // SemanticException("Operation not supported. Table "+likeTableName+" should have been created through HCat. Seems like its not."); + // } + // return ast; + } + break; - case HiveParser.TOK_IFNOTEXISTS: - try { - List tables = db.getTablesByPattern(tableName); - if (tables != null && tables.size() > 0) { // table exists - return ast; - } - } catch (HiveException e) { - throw new SemanticException(e); - } - break; + case HiveParser.TOK_IFNOTEXISTS: + try { + List tables = db.getTablesByPattern(tableName); + if (tables != null && tables.size() > 0) { // table exists + return ast; + } + } catch (HiveException e) { + throw new SemanticException(e); + } + break; - case HiveParser.TOK_TABLEPARTCOLS: - List partCols = BaseSemanticAnalyzer.getColumns((ASTNode) child.getChild(0), false); - for(FieldSchema fs : partCols){ - if(!fs.getType().equalsIgnoreCase("string")){ - throw new SemanticException("Operation not supported. HCatalog only supports partition columns of type string. " + - "For column: "+fs.getName()+" Found type: "+fs.getType()); - } - } - break; + case HiveParser.TOK_TABLEPARTCOLS: + List partCols = BaseSemanticAnalyzer.getColumns( + (ASTNode) child.getChild(0), false); + for (FieldSchema fs : partCols) { + if (!fs.getType().equalsIgnoreCase("string")) { + throw new SemanticException( + "Operation not supported. HCatalog only supports partition columns of type string. " + + "For column: " + + fs.getName() + + " Found type: " + fs.getType()); + } + } + break; - case HiveParser.TOK_TABLEFILEFORMAT: - if(child.getChildCount() < 4) { - throw new SemanticException("Incomplete specification of File Format. You must provide InputFormat, OutputFormat, InputDriver, OutputDriver."); - } - inputFormat = BaseSemanticAnalyzer.unescapeSQLString(child.getChild(0).getText()); - outputFormat = BaseSemanticAnalyzer.unescapeSQLString(child.getChild(1).getText()); - inStorageDriver = BaseSemanticAnalyzer.unescapeSQLString(child.getChild(2).getText()); - outStorageDriver = BaseSemanticAnalyzer.unescapeSQLString(child.getChild(3).getText()); - break; + case HiveParser.TOK_STORAGEHANDLER: + String storageHandler = BaseSemanticAnalyzer + .unescapeSQLString(child.getChild(0).getText()); + if (org.apache.commons.lang.StringUtils + .isNotEmpty(storageHandler)) { + return ast; + } - case HiveParser.TOK_TBLRCFILE: - inputFormat = RCFileInputFormat.class.getName(); - outputFormat = RCFileOutputFormat.class.getName(); - inStorageDriver = RCFileInputDriver.class.getName(); - outStorageDriver = RCFileOutputDriver.class.getName(); - break; + break; - } - } + case HiveParser.TOK_TABLEFILEFORMAT: + if (child.getChildCount() < 4) { + throw new SemanticException( + "Incomplete specification of File Format. You must provide InputFormat, OutputFormat, InputDriver, OutputDriver."); + } + inputFormat = BaseSemanticAnalyzer.unescapeSQLString(child + .getChild(0).getText()); + outputFormat = BaseSemanticAnalyzer.unescapeSQLString(child + .getChild(1).getText()); + inStorageDriver = BaseSemanticAnalyzer.unescapeSQLString(child + .getChild(2).getText()); + outStorageDriver = BaseSemanticAnalyzer.unescapeSQLString(child + .getChild(3).getText()); + break; - if(inputFormat == null || outputFormat == null || inStorageDriver == null || outStorageDriver == null){ - throw new SemanticException("STORED AS specification is either incomplete or incorrect."); - } + case HiveParser.TOK_TBLRCFILE: + inputFormat = RCFileInputFormat.class.getName(); + outputFormat = RCFileOutputFormat.class.getName(); + inStorageDriver = RCFileInputDriver.class.getName(); + outStorageDriver = RCFileOutputDriver.class.getName(); + break; - return ast; - } + } + } - @Override - public void postAnalyze(HiveSemanticAnalyzerHookContext context, List> rootTasks) throws SemanticException { + if (inputFormat == null || outputFormat == null + || inStorageDriver == null || outStorageDriver == null) { + throw new SemanticException( + "STORED AS specification is either incomplete or incorrect."); + } - if(rootTasks.size() == 0){ - // There will be no DDL task created in case if its CREATE TABLE IF NOT EXISTS - return; - } - CreateTableDesc desc = ((DDLTask)rootTasks.get(rootTasks.size()-1)).getWork().getCreateTblDesc(); + return ast; + } - // first check if we will allow the user to create table. - authorize(context, desc.getLocation()); + @Override + public void postAnalyze(HiveSemanticAnalyzerHookContext context, + List> rootTasks) + throws SemanticException { - if(desc == null){ - // Desc will be null if its CREATE TABLE LIKE. Desc will be contained - // in CreateTableLikeDesc. Currently, HCat disallows CTLT in pre-hook. - // So, desc can never be null. - return; - } - Map tblProps = desc.getTblProps(); - if(tblProps == null) { - // tblProps will be null if user didnt use tblprops in his CREATE TABLE cmd. - tblProps = new HashMap(); - } - tblProps.put(HCatConstants.HCAT_ISD_CLASS, inStorageDriver); - tblProps.put(HCatConstants.HCAT_OSD_CLASS, outStorageDriver); - desc.setTblProps(tblProps); - context.getConf().set(HCatConstants.HCAT_CREATE_TBL_NAME, tableName); - } + if (rootTasks.size() == 0) { + // There will be no DDL task created in case if its CREATE TABLE IF + // NOT EXISTS + return; + } + CreateTableDesc desc = ((DDLTask) rootTasks.get(rootTasks.size() - 1)) + .getWork().getCreateTblDesc(); - private void authorize(HiveSemanticAnalyzerHookContext context, String loc) throws SemanticException{ + Map tblProps = desc.getTblProps(); + if (tblProps == null) { + // tblProps will be null if user didnt use tblprops in his CREATE + // TABLE cmd. + tblProps = new HashMap(); - Path tblDir; - Configuration conf = context.getConf(); - try { - Warehouse wh = new Warehouse(conf); - if (loc == null || loc.isEmpty()){ - Hive hive = context.getHive(); - tblDir = wh.getTablePath(hive.getDatabase(hive.getCurrentDatabase()), tableName).getParent(); - } - else{ - tblDir = wh.getDnsPath(new Path(loc)); - } + } - try { - AuthUtils.authorize(tblDir, FsAction.WRITE, conf); - } catch (HCatException e) { - throw new SemanticException(e); - } - } - catch (MetaException e) { - throw new SemanticException(e); - } catch (HiveException e) { - throw new SemanticException(e); - } - } + // first check if we will allow the user to create table. + String storageHandler = desc.getStorageHandler(); + if (StringUtils.isEmpty(storageHandler)) { + + authorize(context, desc.getLocation()); + tblProps.put(HCatConstants.HCAT_ISD_CLASS, inStorageDriver); + tblProps.put(HCatConstants.HCAT_OSD_CLASS, outStorageDriver); + + } else { + // Create instance of HCatStorageHandler and obtain the + // HiveAuthorizationprovider for the handler and use it + // to authorize. + try { + HCatStorageHandler storageHandlerInst = HCatUtil + .getStorageHandler(context.getConf(), storageHandler); + HiveAuthorizationProvider auth = storageHandlerInst.getAuthorizationProvider(); + + // TBD: To pass in the exact read and write privileges. + auth.authorize(context.getHive().getTable(tableName), null, + null); + + tblProps.put(HCatConstants.HCAT_ISD_CLASS, storageHandlerInst + .getInputStorageDriver().toString()); + tblProps.put(HCatConstants.HCAT_OSD_CLASS, storageHandlerInst + .getOutputStorageDriver().toString()); + + } catch (HiveException e) { + new SemanticException(e); + } + + } + + if (desc == null) { + // Desc will be null if its CREATE TABLE LIKE. Desc will be + // contained + // in CreateTableLikeDesc. Currently, HCat disallows CTLT in + // pre-hook. + // So, desc can never be null. + return; + } + + desc.setTblProps(tblProps); + context.getConf().set(HCatConstants.HCAT_CREATE_TBL_NAME, tableName); + } + + private void authorize(HiveSemanticAnalyzerHookContext context, String loc) + throws SemanticException { + + Path tblDir; + Configuration conf = context.getConf(); + try { + Warehouse wh = new Warehouse(conf); + if (loc == null || loc.isEmpty()) { + Hive hive = context.getHive(); + tblDir = wh.getTablePath( + hive.getDatabase(hive.getCurrentDatabase()), tableName) + .getParent(); + } else { + tblDir = wh.getDnsPath(new Path(loc)); + } + + try { + AuthUtils.authorize(tblDir, FsAction.WRITE, conf); + } catch (HCatException e) { + throw new SemanticException(e); + } + } catch (MetaException e) { + throw new SemanticException(e); + } catch (HiveException e) { + throw new SemanticException(e); + } + } } Index: src/java/org/apache/hcatalog/common/HCatUtil.java =================================================================== --- src/java/org/apache/hcatalog/common/HCatUtil.java (revision 1170359) +++ src/java/org/apache/hcatalog/common/HCatUtil.java (working copy) @@ -33,32 +33,33 @@ import java.util.Set; import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsAction; +import org.apache.hadoop.hive.common.JavaUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.security.authorization.HiveAuthorizationProvider; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; import org.apache.hadoop.hive.thrift.DelegationTokenIdentifier; import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapred.JobClient; +import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapreduce.JobContext; -import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier; +import org.apache.hadoop.util.ReflectionUtils; import org.apache.hcatalog.data.schema.HCatFieldSchema; import org.apache.hcatalog.data.schema.HCatSchema; import org.apache.hcatalog.data.schema.HCatSchemaUtils; import org.apache.hcatalog.mapreduce.HCatOutputFormat; -import org.apache.hadoop.mapred.JobClient; -import org.apache.hadoop.mapred.JobConf; +import org.apache.hcatalog.storagehandler.HCatStorageHandler; import org.apache.thrift.TException; public class HCatUtil { @@ -392,4 +393,22 @@ logger.info("\tservice : "+t.getService()); } + public static HCatStorageHandler getStorageHandler(Configuration conf, + String className) throws HiveException { + + if (className == null) { + return null; + } + try { + Class handlerClass = (Class) Class + .forName(className, true, JavaUtils.getClassLoader()); + HCatStorageHandler storageHandler = (HCatStorageHandler) ReflectionUtils + .newInstance(handlerClass, conf); + return storageHandler; + } catch (ClassNotFoundException e) { + throw new HiveException("Error in loading storage handler." + + e.getMessage(), e); + } + } + } Index: src/java/org/apache/hcatalog/storagehandler/DummyInputFormat.java =================================================================== --- src/java/org/apache/hcatalog/storagehandler/DummyInputFormat.java (revision 0) +++ src/java/org/apache/hcatalog/storagehandler/DummyInputFormat.java (revision 0) @@ -0,0 +1,27 @@ +package org.apache.hcatalog.storagehandler; + +import java.io.IOException; + +import org.apache.hadoop.io.WritableComparable; +import org.apache.hadoop.mapred.InputFormat; +import org.apache.hadoop.mapred.InputSplit; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.RecordReader; +import org.apache.hadoop.mapred.Reporter; +import org.apache.hcatalog.data.HCatRecord; + + +class DummyInputFormat implements InputFormat { + + @Override + public RecordReader getRecordReader(InputSplit arg0, JobConf arg1, + Reporter arg2) throws IOException { + throw new IOException("This operation is not supported."); + } + + @Override + public InputSplit[] getSplits(JobConf arg0, int arg1) throws IOException { + throw new IOException("This operation is not supported."); + } + +} Index: src/java/org/apache/hcatalog/storagehandler/DummyOutputFormat.java =================================================================== --- src/java/org/apache/hcatalog/storagehandler/DummyOutputFormat.java (revision 0) +++ src/java/org/apache/hcatalog/storagehandler/DummyOutputFormat.java (revision 0) @@ -0,0 +1,42 @@ +package org.apache.hcatalog.storagehandler; + +import java.io.IOException; +import java.util.Properties; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.ql.io.HiveOutputFormat; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.WritableComparable; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.OutputFormat; +import org.apache.hadoop.mapred.RecordWriter; +import org.apache.hadoop.util.Progressable; +import org.apache.hcatalog.data.HCatRecord; + +class DummyOutputFormat implements OutputFormat, HCatRecord>, +HiveOutputFormat, HCatRecord> { + + @Override + public void checkOutputSpecs(FileSystem arg0, JobConf arg1) + throws IOException { + throw new IOException("This operation is not supported."); + + } + + @Override + public RecordWriter, HCatRecord> getRecordWriter(FileSystem arg0, JobConf arg1, + String arg2, Progressable arg3) throws IOException { + throw new IOException("This operation is not supported."); + } + + @Override + public org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter getHiveRecordWriter( + JobConf jc, Path finalOutPath, + Class valueClass, boolean isCompressed, + Properties tableProperties, Progressable progress) + throws IOException { + throw new IOException("This operation is not supported."); + } + +} Index: src/java/org/apache/hcatalog/storagehandler/HCatStorageHandler.java =================================================================== --- src/java/org/apache/hcatalog/storagehandler/HCatStorageHandler.java (revision 0) +++ src/java/org/apache/hcatalog/storagehandler/HCatStorageHandler.java (revision 0) @@ -0,0 +1,73 @@ +package org.apache.hcatalog.storagehandler; + +import java.util.Map; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.HiveMetaHook; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler; +import org.apache.hadoop.hive.ql.plan.TableDesc; +import org.apache.hadoop.hive.ql.security.authorization.HiveAuthorizationProvider; +import org.apache.hadoop.hive.serde2.SerDe; +import org.apache.hadoop.mapred.InputFormat; +import org.apache.hadoop.mapred.OutputFormat; +import org.apache.hcatalog.mapreduce.HCatInputStorageDriver; +import org.apache.hcatalog.mapreduce.HCatOutputStorageDriver; + +public abstract class HCatStorageHandler implements HiveMetaHook, + HiveStorageHandler { + + public abstract Class getInputStorageDriver(); + + public abstract Class getOutputStorageDriver(); + + public abstract HiveAuthorizationProvider getAuthorizationProvider() throws HiveException; + + @Override + public abstract void commitCreateTable(Table table) throws MetaException; + + @Override + public abstract void commitDropTable(Table table, boolean deleteData) + throws MetaException; + + @Override + public abstract void preCreateTable(Table table) throws MetaException; + + @Override + public abstract void preDropTable(Table table) throws MetaException; + + @Override + public abstract void rollbackCreateTable(Table table) throws MetaException; + + @Override + public abstract void rollbackDropTable(Table table) throws MetaException; + + @Override + public abstract HiveMetaHook getMetaHook(); + + @Override + public abstract void configureTableJobProperties(TableDesc tableDesc, + Map jobProperties); + + @Override + public abstract Configuration getConf(); + + @Override + public abstract void setConf(Configuration conf); + + @Override + public abstract Class getSerDeClass(); + + @Override + public Class getInputFormatClass() { + return DummyInputFormat.class; + } + + @Override + public Class getOutputFormatClass() { + return DummyOutputFormat.class; + } + +}