-- ql/src/java/org/apache/hadoop/hive/ql/Driver.java +++ ql/src/java/org/apache/hadoop/hive/ql/Driver.java @@ -53,6 +53,7 @@ import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.history.HiveHistory.Keys; import org.apache.hadoop.hive.ql.hooks.PostExecute; import org.apache.hadoop.hive.ql.hooks.PreExecute; +import org.apache.hadoop.hive.ql.metadata.HiveUtils; import org.apache.hadoop.hive.ql.parse.ASTNode; import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer; import org.apache.hadoop.hive.ql.parse.ErrorMsg; @@ -60,6 +61,7 @@ import org.apache.hadoop.hive.ql.parse.ParseDriver; import org.apache.hadoop.hive.ql.parse.ParseException; import org.apache.hadoop.hive.ql.parse.ParseUtils; import org.apache.hadoop.hive.ql.parse.SemanticAnalyzerFactory; +import org.apache.hadoop.hive.ql.parse.SemanticAnalyzerFactoryInterface; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.TableDesc; import org.apache.hadoop.hive.ql.processors.CommandProcessor; @@ -271,7 +273,10 @@ public class Driver implements CommandProcessor { ASTNode tree = pd.parse(command, ctx); tree = ParseUtils.findRootNonNullToken(tree); - BaseSemanticAnalyzer sem = SemanticAnalyzerFactory.get(conf, tree); + BaseSemanticAnalyzer sem = HiveUtils.getSemanticAnalyzerFactoryImpl( + conf, conf.get(SemanticAnalyzerFactoryInterface.SEMANTIC_ANALYZER_FACTORY_IMPL_KEY, + SemanticAnalyzerFactory.class.getName())).get(conf, tree); + // Do semantic analysis and plan generation sem.analyze(tree, ctx); LOG.info("Semantic Analysis Completed"); diff --git ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveUtils.java ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveUtils.java index be9f312..2178750 100644 --- ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveUtils.java @@ -18,10 +18,15 @@ package org.apache.hadoop.hive.ql.metadata; +import java.util.HashMap; +import java.util.Map; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.common.JavaUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.index.HiveIndexHandler; +import org.apache.hadoop.hive.ql.parse.HiveParser; +import org.apache.hadoop.hive.ql.parse.SemanticAnalyzerFactoryInterface; import org.apache.hadoop.util.ReflectionUtils; /** @@ -168,4 +173,65 @@ public final class HiveUtils { } } + public static SemanticAnalyzerFactoryInterface getSemanticAnalyzerFactoryImpl + (HiveConf conf, String semAnalyzerFacClass) throws HiveException{ + + if(semAnalyzerFacClass == null) { + return null; + } + + try { + Class factoryImplClass = + (Class) + Class.forName(semAnalyzerFacClass, true, JavaUtils.getClassLoader()); + return (SemanticAnalyzerFactoryInterface) ReflectionUtils.newInstance( + factoryImplClass, conf); + } catch (ClassNotFoundException e) { + throw new HiveException("Error in loading semantic analyzer factory impl." + +e.getMessage(),e); + } + } + + static Map commandType; + + static { + commandType = new HashMap(); + commandType.put(HiveParser.TOK_EXPLAIN, "EXPLAIN"); + commandType.put(HiveParser.TOK_LOAD, "LOAD"); + commandType.put(HiveParser.TOK_CREATETABLE, "CREATETABLE"); + commandType.put(HiveParser.TOK_DROPTABLE, "DROPTABLE"); + commandType.put(HiveParser.TOK_DESCTABLE, "DESCTABLE"); + commandType.put(HiveParser.TOK_DESCFUNCTION, "DESCFUNCTION"); + commandType.put(HiveParser.TOK_MSCK, "MSCK"); + commandType.put(HiveParser.TOK_ALTERTABLE_ADDCOLS, "ALTERTABLE_ADDCOLS"); + commandType.put(HiveParser.TOK_ALTERTABLE_REPLACECOLS, "ALTERTABLE_REPLACECOLS"); + commandType.put(HiveParser.TOK_ALTERTABLE_RENAMECOL, "ALTERTABLE_RENAMECOL"); + commandType.put(HiveParser.TOK_ALTERTABLE_RENAME, "ALTERTABLE_RENAME"); + commandType.put(HiveParser.TOK_ALTERTABLE_DROPPARTS, "ALTERTABLE_DROPPARTS"); + commandType.put(HiveParser.TOK_ALTERTABLE_ADDPARTS, "ALTERTABLE_ADDPARTS"); + commandType.put(HiveParser.TOK_ALTERTABLE_TOUCH, "ALTERTABLE_TOUCH"); + commandType.put(HiveParser.TOK_ALTERTABLE_ARCHIVE, "ALTERTABLE_ARCHIVE"); + commandType.put(HiveParser.TOK_ALTERTABLE_UNARCHIVE, "ALTERTABLE_UNARCHIVE"); + commandType.put(HiveParser.TOK_ALTERTABLE_PROPERTIES, "ALTERTABLE_PROPERTIES"); + commandType.put(HiveParser.TOK_ALTERTABLE_SERIALIZER, "ALTERTABLE_SERIALIZER"); + commandType.put(HiveParser.TOK_ALTERTABLE_SERDEPROPERTIES, "ALTERTABLE_SERDEPROPERTIES"); + commandType.put(HiveParser.TOK_SHOWTABLES, "SHOWTABLES"); + commandType.put(HiveParser.TOK_SHOW_TABLESTATUS, "SHOW_TABLESTATUS"); + commandType.put(HiveParser.TOK_SHOWFUNCTIONS, "SHOWFUNCTIONS"); + commandType.put(HiveParser.TOK_SHOWPARTITIONS, "SHOWPARTITIONS"); + commandType.put(HiveParser.TOK_CREATEFUNCTION, "CREATEFUNCTION"); + commandType.put(HiveParser.TOK_DROPFUNCTION, "DROPFUNCTION"); + commandType.put(HiveParser.TOK_CREATEVIEW, "CREATEVIEW"); + commandType.put(HiveParser.TOK_DROPVIEW, "DROPVIEW"); + commandType.put(HiveParser.TOK_CREATEINDEX, "CREATEINDEX"); + commandType.put(HiveParser.TOK_DROPINDEX, "DROPINDEX"); + commandType.put(HiveParser.TOK_ALTERINDEX_REBUILD, "ALTERINDEX_REBUILD"); + commandType.put(HiveParser.TOK_ALTERVIEW_PROPERTIES, "ALTERVIEW_PROPERTIES"); + commandType.put(HiveParser.TOK_QUERY, "QUERY"); + } + + public static Map getCmdTypeMap(){ + + return commandType; + } } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java index eedf9e3..5236cf0 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java @@ -53,8 +53,6 @@ import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.session.SessionState.LogHelper; import org.apache.hadoop.hive.serde.Constants; import org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.mapred.SequenceFileInputFormat; import org.apache.hadoop.mapred.SequenceFileOutputFormat; import org.apache.hadoop.mapred.TextInputFormat; @@ -447,6 +445,14 @@ public abstract class BaseSemanticAnalyzer { return null; } + protected String[] handleGenericFileFormat(ASTNode node) throws SemanticException{ + + ASTNode child = (ASTNode)node.getChild(0); + + throw new SemanticException("Unrecognized file format in STORED AS clause:"+ + " "+ (child == null ? "" : child.getText())); + } + protected List getColumns(ASTNode ast) throws SemanticException { return getColumns(ast, true); } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java index 1801c10..6fdaa13 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java @@ -421,7 +421,7 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer { alterTblDesc), conf)); } - private void analyzeAlterTableFileFormat(ASTNode ast, String tableName, + protected void analyzeAlterTableFileFormat(ASTNode ast, String tableName, HashMap partSpec) throws SemanticException { @@ -466,6 +466,9 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer { outputFormat = RCFILE_OUTPUT; serde = COLUMNAR_SERDE; break; + case HiveParser.TOK_FILEFORMAT_GENERIC: + handleGenericFileFormat(child); + break; } AlterTableDesc alterTblDesc = new AlterTableDesc(tableName, inputFormat, @@ -815,7 +818,7 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer { * @throws SemanticException * Parsin failed */ - private void analyzeAlterTableAddParts(CommonTree ast) + protected void analyzeAlterTableAddParts(CommonTree ast) throws SemanticException { String tblName = unescapeIdentifier(ast.getChild(0).getText()); diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java index b1d014b..8111bec 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java @@ -43,8 +43,8 @@ public class ExplainSemanticAnalyzer extends BaseSemanticAnalyzer { ctx.setExplain(true); // Create a semantic analyzer for the query - BaseSemanticAnalyzer sem = SemanticAnalyzerFactory.get(conf, (ASTNode) ast - .getChild(0)); + BaseSemanticAnalyzer sem = new SemanticAnalyzerFactory().get(conf, + (ASTNode) ast.getChild(0)); sem.analyze((ASTNode) ast.getChild(0), ctx); boolean extended = false; diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g index b080780..46de1e8 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g +++ ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g @@ -132,6 +132,7 @@ TOK_TBLSEQUENCEFILE; TOK_TBLTEXTFILE; TOK_TBLRCFILE; TOK_TABLEFILEFORMAT; +TOK_FILEFORMAT_GENERIC; TOK_OFFLINE; TOK_ENABLE; TOK_DISABLE; @@ -545,8 +546,9 @@ fileFormat : KW_SEQUENCEFILE -> ^(TOK_TBLSEQUENCEFILE) | KW_TEXTFILE -> ^(TOK_TBLTEXTFILE) | KW_RCFILE -> ^(TOK_TBLRCFILE) - | KW_INPUTFORMAT inFmt=StringLiteral KW_OUTPUTFORMAT outFmt=StringLiteral - -> ^(TOK_TABLEFILEFORMAT $inFmt $outFmt) + | KW_INPUTFORMAT inFmt=StringLiteral KW_OUTPUTFORMAT outFmt=StringLiteral (KW_INPUTDRIVER inDriver=StringLiteral KW_OUTPUTDRIVER outDriver=StringLiteral)? + -> ^(TOK_TABLEFILEFORMAT $inFmt $outFmt $inDriver? $outDriver?) + | genericSpec=Identifier -> ^(TOK_FILEFORMAT_GENERIC $genericSpec) ; tabTypeExpr @@ -768,11 +770,13 @@ tableFileFormat KW_STORED KW_AS KW_SEQUENCEFILE -> TOK_TBLSEQUENCEFILE | KW_STORED KW_AS KW_TEXTFILE -> TOK_TBLTEXTFILE | KW_STORED KW_AS KW_RCFILE -> TOK_TBLRCFILE - | KW_STORED KW_AS KW_INPUTFORMAT inFmt=StringLiteral KW_OUTPUTFORMAT outFmt=StringLiteral - -> ^(TOK_TABLEFILEFORMAT $inFmt $outFmt) + | KW_STORED KW_AS KW_INPUTFORMAT inFmt=StringLiteral KW_OUTPUTFORMAT outFmt=StringLiteral (KW_INPUTDRIVER inDriver=StringLiteral KW_OUTPUTDRIVER outDriver=StringLiteral)? + -> ^(TOK_TABLEFILEFORMAT $inFmt $outFmt $inDriver? $outDriver?) | KW_STORED KW_BY storageHandler=StringLiteral (KW_WITH KW_SERDEPROPERTIES serdeprops=tableProperties)? -> ^(TOK_STORAGEHANDLER $storageHandler $serdeprops?) + | KW_STORED KW_AS genericSpec=Identifier + -> ^(TOK_FILEFORMAT_GENERIC $genericSpec) ; tableLocation @@ -1712,6 +1716,8 @@ KW_TEXTFILE: 'TEXTFILE'; KW_RCFILE: 'RCFILE'; KW_INPUTFORMAT: 'INPUTFORMAT'; KW_OUTPUTFORMAT: 'OUTPUTFORMAT'; +KW_INPUTDRIVER: 'INPUTDRIVER'; +KW_OUTPUTDRIVER: 'OUTPUTDRIVER'; KW_OFFLINE: 'OFFLINE'; KW_ENABLE: 'ENABLE'; KW_DISABLE: 'DISABLE'; diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 682a8b5..28cb2f6 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -6597,7 +6597,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { * the semantic analyzer need to deal with the select statement with respect * to the SerDe and Storage Format. */ - private ASTNode analyzeCreateTable(ASTNode ast, QB qb) + protected ASTNode analyzeCreateTable(ASTNode ast, QB qb) throws SemanticException { String tableName = unescapeIdentifier(ast.getChild(0).getText()); String likeTableName = null; @@ -6718,6 +6718,11 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { shared.serdeProps); } break; + + case HiveParser.TOK_FILEFORMAT_GENERIC: + handleGenericFileFormat(child); + break; + default: assert false; } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java index 6af3b17..24a557d 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java @@ -19,55 +19,22 @@ package org.apache.hadoop.hive.ql.parse; import java.util.HashMap; +import java.util.Map; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.metadata.HiveUtils; import org.apache.hadoop.hive.ql.session.SessionState; /** * SemanticAnalyzerFactory. * */ -public final class SemanticAnalyzerFactory { +public final class SemanticAnalyzerFactory implements SemanticAnalyzerFactoryInterface{ - static HashMap commandType = new HashMap(); + static Map commandType = HiveUtils.getCmdTypeMap(); static HashMap tablePartitionCommandType = new HashMap(); static { - commandType.put(HiveParser.TOK_EXPLAIN, "EXPLAIN"); - commandType.put(HiveParser.TOK_LOAD, "LOAD"); - commandType.put(HiveParser.TOK_CREATETABLE, "CREATETABLE"); - commandType.put(HiveParser.TOK_DROPTABLE, "DROPTABLE"); - commandType.put(HiveParser.TOK_DESCTABLE, "DESCTABLE"); - commandType.put(HiveParser.TOK_DESCFUNCTION, "DESCFUNCTION"); - commandType.put(HiveParser.TOK_MSCK, "MSCK"); - commandType.put(HiveParser.TOK_ALTERTABLE_ADDCOLS, "ALTERTABLE_ADDCOLS"); - commandType.put(HiveParser.TOK_ALTERTABLE_REPLACECOLS, "ALTERTABLE_REPLACECOLS"); - commandType.put(HiveParser.TOK_ALTERTABLE_RENAMECOL, "ALTERTABLE_RENAMECOL"); - commandType.put(HiveParser.TOK_ALTERTABLE_RENAME, "ALTERTABLE_RENAME"); - commandType.put(HiveParser.TOK_ALTERTABLE_DROPPARTS, "ALTERTABLE_DROPPARTS"); - commandType.put(HiveParser.TOK_ALTERTABLE_ADDPARTS, "ALTERTABLE_ADDPARTS"); - commandType.put(HiveParser.TOK_ALTERTABLE_TOUCH, "ALTERTABLE_TOUCH"); - commandType.put(HiveParser.TOK_ALTERTABLE_ARCHIVE, "ALTERTABLE_ARCHIVE"); - commandType.put(HiveParser.TOK_ALTERTABLE_UNARCHIVE, "ALTERTABLE_UNARCHIVE"); - commandType.put(HiveParser.TOK_ALTERTABLE_PROPERTIES, "ALTERTABLE_PROPERTIES"); - commandType.put(HiveParser.TOK_ALTERTABLE_SERIALIZER, "ALTERTABLE_SERIALIZER"); - commandType.put(HiveParser.TOK_ALTERTABLE_SERDEPROPERTIES, "ALTERTABLE_SERDEPROPERTIES"); - commandType.put(HiveParser.TOK_SHOWTABLES, "SHOWTABLES"); - commandType.put(HiveParser.TOK_SHOW_TABLESTATUS, "SHOW_TABLESTATUS"); - commandType.put(HiveParser.TOK_SHOWFUNCTIONS, "SHOWFUNCTIONS"); - commandType.put(HiveParser.TOK_SHOWPARTITIONS, "SHOWPARTITIONS"); - commandType.put(HiveParser.TOK_CREATEFUNCTION, "CREATEFUNCTION"); - commandType.put(HiveParser.TOK_DROPFUNCTION, "DROPFUNCTION"); - commandType.put(HiveParser.TOK_CREATEVIEW, "CREATEVIEW"); - commandType.put(HiveParser.TOK_DROPVIEW, "DROPVIEW"); - commandType.put(HiveParser.TOK_CREATEINDEX, "CREATEINDEX"); - commandType.put(HiveParser.TOK_DROPINDEX, "DROPINDEX"); - commandType.put(HiveParser.TOK_ALTERINDEX_REBUILD, "ALTERINDEX_REBUILD"); - commandType.put(HiveParser.TOK_ALTERVIEW_PROPERTIES, "ALTERVIEW_PROPERTIES"); - commandType.put(HiveParser.TOK_QUERY, "QUERY"); - } - - static { tablePartitionCommandType.put(HiveParser.TOK_ALTERTABLE_ALTERPARTS_PROTECTMODE, new String[] { "ALTERTABLE_PROTECTMODE", "ALTERPARTITION_PROTECTMODE" }); tablePartitionCommandType.put(HiveParser.TOK_ALTERTABLE_FILEFORMAT, @@ -77,7 +44,8 @@ public final class SemanticAnalyzerFactory { } - public static BaseSemanticAnalyzer get(HiveConf conf, ASTNode tree) + @Override + public BaseSemanticAnalyzer get(HiveConf conf, ASTNode tree) throws SemanticException { if (tree.getToken() == null) { throw new RuntimeException("Empty Syntax Tree"); @@ -141,7 +109,6 @@ public final class SemanticAnalyzerFactory { } } - private SemanticAnalyzerFactory() { - // prevent instantiation + public SemanticAnalyzerFactory() { } } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactoryInterface.java ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactoryInterface.java new file mode 100644 index 0000000..a6dd226 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactoryInterface.java @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.hadoop.hive.ql.parse; + +import org.apache.hadoop.hive.conf.HiveConf; + +/** + * + */ +public interface SemanticAnalyzerFactoryInterface { + public static final String SEMANTIC_ANALYZER_FACTORY_IMPL_KEY = "hive.semantic.analyzer.factory.impl"; + public BaseSemanticAnalyzer get(HiveConf conf, ASTNode tree) throws SemanticException; + +}