Index: common/src/java/org/apache/hadoop/hive/conf/HiveConf.java =================================================================== --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (revision 1002844) +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (working copy) @@ -302,7 +302,9 @@ HIVEOUTERJOINSUPPORTSFILTERS("hive.outerjoin.supports.filters", true), // Serde for FetchTask - HIVEFETCHOUTPUTSERDE("hive.fetch.output.serde", "org.apache.hadoop.hive.serde2.DelimitedJSONSerDe") + HIVEFETCHOUTPUTSERDE("hive.fetch.output.serde", "org.apache.hadoop.hive.serde2.DelimitedJSONSerDe"), + + SEMANTIC_ANALYZER_HOOK("hive.semantic.analyzer.hook",null), ; Index: ql/src/test/results/clientnegative/genericFileFormat.q.out =================================================================== --- ql/src/test/results/clientnegative/genericFileFormat.q.out (revision 0) +++ ql/src/test/results/clientnegative/genericFileFormat.q.out (revision 0) @@ -0,0 +1 @@ +FAILED: Error in semantic analysis: Unrecognized file format in STORED AS clause: foo Index: ql/src/test/results/clientpositive/inoutdriver.q.out =================================================================== --- ql/src/test/results/clientpositive/inoutdriver.q.out (revision 0) +++ ql/src/test/results/clientpositive/inoutdriver.q.out (revision 0) @@ -0,0 +1,36 @@ +PREHOOK: query: create table test (a int) stored as inputformat 'org.apache.hadoop.hive.ql.io.RCFileInputFormat' outputformat 'org.apache.hadoop.hive.ql.io.RCFileOutputFormat' inputdriver 'RCFileInDriver' outputdriver 'RCFileOutDriver' +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table test (a int) stored as inputformat 'org.apache.hadoop.hive.ql.io.RCFileInputFormat' outputformat 'org.apache.hadoop.hive.ql.io.RCFileOutputFormat' inputdriver 'RCFileInDriver' outputdriver 'RCFileOutDriver' +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@test +PREHOOK: query: desc extended test +PREHOOK: type: DESCTABLE +POSTHOOK: query: desc extended test +POSTHOOK: type: DESCTABLE +col_name data_type comment + +a int None + +# Detailed Table Information +Database: default +Owner: chauhana +CreateTime: Wed Sep 29 10:52:37 PDT 2010 +LastAccessTime: Wed Dec 31 16:00:00 PST 1969 +Protect Mode: None +Retention: 0 +Location: pfile:/Users/chauhana/workspace/howl-sprint2-hooks/build/ql/test/data/warehouse/test +Table Type: MANAGED_TABLE +Table Parameters: + transient_lastDdlTime 1285782757 + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.hive.ql.io.RCFileInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.RCFileOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 + Index: ql/src/test/org/apache/hadoop/hive/ql/metadata/DummySemanticAnalyzerHook.java =================================================================== --- ql/src/test/org/apache/hadoop/hive/ql/metadata/DummySemanticAnalyzerHook.java (revision 0) +++ ql/src/test/org/apache/hadoop/hive/ql/metadata/DummySemanticAnalyzerHook.java (revision 0) @@ -0,0 +1,104 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.metadata; + +import java.io.Serializable; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.hive.ql.exec.DDLTask; +import org.apache.hadoop.hive.ql.exec.Task; +import org.apache.hadoop.hive.ql.parse.ASTNode; +import org.apache.hadoop.hive.ql.parse.AbstractSemanticAnalyzerHook; +import org.apache.hadoop.hive.ql.parse.HiveParser; +import org.apache.hadoop.hive.ql.parse.HiveSemanticAnalyzerHookContext; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.plan.CreateTableDesc; + +public class DummySemanticAnalyzerHook extends AbstractSemanticAnalyzerHook{ + + private AbstractSemanticAnalyzerHook hook; + + @Override + public ASTNode preAnalyze(HiveSemanticAnalyzerHookContext context, ASTNode ast) + throws SemanticException { + + switch (ast.getToken().getType()) { + + case HiveParser.TOK_CREATETABLE: + hook = new DummyCreateTableHook(); + return hook.preAnalyze(context, ast); + + case HiveParser.TOK_DROPTABLE: + case HiveParser.TOK_DESCTABLE: + return ast; + + default: + throw new SemanticException("Operation not supported."); + } + } + + public DummySemanticAnalyzerHook() { + + } + + @Override + public void postAnalyze(HiveSemanticAnalyzerHookContext context, + List> rootTasks) throws SemanticException { + + if(hook != null) { + hook.postAnalyze(context, rootTasks); + } + } +} + +class DummyCreateTableHook extends AbstractSemanticAnalyzerHook{ + + @Override + public ASTNode preAnalyze(HiveSemanticAnalyzerHookContext context, ASTNode ast) + throws SemanticException { + + int numCh = ast.getChildCount(); + + for (int num = 1; num < numCh; num++) { + ASTNode child = (ASTNode) ast.getChild(num); + + switch (child.getToken().getType()) { + + case HiveParser.TOK_QUERY: + throw new SemanticException("CTAS not supported."); + } + } + return ast; + } + + @Override + public void postAnalyze(HiveSemanticAnalyzerHookContext context, + List> rootTasks) throws SemanticException { + CreateTableDesc desc = ((DDLTask)rootTasks.get(rootTasks.size()-1)).getWork().getCreateTblDesc(); + Map tblProps = desc.getTblProps(); + if(tblProps == null) { + tblProps = new HashMap(); + } + tblProps.put("createdBy", DummyCreateTableHook.class.getName()); + tblProps.put("Message", "Open Source rocks!!"); + desc.setTblProps(tblProps); + } +} Index: ql/src/test/org/apache/hadoop/hive/ql/metadata/TestSemanticAnalyzerHookLoading.java =================================================================== --- ql/src/test/org/apache/hadoop/hive/ql/metadata/TestSemanticAnalyzerHookLoading.java (revision 0) +++ ql/src/test/org/apache/hadoop/hive/ql/metadata/TestSemanticAnalyzerHookLoading.java (revision 0) @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.metadata; + +import java.util.Map; + +import junit.framework.TestCase; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.metastore.MetaStoreUtils; +import org.apache.hadoop.hive.ql.Driver; +import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; + +public class TestSemanticAnalyzerHookLoading extends TestCase { + + public void testHookLoading() throws Exception{ + + HiveConf conf = new HiveConf(this.getClass()); + conf.set(ConfVars.SEMANTIC_ANALYZER_HOOK.varname, DummySemanticAnalyzerHook.class.getName()); + conf.set(ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false"); + Driver driver = new Driver(conf); + + driver.run("drop table testDL"); + CommandProcessorResponse resp = driver.run("create table testDL (a int) as select * from tbl2"); + assertEquals(10, resp.getResponseCode()); + assertTrue(resp.getErrorMessage().contains("CTAS not supported.")); + + resp = driver.run("create table testDL (a int)"); + assertEquals(0, resp.getResponseCode()); + assertNull(resp.getErrorMessage()); + + Map params = Hive.get(conf).getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, "testDL").getParameters(); + + assertEquals(DummyCreateTableHook.class.getName(),params.get("createdBy")); + assertEquals("Open Source rocks!!", params.get("Message")); + + driver.run("drop table testDL"); + } +} Index: ql/src/test/queries/clientnegative/genericFileFormat.q =================================================================== --- ql/src/test/queries/clientnegative/genericFileFormat.q (revision 0) +++ ql/src/test/queries/clientnegative/genericFileFormat.q (revision 0) @@ -0,0 +1 @@ +create table testFail (a int) stored as foo; Index: ql/src/test/queries/clientpositive/inoutdriver.q =================================================================== --- ql/src/test/queries/clientpositive/inoutdriver.q (revision 0) +++ ql/src/test/queries/clientpositive/inoutdriver.q (revision 0) @@ -0,0 +1,2 @@ +create table test (a int) stored as inputformat 'org.apache.hadoop.hive.ql.io.RCFileInputFormat' outputformat 'org.apache.hadoop.hive.ql.io.RCFileOutputFormat' inputdriver 'RCFileInDriver' outputdriver 'RCFileOutDriver'; +desc extended test; Index: ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveUtils.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveUtils.java (revision 1002844) +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveUtils.java (working copy) @@ -22,6 +22,7 @@ import org.apache.hadoop.hive.common.JavaUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.index.HiveIndexHandler; +import org.apache.hadoop.hive.ql.parse.AbstractSemanticAnalyzerHook; import org.apache.hadoop.util.ReflectionUtils; /** @@ -168,4 +169,18 @@ } } + public static AbstractSemanticAnalyzerHook getSemanticAnalyzerHook( + HiveConf conf, String hookName) throws HiveException{ + try { + Class hookClass = + (Class) + Class.forName(hookName, true, JavaUtils.getClassLoader()); + return (AbstractSemanticAnalyzerHook) ReflectionUtils.newInstance( + hookClass, conf); + } catch (ClassNotFoundException e) { + throw new HiveException("Error in loading semantic analyzer hook: "+ + hookName +e.getMessage(),e); + } + + } } Index: ql/src/java/org/apache/hadoop/hive/ql/parse/HiveSemanticAnalyzerHook.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/HiveSemanticAnalyzerHook.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/HiveSemanticAnalyzerHook.java (revision 0) @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.parse; + +import java.io.Serializable; +import java.util.List; + +import org.apache.hadoop.hive.ql.exec.Task; + +/** + * HiveSemanticAnalyzerHook allows Hive to be extended with custom + * logic for semantic analysis of QL statements. This interface + * and any Hive internals it exposes are currently + * "limited private and evolving" (unless otherwise stated elsewhere) + * and intended mainly for use by the Howl project. + * + *

+ * + * Note that the lifetime of an instantiated hook object is scoped to + * the analysis of a single statement; hook instances are never reused. + */ +public interface HiveSemanticAnalyzerHook { + /** + * Invoked before Hive performs its own semantic analysis on + * a statement. The implementation may inspect the statement AST and + * prevent its execution by throwing a SemanticException. + * Optionally, it may also augment/rewrite the AST, but must produce + * a form equivalent to one which could have + * been returned directly from Hive's own parser. + * + * @param context context information for semantic analysis + * + * @param ast AST being analyzed and optionally rewritten + * + * @return replacement AST (typically the same as the original AST unless the + * entire tree had to be replaced; must not be null) + */ + public ASTNode preAnalyze( + HiveSemanticAnalyzerHookContext context, + ASTNode ast) throws SemanticException; + + /** + * Invoked after Hive performs its own semantic analysis on a + * statement (including optimization). + * Hive calls postAnalyze on the same hook object + * as preAnalyze, so the hook can maintain state across the calls. + * + * @param context context information for semantic analysis + * @param rootTasks root tasks produced by semantic analysis; + * the hook is free to modify this list or its contents + */ + public void postAnalyze( + HiveSemanticAnalyzerHookContext context, + List> rootTasks) throws SemanticException; +} Index: ql/src/java/org/apache/hadoop/hive/ql/parse/HiveSemanticAnalyzerHookContext.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/HiveSemanticAnalyzerHookContext.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/HiveSemanticAnalyzerHookContext.java (revision 0) @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.parse; + +import org.apache.hadoop.conf.Configurable; +import org.apache.hadoop.hive.ql.metadata.Hive; +import org.apache.hadoop.hive.ql.metadata.HiveException; + +/** + * Context information provided by Hive to implementations of + * HiveSemanticAnalyzerHook. + */ +public interface HiveSemanticAnalyzerHookContext extends Configurable{ + /** + * @return the Hive db instance; hook implementations can use this for + * purposes such as getting configuration information or making metastore calls + */ + public Hive getHive() throws HiveException; +} Index: ql/src/java/org/apache/hadoop/hive/ql/parse/AbstractSemanticAnalyzerHook.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/AbstractSemanticAnalyzerHook.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/AbstractSemanticAnalyzerHook.java (revision 0) @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.parse; + +import java.io.Serializable; +import java.util.List; + +import org.apache.hadoop.hive.ql.exec.Task; + +public abstract class AbstractSemanticAnalyzerHook implements +HiveSemanticAnalyzerHook { + + public ASTNode preAnalyze(HiveSemanticAnalyzerHookContext context,ASTNode ast) + throws SemanticException { + return ast; + } + + public void postAnalyze(HiveSemanticAnalyzerHookContext context, + List> rootTasks) throws SemanticException { + } +} Index: ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g (revision 1002844) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g (working copy) @@ -140,6 +140,7 @@ TOK_TBLTEXTFILE; TOK_TBLRCFILE; TOK_TABLEFILEFORMAT; +TOK_FILEFORMAT_GENERIC; TOK_OFFLINE; TOK_ENABLE; TOK_DISABLE; @@ -600,8 +601,9 @@ : KW_SEQUENCEFILE -> ^(TOK_TBLSEQUENCEFILE) | KW_TEXTFILE -> ^(TOK_TBLTEXTFILE) | KW_RCFILE -> ^(TOK_TBLRCFILE) - | KW_INPUTFORMAT inFmt=StringLiteral KW_OUTPUTFORMAT outFmt=StringLiteral - -> ^(TOK_TABLEFILEFORMAT $inFmt $outFmt) + | KW_INPUTFORMAT inFmt=StringLiteral KW_OUTPUTFORMAT outFmt=StringLiteral (KW_INPUTDRIVER inDriver=StringLiteral KW_OUTPUTDRIVER outDriver=StringLiteral)? + -> ^(TOK_TABLEFILEFORMAT $inFmt $outFmt $inDriver? $outDriver?) + | genericSpec=Identifier -> ^(TOK_FILEFORMAT_GENERIC $genericSpec) ; tabTypeExpr @@ -849,11 +851,13 @@ KW_STORED KW_AS KW_SEQUENCEFILE -> TOK_TBLSEQUENCEFILE | KW_STORED KW_AS KW_TEXTFILE -> TOK_TBLTEXTFILE | KW_STORED KW_AS KW_RCFILE -> TOK_TBLRCFILE - | KW_STORED KW_AS KW_INPUTFORMAT inFmt=StringLiteral KW_OUTPUTFORMAT outFmt=StringLiteral - -> ^(TOK_TABLEFILEFORMAT $inFmt $outFmt) + | KW_STORED KW_AS KW_INPUTFORMAT inFmt=StringLiteral KW_OUTPUTFORMAT outFmt=StringLiteral (KW_INPUTDRIVER inDriver=StringLiteral KW_OUTPUTDRIVER outDriver=StringLiteral)? + -> ^(TOK_TABLEFILEFORMAT $inFmt $outFmt $inDriver? $outDriver?) | KW_STORED KW_BY storageHandler=StringLiteral (KW_WITH KW_SERDEPROPERTIES serdeprops=tableProperties)? -> ^(TOK_STORAGEHANDLER $storageHandler $serdeprops?) + | KW_STORED KW_AS genericSpec=Identifier + -> ^(TOK_FILEFORMAT_GENERIC $genericSpec) ; tableLocation @@ -1793,6 +1797,8 @@ KW_RCFILE: 'RCFILE'; KW_INPUTFORMAT: 'INPUTFORMAT'; KW_OUTPUTFORMAT: 'OUTPUTFORMAT'; +KW_INPUTDRIVER: 'INPUTDRIVER'; +KW_OUTPUTDRIVER: 'OUTPUTDRIVER'; KW_OFFLINE: 'OFFLINE'; KW_ENABLE: 'ENABLE'; KW_DISABLE: 'DISABLE'; Index: ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java (revision 1002844) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java (working copy) @@ -448,10 +448,17 @@ return getColumns(ast, true); } + protected void handleGenericFileFormat(ASTNode node) throws SemanticException{ + + ASTNode child = (ASTNode)node.getChild(0); + throw new SemanticException("Unrecognized file format in STORED AS clause:"+ + " "+ (child == null ? "" : child.getText())); + } + /** * Get the list of FieldSchema out of the ASTNode. */ - protected List getColumns(ASTNode ast, boolean lowerCase) throws SemanticException { + public static List getColumns(ASTNode ast, boolean lowerCase) throws SemanticException { List colList = new ArrayList(); int numCh = ast.getChildCount(); for (int i = 0; i < numCh; i++) { Index: ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (revision 1002844) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (working copy) @@ -29,9 +29,9 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.Set; import java.util.TreeSet; +import java.util.Map.Entry; import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; @@ -92,7 +92,6 @@ import org.apache.hadoop.hive.ql.optimizer.GenMRFileSink1; import org.apache.hadoop.hive.ql.optimizer.GenMROperator; import org.apache.hadoop.hive.ql.optimizer.GenMRProcContext; -import org.apache.hadoop.hive.ql.optimizer.GenMRProcContext.GenMapRedCtx; import org.apache.hadoop.hive.ql.optimizer.GenMRRedSink1; import org.apache.hadoop.hive.ql.optimizer.GenMRRedSink2; import org.apache.hadoop.hive.ql.optimizer.GenMRRedSink3; @@ -102,6 +101,7 @@ import org.apache.hadoop.hive.ql.optimizer.GenMapRedUtils; import org.apache.hadoop.hive.ql.optimizer.MapJoinFactory; import org.apache.hadoop.hive.ql.optimizer.Optimizer; +import org.apache.hadoop.hive.ql.optimizer.GenMRProcContext.GenMapRedCtx; import org.apache.hadoop.hive.ql.optimizer.physical.PhysicalContext; import org.apache.hadoop.hive.ql.optimizer.physical.PhysicalOptimizer; import org.apache.hadoop.hive.ql.optimizer.ppr.PartitionPruner; @@ -121,7 +121,6 @@ import org.apache.hadoop.hive.ql.plan.FetchWork; import org.apache.hadoop.hive.ql.plan.FileSinkDesc; import org.apache.hadoop.hive.ql.plan.FilterDesc; -import org.apache.hadoop.hive.ql.plan.FilterDesc.sampleDesc; import org.apache.hadoop.hive.ql.plan.ForwardDesc; import org.apache.hadoop.hive.ql.plan.GroupByDesc; import org.apache.hadoop.hive.ql.plan.JoinCondDesc; @@ -143,11 +142,12 @@ import org.apache.hadoop.hive.ql.plan.TableScanDesc; import org.apache.hadoop.hive.ql.plan.UDTFDesc; import org.apache.hadoop.hive.ql.plan.UnionDesc; +import org.apache.hadoop.hive.ql.plan.FilterDesc.sampleDesc; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator; -import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.Mode; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFHash; import org.apache.hadoop.hive.ql.udf.generic.GenericUDTF; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.Mode; import org.apache.hadoop.hive.serde.Constants; import org.apache.hadoop.hive.serde2.Deserializer; import org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe; @@ -155,9 +155,9 @@ import org.apache.hadoop.hive.serde2.SerDeUtils; import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; @@ -6890,6 +6890,11 @@ shared.serdeProps); } break; + + case HiveParser.TOK_FILEFORMAT_GENERIC: + handleGenericFileFormat(child); + break; + default: assert false; } Index: ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java (revision 1002844) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java (working copy) @@ -75,17 +75,17 @@ import org.apache.hadoop.hive.ql.plan.DropIndexDesc; import org.apache.hadoop.hive.ql.plan.DropTableDesc; import org.apache.hadoop.hive.ql.plan.FetchWork; +import org.apache.hadoop.hive.ql.plan.LockTableDesc; import org.apache.hadoop.hive.ql.plan.MsckDesc; import org.apache.hadoop.hive.ql.plan.ShowDatabasesDesc; import org.apache.hadoop.hive.ql.plan.ShowFunctionsDesc; +import org.apache.hadoop.hive.ql.plan.ShowLocksDesc; import org.apache.hadoop.hive.ql.plan.ShowPartitionsDesc; import org.apache.hadoop.hive.ql.plan.ShowTableStatusDesc; import org.apache.hadoop.hive.ql.plan.ShowTablesDesc; -import org.apache.hadoop.hive.ql.plan.ShowLocksDesc; -import org.apache.hadoop.hive.ql.plan.LockTableDesc; -import org.apache.hadoop.hive.ql.plan.UnlockTableDesc; import org.apache.hadoop.hive.ql.plan.SwitchDatabaseDesc; import org.apache.hadoop.hive.ql.plan.TableDesc; +import org.apache.hadoop.hive.ql.plan.UnlockTableDesc; import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes; import org.apache.hadoop.hive.serde.Constants; import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; @@ -593,6 +593,9 @@ outputFormat = RCFILE_OUTPUT; serde = COLUMNAR_SERDE; break; + case HiveParser.TOK_FILEFORMAT_GENERIC: + handleGenericFileFormat(child); + break; } AlterTableDesc alterTblDesc = new AlterTableDesc(tableName, inputFormat, Index: ql/src/java/org/apache/hadoop/hive/ql/parse/HiveSemanticAnalyzerHookContextImpl.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/HiveSemanticAnalyzerHookContextImpl.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/HiveSemanticAnalyzerHookContextImpl.java (revision 0) @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.parse; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.metadata.Hive; +import org.apache.hadoop.hive.ql.metadata.HiveException; + +public class HiveSemanticAnalyzerHookContextImpl implements HiveSemanticAnalyzerHookContext { + + Configuration conf; + + @Override + public Hive getHive() throws HiveException { + + return Hive.get((HiveConf)conf); + } + + @Override + public Configuration getConf() { + return conf; + } + + @Override + public void setConf(Configuration conf) { + this.conf = conf; + } + +} Index: ql/src/java/org/apache/hadoop/hive/ql/Driver.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/Driver.java (revision 1002844) +++ ql/src/java/org/apache/hadoop/hive/ql/Driver.java (working copy) @@ -40,6 +40,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.JavaUtils; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.metastore.MetaStoreUtils; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Schema; @@ -63,10 +64,16 @@ import org.apache.hadoop.hive.ql.lockmgr.HiveLockMode; import org.apache.hadoop.hive.ql.lockmgr.HiveLockObject; import org.apache.hadoop.hive.ql.lockmgr.LockException; -import org.apache.hadoop.hive.ql.io.IOPrepareCache; +import org.apache.hadoop.hive.ql.metadata.DummyPartition; +import org.apache.hadoop.hive.ql.metadata.HiveUtils; +import org.apache.hadoop.hive.ql.metadata.Partition; +import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.parse.ASTNode; +import org.apache.hadoop.hive.ql.parse.AbstractSemanticAnalyzerHook; import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer; import org.apache.hadoop.hive.ql.parse.ErrorMsg; +import org.apache.hadoop.hive.ql.parse.HiveSemanticAnalyzerHookContext; +import org.apache.hadoop.hive.ql.parse.HiveSemanticAnalyzerHookContextImpl; import org.apache.hadoop.hive.ql.parse.ParseDriver; import org.apache.hadoop.hive.ql.parse.ParseException; import org.apache.hadoop.hive.ql.parse.ParseUtils; @@ -83,9 +90,6 @@ import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.security.UnixUserGroupInformation; import org.apache.hadoop.util.ReflectionUtils; -import org.apache.hadoop.hive.ql.metadata.Partition; -import org.apache.hadoop.hive.ql.metadata.DummyPartition; -import org.apache.hadoop.hive.ql.metadata.Table; public class Driver implements CommandProcessor { @@ -322,8 +326,21 @@ tree = ParseUtils.findRootNonNullToken(tree); BaseSemanticAnalyzer sem = SemanticAnalyzerFactory.get(conf, tree); + String hookName = HiveConf.getVar(conf, ConfVars.SEMANTIC_ANALYZER_HOOK); + // Do semantic analysis and plan generation - sem.analyze(tree, ctx); + if (hookName != null){ + AbstractSemanticAnalyzerHook hook = HiveUtils.getSemanticAnalyzerHook(conf, hookName); + HiveSemanticAnalyzerHookContext hookCtx = new HiveSemanticAnalyzerHookContextImpl(); + hookCtx.setConf(conf); + hook.preAnalyze(hookCtx, tree); + sem.analyze(tree, ctx); + hook.postAnalyze(hookCtx, sem.getRootTasks()); + } + else{ + sem.analyze(tree, ctx); + } + LOG.info("Semantic Analysis Completed"); // validate the plan