diff --git a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/cli/SemanticAnalysis/CreateTableHook.java b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/cli/SemanticAnalysis/CreateTableHook.java index ec24531..ff0f210 100644 --- a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/cli/SemanticAnalysis/CreateTableHook.java +++ b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/cli/SemanticAnalysis/CreateTableHook.java @@ -30,8 +30,6 @@ import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.exec.DDLTask; import org.apache.hadoop.hive.ql.exec.Task; -import org.apache.hadoop.hive.ql.io.RCFileInputFormat; -import org.apache.hadoop.hive.ql.io.RCFileOutputFormat; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler; @@ -41,6 +39,7 @@ import org.apache.hadoop.hive.ql.parse.HiveParser; import org.apache.hadoop.hive.ql.parse.HiveSemanticAnalyzerHookContext; import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.parse.StorageFormat; import org.apache.hadoop.hive.ql.plan.CreateTableDesc; import org.apache.hadoop.hive.ql.security.authorization.Privilege; import org.apache.hive.hcatalog.common.HCatConstants; @@ -66,14 +65,20 @@ public ASTNode preAnalyze(HiveSemanticAnalyzerHookContext context, // Analyze and create tbl properties object int numCh = ast.getChildCount(); - String inputFormat = null, outputFormat = null; tableName = BaseSemanticAnalyzer.getUnescapedName((ASTNode) ast .getChild(0)); boolean likeTable = false; + StorageFormat format = new StorageFormat(context.getConf()); for (int num = 1; num < numCh; num++) { ASTNode child = (ASTNode) ast.getChild(num); - + if (format.fillStorageFormat(child)) { + if (org.apache.commons.lang.StringUtils + .isNotEmpty(format.getStorageHandler())) { + return ast; + } + continue; + } switch (child.getToken().getType()) { case HiveParser.TOK_QUERY: // CTAS @@ -84,17 +89,6 @@ public ASTNode preAnalyze(HiveSemanticAnalyzerHookContext context, case HiveParser.TOK_TABLEBUCKETS: break; - case HiveParser.TOK_TBLSEQUENCEFILE: - inputFormat = HCatConstants.SEQUENCEFILE_INPUT; - outputFormat = HCatConstants.SEQUENCEFILE_OUTPUT; - break; - - case HiveParser.TOK_TBLTEXTFILE: - inputFormat = org.apache.hadoop.mapred.TextInputFormat.class.getName(); - outputFormat = org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat.class.getName(); - - break; - case HiveParser.TOK_LIKETABLE: likeTable = true; break; @@ -125,43 +119,14 @@ public ASTNode preAnalyze(HiveSemanticAnalyzerHookContext context, } } break; - - case HiveParser.TOK_STORAGEHANDLER: - String storageHandler = BaseSemanticAnalyzer - .unescapeSQLString(child.getChild(0).getText()); - if (org.apache.commons.lang.StringUtils - .isNotEmpty(storageHandler)) { - return ast; - } - - break; - - case HiveParser.TOK_TABLEFILEFORMAT: - if (child.getChildCount() < 2) { - throw new SemanticException( - "Incomplete specification of File Format. " + - "You must provide InputFormat, OutputFormat."); - } - inputFormat = BaseSemanticAnalyzer.unescapeSQLString(child - .getChild(0).getText()); - outputFormat = BaseSemanticAnalyzer.unescapeSQLString(child - .getChild(1).getText()); - break; - - case HiveParser.TOK_TBLRCFILE: - inputFormat = RCFileInputFormat.class.getName(); - outputFormat = RCFileOutputFormat.class.getName(); - break; - } } - if (!likeTable && (inputFormat == null || outputFormat == null)) { + if (!likeTable && (format.getInputFormat() == null || format.getOutputFormat() == null)) { throw new SemanticException( "STORED AS specification is either incomplete or incorrect."); } - return ast; } diff --git a/itests/custom-serde/src/main/java/org/apache/hadoop/hive/serde2/CustomTextSerDe.java b/itests/custom-serde/src/main/java/org/apache/hadoop/hive/serde2/CustomTextSerDe.java new file mode 100644 index 0000000..6293678 --- /dev/null +++ b/itests/custom-serde/src/main/java/org/apache/hadoop/hive/serde2/CustomTextSerDe.java @@ -0,0 +1,27 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.serde2; + +import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; + +public class CustomTextSerDe extends LazySimpleSerDe { + public CustomTextSerDe() throws SerDeException { + } +} diff --git a/itests/custom-serde/src/main/java/org/apache/hadoop/hive/serde2/CustomTextStorageFormatDescriptor.java b/itests/custom-serde/src/main/java/org/apache/hadoop/hive/serde2/CustomTextStorageFormatDescriptor.java new file mode 100644 index 0000000..1f4ab14 --- /dev/null +++ b/itests/custom-serde/src/main/java/org/apache/hadoop/hive/serde2/CustomTextStorageFormatDescriptor.java @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.serde2; + +import java.util.Set; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.io.IOConstants; +import org.apache.hadoop.hive.ql.io.TextFileStorageFormatDescriptor; + +import com.google.common.collect.ImmutableSet; + +public class CustomTextStorageFormatDescriptor extends TextFileStorageFormatDescriptor { + + @Override + public Set getNames() { + return ImmutableSet.of(IOConstants.CUSTOM_TEXT_SERDE); + } + + @Override + public String getSerde() { + return CustomTextSerDe.class.getName(); + } +} diff --git a/itests/custom-serde/src/main/resources/META-INF/services/org.apache.hadoop.hive.ql.io.StorageFormatDescriptor b/itests/custom-serde/src/main/resources/META-INF/services/org.apache.hadoop.hive.ql.io.StorageFormatDescriptor new file mode 100644 index 0000000..c25c1c6 --- /dev/null +++ b/itests/custom-serde/src/main/resources/META-INF/services/org.apache.hadoop.hive.ql.io.StorageFormatDescriptor @@ -0,0 +1 @@ +org.apache.hadoop.hive.serde2.CustomTextStorageFormatDescriptor diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/AbstractStorageFormatDescriptor.java b/ql/src/java/org/apache/hadoop/hive/ql/io/AbstractStorageFormatDescriptor.java new file mode 100644 index 0000000..854a7a3 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/AbstractStorageFormatDescriptor.java @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.io; + + +public abstract class AbstractStorageFormatDescriptor implements StorageFormatDescriptor { + + @Override + public String getSerde() { + return null; + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/IOConstants.java b/ql/src/java/org/apache/hadoop/hive/ql/io/IOConstants.java index 4131066..1bae0a8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/IOConstants.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/IOConstants.java @@ -17,11 +17,32 @@ */ package org.apache.hadoop.hive.ql.io; +import org.apache.hadoop.mapred.TextInputFormat; + +import com.google.common.annotations.VisibleForTesting; + public final class IOConstants { public static final String COLUMNS = "columns"; public static final String COLUMNS_TYPES = "columns.types"; public static final String MAPRED_TASK_ID = "mapred.task.id"; + public static final String TEXTFILE = "TEXTFILE"; + public static final String SEQUENCEFILE = "SEQUENCEFILE"; + public static final String RCFILE = "RCFILE"; + public static final String ORC = "ORC"; + public static final String ORCFILE = "ORCFILE"; + public static final String PARQUET = "PARQUET"; + public static final String PARQUETFILE = "PARQUETFILE"; + + @VisibleForTesting + public static final String CUSTOM_TEXT_SERDE = "CustomTextSerde"; + + public static final String TEXTFILE_INPUT = TextInputFormat.class + .getName(); + @SuppressWarnings("deprecation") + public static final String TEXTFILE_OUTPUT = IgnoreKeyTextOutputFormat.class + .getName(); + private IOConstants() { // prevent instantiation } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/ORCFileStorageFormatDescriptor.java b/ql/src/java/org/apache/hadoop/hive/ql/io/ORCFileStorageFormatDescriptor.java new file mode 100644 index 0000000..38aafac --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/ORCFileStorageFormatDescriptor.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.io; + +import java.util.Set; + +import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat; +import org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat; +import org.apache.hadoop.hive.ql.io.orc.OrcSerde; + +import com.google.common.collect.ImmutableSet; + +public class ORCFileStorageFormatDescriptor extends AbstractStorageFormatDescriptor { + @Override + public Set getNames() { + return ImmutableSet.of(IOConstants.ORCFILE, IOConstants.ORC); + } + @Override + public String getInputFormat() { + return OrcInputFormat.class.getName(); + } + @Override + public String getOutputFormat() { + return OrcOutputFormat.class.getName(); + } + @Override + public String getSerde() { + return OrcSerde.class.getName(); + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/ParquetFileStorageFormatDescriptor.java b/ql/src/java/org/apache/hadoop/hive/ql/io/ParquetFileStorageFormatDescriptor.java new file mode 100644 index 0000000..166775f --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/ParquetFileStorageFormatDescriptor.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.io; + +import java.util.Set; + +import org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat; +import org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat; +import org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe; + +import com.google.common.collect.ImmutableSet; + +public class ParquetFileStorageFormatDescriptor extends AbstractStorageFormatDescriptor { + @Override + public Set getNames() { + return ImmutableSet.of(IOConstants.PARQUETFILE, IOConstants.PARQUET); + } + @Override + public String getInputFormat() { + return MapredParquetInputFormat.class.getName(); + } + @Override + public String getOutputFormat() { + return MapredParquetOutputFormat.class.getName(); + } + @Override + public String getSerde() { + return ParquetHiveSerDe.class.getName(); + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/RCFileStorageFormatDescriptor.java b/ql/src/java/org/apache/hadoop/hive/ql/io/RCFileStorageFormatDescriptor.java new file mode 100644 index 0000000..9a52939 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/RCFileStorageFormatDescriptor.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.io; + +import java.util.Set; + +import com.google.common.collect.ImmutableSet; + +public class RCFileStorageFormatDescriptor extends AbstractStorageFormatDescriptor { + @Override + public Set getNames() { + return ImmutableSet.of(IOConstants.RCFILE); + } + @Override + public String getInputFormat() { + return RCFileInputFormat.class.getName(); + } + @Override + public String getOutputFormat() { + return RCFileOutputFormat.class.getName(); + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/SequenceFileStorageFormatDescriptor.java b/ql/src/java/org/apache/hadoop/hive/ql/io/SequenceFileStorageFormatDescriptor.java new file mode 100644 index 0000000..166ae81 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/SequenceFileStorageFormatDescriptor.java @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.io; + +import java.util.Set; + +import org.apache.hadoop.mapred.SequenceFileInputFormat; +import org.apache.hadoop.mapred.SequenceFileOutputFormat; + +import com.google.common.collect.ImmutableSet; + +public class SequenceFileStorageFormatDescriptor extends AbstractStorageFormatDescriptor { + @Override + public Set getNames() { + return ImmutableSet.of(IOConstants.SEQUENCEFILE); + } + @Override + public String getInputFormat() { + return SequenceFileInputFormat.class.getName(); + } + @Override + public String getOutputFormat() { + return SequenceFileOutputFormat.class.getName(); + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/StorageFormatDescriptor.java b/ql/src/java/org/apache/hadoop/hive/ql/io/StorageFormatDescriptor.java new file mode 100644 index 0000000..604abf8 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/StorageFormatDescriptor.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.io; + +import java.util.Set; + +import javax.annotation.Nullable; + +/** + * Subclasses represent a storage format for the + * CREATE TABLE ... STORED AS ... command. Subclasses are + * found via the ServiceLoader facility. + */ +public interface StorageFormatDescriptor { + /** + * Return the set of names this storage format is known as. + */ + Set getNames(); + /** + * Return the name of the input format as a string + */ + String getInputFormat(); + /** + * Return the name of the output format as a string + */ + String getOutputFormat(); + /** + * Return the name of the serde as a string or null + */ + @Nullable String getSerde(); + +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/StorageFormatFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/io/StorageFormatFactory.java new file mode 100644 index 0000000..e0bf153 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/StorageFormatFactory.java @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.io; + +import java.util.HashMap; +import java.util.Map; +import java.util.ServiceLoader; + +import javax.annotation.Nullable; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import com.google.common.collect.ImmutableMap; + +public class StorageFormatFactory { + private static final Log LOG = LogFactory.getLog(StorageFormatFactory.class); + + private final Map storageFormats; + + public StorageFormatFactory() { + Map localStorageFormats = + new HashMap(); + for (StorageFormatDescriptor descriptor : ServiceLoader.load(StorageFormatDescriptor.class)) { + for (String name : descriptor.getNames()) { + name = name.trim().toUpperCase(); + StorageFormatDescriptor oldDescriptor = localStorageFormats.put(name, descriptor); + if (oldDescriptor != null) { + String msg = "Storage Format Descriptor conflict at name '" + name + "', " + + "the descriptor " + descriptor + " is overriding " + oldDescriptor; + LOG.warn(msg); + } + } + } + this.storageFormats = ImmutableMap.copyOf(localStorageFormats); + } + + public @Nullable StorageFormatDescriptor get(String name) { + name = name.trim().toUpperCase(); + return storageFormats.get(name); + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/TextFileStorageFormatDescriptor.java b/ql/src/java/org/apache/hadoop/hive/ql/io/TextFileStorageFormatDescriptor.java new file mode 100644 index 0000000..86d1939 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/TextFileStorageFormatDescriptor.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.io; + +import java.util.Set; + +import com.google.common.collect.ImmutableSet; + +public class TextFileStorageFormatDescriptor extends AbstractStorageFormatDescriptor { + @Override + public Set getNames() { + return ImmutableSet.of(IOConstants.TEXTFILE); + } + @Override + public String getInputFormat() { + return IOConstants.TEXTFILE_INPUT; + } + @Override + public String getOutputFormat() { + return IOConstants.TEXTFILE_OUTPUT; + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java index 60d54b6..7d35de1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java @@ -50,15 +50,6 @@ import org.apache.hadoop.hive.ql.hooks.LineageInfo; import org.apache.hadoop.hive.ql.hooks.ReadEntity; import org.apache.hadoop.hive.ql.hooks.WriteEntity; -import org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat; -import org.apache.hadoop.hive.ql.io.RCFileInputFormat; -import org.apache.hadoop.hive.ql.io.RCFileOutputFormat; -import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat; -import org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat; -import org.apache.hadoop.hive.ql.io.orc.OrcSerde; -import org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat; -import org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat; -import org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe; import org.apache.hadoop.hive.ql.lib.Node; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; @@ -77,9 +68,6 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; -import org.apache.hadoop.mapred.SequenceFileInputFormat; -import org.apache.hadoop.mapred.SequenceFileOutputFormat; -import org.apache.hadoop.mapred.TextInputFormat; import com.google.common.annotations.VisibleForTesting; @@ -87,7 +75,6 @@ * BaseSemanticAnalyzer. * */ -@SuppressWarnings("deprecation") public abstract class BaseSemanticAnalyzer { private static final Log STATIC_LOG = LogFactory.getLog(BaseSemanticAnalyzer.class.getName()); protected final Hive db; @@ -119,28 +106,6 @@ protected TableAccessInfo tableAccessInfo; protected ColumnAccessInfo columnAccessInfo; - protected static final String TEXTFILE_INPUT = TextInputFormat.class - .getName(); - protected static final String TEXTFILE_OUTPUT = IgnoreKeyTextOutputFormat.class - .getName(); - protected static final String SEQUENCEFILE_INPUT = SequenceFileInputFormat.class - .getName(); - protected static final String SEQUENCEFILE_OUTPUT = SequenceFileOutputFormat.class - .getName(); - protected static final String RCFILE_INPUT = RCFileInputFormat.class - .getName(); - protected static final String RCFILE_OUTPUT = RCFileOutputFormat.class - .getName(); - protected static final String ORCFILE_INPUT = OrcInputFormat.class - .getName(); - protected static final String ORCFILE_OUTPUT = OrcOutputFormat.class - .getName(); - protected static final String ORCFILE_SERDE = OrcSerde.class - .getName(); - - protected static final String PARQUETFILE_INPUT = MapredParquetInputFormat.class.getName(); - protected static final String PARQUETFILE_OUTPUT = MapredParquetOutputFormat.class.getName(); - protected static final String PARQUETFILE_SERDE = ParquetHiveSerDe.class.getName(); public boolean skipAuthorization() { return false; @@ -154,7 +119,7 @@ public boolean skipAuthorization() { String lineDelim = null; String nullFormat = null; - protected void analyzeRowFormat(AnalyzeCreateCommonVars shared, ASTNode child) throws SemanticException { + protected void analyzeRowFormat(ASTNode child) throws SemanticException { child = (ASTNode) child.getChild(0); int numChildRowFormat = child.getChildCount(); for (int numC = 0; numC < numChildRowFormat; numC++) { @@ -190,93 +155,7 @@ protected void analyzeRowFormat(AnalyzeCreateCommonVars shared, ASTNode child) t .getText()); break; default: - assert false; - } - } - } - } - - class AnalyzeCreateCommonVars { - String serde = null; - Map serdeProps = new HashMap(); - } - - class StorageFormat { - String inputFormat = null; - String outputFormat = null; - String storageHandler = null; - - protected boolean fillStorageFormat(ASTNode child, AnalyzeCreateCommonVars shared) { - boolean storageFormat = false; - switch(child.getToken().getType()) { - case HiveParser.TOK_TBLSEQUENCEFILE: - inputFormat = SEQUENCEFILE_INPUT; - outputFormat = SEQUENCEFILE_OUTPUT; - storageFormat = true; - break; - case HiveParser.TOK_TBLTEXTFILE: - inputFormat = TEXTFILE_INPUT; - outputFormat = TEXTFILE_OUTPUT; - storageFormat = true; - break; - case HiveParser.TOK_TBLRCFILE: - inputFormat = RCFILE_INPUT; - outputFormat = RCFILE_OUTPUT; - if (shared.serde == null) { - shared.serde = conf.getVar(HiveConf.ConfVars.HIVEDEFAULTRCFILESERDE); - } - storageFormat = true; - break; - case HiveParser.TOK_TBLORCFILE: - inputFormat = ORCFILE_INPUT; - outputFormat = ORCFILE_OUTPUT; - shared.serde = ORCFILE_SERDE; - storageFormat = true; - break; - case HiveParser.TOK_TBLPARQUETFILE: - inputFormat = PARQUETFILE_INPUT; - outputFormat = PARQUETFILE_OUTPUT; - shared.serde = PARQUETFILE_SERDE; - storageFormat = true; - break; - case HiveParser.TOK_TABLEFILEFORMAT: - inputFormat = unescapeSQLString(child.getChild(0).getText()); - outputFormat = unescapeSQLString(child.getChild(1).getText()); - storageFormat = true; - break; - case HiveParser.TOK_STORAGEHANDLER: - storageHandler = unescapeSQLString(child.getChild(0).getText()); - if (child.getChildCount() == 2) { - readProps( - (ASTNode) (child.getChild(1).getChild(0)), - shared.serdeProps); - } - storageFormat = true; - break; - } - return storageFormat; - } - - protected void fillDefaultStorageFormat(AnalyzeCreateCommonVars shared) { - if ((inputFormat == null) && (storageHandler == null)) { - if ("SequenceFile".equalsIgnoreCase(conf.getVar(HiveConf.ConfVars.HIVEDEFAULTFILEFORMAT))) { - inputFormat = SEQUENCEFILE_INPUT; - outputFormat = SEQUENCEFILE_OUTPUT; - } else if ("RCFile".equalsIgnoreCase(conf.getVar(HiveConf.ConfVars.HIVEDEFAULTFILEFORMAT))) { - inputFormat = RCFILE_INPUT; - outputFormat = RCFILE_OUTPUT; - shared.serde = conf.getVar(HiveConf.ConfVars.HIVEDEFAULTRCFILESERDE); - } else if ("ORC".equalsIgnoreCase(conf.getVar(HiveConf.ConfVars.HIVEDEFAULTFILEFORMAT))) { - inputFormat = ORCFILE_INPUT; - outputFormat = ORCFILE_OUTPUT; - shared.serde = ORCFILE_SERDE; - } else if ("PARQUET".equalsIgnoreCase(conf.getVar(HiveConf.ConfVars.HIVEDEFAULTFILEFORMAT))) { - inputFormat = PARQUETFILE_INPUT; - outputFormat = PARQUETFILE_OUTPUT; - shared.serde = PARQUETFILE_SERDE; - } else { - inputFormat = TEXTFILE_INPUT; - outputFormat = TEXTFILE_OUTPUT; + throw new AssertionError("Unkown Token: " + rowChild); } } } @@ -607,13 +486,6 @@ public static String unescapeSQLString(String b) { return getColumns(ast, true); } - protected void handleGenericFileFormat(ASTNode node) throws SemanticException{ - - ASTNode child = (ASTNode)node.getChild(0); - throw new SemanticException("Unrecognized file format in STORED AS clause:"+ - " "+ (child == null ? "" : child.getText())); - } - /** * Get the list of FieldSchema out of the ASTNode. */ @@ -966,7 +838,7 @@ public void setColumnAccessInfo(ColumnAccessInfo columnAccessInfo) { * @throws HiveException */ public final boolean isValidPrefixSpec(Table tTable, Map spec) - throws HiveException { + throws HiveException { // TODO - types need to be checked. List partCols = tTable.getPartitionKeys(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java index 640b6b3..a75fd6a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java @@ -1032,17 +1032,16 @@ private void analyzeCreateIndex(ASTNode ast) throws SemanticException { String indexComment = null; RowFormatParams rowFormatParams = new RowFormatParams(); - StorageFormat storageFormat = new StorageFormat(); - AnalyzeCreateCommonVars shared = new AnalyzeCreateCommonVars(); + StorageFormat storageFormat = new StorageFormat(conf); for (int idx = 4; idx < ast.getChildCount(); idx++) { ASTNode child = (ASTNode) ast.getChild(idx); - if (storageFormat.fillStorageFormat(child, shared)) { + if (storageFormat.fillStorageFormat(child)) { continue; } switch (child.getToken().getType()) { case HiveParser.TOK_TABLEROWFORMAT: - rowFormatParams.analyzeRowFormat(shared, child); + rowFormatParams.analyzeRowFormat(child); break; case HiveParser.TOK_CREATEINDEX_INDEXTBLNAME: ASTNode ch = (ASTNode) child.getChild(0); @@ -1063,10 +1062,10 @@ private void analyzeCreateIndex(ASTNode ast) throws SemanticException { break; case HiveParser.TOK_TABLESERIALIZER: child = (ASTNode) child.getChild(0); - shared.serde = unescapeSQLString(child.getChild(0).getText()); + storageFormat.setSerde(unescapeSQLString(child.getChild(0).getText())); if (child.getChildCount() == 2) { readProps((ASTNode) (child.getChild(1).getChild(0)), - shared.serdeProps); + storageFormat.getSerdeProps()); } break; case HiveParser.TOK_INDEXCOMMENT: @@ -1075,14 +1074,14 @@ private void analyzeCreateIndex(ASTNode ast) throws SemanticException { } } - storageFormat.fillDefaultStorageFormat(shared); + storageFormat.fillDefaultStorageFormat(); CreateIndexDesc crtIndexDesc = new CreateIndexDesc(tableName, indexName, - indexedCols, indexTableName, deferredRebuild, storageFormat.inputFormat, - storageFormat.outputFormat, - storageFormat.storageHandler, typeName, location, idxProps, tblProps, - shared.serde, shared.serdeProps, rowFormatParams.collItemDelim, + indexedCols, indexTableName, deferredRebuild, storageFormat.getInputFormat(), + storageFormat.getOutputFormat(), + storageFormat.getStorageHandler(), typeName, location, idxProps, tblProps, + storageFormat.getSerde(), storageFormat.getSerdeProps(), rowFormatParams.collItemDelim, rowFormatParams.fieldDelim, rowFormatParams.fieldEscape, rowFormatParams.lineDelim, rowFormatParams.mapKeyDelim, indexComment); Task createIndex = @@ -1317,69 +1316,15 @@ private void analyzeAlterTableFileFormat(ASTNode ast, String tableName, HashMap partSpec) throws SemanticException { - String inputFormat = null; - String outputFormat = null; - String storageHandler = null; - String serde = null; + StorageFormat format = new StorageFormat(conf); ASTNode child = (ASTNode) ast.getChild(0); - switch (child.getToken().getType()) { - case HiveParser.TOK_TABLEFILEFORMAT: - inputFormat = unescapeSQLString(((ASTNode) child.getChild(0)).getToken() - .getText()); - outputFormat = unescapeSQLString(((ASTNode) child.getChild(1)).getToken() - .getText()); - serde = unescapeSQLString(((ASTNode) child.getChild(2)).getToken() - .getText()); - try { - Class.forName(inputFormat); - Class.forName(outputFormat); - Class.forName(serde); - } catch (ClassNotFoundException e) { - throw new SemanticException(e); - } - break; - case HiveParser.TOK_STORAGEHANDLER: - storageHandler = - unescapeSQLString(((ASTNode) child.getChild(1)).getToken().getText()); - try { - Class.forName(storageHandler); - } catch (ClassNotFoundException e) { - throw new SemanticException(e); - } - break; - case HiveParser.TOK_TBLSEQUENCEFILE: - inputFormat = SEQUENCEFILE_INPUT; - outputFormat = SEQUENCEFILE_OUTPUT; - serde = org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class.getName(); - break; - case HiveParser.TOK_TBLTEXTFILE: - inputFormat = TEXTFILE_INPUT; - outputFormat = TEXTFILE_OUTPUT; - serde = org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class.getName(); - break; - case HiveParser.TOK_TBLRCFILE: - inputFormat = RCFILE_INPUT; - outputFormat = RCFILE_OUTPUT; - serde = conf.getVar(HiveConf.ConfVars.HIVEDEFAULTRCFILESERDE); - break; - case HiveParser.TOK_TBLORCFILE: - inputFormat = ORCFILE_INPUT; - outputFormat = ORCFILE_OUTPUT; - serde = ORCFILE_SERDE; - break; - case HiveParser.TOK_TBLPARQUETFILE: - inputFormat = PARQUETFILE_INPUT; - outputFormat = PARQUETFILE_OUTPUT; - serde = PARQUETFILE_SERDE; - break; - case HiveParser.TOK_FILEFORMAT_GENERIC: - handleGenericFileFormat(child); - break; + if (!format.fillStorageFormat(child)) { + throw new AssertionError("Unknown token " + child.getText()); } - AlterTableDesc alterTblDesc = new AlterTableDesc(tableName, inputFormat, - outputFormat, serde, storageHandler, partSpec); + AlterTableDesc alterTblDesc = new AlterTableDesc(tableName, format.getInputFormat(), + format.getOutputFormat(), format.getSerde(), format.getStorageHandler(), partSpec); addInputsOutputsAlterTable(tableName, partSpec, alterTblDesc); rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g index 412a046..ba8481c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g @@ -151,11 +151,6 @@ KW_KEY_TYPE: '$KEY$'; KW_LINES: 'LINES'; KW_STORED: 'STORED'; KW_FILEFORMAT: 'FILEFORMAT'; -KW_SEQUENCEFILE: 'SEQUENCEFILE'; -KW_TEXTFILE: 'TEXTFILE'; -KW_RCFILE: 'RCFILE'; -KW_ORCFILE: 'ORC'; -KW_PARQUETFILE: 'PARQUET'; KW_INPUTFORMAT: 'INPUTFORMAT'; KW_OUTPUTFORMAT: 'OUTPUTFORMAT'; KW_INPUTDRIVER: 'INPUTDRIVER'; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g index f934ac4..8247e61 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g @@ -182,11 +182,6 @@ TOK_TABLEROWFORMATCOLLITEMS; TOK_TABLEROWFORMATMAPKEYS; TOK_TABLEROWFORMATLINES; TOK_TABLEROWFORMATNULL; -TOK_TBLORCFILE; -TOK_TBLPARQUETFILE; -TOK_TBLSEQUENCEFILE; -TOK_TBLTEXTFILE; -TOK_TBLRCFILE; TOK_TABLEFILEFORMAT; TOK_FILEFORMAT_GENERIC; TOK_OFFLINE; @@ -1265,12 +1260,7 @@ alterStatementSuffixCompact fileFormat @init { pushMsg("file format specification", state); } @after { popMsg(state); } - : KW_SEQUENCEFILE -> ^(TOK_TBLSEQUENCEFILE) - | KW_TEXTFILE -> ^(TOK_TBLTEXTFILE) - | KW_RCFILE -> ^(TOK_TBLRCFILE) - | KW_ORCFILE -> ^(TOK_TBLORCFILE) - | KW_PARQUETFILE -> ^(TOK_TBLPARQUETFILE) - | KW_INPUTFORMAT inFmt=StringLiteral KW_OUTPUTFORMAT outFmt=StringLiteral KW_SERDE serdeCls=StringLiteral (KW_INPUTDRIVER inDriver=StringLiteral KW_OUTPUTDRIVER outDriver=StringLiteral)? + : KW_INPUTFORMAT inFmt=StringLiteral KW_OUTPUTFORMAT outFmt=StringLiteral KW_SERDE serdeCls=StringLiteral (KW_INPUTDRIVER inDriver=StringLiteral KW_OUTPUTDRIVER outDriver=StringLiteral)? -> ^(TOK_TABLEFILEFORMAT $inFmt $outFmt $serdeCls $inDriver? $outDriver?) | genericSpec=identifier -> ^(TOK_FILEFORMAT_GENERIC $genericSpec) ; @@ -1808,12 +1798,7 @@ tableFileFormat @init { pushMsg("table file format specification", state); } @after { popMsg(state); } : - KW_STORED KW_AS KW_SEQUENCEFILE -> TOK_TBLSEQUENCEFILE - | KW_STORED KW_AS KW_TEXTFILE -> TOK_TBLTEXTFILE - | KW_STORED KW_AS KW_RCFILE -> TOK_TBLRCFILE - | KW_STORED KW_AS KW_ORCFILE -> TOK_TBLORCFILE - | KW_STORED KW_AS KW_PARQUETFILE -> TOK_TBLPARQUETFILE - | KW_STORED KW_AS KW_INPUTFORMAT inFmt=StringLiteral KW_OUTPUTFORMAT outFmt=StringLiteral (KW_INPUTDRIVER inDriver=StringLiteral KW_OUTPUTDRIVER outDriver=StringLiteral)? + KW_STORED KW_AS KW_INPUTFORMAT inFmt=StringLiteral KW_OUTPUTFORMAT outFmt=StringLiteral (KW_INPUTDRIVER inDriver=StringLiteral KW_OUTPUTDRIVER outDriver=StringLiteral)? -> ^(TOK_TABLEFILEFORMAT $inFmt $outFmt $inDriver? $outDriver?) | KW_STORED KW_BY storageHandler=StringLiteral (KW_WITH KW_SERDEPROPERTIES serdeprops=tableProperties)? diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g index 9c001c1..75897b8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g @@ -538,5 +538,5 @@ functionIdentifier nonReserved : - KW_TRUE | KW_FALSE | KW_LIKE | KW_EXISTS | KW_ASC | KW_DESC | KW_ORDER | KW_GROUP | KW_BY | KW_AS | KW_INSERT | KW_OVERWRITE | KW_OUTER | KW_LEFT | KW_RIGHT | KW_FULL | KW_PARTITION | KW_PARTITIONS | KW_TABLE | KW_TABLES | KW_COLUMNS | KW_INDEX | KW_INDEXES | KW_REBUILD | KW_FUNCTIONS | KW_SHOW | KW_MSCK | KW_REPAIR | KW_DIRECTORY | KW_LOCAL | KW_USING | KW_CLUSTER | KW_DISTRIBUTE | KW_SORT | KW_UNION | KW_LOAD | KW_EXPORT | KW_IMPORT | KW_DATA | KW_INPATH | KW_IS | KW_NULL | KW_CREATE | KW_EXTERNAL | KW_ALTER | KW_CHANGE | KW_FIRST | KW_AFTER | KW_DESCRIBE | KW_DROP | KW_RENAME | KW_IGNORE | KW_PROTECTION | KW_TO | KW_COMMENT | KW_BOOLEAN | KW_TINYINT | KW_SMALLINT | KW_INT | KW_BIGINT | KW_FLOAT | KW_DOUBLE | KW_DATE | KW_DATETIME | KW_TIMESTAMP | KW_DECIMAL | KW_STRING | KW_ARRAY | KW_STRUCT | KW_UNIONTYPE | KW_PARTITIONED | KW_CLUSTERED | KW_SORTED | KW_INTO | KW_BUCKETS | KW_ROW | KW_ROWS | KW_FORMAT | KW_DELIMITED | KW_FIELDS | KW_TERMINATED | KW_ESCAPED | KW_COLLECTION | KW_ITEMS | KW_KEYS | KW_KEY_TYPE | KW_LINES | KW_STORED | KW_FILEFORMAT | KW_SEQUENCEFILE | KW_TEXTFILE | KW_RCFILE | KW_ORCFILE | KW_PARQUETFILE | KW_INPUTFORMAT | KW_OUTPUTFORMAT | KW_INPUTDRIVER | KW_OUTPUTDRIVER | KW_OFFLINE | KW_ENABLE | KW_DISABLE | KW_READONLY | KW_NO_DROP | KW_LOCATION | KW_BUCKET | KW_OUT | KW_OF | KW_PERCENT | KW_ADD | KW_REPLACE | KW_RLIKE | KW_REGEXP | KW_TEMPORARY | KW_EXPLAIN | KW_FORMATTED | KW_PRETTY | KW_DEPENDENCY | KW_LOGICAL | KW_SERDE | KW_WITH | KW_DEFERRED | KW_SERDEPROPERTIES | KW_DBPROPERTIES | KW_LIMIT | KW_SET | KW_UNSET | KW_TBLPROPERTIES | KW_IDXPROPERTIES | KW_VALUE_TYPE | KW_ELEM_TYPE | KW_MAPJOIN | KW_STREAMTABLE | KW_HOLD_DDLTIME | KW_CLUSTERSTATUS | KW_UTC | KW_UTCTIMESTAMP | KW_LONG | KW_DELETE | KW_PLUS | KW_MINUS | KW_FETCH | KW_INTERSECT | KW_VIEW | KW_IN | KW_DATABASES | KW_MATERIALIZED | KW_SCHEMA | KW_SCHEMAS | KW_GRANT | KW_REVOKE | KW_SSL | KW_UNDO | KW_LOCK | KW_LOCKS | KW_UNLOCK | KW_SHARED | KW_EXCLUSIVE | KW_PROCEDURE | KW_UNSIGNED | KW_WHILE | KW_READ | KW_READS | KW_PURGE | KW_RANGE | KW_ANALYZE | KW_BEFORE | KW_BETWEEN | KW_BOTH | KW_BINARY | KW_CONTINUE | KW_CURSOR | KW_TRIGGER | KW_RECORDREADER | KW_RECORDWRITER | KW_SEMI | KW_LATERAL | KW_TOUCH | KW_ARCHIVE | KW_UNARCHIVE | KW_COMPUTE | KW_STATISTICS | KW_USE | KW_OPTION | KW_CONCATENATE | KW_SHOW_DATABASE | KW_UPDATE | KW_RESTRICT | KW_CASCADE | KW_SKEWED | KW_ROLLUP | KW_CUBE | KW_DIRECTORIES | KW_FOR | KW_GROUPING | KW_SETS | KW_TRUNCATE | KW_NOSCAN | KW_USER | KW_ROLE | KW_ROLES | KW_INNER | KW_DEFINED | KW_ADMIN | KW_JAR | KW_FILE | KW_OWNER | KW_PRINCIPALS | KW_ALL | KW_DEFAULT | KW_NONE | KW_COMPACT | KW_COMPACTIONS | KW_TRANSACTIONS | KW_REWRITE | KW_AUTHORIZATION + KW_TRUE | KW_FALSE | KW_LIKE | KW_EXISTS | KW_ASC | KW_DESC | KW_ORDER | KW_GROUP | KW_BY | KW_AS | KW_INSERT | KW_OVERWRITE | KW_OUTER | KW_LEFT | KW_RIGHT | KW_FULL | KW_PARTITION | KW_PARTITIONS | KW_TABLE | KW_TABLES | KW_COLUMNS | KW_INDEX | KW_INDEXES | KW_REBUILD | KW_FUNCTIONS | KW_SHOW | KW_MSCK | KW_REPAIR | KW_DIRECTORY | KW_LOCAL | KW_USING | KW_CLUSTER | KW_DISTRIBUTE | KW_SORT | KW_UNION | KW_LOAD | KW_EXPORT | KW_IMPORT | KW_DATA | KW_INPATH | KW_IS | KW_NULL | KW_CREATE | KW_EXTERNAL | KW_ALTER | KW_CHANGE | KW_FIRST | KW_AFTER | KW_DESCRIBE | KW_DROP | KW_RENAME | KW_IGNORE | KW_PROTECTION | KW_TO | KW_COMMENT | KW_BOOLEAN | KW_TINYINT | KW_SMALLINT | KW_INT | KW_BIGINT | KW_FLOAT | KW_DOUBLE | KW_DATE | KW_DATETIME | KW_TIMESTAMP | KW_DECIMAL | KW_STRING | KW_ARRAY | KW_STRUCT | KW_UNIONTYPE | KW_PARTITIONED | KW_CLUSTERED | KW_SORTED | KW_INTO | KW_BUCKETS | KW_ROW | KW_ROWS | KW_FORMAT | KW_DELIMITED | KW_FIELDS | KW_TERMINATED | KW_ESCAPED | KW_COLLECTION | KW_ITEMS | KW_KEYS | KW_KEY_TYPE | KW_LINES | KW_STORED | KW_FILEFORMAT | KW_INPUTFORMAT | KW_OUTPUTFORMAT | KW_INPUTDRIVER | KW_OUTPUTDRIVER | KW_OFFLINE | KW_ENABLE | KW_DISABLE | KW_READONLY | KW_NO_DROP | KW_LOCATION | KW_BUCKET | KW_OUT | KW_OF | KW_PERCENT | KW_ADD | KW_REPLACE | KW_RLIKE | KW_REGEXP | KW_TEMPORARY | KW_EXPLAIN | KW_FORMATTED | KW_PRETTY | KW_DEPENDENCY | KW_LOGICAL | KW_SERDE | KW_WITH | KW_DEFERRED | KW_SERDEPROPERTIES | KW_DBPROPERTIES | KW_LIMIT | KW_SET | KW_UNSET | KW_TBLPROPERTIES | KW_IDXPROPERTIES | KW_VALUE_TYPE | KW_ELEM_TYPE | KW_MAPJOIN | KW_STREAMTABLE | KW_HOLD_DDLTIME | KW_CLUSTERSTATUS | KW_UTC | KW_UTCTIMESTAMP | KW_LONG | KW_DELETE | KW_PLUS | KW_MINUS | KW_FETCH | KW_INTERSECT | KW_VIEW | KW_IN | KW_DATABASES | KW_MATERIALIZED | KW_SCHEMA | KW_SCHEMAS | KW_GRANT | KW_REVOKE | KW_SSL | KW_UNDO | KW_LOCK | KW_LOCKS | KW_UNLOCK | KW_SHARED | KW_EXCLUSIVE | KW_PROCEDURE | KW_UNSIGNED | KW_WHILE | KW_READ | KW_READS | KW_PURGE | KW_RANGE | KW_ANALYZE | KW_BEFORE | KW_BETWEEN | KW_BOTH | KW_BINARY | KW_CONTINUE | KW_CURSOR | KW_TRIGGER | KW_RECORDREADER | KW_RECORDWRITER | KW_SEMI | KW_LATERAL | KW_TOUCH | KW_ARCHIVE | KW_UNARCHIVE | KW_COMPUTE | KW_STATISTICS | KW_USE | KW_OPTION | KW_CONCATENATE | KW_SHOW_DATABASE | KW_UPDATE | KW_RESTRICT | KW_CASCADE | KW_SKEWED | KW_ROLLUP | KW_CUBE | KW_DIRECTORIES | KW_FOR | KW_GROUPING | KW_SETS | KW_TRUNCATE | KW_NOSCAN | KW_USER | KW_ROLE | KW_ROLES | KW_INNER | KW_DEFINED | KW_ADMIN | KW_JAR | KW_FILE | KW_OWNER | KW_PRINCIPALS | KW_ALL | KW_DEFAULT | KW_NONE | KW_COMPACT | KW_COMPACTIONS | KW_TRANSACTIONS | KW_REWRITE | KW_AUTHORIZATION ; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java index 0af2536..0a1c660 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java @@ -22,12 +22,11 @@ import java.util.Iterator; import java.util.List; +import org.apache.hadoop.hive.common.JavaUtils; import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; -import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; -import org.apache.hadoop.hive.serde2.typeinfo.BaseCharTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; @@ -216,4 +215,18 @@ public static DecimalTypeInfo getDecimalTypeTypeInfo(ASTNode node) return TypeInfoFactory.getDecimalTypeInfo(precision, scale); } + public static String ensureClassExists(String className) + throws SemanticException { + if (className == null) { + return null; + } + try { + Class.forName(className, true, JavaUtils.getClassLoader()); + } catch (ClassNotFoundException e) { + throw new SemanticException("Cannot find class '" + className + "'", e); + } + return className; + } + + } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 399f92a..34ffe10 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -1177,10 +1177,6 @@ public boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1) return phase1Result; } - private void getMetaData(QBExpr qbexpr) throws SemanticException { - getMetaData(qbexpr, null); - } - private void getMetaData(QBExpr qbexpr, ReadEntity parentInput) throws SemanticException { if (qbexpr.getOpcode() == QBExpr.Opcode.NULLOP) { @@ -1358,8 +1354,7 @@ public void getMetaData(QB qb, ReadEntity parentInput) throws SemanticException } RowFormatParams rowFormatParams = new RowFormatParams(); - AnalyzeCreateCommonVars shared = new AnalyzeCreateCommonVars(); - StorageFormat storageFormat = new StorageFormat(); + StorageFormat storageFormat = new StorageFormat(conf); LOG.info("Get metadata for destination tables"); // Go over all the destination structures and populate the related @@ -1454,10 +1449,16 @@ public void getMetaData(QB qb, ReadEntity parentInput) throws SemanticException int numCh = ast.getChildCount(); for (int num = 1; num < numCh ; num++){ ASTNode child = (ASTNode) ast.getChild(num); - if (ast.getChild(num) != null){ + if (child != null) { + if (storageFormat.fillStorageFormat(child)) { + localDirectoryDesc.setOutputFormat(storageFormat.getOutputFormat()); + localDirectoryDesc.setSerName(storageFormat.getSerde()); + localDirectoryDescIsSet = true; + continue; + } switch (child.getToken().getType()) { case HiveParser.TOK_TABLEROWFORMAT: - rowFormatParams.analyzeRowFormat(shared, child); + rowFormatParams.analyzeRowFormat(child); localDirectoryDesc.setFieldDelim(rowFormatParams.fieldDelim); localDirectoryDesc.setLineDelim(rowFormatParams.lineDelim); localDirectoryDesc.setCollItemDelim(rowFormatParams.collItemDelim); @@ -1468,18 +1469,8 @@ public void getMetaData(QB qb, ReadEntity parentInput) throws SemanticException break; case HiveParser.TOK_TABLESERIALIZER: ASTNode serdeChild = (ASTNode) child.getChild(0); - shared.serde = unescapeSQLString(serdeChild.getChild(0).getText()); - localDirectoryDesc.setSerName(shared.serde); - localDirectoryDescIsSet=true; - break; - case HiveParser.TOK_TBLSEQUENCEFILE: - case HiveParser.TOK_TBLTEXTFILE: - case HiveParser.TOK_TBLRCFILE: - case HiveParser.TOK_TBLORCFILE: - case HiveParser.TOK_TABLEFILEFORMAT: - storageFormat.fillStorageFormat(child, shared); - localDirectoryDesc.setOutputFormat(storageFormat.outputFormat); - localDirectoryDesc.setSerName(shared.serde); + storageFormat.setSerde(unescapeSQLString(serdeChild.getChild(0).getText())); + localDirectoryDesc.setSerName(storageFormat.getSerde()); localDirectoryDescIsSet=true; break; } @@ -10021,8 +10012,7 @@ private ASTNode analyzeCreateTable(ASTNode ast, QB qb) boolean storedAsDirs = false; RowFormatParams rowFormatParams = new RowFormatParams(); - StorageFormat storageFormat = new StorageFormat(); - AnalyzeCreateCommonVars shared = new AnalyzeCreateCommonVars(); + StorageFormat storageFormat = new StorageFormat(conf); LOG.info("Creating table " + tableName + " position=" + ast.getCharPositionInLine()); @@ -10036,7 +10026,7 @@ private ASTNode analyzeCreateTable(ASTNode ast, QB qb) */ for (int num = 1; num < numCh; num++) { ASTNode child = (ASTNode) ast.getChild(num); - if (storageFormat.fillStorageFormat(child, shared)) { + if (storageFormat.fillStorageFormat(child)) { continue; } switch (child.getToken().getType()) { @@ -10105,7 +10095,7 @@ private ASTNode analyzeCreateTable(ASTNode ast, QB qb) } break; case HiveParser.TOK_TABLEROWFORMAT: - rowFormatParams.analyzeRowFormat(shared, child); + rowFormatParams.analyzeRowFormat(child); break; case HiveParser.TOK_TABLELOCATION: location = unescapeSQLString(child.getChild(0).getText()); @@ -10117,16 +10107,12 @@ private ASTNode analyzeCreateTable(ASTNode ast, QB qb) break; case HiveParser.TOK_TABLESERIALIZER: child = (ASTNode) child.getChild(0); - shared.serde = unescapeSQLString(child.getChild(0).getText()); + storageFormat.setSerde(unescapeSQLString(child.getChild(0).getText())); if (child.getChildCount() == 2) { readProps((ASTNode) (child.getChild(1).getChild(0)), - shared.serdeProps); + storageFormat.getSerdeProps()); } break; - - case HiveParser.TOK_FILEFORMAT_GENERIC: - handleGenericFileFormat(child); - break; case HiveParser.TOK_TABLESKEWED: /** * Throw an error if the user tries to use the DDL with @@ -10147,9 +10133,9 @@ private ASTNode analyzeCreateTable(ASTNode ast, QB qb) } } - storageFormat.fillDefaultStorageFormat(shared); + storageFormat.fillDefaultStorageFormat(); - if ((command_type == CTAS) && (storageFormat.storageHandler != null)) { + if ((command_type == CTAS) && (storageFormat.getStorageHandler() != null)) { throw new SemanticException(ErrorMsg.CREATE_NON_NATIVE_AS.getMsg()); } @@ -10161,7 +10147,8 @@ private ASTNode analyzeCreateTable(ASTNode ast, QB qb) return null; } } catch (HiveException e) { - e.printStackTrace(); + // should not occur since second parameter to getTableWithQN is false + throw new IllegalStateException("Unxpected Exception thrown: " + e.getMessage(), e); } } @@ -10181,8 +10168,8 @@ private ASTNode analyzeCreateTable(ASTNode ast, QB qb) rowFormatParams.fieldEscape, rowFormatParams.collItemDelim, rowFormatParams.mapKeyDelim, rowFormatParams.lineDelim, comment, - storageFormat.inputFormat, storageFormat.outputFormat, location, shared.serde, - storageFormat.storageHandler, shared.serdeProps, tblProps, ifNotExists, skewedColNames, + storageFormat.getInputFormat(), storageFormat.getOutputFormat(), location, storageFormat.getSerde(), + storageFormat.getStorageHandler(), storageFormat.getSerdeProps(), tblProps, ifNotExists, skewedColNames, skewedValues); crtTblDesc.setStoredAsSubDirectories(storedAsDirs); crtTblDesc.setNullFormat(rowFormatParams.nullFormat); @@ -10199,8 +10186,9 @@ private ASTNode analyzeCreateTable(ASTNode ast, QB qb) tblProps = addDefaultProperties(tblProps); CreateTableLikeDesc crtTblLikeDesc = new CreateTableLikeDesc(tableName, isExt, - storageFormat.inputFormat, storageFormat.outputFormat, location, - shared.serde, shared.serdeProps, tblProps, ifNotExists, likeTableName); + storageFormat.getInputFormat(), storageFormat.getOutputFormat(), location, + storageFormat.getSerde(), storageFormat.getSerdeProps(), tblProps, ifNotExists, + likeTableName); SessionState.get().setCommandType(HiveOperation.CREATETABLE); rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), crtTblLikeDesc), conf)); @@ -10224,10 +10212,10 @@ private ASTNode analyzeCreateTable(ASTNode ast, QB qb) bucketCols, sortCols, numBuckets, rowFormatParams.fieldDelim, rowFormatParams.fieldEscape, rowFormatParams.collItemDelim, rowFormatParams.mapKeyDelim, rowFormatParams.lineDelim, - comment, storageFormat.inputFormat, - storageFormat.outputFormat, location, shared.serde, storageFormat.storageHandler, - shared.serdeProps, - tblProps, ifNotExists, skewedColNames, skewedValues); + comment, storageFormat.getInputFormat(), + storageFormat.getOutputFormat(), location, storageFormat.getSerde(), + storageFormat.getStorageHandler(), storageFormat.getSerdeProps(), tblProps, ifNotExists, + skewedColNames, skewedValues); crtTblDesc.setStoredAsSubDirectories(storedAsDirs); crtTblDesc.setNullFormat(rowFormatParams.nullFormat); qb.setTableDesc(crtTblDesc); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/StorageFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/StorageFormat.java new file mode 100644 index 0000000..f210c22 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/StorageFormat.java @@ -0,0 +1,135 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.parse; + +import static org.apache.hadoop.hive.ql.parse.ParseUtils.ensureClassExists; +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.io.IOConstants; +import org.apache.hadoop.hive.ql.io.StorageFormatDescriptor; +import org.apache.hadoop.hive.ql.io.StorageFormatFactory; + +public class StorageFormat { + private static final StorageFormatFactory storageFormatFactory = new StorageFormatFactory(); + private final Configuration conf; + private String inputFormat; + private String outputFormat; + private String storageHandler; + private String serde; + private final Map serdeProps; + + public StorageFormat(Configuration conf) { + this.conf = conf; + this.serdeProps = new HashMap(); + } + /** + * Returns true if the passed token was a storage format token + * and thus was processed accordingly. + */ + public boolean fillStorageFormat(ASTNode child) + throws SemanticException { + switch(child.getToken().getType()) { + case HiveParser.TOK_TABLEFILEFORMAT: + if (child.getChildCount() < 2) { + throw new SemanticException( + "Incomplete specification of File Format. " + + "You must provide InputFormat, OutputFormat."); + } + inputFormat = ensureClassExists(BaseSemanticAnalyzer.unescapeSQLString(child.getChild(0).getText())); + outputFormat = ensureClassExists(BaseSemanticAnalyzer.unescapeSQLString(child.getChild(1).getText())); + break; + case HiveParser.TOK_STORAGEHANDLER: + storageHandler = ensureClassExists(BaseSemanticAnalyzer.unescapeSQLString(child.getChild(0).getText())); + if (child.getChildCount() == 2) { + BaseSemanticAnalyzer.readProps( + (ASTNode) (child.getChild(1).getChild(0)), + serdeProps); + } + break; + case HiveParser.TOK_FILEFORMAT_GENERIC: + ASTNode grandChild = (ASTNode)child.getChild(0); + String name = (grandChild == null ? "" : grandChild.getText()).trim().toUpperCase(); + processStorageFormat(name); + break; + default: + // token was not a storage format token + return false; + } + return true; + } + + private void processStorageFormat(String name) + throws SemanticException { + if (name.isEmpty()) { + throw new SemanticException("File format in STORED AS clause cannot be empty"); + } + StorageFormatDescriptor descriptor = storageFormatFactory.get(name); + if (descriptor == null) { + throw new SemanticException("Unrecognized file format in STORED AS clause:" + + " '" + name + "'"); + } + inputFormat = ensureClassExists(descriptor.getInputFormat()); + outputFormat = ensureClassExists(descriptor.getOutputFormat()); + if (serde == null) { + serde = ensureClassExists(descriptor.getSerde()); + } + // RCFile supports a configurable SerDe + if (name.equalsIgnoreCase(IOConstants.RCFILE) && serde == null) { + serde = ensureClassExists(HiveConf.getVar(conf, HiveConf.ConfVars.HIVEDEFAULTRCFILESERDE)); + } + } + + protected void fillDefaultStorageFormat() + throws SemanticException { + if ((inputFormat == null) && (storageHandler == null)) { + String defaultFormat = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEDEFAULTFILEFORMAT); + if (StringUtils.isBlank(defaultFormat)) { + inputFormat = IOConstants.TEXTFILE_INPUT; + outputFormat = IOConstants.TEXTFILE_OUTPUT; + } else { + processStorageFormat(defaultFormat); + if (defaultFormat.equalsIgnoreCase(IOConstants.RCFILE)) { + serde = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEDEFAULTRCFILESERDE); + } + } + } + } + + public void setSerde(String serde) { + this.serde = serde; + } + public String getInputFormat() { + return inputFormat; + } + public String getOutputFormat() { + return outputFormat; + } + public String getStorageHandler() { + return storageHandler; + } + public String getSerde() { + return serde; + } + public Map getSerdeProps() { + return serdeProps; + } +} \ No newline at end of file diff --git a/ql/src/main/resources/META-INF/services/org.apache.hadoop.hive.ql.io.StorageFormatDescriptor b/ql/src/main/resources/META-INF/services/org.apache.hadoop.hive.ql.io.StorageFormatDescriptor new file mode 100644 index 0000000..a23ff11 --- /dev/null +++ b/ql/src/main/resources/META-INF/services/org.apache.hadoop.hive.ql.io.StorageFormatDescriptor @@ -0,0 +1,5 @@ +org.apache.hadoop.hive.ql.io.TextFileStorageFormatDescriptor +org.apache.hadoop.hive.ql.io.SequenceFileStorageFormatDescriptor +org.apache.hadoop.hive.ql.io.RCFileStorageFormatDescriptor +org.apache.hadoop.hive.ql.io.ORCFileStorageFormatDescriptor +org.apache.hadoop.hive.ql.io.ParquetFileStorageFormatDescriptor diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/TestStorageFormatDescriptor.java b/ql/src/test/org/apache/hadoop/hive/ql/io/TestStorageFormatDescriptor.java new file mode 100644 index 0000000..d53ebc6 --- /dev/null +++ b/ql/src/test/org/apache/hadoop/hive/ql/io/TestStorageFormatDescriptor.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.io; + +import junit.framework.Assert; + +import org.junit.Test; + +import com.google.common.collect.Sets; + +public class TestStorageFormatDescriptor { + + @Test + public void testNames() { + Assert.assertEquals(Sets.newHashSet(IOConstants.TEXTFILE), + (new TextFileStorageFormatDescriptor()).getNames()); + Assert.assertEquals(Sets.newHashSet(IOConstants.SEQUENCEFILE), + (new SequenceFileStorageFormatDescriptor()).getNames()); + Assert.assertEquals(Sets.newHashSet(IOConstants.RCFILE), + (new RCFileStorageFormatDescriptor()).getNames()); + Assert.assertEquals(Sets.newHashSet(IOConstants.ORC, IOConstants.ORCFILE), + (new ORCFileStorageFormatDescriptor()).getNames()); + Assert.assertEquals(Sets.newHashSet(IOConstants.PARQUET, IOConstants.PARQUETFILE), + (new ParquetFileStorageFormatDescriptor()).getNames()); + } +} diff --git a/ql/src/test/queries/clientpositive/storage_format_descriptor.q b/ql/src/test/queries/clientpositive/storage_format_descriptor.q new file mode 100644 index 0000000..548890f --- /dev/null +++ b/ql/src/test/queries/clientpositive/storage_format_descriptor.q @@ -0,0 +1,5 @@ +DROP TABLE stored_as_custom_text_serde; +CREATE TABLE stored_as_custom_text_serde(key string, value string) STORED AS customtextserde; +LOAD DATA LOCAL INPATH '../../data/files/kv1.txt' INTO TABLE stored_as_custom_text_serde; +SELECT * FROM stored_as_custom_text_serde ORDER BY key, value; +DROP TABLE stored_as_custom_text_serde; diff --git a/ql/src/test/results/clientnegative/fileformat_bad_class.q.out b/ql/src/test/results/clientnegative/fileformat_bad_class.q.out index ab1e935..8521ccf 100644 --- a/ql/src/test/results/clientnegative/fileformat_bad_class.q.out +++ b/ql/src/test/results/clientnegative/fileformat_bad_class.q.out @@ -1 +1 @@ -FAILED: SemanticException [Error 10055]: Output Format must implement HiveOutputFormat, otherwise it should be either IgnoreKeyTextOutputFormat or SequenceFileOutputFormat +FAILED: SemanticException Cannot find class 'ClassDoesNotExist' diff --git a/ql/src/test/results/clientnegative/genericFileFormat.q.out b/ql/src/test/results/clientnegative/genericFileFormat.q.out index 9613df9..e4bbc14 100644 --- a/ql/src/test/results/clientnegative/genericFileFormat.q.out +++ b/ql/src/test/results/clientnegative/genericFileFormat.q.out @@ -1 +1 @@ -FAILED: SemanticException Unrecognized file format in STORED AS clause: foo +FAILED: SemanticException Unrecognized file format in STORED AS clause: 'FOO' diff --git a/ql/src/test/results/clientpositive/ctas.q.out b/ql/src/test/results/clientpositive/ctas.q.out index 0040f3c..3999932 100644 --- a/ql/src/test/results/clientpositive/ctas.q.out +++ b/ql/src/test/results/clientpositive/ctas.q.out @@ -677,7 +677,8 @@ TOK_CREATETABLE ',' TOK_TABLEROWFORMATLINES '\012' - TOK_TBLTEXTFILE + TOK_FILEFORMAT_GENERIC + textfile TOK_QUERY TOK_FROM TOK_TABREF diff --git a/ql/src/test/results/clientpositive/storage_format_descriptor.q.out b/ql/src/test/results/clientpositive/storage_format_descriptor.q.out new file mode 100644 index 0000000..b764ddb --- /dev/null +++ b/ql/src/test/results/clientpositive/storage_format_descriptor.q.out @@ -0,0 +1,531 @@ +PREHOOK: query: DROP TABLE stored_as_custom_text_serde +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE stored_as_custom_text_serde +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE stored_as_custom_text_serde(key string, value string) STORED AS customtextserde +PREHOOK: type: CREATETABLE +POSTHOOK: query: CREATE TABLE stored_as_custom_text_serde(key string, value string) STORED AS customtextserde +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@stored_as_custom_text_serde +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/kv1.txt' INTO TABLE stored_as_custom_text_serde +PREHOOK: type: LOAD +PREHOOK: Output: default@stored_as_custom_text_serde +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/kv1.txt' INTO TABLE stored_as_custom_text_serde +POSTHOOK: type: LOAD +POSTHOOK: Output: default@stored_as_custom_text_serde +PREHOOK: query: SELECT * FROM stored_as_custom_text_serde ORDER BY key, value +PREHOOK: type: QUERY +PREHOOK: Input: default@stored_as_custom_text_serde +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM stored_as_custom_text_serde ORDER BY key, value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@stored_as_custom_text_serde +#### A masked pattern was here #### +0 val_0 +0 val_0 +0 val_0 +10 val_10 +100 val_100 +100 val_100 +103 val_103 +103 val_103 +104 val_104 +104 val_104 +105 val_105 +11 val_11 +111 val_111 +113 val_113 +113 val_113 +114 val_114 +116 val_116 +118 val_118 +118 val_118 +119 val_119 +119 val_119 +119 val_119 +12 val_12 +12 val_12 +120 val_120 +120 val_120 +125 val_125 +125 val_125 +126 val_126 +128 val_128 +128 val_128 +128 val_128 +129 val_129 +129 val_129 +131 val_131 +133 val_133 +134 val_134 +134 val_134 +136 val_136 +137 val_137 +137 val_137 +138 val_138 +138 val_138 +138 val_138 +138 val_138 +143 val_143 +145 val_145 +146 val_146 +146 val_146 +149 val_149 +149 val_149 +15 val_15 +15 val_15 +150 val_150 +152 val_152 +152 val_152 +153 val_153 +155 val_155 +156 val_156 +157 val_157 +158 val_158 +160 val_160 +162 val_162 +163 val_163 +164 val_164 +164 val_164 +165 val_165 +165 val_165 +166 val_166 +167 val_167 +167 val_167 +167 val_167 +168 val_168 +169 val_169 +169 val_169 +169 val_169 +169 val_169 +17 val_17 +170 val_170 +172 val_172 +172 val_172 +174 val_174 +174 val_174 +175 val_175 +175 val_175 +176 val_176 +176 val_176 +177 val_177 +178 val_178 +179 val_179 +179 val_179 +18 val_18 +18 val_18 +180 val_180 +181 val_181 +183 val_183 +186 val_186 +187 val_187 +187 val_187 +187 val_187 +189 val_189 +19 val_19 +190 val_190 +191 val_191 +191 val_191 +192 val_192 +193 val_193 +193 val_193 +193 val_193 +194 val_194 +195 val_195 +195 val_195 +196 val_196 +197 val_197 +197 val_197 +199 val_199 +199 val_199 +199 val_199 +2 val_2 +20 val_20 +200 val_200 +200 val_200 +201 val_201 +202 val_202 +203 val_203 +203 val_203 +205 val_205 +205 val_205 +207 val_207 +207 val_207 +208 val_208 +208 val_208 +208 val_208 +209 val_209 +209 val_209 +213 val_213 +213 val_213 +214 val_214 +216 val_216 +216 val_216 +217 val_217 +217 val_217 +218 val_218 +219 val_219 +219 val_219 +221 val_221 +221 val_221 +222 val_222 +223 val_223 +223 val_223 +224 val_224 +224 val_224 +226 val_226 +228 val_228 +229 val_229 +229 val_229 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +233 val_233 +233 val_233 +235 val_235 +237 val_237 +237 val_237 +238 val_238 +238 val_238 +239 val_239 +239 val_239 +24 val_24 +24 val_24 +241 val_241 +242 val_242 +242 val_242 +244 val_244 +247 val_247 +248 val_248 +249 val_249 +252 val_252 +255 val_255 +255 val_255 +256 val_256 +256 val_256 +257 val_257 +258 val_258 +26 val_26 +26 val_26 +260 val_260 +262 val_262 +263 val_263 +265 val_265 +265 val_265 +266 val_266 +27 val_27 +272 val_272 +272 val_272 +273 val_273 +273 val_273 +273 val_273 +274 val_274 +275 val_275 +277 val_277 +277 val_277 +277 val_277 +277 val_277 +278 val_278 +278 val_278 +28 val_28 +280 val_280 +280 val_280 +281 val_281 +281 val_281 +282 val_282 +282 val_282 +283 val_283 +284 val_284 +285 val_285 +286 val_286 +287 val_287 +288 val_288 +288 val_288 +289 val_289 +291 val_291 +292 val_292 +296 val_296 +298 val_298 +298 val_298 +298 val_298 +30 val_30 +302 val_302 +305 val_305 +306 val_306 +307 val_307 +307 val_307 +308 val_308 +309 val_309 +309 val_309 +310 val_310 +311 val_311 +311 val_311 +311 val_311 +315 val_315 +316 val_316 +316 val_316 +316 val_316 +317 val_317 +317 val_317 +318 val_318 +318 val_318 +318 val_318 +321 val_321 +321 val_321 +322 val_322 +322 val_322 +323 val_323 +325 val_325 +325 val_325 +327 val_327 +327 val_327 +327 val_327 +33 val_33 +331 val_331 +331 val_331 +332 val_332 +333 val_333 +333 val_333 +335 val_335 +336 val_336 +338 val_338 +339 val_339 +34 val_34 +341 val_341 +342 val_342 +342 val_342 +344 val_344 +344 val_344 +345 val_345 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +35 val_35 +35 val_35 +35 val_35 +351 val_351 +353 val_353 +353 val_353 +356 val_356 +360 val_360 +362 val_362 +364 val_364 +365 val_365 +366 val_366 +367 val_367 +367 val_367 +368 val_368 +369 val_369 +369 val_369 +369 val_369 +37 val_37 +37 val_37 +373 val_373 +374 val_374 +375 val_375 +377 val_377 +378 val_378 +379 val_379 +382 val_382 +382 val_382 +384 val_384 +384 val_384 +384 val_384 +386 val_386 +389 val_389 +392 val_392 +393 val_393 +394 val_394 +395 val_395 +395 val_395 +396 val_396 +396 val_396 +396 val_396 +397 val_397 +397 val_397 +399 val_399 +399 val_399 +4 val_4 +400 val_400 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +402 val_402 +403 val_403 +403 val_403 +403 val_403 +404 val_404 +404 val_404 +406 val_406 +406 val_406 +406 val_406 +406 val_406 +407 val_407 +409 val_409 +409 val_409 +409 val_409 +41 val_41 +411 val_411 +413 val_413 +413 val_413 +414 val_414 +414 val_414 +417 val_417 +417 val_417 +417 val_417 +418 val_418 +419 val_419 +42 val_42 +42 val_42 +421 val_421 +424 val_424 +424 val_424 +427 val_427 +429 val_429 +429 val_429 +43 val_43 +430 val_430 +430 val_430 +430 val_430 +431 val_431 +431 val_431 +431 val_431 +432 val_432 +435 val_435 +436 val_436 +437 val_437 +438 val_438 +438 val_438 +438 val_438 +439 val_439 +439 val_439 +44 val_44 +443 val_443 +444 val_444 +446 val_446 +448 val_448 +449 val_449 +452 val_452 +453 val_453 +454 val_454 +454 val_454 +454 val_454 +455 val_455 +457 val_457 +458 val_458 +458 val_458 +459 val_459 +459 val_459 +460 val_460 +462 val_462 +462 val_462 +463 val_463 +463 val_463 +466 val_466 +466 val_466 +466 val_466 +467 val_467 +468 val_468 +468 val_468 +468 val_468 +468 val_468 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +47 val_47 +470 val_470 +472 val_472 +475 val_475 +477 val_477 +478 val_478 +478 val_478 +479 val_479 +480 val_480 +480 val_480 +480 val_480 +481 val_481 +482 val_482 +483 val_483 +484 val_484 +485 val_485 +487 val_487 +489 val_489 +489 val_489 +489 val_489 +489 val_489 +490 val_490 +491 val_491 +492 val_492 +492 val_492 +493 val_493 +494 val_494 +495 val_495 +496 val_496 +497 val_497 +498 val_498 +498 val_498 +498 val_498 +5 val_5 +5 val_5 +5 val_5 +51 val_51 +51 val_51 +53 val_53 +54 val_54 +57 val_57 +58 val_58 +58 val_58 +64 val_64 +65 val_65 +66 val_66 +67 val_67 +67 val_67 +69 val_69 +70 val_70 +70 val_70 +70 val_70 +72 val_72 +72 val_72 +74 val_74 +76 val_76 +76 val_76 +77 val_77 +78 val_78 +8 val_8 +80 val_80 +82 val_82 +83 val_83 +83 val_83 +84 val_84 +84 val_84 +85 val_85 +86 val_86 +87 val_87 +9 val_9 +90 val_90 +90 val_90 +90 val_90 +92 val_92 +95 val_95 +95 val_95 +96 val_96 +97 val_97 +97 val_97 +98 val_98 +98 val_98 +PREHOOK: query: DROP TABLE stored_as_custom_text_serde +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@stored_as_custom_text_serde +PREHOOK: Output: default@stored_as_custom_text_serde +POSTHOOK: query: DROP TABLE stored_as_custom_text_serde +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@stored_as_custom_text_serde +POSTHOOK: Output: default@stored_as_custom_text_serde diff --git a/ql/src/test/results/clientpositive/tez/ctas.q.out b/ql/src/test/results/clientpositive/tez/ctas.q.out index a58e166..d1aa48d 100644 --- a/ql/src/test/results/clientpositive/tez/ctas.q.out +++ b/ql/src/test/results/clientpositive/tez/ctas.q.out @@ -677,7 +677,8 @@ TOK_CREATETABLE ',' TOK_TABLEROWFORMATLINES '\012' - TOK_TBLTEXTFILE + TOK_FILEFORMAT_GENERIC + textfile TOK_QUERY TOK_FROM TOK_TABREF