diff --git common/src/java/org/apache/hadoop/hive/common/FileUtils.java common/src/java/org/apache/hadoop/hive/common/FileUtils.java index 16d7c80..bdff419 100644 --- common/src/java/org/apache/hadoop/hive/common/FileUtils.java +++ common/src/java/org/apache/hadoop/hive/common/FileUtils.java @@ -20,7 +20,6 @@ import java.io.IOException; import java.net.URI; -import java.net.URISyntaxException; import java.util.BitSet; import java.util.List; @@ -402,21 +401,29 @@ public static boolean isActionPermittedForFileHierarchy(FileSystem fs, FileStatu return true; } + public static boolean isLocalFile(String fileName, HiveConf conf) { + try { + return isLocalFile(new Path(fileName), conf); + } catch (IllegalArgumentException e) { + LOG.warn("Unable to create URI from " + fileName, e); + } + return false; + } + /** * A best effort attempt to determine if if the file is a local file * @param conf * @param fileName * @return true if it was successfully able to determine that it is a local file */ - public static boolean isLocalFile(HiveConf conf, String fileName) { + public static boolean isLocalFile(Path path, HiveConf conf) { try { - // do best effor to determine if this is a local file - FileSystem fsForFile = FileSystem.get(new URI(fileName), conf); - return LocalFileSystem.class.isInstance(fsForFile); - } catch (URISyntaxException e) { - LOG.warn("Unable to create URI from " + fileName, e); + // do best effort to determine if this is a local file + FileSystem fsForFile = path.getFileSystem(conf); + return LocalFileSystem.class.isInstance(fsForFile) || + "file".equals(path.toUri().getScheme()); } catch (IOException e) { - LOG.warn("Unable to get FileSystem for " + fileName, e); + LOG.warn("Unable to get FileSystem for " + path, e); } return false; } diff --git ql/src/java/org/apache/hadoop/hive/ql/Driver.java ql/src/java/org/apache/hadoop/hive/ql/Driver.java index 7dbb8be..d9e2650 100644 --- ql/src/java/org/apache/hadoop/hive/ql/Driver.java +++ ql/src/java/org/apache/hadoop/hive/ql/Driver.java @@ -514,6 +514,9 @@ private void doAuthorization(BaseSemanticAnalyzer sem) null, op.getOutputRequiredPrivileges()); continue; } + if (write.getType() == Entity.Type.DFS_DIR || write.getType() == Entity.Type.LOCAL_DIR) { + continue; + } if (write.getType() == WriteEntity.Type.PARTITION) { Partition part = db.getPartition(write.getTable(), write @@ -539,7 +542,8 @@ private void doAuthorization(BaseSemanticAnalyzer sem) //determine if partition level privileges should be checked for input tables Map tableUsePartLevelAuth = new HashMap(); for (ReadEntity read : inputs) { - if (read.isDummy() || read.getType() == Entity.Type.DATABASE) { + if (read.isDummy() || + (read.getType() != Entity.Type.TABLE && read.getType() != Entity.Type.PARTITION)) { continue; } Table tbl = read.getTable(); @@ -620,6 +624,9 @@ private void doAuthorization(BaseSemanticAnalyzer sem) ss.getAuthorizer().authorize(read.getDatabase(), op.getInputRequiredPrivileges(), null); continue; } + if (read.getType() == Entity.Type.DFS_DIR || read.getType() == Entity.Type.LOCAL_DIR) { + continue; + } Table tbl = read.getTable(); if (read.getPartition() != null) { Partition partition = read.getPartition(); @@ -702,7 +709,7 @@ private void doAuthorizationV2(SessionState ss, HiveOperation op, HashSet getParents() { diff --git ql/src/java/org/apache/hadoop/hive/ql/hooks/WriteEntity.java ql/src/java/org/apache/hadoop/hive/ql/hooks/WriteEntity.java index 44a3924..8779c26 100644 --- ql/src/java/org/apache/hadoop/hive/ql/hooks/WriteEntity.java +++ ql/src/java/org/apache/hadoop/hive/ql/hooks/WriteEntity.java @@ -113,7 +113,7 @@ public WriteEntity(Path d, boolean islocal) { * True if this is a temporary location such as scratch dir */ public WriteEntity(Path d, boolean islocal, boolean isTemp) { - super(d.toString(), islocal, true); + super(d, islocal, true); this.isTempURI = isTemp; } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java index db9fa74..b2168f6 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java @@ -18,11 +18,11 @@ package org.apache.hadoop.hive.ql.parse; +import java.io.IOException; import java.io.Serializable; import java.io.UnsupportedEncodingException; +import java.net.URI; import java.sql.Date; -import java.text.DateFormat; -import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -38,6 +38,8 @@ import org.antlr.runtime.tree.Tree; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.HiveMetaStore; import org.apache.hadoop.hive.metastore.api.Database; @@ -49,7 +51,6 @@ import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluatorFactory; import org.apache.hadoop.hive.ql.exec.FetchTask; import org.apache.hadoop.hive.ql.exec.Task; -import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.hooks.LineageInfo; import org.apache.hadoop.hive.ql.hooks.ReadEntity; import org.apache.hadoop.hive.ql.hooks.WriteEntity; @@ -1287,6 +1288,32 @@ private static String normalizeDateCol( return HiveMetaStore.PARTITION_DATE_FORMAT.format(value); } + protected WriteEntity toWriteEntity(String location) throws SemanticException { + try { + Path path = tryQualifyPath(new Path(location), conf); + return new WriteEntity(path, FileUtils.isLocalFile(path, conf)); + } catch (Exception e) { + throw new SemanticException(e); + } + } + + protected ReadEntity toReadEntity(URI location) throws SemanticException { + try { + Path path = tryQualifyPath(new Path(location), conf); + return new ReadEntity(path, FileUtils.isLocalFile(path, conf)); + } catch (Exception e) { + throw new SemanticException(e); + } + } + + private Path tryQualifyPath(Path path, HiveConf conf) throws IOException { + try { + return path.getFileSystem(conf).makeQualified(path); + } catch (IOException e) { + return path; // some tests expected to pass invalid schema + } + } + protected Database getDatabase(String dbName) throws SemanticException { return getDatabase(dbName, true); } @@ -1312,7 +1339,7 @@ protected Table getTable(String tblName, boolean throwException) throws Semantic return getTable(currentDb, tblName, throwException); } - // qnName : possibly contains database name (dot seperated) + // qnName : possibly contains database name (dot separated) protected Table getTableWithQN(String qnName, boolean throwException) throws SemanticException { int dot = qnName.indexOf('.'); if (dot < 0) { diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java index e642919..ea8f0ac 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java @@ -42,7 +42,6 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.metastore.MetaStoreUtils; @@ -2608,17 +2607,7 @@ private void analyzeAlterTableAddParts(CommonTree ast, boolean expectView) throw new SemanticException("LOCATION clause illegal for view partition"); } currentLocation = unescapeSQLString(child.getChild(0).getText()); - boolean isLocal = false; - try { - // do best effor to determine if this is a local file - String scheme = new URI(currentLocation).getScheme(); - if (scheme != null) { - isLocal = FileUtils.isLocalFile(conf, currentLocation); - } - } catch (URISyntaxException e) { - LOG.warn("Unable to create URI from " + currentLocation, e); - } - inputs.add(new ReadEntity(new Path(currentLocation), isLocal)); + addLocationToOutputs(currentLocation); break; default: throw new SemanticException("Unknown child: " + child); @@ -3248,8 +3237,8 @@ private void analyzeAlterTableSkewedLocation(ASTNode ast, String tableName, alterTblDesc), conf)); } - private void addLocationToOutputs(String newLocation) { - outputs.add(new WriteEntity(new Path(newLocation), FileUtils.isLocalFile(conf, newLocation))); + private void addLocationToOutputs(String newLocation) throws SemanticException { + outputs.add(toWriteEntity(newLocation)); } /** diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java index 92ec334..17e9f98 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java @@ -79,6 +79,8 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { List partitionDescs = new ArrayList(); Path fromPath = new Path(fromURI.getScheme(), fromURI.getAuthority(), fromURI.getPath()); + inputs.add(toReadEntity(fromURI)); + try { Path metadataPath = new Path(fromPath, METADATA_NAME); Map.Entry' '' '' diff --git ql/src/test/results/clientnegative/authorization_uri_create_table_ext.q.out ql/src/test/results/clientnegative/authorization_uri_create_table_ext.q.out index 0b8182a..6ddd742 100644 --- ql/src/test/results/clientnegative/authorization_uri_create_table_ext.q.out +++ ql/src/test/results/clientnegative/authorization_uri_create_table_ext.q.out @@ -1 +1,9 @@ #### A masked pattern was here #### +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t1 +#### A masked pattern was here #### +FAILED: ParseException line 4:0 cannot recognize input near '' '' '' diff --git ql/src/test/results/clientnegative/deletejar.q.out ql/src/test/results/clientnegative/deletejar.q.out index 91560ee..2175edb 100644 --- ql/src/test/results/clientnegative/deletejar.q.out +++ ql/src/test/results/clientnegative/deletejar.q.out @@ -1,4 +1,2 @@ -PREHOOK: query: CREATE TABLE DELETEJAR(KEY STRING, VALUE STRING) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.TestSerDe' STORED AS TEXTFILE -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Cannot validate serde: org.apache.hadoop.hive.serde2.TestSerDe +/home/navis/.m2/repository/org/apache/hive/hive-it-test-serde/0.14.0-SNAPSHOT/hive-it-test-serde-0.14.0-SNAPSHOT.jar does not exist +Query returned non-zero code: 1, cause: /home/navis/.m2/repository/org/apache/hive/hive-it-test-serde/0.14.0-SNAPSHOT/hive-it-test-serde-0.14.0-SNAPSHOT.jar does not exist. diff --git ql/src/test/results/clientnegative/exim_20_managed_location_over_existing.q.out ql/src/test/results/clientnegative/exim_20_managed_location_over_existing.q.out index fd4a418..dc64d8c 100644 --- ql/src/test/results/clientnegative/exim_20_managed_location_over_existing.q.out +++ ql/src/test/results/clientnegative/exim_20_managed_location_over_existing.q.out @@ -48,14 +48,13 @@ PREHOOK: query: create table exim_department ( dep_id int comment "department id location 'ql/test/data/tablestore/exim_department' tblproperties("creator"="krishna") PREHOOK: type: CREATETABLE -#### A masked pattern was here #### PREHOOK: Output: database:importer +#### A masked pattern was here #### POSTHOOK: query: create table exim_department ( dep_id int comment "department id") stored as textfile location 'ql/test/data/tablestore/exim_department' tblproperties("creator"="krishna") POSTHOOK: type: CREATETABLE -#### A masked pattern was here #### POSTHOOK: Output: database:importer POSTHOOK: Output: importer@exim_department #### A masked pattern was here #### diff --git ql/src/test/results/clientnegative/external1.q.out ql/src/test/results/clientnegative/external1.q.out index 696beaa..5d7160f 100644 --- ql/src/test/results/clientnegative/external1.q.out +++ ql/src/test/results/clientnegative/external1.q.out @@ -1,5 +1,5 @@ #### A masked pattern was here #### PREHOOK: type: CREATETABLE -#### A masked pattern was here #### PREHOOK: Output: database:default +#### A masked pattern was here #### FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. MetaException(message:Got exception: java.io.IOException No FileSystem for scheme: invalidscheme) diff --git ql/src/test/results/clientnegative/external2.q.out ql/src/test/results/clientnegative/external2.q.out index a604885..ff5d657 100644 --- ql/src/test/results/clientnegative/external2.q.out +++ ql/src/test/results/clientnegative/external2.q.out @@ -7,6 +7,6 @@ POSTHOOK: Output: database:default POSTHOOK: Output: default@external2 #### A masked pattern was here #### PREHOOK: type: ALTERTABLE_ADDPARTS -#### A masked pattern was here #### PREHOOK: Output: default@external2 +#### A masked pattern was here #### FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. MetaException(message:Got exception: java.io.IOException No FileSystem for scheme: invalidscheme) diff --git ql/src/test/results/clientnegative/insertexternal1.q.out ql/src/test/results/clientnegative/insertexternal1.q.out index 3df5013..c0678ed 100644 --- ql/src/test/results/clientnegative/insertexternal1.q.out +++ ql/src/test/results/clientnegative/insertexternal1.q.out @@ -7,11 +7,10 @@ POSTHOOK: Output: database:default POSTHOOK: Output: default@texternal #### A masked pattern was here #### PREHOOK: type: ALTERTABLE_ADDPARTS -#### A masked pattern was here #### PREHOOK: Output: default@texternal #### A masked pattern was here #### POSTHOOK: type: ALTERTABLE_ADDPARTS -#### A masked pattern was here #### POSTHOOK: Output: default@texternal POSTHOOK: Output: default@texternal@insertdate=2008-01-01 +#### A masked pattern was here #### FAILED: SemanticException [Error 10071]: Inserting into a external table is not allowed texternal