Index: ql/src/test/results/clientpositive/ctas_uses_database_location.q.out =================================================================== --- ql/src/test/results/clientpositive/ctas_uses_database_location.q.out (revision 0) +++ ql/src/test/results/clientpositive/ctas_uses_database_location.q.out (revision 0) @@ -0,0 +1,159 @@ +PREHOOK: query: -- Tests that CTAS queries in non-default databases use the location of the database +-- not the hive.metastore.warehouse.dir for intermediate files (FileSinkOperator output). +-- If hive.metastore.warehouse.dir were used this would fail because the scheme is invalid. + +CREATE DATABASE db1 +#### A masked pattern was here #### +PREHOOK: type: CREATEDATABASE +POSTHOOK: query: -- Tests that CTAS queries in non-default databases use the location of the database +-- not the hive.metastore.warehouse.dir for intermediate files (FileSinkOperator output). +-- If hive.metastore.warehouse.dir were used this would fail because the scheme is invalid. + +CREATE DATABASE db1 +#### A masked pattern was here #### +POSTHOOK: type: CREATEDATABASE +PREHOOK: query: USE db1 +PREHOOK: type: SWITCHDATABASE +POSTHOOK: query: USE db1 +POSTHOOK: type: SWITCHDATABASE +PREHOOK: query: EXPLAIN CREATE TABLE table_db1 AS SELECT * FROM default.src +PREHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: query: EXPLAIN CREATE TABLE table_db1 AS SELECT * FROM default.src +POSTHOOK: type: CREATETABLE_AS_SELECT +ABSTRACT SYNTAX TREE: + (TOK_CREATETABLE (TOK_TABNAME table_db1) TOK_LIKETABLE (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME default src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF))))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 + Stage-4 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 + Stage-8 depends on stages: Stage-0 + Stage-2 depends on stages: Stage-8 + Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + src + TableScan + alias: src + Select Operator + expressions: + expr: key + type: string + expr: value + type: string + outputColumnNames: _col0, _col1 + File Output Operator + compressed: false + GlobalTableId: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + name: db1.table_db1 + + Stage: Stage-7 + Conditional Operator + + Stage: Stage-4 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-0 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-8 + Create Table Operator: + Create Table + columns: key string, value string + if not exists: false + input format: org.apache.hadoop.mapred.TextInputFormat + # buckets: -1 + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + name: table_db1 + isExternal: false + + Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + name: db1.table_db1 + + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + name: db1.table_db1 + + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + +PREHOOK: query: CREATE TABLE table_db1 AS SELECT * FROM default.src +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +POSTHOOK: query: CREATE TABLE table_db1 AS SELECT * FROM default.src +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: db1@table_db1 +PREHOOK: query: DESCRIBE FORMATTED table_db1 +PREHOOK: type: DESCTABLE +POSTHOOK: query: DESCRIBE FORMATTED table_db1 +POSTHOOK: type: DESCTABLE +# col_name data_type comment + +key string None +value string None + +# Detailed Table Information +Database: db1 +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + numFiles 1 + numPartitions 0 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 Index: ql/src/test/queries/clientpositive/ctas_uses_database_location.q =================================================================== --- ql/src/test/queries/clientpositive/ctas_uses_database_location.q (revision 0) +++ ql/src/test/queries/clientpositive/ctas_uses_database_location.q (revision 0) @@ -0,0 +1,14 @@ +set hive.metastore.warehouse.dir=invalid_scheme://${system:test.tmp.dir}; + +-- Tests that CTAS queries in non-default databases use the location of the database +-- not the hive.metastore.warehouse.dir for intermediate files (FileSinkOperator output). +-- If hive.metastore.warehouse.dir were used this would fail because the scheme is invalid. + +CREATE DATABASE db1 +LOCATION 'pfile://${system:test.tmp.dir}/db1'; + +USE db1; +EXPLAIN CREATE TABLE table_db1 AS SELECT * FROM default.src; +CREATE TABLE table_db1 AS SELECT * FROM default.src; + +DESCRIBE FORMATTED table_db1; Index: ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (revision 1386751) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (working copy) @@ -70,8 +70,8 @@ import org.apache.hadoop.hive.ql.exec.RecordWriter; import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; import org.apache.hadoop.hive.ql.exec.RowSchema; +import org.apache.hadoop.hive.ql.exec.SelectOperator; import org.apache.hadoop.hive.ql.exec.StatsTask; -import org.apache.hadoop.hive.ql.exec.SelectOperator; import org.apache.hadoop.hive.ql.exec.TableScanOperator; import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.exec.TaskFactory; @@ -1054,13 +1054,21 @@ ctx.setResFile(null); // allocate a temporary output dir on the location of the table - String location = conf.getVar(HiveConf.ConfVars.METASTOREWAREHOUSE); + String tableName = getUnescapedName((ASTNode)ast.getChild(0)); + Table newTable = db.newTable(tableName); + Path location; try { + Warehouse wh = new Warehouse(conf); + location = wh.getDatabasePath(db.getDatabase(newTable.getDbName())); + } catch (MetaException e) { + throw new SemanticException(e); + } + try { fname = ctx.getExternalTmpFileURI( - FileUtils.makeQualified(new Path(location), conf).toUri()); + FileUtils.makeQualified(location, conf).toUri()); } catch (Exception e) { throw new SemanticException(generateErrorMessage(ast, - "Error creating temporary folder on: " + location), e); + "Error creating temporary folder on: " + location.toString()), e); } if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVESTATSAUTOGATHER)) { tableSpec ts = new tableSpec(db, conf, this.ast);