diff --git hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatMapReduceTest.java hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatMapReduceTest.java index ae56ff7c81..29a006cd5e 100644 --- hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatMapReduceTest.java +++ hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatMapReduceTest.java @@ -217,6 +217,7 @@ public void createTable() throws Exception { tableParams.put(hive_metastoreConstants.IS_IMMUTABLE,"true"); } StatsSetupConst.setBasicStatsState(tableParams, StatsSetupConst.TRUE); + tableParams.put(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL, "false"); tbl.setParameters(tableParams); client.createTable(tbl); diff --git hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/AbstractHCatLoaderTest.java hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/AbstractHCatLoaderTest.java index 59d2efb156..124a5bcbf7 100644 --- hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/AbstractHCatLoaderTest.java +++ hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/AbstractHCatLoaderTest.java @@ -112,6 +112,8 @@ static void createTable(String tablename, String schema, String partitionedBy, D createTable = createTable + "partitioned by (" + partitionedBy + ") "; } createTable = createTable + "stored as " +storageFormat; + //HCat doesn't support transactional tables + createTable += " TBLPROPERTIES ('transactional'='false')"; executeStatementOnDriver(createTable, driver); } diff --git hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/AbstractHCatStorerTest.java hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/AbstractHCatStorerTest.java index 4f7cf2b5e5..4e4638ae07 100644 --- hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/AbstractHCatStorerTest.java +++ hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/AbstractHCatStorerTest.java @@ -417,14 +417,9 @@ static void dumpFile(String fileName) throws Exception { @Test public void testPartColsInData() throws IOException, CommandNeedRetryException { + AbstractHCatLoaderTest.dropTable("junit_unparted", driver); + AbstractHCatLoaderTest.createTable("junit_unparted","a int", "b string", driver, storageFormat); - driver.run("drop table junit_unparted"); - String createTable = - "create table junit_unparted(a int) partitioned by (b string) stored as " + storageFormat; - int retCode = driver.run(createTable).getResponseCode(); - if (retCode != 0) { - throw new IOException("Failed to create table."); - } int LOOP_SIZE = 11; String[] input = new String[LOOP_SIZE]; for (int i = 0; i < LOOP_SIZE; i++) { @@ -456,15 +451,10 @@ public void testPartColsInData() throws IOException, CommandNeedRetryException { @Test public void testMultiPartColsInData() throws Exception { - driver.run("drop table employee"); - String createTable = - "CREATE TABLE employee (emp_id INT, emp_name STRING, emp_start_date STRING , emp_gender STRING ) " - + " PARTITIONED BY (emp_country STRING , emp_state STRING ) STORED AS " + storageFormat; - - int retCode = driver.run(createTable).getResponseCode(); - if (retCode != 0) { - throw new IOException("Failed to create table."); - } + AbstractHCatLoaderTest.dropTable("employee", driver); + AbstractHCatLoaderTest.createTable("employee", + "emp_id INT, emp_name STRING, emp_start_date STRING , emp_gender STRING", + "emp_country STRING , emp_state STRING", driver, storageFormat); String[] inputData = { "111237\tKrishna\t01/01/1990\tM\tIN\tTN", "111238\tKalpana\t01/01/2000\tF\tIN\tKA", @@ -512,13 +502,10 @@ public void testMultiPartColsInData() throws Exception { @Test public void testStoreInPartiitonedTbl() throws Exception { - driver.run("drop table junit_unparted"); - String createTable = - "create table junit_unparted(a int) partitioned by (b string) stored as " + storageFormat; - int retCode = driver.run(createTable).getResponseCode(); - if (retCode != 0) { - throw new IOException("Failed to create table."); - } + AbstractHCatLoaderTest.dropTable("junit_unparted", driver); + AbstractHCatLoaderTest.createTable("junit_unparted","a int", "b string", + driver, storageFormat); + int LOOP_SIZE = 11; String[] input = new String[LOOP_SIZE]; for (int i = 0; i < LOOP_SIZE; i++) { @@ -553,14 +540,8 @@ public void testStoreInPartiitonedTbl() throws Exception { @Test public void testNoAlias() throws IOException, CommandNeedRetryException { - driver.run("drop table junit_parted"); - String createTable = - "create table junit_parted(a int, b string) partitioned by (ds string) stored as " - + storageFormat; - int retCode = driver.run(createTable).getResponseCode(); - if (retCode != 0) { - throw new IOException("Failed to create table."); - } + AbstractHCatLoaderTest.dropTable("junit_parted", driver); + AbstractHCatLoaderTest.createTable("junit_parted","a int, b string", "ds string", driver, storageFormat); PigServer server = new PigServer(ExecType.LOCAL); boolean errCaught = false; try { @@ -603,19 +584,13 @@ public void testNoAlias() throws IOException, CommandNeedRetryException { @Test public void testStoreMultiTables() throws IOException, CommandNeedRetryException { + AbstractHCatLoaderTest.dropTable("junit_unparted", driver); + AbstractHCatLoaderTest.createTable("junit_unparted","a int, b string", null, + driver, storageFormat); - driver.run("drop table junit_unparted"); - String createTable = "create table junit_unparted(a int, b string) stored as " + storageFormat; - int retCode = driver.run(createTable).getResponseCode(); - if (retCode != 0) { - throw new IOException("Failed to create table."); - } - driver.run("drop table junit_unparted2"); - createTable = "create table junit_unparted2(a int, b string) stored as RCFILE"; - retCode = driver.run(createTable).getResponseCode(); - if (retCode != 0) { - throw new IOException("Failed to create table."); - } + AbstractHCatLoaderTest.dropTable("junit_unparted2", driver); + AbstractHCatLoaderTest.createTable("junit_unparted2","a int, b string", null, + driver, "RCFILE"); int LOOP_SIZE = 3; String[] input = new String[LOOP_SIZE * LOOP_SIZE]; @@ -660,13 +635,9 @@ public void testStoreMultiTables() throws IOException, CommandNeedRetryException @Test public void testStoreWithNoSchema() throws IOException, CommandNeedRetryException { - - driver.run("drop table junit_unparted"); - String createTable = "create table junit_unparted(a int, b string) stored as " + storageFormat; - int retCode = driver.run(createTable).getResponseCode(); - if (retCode != 0) { - throw new IOException("Failed to create table."); - } + AbstractHCatLoaderTest.dropTable("junit_unparted", driver); + AbstractHCatLoaderTest.createTable("junit_unparted","a int, b string", null, + driver, storageFormat); int LOOP_SIZE = 3; String[] input = new String[LOOP_SIZE * LOOP_SIZE]; @@ -700,13 +671,9 @@ public void testStoreWithNoSchema() throws IOException, CommandNeedRetryExceptio @Test public void testStoreWithNoCtorArgs() throws IOException, CommandNeedRetryException { - - driver.run("drop table junit_unparted"); - String createTable = "create table junit_unparted(a int, b string) stored as " + storageFormat; - int retCode = driver.run(createTable).getResponseCode(); - if (retCode != 0) { - throw new IOException("Failed to create table."); - } + AbstractHCatLoaderTest.dropTable("junit_unparted", driver); + AbstractHCatLoaderTest.createTable("junit_unparted","a int, b string", null, + driver, storageFormat); int LOOP_SIZE = 3; String[] input = new String[LOOP_SIZE * LOOP_SIZE]; @@ -741,12 +708,8 @@ public void testStoreWithNoCtorArgs() throws IOException, CommandNeedRetryExcept @Test public void testEmptyStore() throws IOException, CommandNeedRetryException { - driver.run("drop table junit_unparted"); - String createTable = "create table junit_unparted(a int, b string) stored as " + storageFormat; - int retCode = driver.run(createTable).getResponseCode(); - if (retCode != 0) { - throw new IOException("Failed to create table."); - } + AbstractHCatLoaderTest.dropTable("junit_unparted", driver); + AbstractHCatLoaderTest.createTable("junit_unparted","a int, b string", null, driver, storageFormat); int LOOP_SIZE = 3; String[] input = new String[LOOP_SIZE * LOOP_SIZE]; @@ -777,15 +740,11 @@ public void testEmptyStore() throws IOException, CommandNeedRetryException { @Test public void testBagNStruct() throws IOException, CommandNeedRetryException { - driver.run("drop table junit_unparted"); - String createTable = - "create table junit_unparted(b string,a struct, arr_of_struct array, " - + "arr_of_struct2 array>, arr_of_struct3 array>) stored as " - + storageFormat; - int retCode = driver.run(createTable).getResponseCode(); - if (retCode != 0) { - throw new IOException("Failed to create table."); - } + AbstractHCatLoaderTest.dropTable("junit_unparted", driver); + AbstractHCatLoaderTest.createTable("junit_unparted", + "b string,a struct, arr_of_struct array, " + + "arr_of_struct2 array>, arr_of_struct3 array>", + null, driver, storageFormat); String[] inputData = new String[] { "zookeeper\t(2)\t{(pig)}\t{(pnuts,hdfs)}\t{(hadoop),(hcat)}", @@ -823,15 +782,10 @@ public void testBagNStruct() throws IOException, CommandNeedRetryException { @Test public void testStoreFuncAllSimpleTypes() throws IOException, CommandNeedRetryException { - - driver.run("drop table junit_unparted"); - String createTable = - "create table junit_unparted(a int, b float, c double, d bigint, e string, h boolean, f binary, g binary) stored as " - + storageFormat; - int retCode = driver.run(createTable).getResponseCode(); - if (retCode != 0) { - throw new IOException("Failed to create table."); - } + AbstractHCatLoaderTest.dropTable("junit_unparted", driver); + AbstractHCatLoaderTest.createTable("junit_unparted", + "a int, b float, c double, d bigint, e string, h boolean, f binary, g binary", null, + driver, storageFormat); int i = 0; String[] input = new String[3]; @@ -887,13 +841,9 @@ public void testStoreFuncAllSimpleTypes() throws IOException, CommandNeedRetryEx @Test public void testStoreFuncSimple() throws IOException, CommandNeedRetryException { - - driver.run("drop table junit_unparted"); - String createTable = "create table junit_unparted(a int, b string) stored as " + storageFormat; - int retCode = driver.run(createTable).getResponseCode(); - if (retCode != 0) { - throw new IOException("Failed to create table."); - } + AbstractHCatLoaderTest.dropTable("junit_unparted", driver); + AbstractHCatLoaderTest.createTable("junit_unparted","a int, b string", null, + driver, storageFormat); int LOOP_SIZE = 3; String[] inputData = new String[LOOP_SIZE * LOOP_SIZE]; @@ -930,16 +880,10 @@ public void testStoreFuncSimple() throws IOException, CommandNeedRetryException @Test public void testDynamicPartitioningMultiPartColsInDataPartialSpec() throws IOException, CommandNeedRetryException { - - driver.run("drop table if exists employee"); - String createTable = - "CREATE TABLE employee (emp_id INT, emp_name STRING, emp_start_date STRING , emp_gender STRING ) " - + " PARTITIONED BY (emp_country STRING , emp_state STRING ) STORED AS " + storageFormat; - - int retCode = driver.run(createTable).getResponseCode(); - if (retCode != 0) { - throw new IOException("Failed to create table."); - } + AbstractHCatLoaderTest.dropTable("employee", driver); + AbstractHCatLoaderTest.createTable("employee", + "emp_id INT, emp_name STRING, emp_start_date STRING , emp_gender STRING", + "emp_country STRING , emp_state STRING", driver, storageFormat); String[] inputData = { "111237\tKrishna\t01/01/1990\tM\tIN\tTN", "111238\tKalpana\t01/01/2000\tF\tIN\tKA", @@ -970,16 +914,10 @@ public void testDynamicPartitioningMultiPartColsInDataPartialSpec() throws IOExc @Test public void testDynamicPartitioningMultiPartColsInDataNoSpec() throws IOException, CommandNeedRetryException { - - driver.run("drop table if exists employee"); - String createTable = - "CREATE TABLE employee (emp_id INT, emp_name STRING, emp_start_date STRING , emp_gender STRING ) " - + " PARTITIONED BY (emp_country STRING , emp_state STRING ) STORED AS " + storageFormat; - - int retCode = driver.run(createTable).getResponseCode(); - if (retCode != 0) { - throw new IOException("Failed to create table."); - } + AbstractHCatLoaderTest.dropTable("employee", driver); + AbstractHCatLoaderTest.createTable("employee", + "emp_id INT, emp_name STRING, emp_start_date STRING , emp_gender STRING", + "emp_country STRING , emp_state STRING", driver, storageFormat); String[] inputData = { "111237\tKrishna\t01/01/1990\tM\tIN\tTN", "111238\tKalpana\t01/01/2000\tF\tIN\tKA", @@ -1009,16 +947,11 @@ public void testDynamicPartitioningMultiPartColsInDataNoSpec() throws IOExceptio @Test public void testDynamicPartitioningMultiPartColsNoDataInDataNoSpec() throws IOException, CommandNeedRetryException { + AbstractHCatLoaderTest.dropTable("employee", driver); + AbstractHCatLoaderTest.createTable("employee", + "emp_id INT, emp_name STRING, emp_start_date STRING , emp_gender STRING", + "emp_country STRING , emp_state STRING", driver, storageFormat); - driver.run("drop table if exists employee"); - String createTable = - "CREATE TABLE employee (emp_id INT, emp_name STRING, emp_start_date STRING , emp_gender STRING ) " - + " PARTITIONED BY (emp_country STRING , emp_state STRING ) STORED AS " + storageFormat; - - int retCode = driver.run(createTable).getResponseCode(); - if (retCode != 0) { - throw new IOException("Failed to create table."); - } String[] inputData = {}; HcatTestUtils.createTestDataFile(INPUT_FILE_NAME, inputData); @@ -1040,15 +973,10 @@ public void testDynamicPartitioningMultiPartColsNoDataInDataNoSpec() throws IOEx @Test public void testPartitionPublish() throws IOException, CommandNeedRetryException { + AbstractHCatLoaderTest.dropTable("ptn_fail", driver); + AbstractHCatLoaderTest.createTable("ptn_fail","a int, c string", "b string", + driver, storageFormat); - driver.run("drop table ptn_fail"); - String createTable = - "create table ptn_fail(a int, c string) partitioned by (b string) stored as " - + storageFormat; - int retCode = driver.run(createTable).getResponseCode(); - if (retCode != 0) { - throw new IOException("Failed to create table."); - } int LOOP_SIZE = 11; String[] input = new String[LOOP_SIZE]; @@ -1065,7 +993,7 @@ public void testPartitionPublish() throws IOException, CommandNeedRetryException server.executeBatch(); String query = "show partitions ptn_fail"; - retCode = driver.run(query).getResponseCode(); + int retCode = driver.run(query).getResponseCode(); if (retCode != 0) { throw new IOException("Error " + retCode + " running query " + query); diff --git hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestE2EScenarios.java hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestE2EScenarios.java index 4a6c6a3a6f..95a8164dfe 100644 --- hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestE2EScenarios.java +++ hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestE2EScenarios.java @@ -112,15 +112,7 @@ private void dropTable(String tablename) throws IOException, CommandNeedRetryExc } private void createTable(String tablename, String schema, String partitionedBy, String storageFormat) throws IOException, CommandNeedRetryException { - String createTable; - createTable = "create table " + tablename + "(" + schema + ") "; - if ((partitionedBy != null) && (!partitionedBy.trim().isEmpty())) { - createTable = createTable + "partitioned by (" + partitionedBy + ") "; - } - if (storageFormat != null){ - createTable = createTable + "stored as " +storageFormat; - } - driverRun(createTable); + AbstractHCatLoaderTest.createTable(tablename, schema, partitionedBy, driver, storageFormat); } private void driverRun(String cmd) throws IOException, CommandNeedRetryException { diff --git hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java index ea9cdda31c..5c9bae48e5 100644 --- hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java +++ hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java @@ -97,18 +97,7 @@ private void dropTable(String tablename) throws IOException, CommandNeedRetryExc } private void createTable(String tablename, String schema, String partitionedBy) throws IOException, CommandNeedRetryException { - String createTable; - createTable = "create table " + tablename + "(" + schema + ") "; - if ((partitionedBy != null) && (!partitionedBy.trim().isEmpty())) { - createTable = createTable + "partitioned by (" + partitionedBy + ") "; - } - createTable = createTable + "stored as " + storageFormat; - LOG.info("Creating table:\n {}", createTable); - CommandProcessorResponse result = driver.run(createTable); - int retCode = result.getResponseCode(); - if (retCode != 0) { - throw new IOException("Failed to create table. [" + createTable + "], return code from hive driver : [" + retCode + " " + result.getErrorMessage() + "]"); - } + AbstractHCatLoaderTest.createTable(tablename, schema, partitionedBy, driver, storageFormat); } private void createTable(String tablename, String schema) throws IOException, CommandNeedRetryException { diff --git hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java index 40ea923858..e2c9b2e33b 100644 --- hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java +++ hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java @@ -83,16 +83,7 @@ private void dropTable(String tablename) throws IOException, CommandNeedRetryExc } private void createTable(String tablename, String schema, String partitionedBy) throws IOException, CommandNeedRetryException { - String createTable; - createTable = "create table " + tablename + "(" + schema + ") "; - if ((partitionedBy != null) && (!partitionedBy.trim().isEmpty())) { - createTable = createTable + "partitioned by (" + partitionedBy + ") "; - } - createTable = createTable + "stored as " + storageFormat; - int retCode = driver.run(createTable).getResponseCode(); - if (retCode != 0) { - throw new IOException("Failed to create table. [" + createTable + "], return code from hive driver : [" + retCode + "]"); - } + AbstractHCatLoaderTest.createTable(tablename, schema, partitionedBy, driver, storageFormat); } private void createTable(String tablename, String schema) throws IOException, CommandNeedRetryException { diff --git hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java index dc8eee1aac..101ba1aa22 100644 --- hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java +++ hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java @@ -511,7 +511,7 @@ public void testTableValidation() throws Exception { runDDL(driver, "use testBucketing3"); runDDL(driver, "create table " + tbl1 + " ( key1 string, data string ) clustered by ( key1 ) into " - + bucketCount + " buckets stored as orc location " + tableLoc) ; + + bucketCount + " buckets stored as orc location " + tableLoc + " TBLPROPERTIES ('transactional'='false')") ; runDDL(driver, "create table " + tbl2 + " ( key1 string, data string ) clustered by ( key1 ) into " + bucketCount + " buckets stored as orc location " + tableLoc2 + " TBLPROPERTIES ('transactional'='false')") ; diff --git ql/src/test/org/apache/hadoop/hive/ql/TestTxnLoadData.java ql/src/test/org/apache/hadoop/hive/ql/TestTxnLoadData.java index b0dcc7d1a1..c4911bbe7c 100644 --- ql/src/test/org/apache/hadoop/hive/ql/TestTxnLoadData.java +++ ql/src/test/org/apache/hadoop/hive/ql/TestTxnLoadData.java @@ -92,7 +92,7 @@ private void loadDataUpdate(boolean isVectorized) throws Exception { runStatementOnDriver( "create table T (a int, b int) stored as orc tblproperties('transactional'='true')"); //Tstage is just a simple way to generate test data - runStatementOnDriver("create table Tstage (a int, b int) stored as orc"); + runStatementOnDriver("create table Tstage (a int, b int) stored as orc tblproperties('transactional'='false')"); runStatementOnDriver("insert into Tstage values(1,2),(3,4)"); //this creates an ORC data file with correct schema under table root runStatementOnDriver("export table Tstage to '" + getWarehouseDir() + "/1'"); @@ -160,7 +160,7 @@ private void loadData(boolean isVectorized) throws Exception { runStatementOnDriver("create table T (a int, b int) stored as orc tblproperties('transactional'='true')"); runStatementOnDriver("insert into T values(0,2),(0,4)"); //Tstage is just a simple way to generate test data - runStatementOnDriver("create table Tstage (a int, b int) stored as orc"); + runStatementOnDriver("create table Tstage (a int, b int) stored as orc tblproperties('transactional'='false')"); runStatementOnDriver("insert into Tstage values(1,2),(3,4)"); //this creates an ORC data file with correct schema under table root runStatementOnDriver("export table Tstage to '" + getWarehouseDir() +"/1'"); @@ -230,10 +230,10 @@ private void loadData(boolean isVectorized) throws Exception { private void loadDataNonAcid2AcidConversion(boolean isVectorized) throws Exception { runStatementOnDriver("drop table if exists T"); runStatementOnDriver("drop table if exists Tstage"); - runStatementOnDriver("create table T (a int, b int) stored as orc"); + runStatementOnDriver("create table T (a int, b int) stored as orc tblproperties('transactional'='false')"); //per acid write to test nonAcid2acid conversion mixed with load data runStatementOnDriver("insert into T values(0,2),(0,4)"); - runStatementOnDriver("create table Tstage (a int, b int) stored as orc"); + runStatementOnDriver("create table Tstage (a int, b int) stored as orc tblproperties('transactional'='false')"); runStatementOnDriver("insert into Tstage values(1,2),(3,4)"); //make 2 more inserts so that we have 000000_0_copy_1, 000000_0_copy_2 files in export //export works at file level so if you have copy_N in the table dir, you'll have those in output @@ -307,7 +307,7 @@ public void loadDataPartitioned() throws Exception { runStatementOnDriver("drop table if exists T"); runStatementOnDriver("drop table if exists Tstage"); runStatementOnDriver("create table T (a int, b int) partitioned by (p int) stored as orc tblproperties('transactional'='true')"); - runStatementOnDriver("create table Tstage (a int, b int) stored as orc"); + runStatementOnDriver("create table Tstage (a int, b int) stored as orc tblproperties('transactional'='false')"); runStatementOnDriver("insert into Tstage values(0,2),(0,4)"); runStatementOnDriver("export table Tstage to '" + getWarehouseDir() +"/1'"); @@ -361,7 +361,7 @@ public void testValidations() throws Exception { runStatementOnDriver("create table T (a int, b int) clustered by (a) into 2 buckets stored as orc tblproperties('transactional'='true')"); File createdFile= folder.newFile("myfile.txt"); FileUtils.writeStringToFile(createdFile, "hello world"); - runStatementOnDriver("create table Tstage (a int, b int) stored as orc"); + runStatementOnDriver("create table Tstage (a int, b int) stored as orc tblproperties('transactional'='false')"); //this creates an ORC data file with correct schema under table root runStatementOnDriver("insert into Tstage values(1,2),(3,4)"); CommandProcessorResponse cpr = runStatementOnDriverNegative("load data local inpath '" + getWarehouseDir() + "' into table T"); @@ -389,7 +389,7 @@ private void testMultiStatement(boolean isVectorized) throws Exception { runStatementOnDriver("drop table if exists Tstage"); runStatementOnDriver("create table T (a int, b int) stored as orc tblproperties('transactional'='true')"); //Tstage is just a simple way to generate test data - runStatementOnDriver("create table Tstage (a int, b int) stored as orc"); + runStatementOnDriver("create table Tstage (a int, b int) stored as orc tblproperties('transactional'='false')"); runStatementOnDriver("insert into Tstage values(5,5),(6,6)"); //this creates an ORC data file with correct schema under table root runStatementOnDriver("export table Tstage to '" + getWarehouseDir() + "/1'"); @@ -430,7 +430,7 @@ public void testAbort() throws Exception { runStatementOnDriver("drop table if exists Tstage"); runStatementOnDriver("create table T (a int, b int) stored as orc tblproperties('transactional'='true')"); //Tstage is just a simple way to generate test data - runStatementOnDriver("create table Tstage (a int, b int) stored as orc"); + runStatementOnDriver("create table Tstage (a int, b int) stored as orc tblproperties('transactional'='false')"); runStatementOnDriver("insert into Tstage values(5,5),(6,6)"); //this creates an ORC data file with correct schema under table root runStatementOnDriver("export table Tstage to '" + getWarehouseDir() + "/1'"); diff --git ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java index 780b97cb51..ac447797d5 100644 --- ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java +++ ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java @@ -30,6 +30,7 @@ import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; import org.junit.Assert; import org.junit.Before; +import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TestName; @@ -67,7 +68,7 @@ public void testNoBuckets() throws Exception { int[][] sourceVals1 = {{0,0,0},{3,3,3}}; int[][] sourceVals2 = {{1,1,1},{2,2,2}}; runStatementOnDriver("drop table if exists tmp"); - runStatementOnDriver("create table tmp (c1 integer, c2 integer, c3 integer) stored as orc"); + runStatementOnDriver("create table tmp (c1 integer, c2 integer, c3 integer) stored as orc tblproperties('transactional'='false')"); runStatementOnDriver("insert into tmp " + makeValuesClause(sourceVals1)); runStatementOnDriver("insert into tmp " + makeValuesClause(sourceVals2)); runStatementOnDriver("drop table if exists nobuckets"); @@ -187,7 +188,7 @@ public void testCTAS() throws Exception { runStatementOnDriver("insert into " + Table.ACIDTBL + makeValuesClause(values)); runStatementOnDriver("create table myctas2 stored as ORC TBLPROPERTIES ('transactional" + - "'='true', 'transactional_properties'='default') as select a, b from " + Table.ACIDTBL); + "'='true', 'transactional_properties'='default') as select a, b from " + Table.ACIDTBL);//todo: try this with acid default - it seem makeing table acid in listener is too late rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from myctas2 order by ROW__ID"); String expected2[][] = { {"{\"transactionid\":17,\"bucketid\":536870912,\"rowid\":0}\t3\t4", "warehouse/myctas2/delta_0000017_0000017_0000/bucket_00000"}, @@ -531,7 +532,7 @@ public void testNonAcidToAcidVectorzied() throws Exception { //this enables vectorization of ROW__ID hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_VECTORIZATION_ROW_IDENTIFIER_ENABLED, true);//HIVE-12631 runStatementOnDriver("drop table if exists T"); - runStatementOnDriver("create table T(a int, b int) stored as orc"); + runStatementOnDriver("create table T(a int, b int) stored as orc tblproperties('transactional'='false')"); int[][] values = {{1,2},{2,4},{5,6},{6,8},{9,10}}; runStatementOnDriver("insert into T(a, b) " + makeValuesClause(values)); //, 'transactional_properties'='default' @@ -696,5 +697,21 @@ public void testCompactStatsGather() throws Exception { map = hms.getPartitionColumnStatistics("default","T", partNames, colNames); Assert.assertEquals("", 5, map.get(partNames.get(0)).get(0).getStatsData().getLongStats().getHighValue()); } + @Ignore("enable after HIVE-18294") + @Test + public void testDefault() throws Exception { + runStatementOnDriver("drop table if exists T"); + runStatementOnDriver("create table T (a int, b int) stored as orc"); + runStatementOnDriver("insert into T values(1,2),(3,4)"); + String query = "select ROW__ID, a, b, INPUT__FILE__NAME from T order by a, b"; + List rs = runStatementOnDriver(query); + String[][] expected = { + //this proves data is written in Acid layout so T was made Acid + {"{\"transactionid\":15,\"bucketid\":536870912,\"rowid\":0}\t1\t2", "t/delta_0000015_0000015_0000/bucket_00000"}, + {"{\"transactionid\":15,\"bucketid\":536870912,\"rowid\":1}\t3\t4", "t/delta_0000015_0000015_0000/bucket_00000"} + }; + checkExpected(rs, expected, "insert data"); + + } } diff --git ql/src/test/org/apache/hadoop/hive/ql/TxnCommandsBaseForTests.java ql/src/test/org/apache/hadoop/hive/ql/TxnCommandsBaseForTests.java index 32a24898ee..30bbb3f149 100644 --- ql/src/test/org/apache/hadoop/hive/ql/TxnCommandsBaseForTests.java +++ ql/src/test/org/apache/hadoop/hive/ql/TxnCommandsBaseForTests.java @@ -25,6 +25,7 @@ import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.txn.TxnDbUtil; import org.apache.hadoop.hive.ql.io.HiveInputFormat; import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; @@ -71,7 +72,7 @@ public String toString() { public void setUp() throws Exception { setUpInternal(); } - protected void setUpInternal() throws Exception { + void setUpInternal() throws Exception { hiveConf = new HiveConf(this.getClass()); hiveConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, ""); hiveConf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, ""); @@ -100,7 +101,7 @@ protected void setUpInternal() throws Exception { runStatementOnDriver("create table " + Table.NONACIDORCTBL + "(a int, b int) clustered by (a) into " + BUCKET_COUNT + " buckets stored as orc TBLPROPERTIES ('transactional'='false')"); runStatementOnDriver("create table " + Table.NONACIDORCTBL2 + "(a int, b int) clustered by (a) into " + BUCKET_COUNT + " buckets stored as orc TBLPROPERTIES ('transactional'='false')"); runStatementOnDriver("create temporary table " + Table.ACIDTBL2 + "(a int, b int, c int) clustered by (c) into " + BUCKET_COUNT + " buckets stored as orc TBLPROPERTIES ('transactional'='true')"); - runStatementOnDriver("create table " + Table.NONACIDNONBUCKET + "(a int, b int) stored as orc"); + runStatementOnDriver("create table " + Table.NONACIDNONBUCKET + "(a int, b int) stored as orc TBLPROPERTIES ('transactional'='false')"); } protected void dropTables() throws Exception { for(TxnCommandsBaseForTests.Table t : TxnCommandsBaseForTests.Table.values()) { diff --git ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java index 406bdea96a..e84f63d798 100644 --- ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java +++ ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java @@ -388,6 +388,11 @@ public void testDDLNoLock() throws Exception { @Test public void concurrencyFalse() throws Exception { HiveConf badConf = new HiveConf(); + if(badConf.getBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY)) { + //TxnManagerFactory is a singleton, so if the default is true, it has already been + //created and won't throw + return; + } badConf.setVar(HiveConf.ConfVars.HIVE_TXN_MANAGER, "org.apache.hadoop.hive.ql.lockmgr.DbTxnManager"); badConf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false); diff --git ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java index d309e3d06f..28131a04e9 100644 --- ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java +++ ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java @@ -623,7 +623,7 @@ public void checkExpectedLocks() throws Exception { CommandProcessorResponse cpr = null; cpr = driver.run("create table acidPart(a int, b int) partitioned by (p string) clustered by (a) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')"); checkCmdOnDriver(cpr); - cpr = driver.run("create table nonAcidPart(a int, b int) partitioned by (p string) stored as orc"); + cpr = driver.run("create table nonAcidPart(a int, b int) partitioned by (p string) stored as orc TBLPROPERTIES ('transactional'='false')"); checkCmdOnDriver(cpr); cpr = driver.compileAndRespond("insert into nonAcidPart partition(p) values(1,2,3)");