diff --git itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index d2c2ccd5ea..0357213ccc 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -158,6 +158,9 @@ public void initConf() throws Exception { // Plug verifying metastore in for testing DirectSQL. conf.setVar(ConfVars.METASTORE_RAW_STORE_IMPL, "org.apache.hadoop.hive.metastore.VerifyingObjectStore"); + if (System.getProperty("hive.metastore.rawstore.impl") != null){ + conf.setVar(ConfVars.METASTORE_RAW_STORE_IMPL, System.getProperty("hive.metastore.rawstore.impl")); + } miniClusters.initConf(conf); } diff --git ql/src/test/queries/clientpositive/partition_params_postgres.q ql/src/test/queries/clientpositive/partition_params_postgres.q new file mode 100644 index 0000000000..7e02e0b5a8 --- /dev/null +++ ql/src/test/queries/clientpositive/partition_params_postgres.q @@ -0,0 +1,5 @@ +--TODO: this test is intended to be run against postgres metastore, see HIVE-21940 for details +drop table if exists my_table; +create external table my_table (col1 int, col3 int) partitioned by (col2 string) STORED AS TEXTFILE TBLPROPERTIES ("serialization.format" = "1"); +insert into my_table VALUES(11, 201, "F"); +describe formatted my_table; diff --git ql/src/test/results/clientpositive/partition_params_postgres.q.out ql/src/test/results/clientpositive/partition_params_postgres.q.out new file mode 100644 index 0000000000..5e59f5eb18 --- /dev/null +++ ql/src/test/results/clientpositive/partition_params_postgres.q.out @@ -0,0 +1,65 @@ +PREHOOK: query: drop table if exists my_table +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table if exists my_table +POSTHOOK: type: DROPTABLE +PREHOOK: query: create external table my_table (col1 int, col3 int) partitioned by (col2 string) STORED AS TEXTFILE TBLPROPERTIES ("serialization.format" = "1") +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@my_table +POSTHOOK: query: create external table my_table (col1 int, col3 int) partitioned by (col2 string) STORED AS TEXTFILE TBLPROPERTIES ("serialization.format" = "1") +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@my_table +PREHOOK: query: insert into my_table VALUES(11, 201, "F") +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@my_table +POSTHOOK: query: insert into my_table VALUES(11, 201, "F") +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@my_table@col2=F +POSTHOOK: Lineage: my_table PARTITION(col2=F).col1 SCRIPT [] +POSTHOOK: Lineage: my_table PARTITION(col2=F).col3 SCRIPT [] +PREHOOK: query: describe formatted my_table +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@my_table +POSTHOOK: query: describe formatted my_table +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@my_table +# col_name data_type comment +col1 int +col3 int + +# Partition Information +# col_name data_type comment +col2 string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: EXTERNAL_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + EXTERNAL TRUE + bucketing_version 2 + discover.partitions true + numFiles 1 + numPartitions 1 + numRows 1 + rawDataSize 6 + serialization.format 1 + totalSize 7 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 diff --git standalone-metastore/DEV-README standalone-metastore/DEV-README index 9c261171fb..ab5df26590 100644 --- standalone-metastore/DEV-README +++ standalone-metastore/DEV-README @@ -45,6 +45,12 @@ To run just one test, do mvn verify -DskipITests=false -Dit.test=ITestMysql -Dtest=nosuch +Supported databases for testing: +-Dit.test=ITestMysql +-Dit.test=ITestOracle +-Dit.test=ITestPostgres +-Dit.test=ITestSqlServer + You can download the Oracle driver at http://www.oracle.com/technetwork/database/features/jdbc/index-091264.html You should download Oracle 11g Release 1, ojdbc6.jar diff --git standalone-metastore/metastore-server/src/main/resources/package.jdo standalone-metastore/metastore-server/src/main/resources/package.jdo index 8ad2f21ea4..4586f22429 100644 --- standalone-metastore/metastore-server/src/main/resources/package.jdo +++ standalone-metastore/metastore-server/src/main/resources/package.jdo @@ -503,7 +503,7 @@ - + @@ -553,7 +553,7 @@ - + diff --git standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/DbInstallBase.java standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/DbInstallBase.java index 6d8fd46f2a..9d3912ee4a 100644 --- standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/DbInstallBase.java +++ standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/DbInstallBase.java @@ -40,11 +40,11 @@ private static final Logger LOG = LoggerFactory.getLogger(DbInstallBase.class); private static final String HIVE_USER = "hiveuser"; + protected static final String HIVE_PASSWORD = "hivepassword"; // used in most of the RDBMS configs, except MSSQL protected static final String HIVE_DB = "hivedb"; private static final String FIRST_VERSION = "1.2.0"; private static final int MAX_STARTUP_WAIT = 5 * 60 * 1000; - protected abstract String getDockerContainerName(); protected abstract String getDockerImageName(); protected abstract String[] getDockerAdditionalArgs(); protected abstract String getDbType(); @@ -52,6 +52,11 @@ protected abstract String getDbRootPassword(); protected abstract String getJdbcDriver(); protected abstract String getJdbcUrl(); + + protected String getDockerContainerName(){ + return String.format("metastore-test-%s-install", getDbType()); + }; + /** * URL to use when connecting as root rather than Hive * @return URL diff --git standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/ITestMysql.java standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/ITestMysql.java index 9999d8d705..7a8d8a9362 100644 --- standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/ITestMysql.java +++ standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/ITestMysql.java @@ -77,6 +77,6 @@ protected String getDockerContainerName() { @Override protected String getHivePassword() { - return "hivepassword"; + return HIVE_PASSWORD; } } diff --git standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/ITestOracle.java standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/ITestOracle.java index 5b93e0ffae..c580017df5 100644 --- standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/ITestOracle.java +++ standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/ITestOracle.java @@ -18,11 +18,6 @@ package org.apache.hadoop.hive.metastore.dbinstall; public class ITestOracle extends DbInstallBase { - @Override - protected String getDockerContainerName() { - return "metastore-test-oracle-install"; - } - @Override protected String getDockerImageName() { return "orangehrm/oracle-xe-11g"; @@ -78,6 +73,6 @@ protected boolean isContainerReady(String logOutput) { @Override protected String getHivePassword() { - return "hivepassword"; + return HIVE_PASSWORD; } } diff --git standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/ITestPostgres.java standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/ITestPostgres.java index 9151ac766e..8fecc3756a 100644 --- standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/ITestPostgres.java +++ standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/ITestPostgres.java @@ -18,11 +18,6 @@ package org.apache.hadoop.hive.metastore.dbinstall; public class ITestPostgres extends DbInstallBase { - @Override - protected String getDockerContainerName() { - return "metastore-test-postgres-install"; - } - @Override protected String getDockerImageName() { return "postgres:9.3"; @@ -77,6 +72,6 @@ protected boolean isContainerReady(String logOutput) { @Override protected String getHivePassword() { - return "hivepassword"; + return HIVE_PASSWORD; } } diff --git standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/ITestSqlServer.java standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/ITestSqlServer.java index 67b6eeeab2..d199e6b809 100644 --- standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/ITestSqlServer.java +++ standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/ITestSqlServer.java @@ -18,11 +18,6 @@ package org.apache.hadoop.hive.metastore.dbinstall; public class ITestSqlServer extends DbInstallBase { - @Override - protected String getDockerContainerName() { - return "metastore-test-mssql-install"; - } - @Override protected String getDockerImageName() { return "microsoft/mssql-server-linux:2017-GA"; diff --git standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/tools/schematool/TestMetastoreSchemaTool.java standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/tools/schematool/TestMetastoreSchemaTool.java index b4a0844be3..a34190c3e7 100644 --- standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/tools/schematool/TestMetastoreSchemaTool.java +++ standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/tools/schematool/TestMetastoreSchemaTool.java @@ -40,7 +40,7 @@ @Mock private Configuration conf; private MetastoreSchemaTool.CommandBuilder builder; - private String pasword = "reallySimplePassword"; + private String password = "reallySimplePassword"; @Before public void setup() throws IOException { @@ -49,7 +49,7 @@ public void setup() throws IOException { if (!file.exists()) { file.createNewFile(); } - builder = new MetastoreSchemaTool.CommandBuilder(conf, null, null, "testUser", pasword, scriptFile); + builder = new MetastoreSchemaTool.CommandBuilder(conf, null, null, "testUser", password, scriptFile); } @After @@ -59,12 +59,12 @@ public void globalAssert() throws IOException { @Test public void shouldReturnStrippedPassword() throws IOException { - assertFalse(builder.buildToLog().contains(pasword)); + assertFalse(builder.buildToLog().contains(password)); } @Test public void shouldReturnActualPassword() throws IOException { String[] strings = builder.buildToRun(); - assertTrue(Arrays.asList(strings).contains(pasword)); + assertTrue(Arrays.asList(strings).contains(password)); } }