diff --git data/conf/hive-site.xml data/conf/hive-site.xml index d504e17f54..5686bb1038 100644 --- data/conf/hive-site.xml +++ data/conf/hive-site.xml @@ -76,22 +76,22 @@ javax.jdo.option.ConnectionURL - jdbc:derby:memory:${test.tmp.dir}/junit_metastore_db;create=true + jdbc:postgresql:hivedb javax.jdo.option.ConnectionDriverName - org.apache.derby.jdbc.EmbeddedDriver + org.postgresql.Driver javax.jdo.option.ConnectionUserName - APP + hiveuser javax.jdo.option.ConnectionPassword - mine + hivepassword diff --git itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index d2c2ccd5ea..0357213ccc 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -158,6 +158,9 @@ public void initConf() throws Exception { // Plug verifying metastore in for testing DirectSQL. conf.setVar(ConfVars.METASTORE_RAW_STORE_IMPL, "org.apache.hadoop.hive.metastore.VerifyingObjectStore"); + if (System.getProperty("hive.metastore.rawstore.impl") != null){ + conf.setVar(ConfVars.METASTORE_RAW_STORE_IMPL, System.getProperty("hive.metastore.rawstore.impl")); + } miniClusters.initConf(conf); } diff --git ql/src/test/queries/clientpositive/partition_params_postgres.q ql/src/test/queries/clientpositive/partition_params_postgres.q new file mode 100644 index 0000000000..e2ad5302c7 --- /dev/null +++ ql/src/test/queries/clientpositive/partition_params_postgres.q @@ -0,0 +1,5 @@ +drop table if exists my_table; +create external table my_table (col1 int, col3 int) partitioned by (col2 string) STORED AS TEXTFILE TBLPROPERTIES ("serialization.format" = "1"); +insert into my_table VALUES(11, 201, "F"); +SELECT * from my_table; +describe formatted my_table; diff --git ql/src/test/results/clientpositive/partition_params_postgres.q.out ql/src/test/results/clientpositive/partition_params_postgres.q.out new file mode 100644 index 0000000000..c078459707 --- /dev/null +++ ql/src/test/results/clientpositive/partition_params_postgres.q.out @@ -0,0 +1,80 @@ +PREHOOK: query: drop table if exists my_table +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@my_table +PREHOOK: Output: default@my_table +POSTHOOK: query: drop table if exists my_table +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@my_table +POSTHOOK: Output: default@my_table +PREHOOK: query: create external table my_table (col1 int, col3 int) partitioned by (col2 string) STORED AS TEXTFILE TBLPROPERTIES ("serialization.format" = "1") +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@my_table +POSTHOOK: query: create external table my_table (col1 int, col3 int) partitioned by (col2 string) STORED AS TEXTFILE TBLPROPERTIES ("serialization.format" = "1") +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@my_table +PREHOOK: query: insert into my_table VALUES(11, 201, "F") +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@my_table +POSTHOOK: query: insert into my_table VALUES(11, 201, "F") +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@my_table@col2=F +POSTHOOK: Lineage: my_table PARTITION(col2=F).col1 SCRIPT [] +POSTHOOK: Lineage: my_table PARTITION(col2=F).col3 SCRIPT [] +PREHOOK: query: SELECT * from my_table +PREHOOK: type: QUERY +PREHOOK: Input: default@my_table +PREHOOK: Input: default@my_table@col2=F +#### A masked pattern was here #### +POSTHOOK: query: SELECT * from my_table +POSTHOOK: type: QUERY +POSTHOOK: Input: default@my_table +POSTHOOK: Input: default@my_table@col2=F +#### A masked pattern was here #### +11 201 F +PREHOOK: query: describe formatted my_table +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@my_table +POSTHOOK: query: describe formatted my_table +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@my_table +# col_name data_type comment +col1 int +col3 int + +# Partition Information +# col_name data_type comment +col2 string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: EXTERNAL_TABLE +Table Parameters: + EXTERNAL TRUE + bucketing_version 2 + discover.partitions true + numFiles 17446 + numFilesErasureCoded 17448 + numPartitions 1 + numRows 17449 + rawDataSize 17450 + serialization.format 1 + totalSize 17443 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 diff --git standalone-metastore/DEV-README standalone-metastore/DEV-README index 9c261171fb..ab5df26590 100644 --- standalone-metastore/DEV-README +++ standalone-metastore/DEV-README @@ -45,6 +45,12 @@ To run just one test, do mvn verify -DskipITests=false -Dit.test=ITestMysql -Dtest=nosuch +Supported databases for testing: +-Dit.test=ITestMysql +-Dit.test=ITestOracle +-Dit.test=ITestPostgres +-Dit.test=ITestSqlServer + You can download the Oracle driver at http://www.oracle.com/technetwork/database/features/jdbc/index-091264.html You should download Oracle 11g Release 1, ojdbc6.jar diff --git standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/DbInstallBase.java standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/DbInstallBase.java index 6d8fd46f2a..9d3912ee4a 100644 --- standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/DbInstallBase.java +++ standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/DbInstallBase.java @@ -40,11 +40,11 @@ private static final Logger LOG = LoggerFactory.getLogger(DbInstallBase.class); private static final String HIVE_USER = "hiveuser"; + protected static final String HIVE_PASSWORD = "hivepassword"; // used in most of the RDBMS configs, except MSSQL protected static final String HIVE_DB = "hivedb"; private static final String FIRST_VERSION = "1.2.0"; private static final int MAX_STARTUP_WAIT = 5 * 60 * 1000; - protected abstract String getDockerContainerName(); protected abstract String getDockerImageName(); protected abstract String[] getDockerAdditionalArgs(); protected abstract String getDbType(); @@ -52,6 +52,11 @@ protected abstract String getDbRootPassword(); protected abstract String getJdbcDriver(); protected abstract String getJdbcUrl(); + + protected String getDockerContainerName(){ + return String.format("metastore-test-%s-install", getDbType()); + }; + /** * URL to use when connecting as root rather than Hive * @return URL diff --git standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/ITestMysql.java standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/ITestMysql.java index 9999d8d705..7a8d8a9362 100644 --- standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/ITestMysql.java +++ standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/ITestMysql.java @@ -77,6 +77,6 @@ protected String getDockerContainerName() { @Override protected String getHivePassword() { - return "hivepassword"; + return HIVE_PASSWORD; } } diff --git standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/ITestOracle.java standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/ITestOracle.java index 5b93e0ffae..c580017df5 100644 --- standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/ITestOracle.java +++ standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/ITestOracle.java @@ -18,11 +18,6 @@ package org.apache.hadoop.hive.metastore.dbinstall; public class ITestOracle extends DbInstallBase { - @Override - protected String getDockerContainerName() { - return "metastore-test-oracle-install"; - } - @Override protected String getDockerImageName() { return "orangehrm/oracle-xe-11g"; @@ -78,6 +73,6 @@ protected boolean isContainerReady(String logOutput) { @Override protected String getHivePassword() { - return "hivepassword"; + return HIVE_PASSWORD; } } diff --git standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/ITestPostgres.java standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/ITestPostgres.java index 9151ac766e..8fecc3756a 100644 --- standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/ITestPostgres.java +++ standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/ITestPostgres.java @@ -18,11 +18,6 @@ package org.apache.hadoop.hive.metastore.dbinstall; public class ITestPostgres extends DbInstallBase { - @Override - protected String getDockerContainerName() { - return "metastore-test-postgres-install"; - } - @Override protected String getDockerImageName() { return "postgres:9.3"; @@ -77,6 +72,6 @@ protected boolean isContainerReady(String logOutput) { @Override protected String getHivePassword() { - return "hivepassword"; + return HIVE_PASSWORD; } } diff --git standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/ITestSqlServer.java standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/ITestSqlServer.java index 67b6eeeab2..d199e6b809 100644 --- standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/ITestSqlServer.java +++ standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/ITestSqlServer.java @@ -18,11 +18,6 @@ package org.apache.hadoop.hive.metastore.dbinstall; public class ITestSqlServer extends DbInstallBase { - @Override - protected String getDockerContainerName() { - return "metastore-test-mssql-install"; - } - @Override protected String getDockerImageName() { return "microsoft/mssql-server-linux:2017-GA"; diff --git standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/tools/schematool/TestMetastoreSchemaTool.java standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/tools/schematool/TestMetastoreSchemaTool.java index b4a0844be3..a34190c3e7 100644 --- standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/tools/schematool/TestMetastoreSchemaTool.java +++ standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/tools/schematool/TestMetastoreSchemaTool.java @@ -40,7 +40,7 @@ @Mock private Configuration conf; private MetastoreSchemaTool.CommandBuilder builder; - private String pasword = "reallySimplePassword"; + private String password = "reallySimplePassword"; @Before public void setup() throws IOException { @@ -49,7 +49,7 @@ public void setup() throws IOException { if (!file.exists()) { file.createNewFile(); } - builder = new MetastoreSchemaTool.CommandBuilder(conf, null, null, "testUser", pasword, scriptFile); + builder = new MetastoreSchemaTool.CommandBuilder(conf, null, null, "testUser", password, scriptFile); } @After @@ -59,12 +59,12 @@ public void globalAssert() throws IOException { @Test public void shouldReturnStrippedPassword() throws IOException { - assertFalse(builder.buildToLog().contains(pasword)); + assertFalse(builder.buildToLog().contains(password)); } @Test public void shouldReturnActualPassword() throws IOException { String[] strings = builder.buildToRun(); - assertTrue(Arrays.asList(strings).contains(pasword)); + assertTrue(Arrays.asList(strings).contains(password)); } }