From 4e34e3d88a20a1265948d52e3682a59debcbd8fb Mon Sep 17 00:00:00 2001 From: Ashutosh Chauhan Date: Fri, 28 Jul 2017 13:22:29 -0700 Subject: [PATCH] HIVE-17190 : Schema changes for bitvectors for unpartitioned tables --- data/conf/perf-reg/hive-site.xml | 5 ----- itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java | 3 ++- metastore/scripts/upgrade/derby/022-HIVE-11107.derby.sql | 2 +- metastore/scripts/upgrade/derby/044-HIVE-16997.derby.sql | 1 + metastore/scripts/upgrade/mssql/029-HIVE-16997.mssql.sql | 1 + metastore/scripts/upgrade/mssql/hive-schema-3.0.0.mssql.sql | 1 + metastore/scripts/upgrade/mysql/044-HIVE-16997.mysql.sql | 1 + metastore/scripts/upgrade/mysql/hive-schema-3.0.0.mysql.sql | 1 + metastore/scripts/upgrade/oracle/044-HIVE-16997.oracle.sql | 1 + metastore/scripts/upgrade/oracle/hive-schema-3.0.0.oracle.sql | 1 + metastore/scripts/upgrade/postgres/043-HIVE-16997.postgres.sql | 1 + metastore/scripts/upgrade/postgres/hive-schema-3.0.0.postgres.sql | 1 + 12 files changed, 12 insertions(+), 7 deletions(-) diff --git a/data/conf/perf-reg/hive-site.xml b/data/conf/perf-reg/hive-site.xml index 331a50056a..012369fdd1 100644 --- a/data/conf/perf-reg/hive-site.xml +++ b/data/conf/perf-reg/hive-site.xml @@ -282,9 +282,4 @@ true - - hive.stats.fetch.bitvector - true - - diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index cd6c41c3b9..535cfd9399 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -2386,7 +2386,8 @@ public int compare(String str1, String str2) { for (int i = 2; i < splits.length; i++) { sb.append(splits[i]+"@"); } - return sb.append(tblID).toString(); + // Add tbl_id and empty bitvector + return sb.append(tblID).append("@").toString(); }); Files.write(tmpFileLoc1, (Iterable)replaced::iterator); diff --git a/metastore/scripts/upgrade/derby/022-HIVE-11107.derby.sql b/metastore/scripts/upgrade/derby/022-HIVE-11107.derby.sql index 4a13ea96c8..5806a19489 100644 --- a/metastore/scripts/upgrade/derby/022-HIVE-11107.derby.sql +++ b/metastore/scripts/upgrade/derby/022-HIVE-11107.derby.sql @@ -1,2 +1,2 @@ -CREATE TABLE "APP"."TAB_COL_STATS"("DB_NAME" VARCHAR(128) NOT NULL,"TABLE_NAME" VARCHAR(256) NOT NULL, "COLUMN_NAME" VARCHAR(767) NOT NULL, "COLUMN_TYPE" VARCHAR(128) NOT NULL, "LONG_LOW_VALUE" BIGINT, "LONG_HIGH_VALUE" BIGINT, "DOUBLE_LOW_VALUE" DOUBLE, "DOUBLE_HIGH_VALUE" DOUBLE, "BIG_DECIMAL_LOW_VALUE" VARCHAR(4000),"BIG_DECIMAL_HIGH_VALUE" VARCHAR(4000),"NUM_DISTINCTS" BIGINT, "NUM_NULLS" BIGINT NOT NULL, "AVG_COL_LEN" DOUBLE, "MAX_COL_LEN" BIGINT, "NUM_TRUES" BIGINT, "NUM_FALSES" BIGINT, "LAST_ANALYZED" BIGINT, "CS_ID" BIGINT NOT NULL, "TBL_ID" BIGINT NOT NULL); +CREATE TABLE "APP"."TAB_COL_STATS"("DB_NAME" VARCHAR(128) NOT NULL,"TABLE_NAME" VARCHAR(256) NOT NULL, "COLUMN_NAME" VARCHAR(767) NOT NULL, "COLUMN_TYPE" VARCHAR(128) NOT NULL, "LONG_LOW_VALUE" BIGINT, "LONG_HIGH_VALUE" BIGINT, "DOUBLE_LOW_VALUE" DOUBLE, "DOUBLE_HIGH_VALUE" DOUBLE, "BIG_DECIMAL_LOW_VALUE" VARCHAR(4000),"BIG_DECIMAL_HIGH_VALUE" VARCHAR(4000),"NUM_DISTINCTS" BIGINT, "NUM_NULLS" BIGINT NOT NULL, "AVG_COL_LEN" DOUBLE, "MAX_COL_LEN" BIGINT, "NUM_TRUES" BIGINT, "NUM_FALSES" BIGINT, "LAST_ANALYZED" BIGINT, "CS_ID" BIGINT NOT NULL, "TBL_ID" BIGINT NOT NULL, "BIT_VECTOR" BLOB); CREATE TABLE "APP"."TABLE_PARAMS" ("TBL_ID" BIGINT NOT NULL, "PARAM_KEY" VARCHAR(256) NOT NULL, "PARAM_VALUE" CLOB); diff --git a/metastore/scripts/upgrade/derby/044-HIVE-16997.derby.sql b/metastore/scripts/upgrade/derby/044-HIVE-16997.derby.sql index 2c2177bd0e..ae9649c134 100644 --- a/metastore/scripts/upgrade/derby/044-HIVE-16997.derby.sql +++ b/metastore/scripts/upgrade/derby/044-HIVE-16997.derby.sql @@ -1 +1,2 @@ ALTER TABLE "APP"."PART_COL_STATS" ADD COLUMN "BIT_VECTOR" BLOB; +ALTER TABLE "APP"."TAB_COL_STATS" ADD COLUMN "BIT_VECTOR" BLOB; diff --git a/metastore/scripts/upgrade/mssql/029-HIVE-16997.mssql.sql b/metastore/scripts/upgrade/mssql/029-HIVE-16997.mssql.sql index 1882c59691..e1b8a57e9b 100644 --- a/metastore/scripts/upgrade/mssql/029-HIVE-16997.mssql.sql +++ b/metastore/scripts/upgrade/mssql/029-HIVE-16997.mssql.sql @@ -1 +1,2 @@ ALTER TABLE PART_COL_STATS ADD BIT_VECTOR VARBINARY(MAX); +ALTER TABLE TAB_COL_STATS ADD BIT_VECTOR VARBINARY(MAX); diff --git a/metastore/scripts/upgrade/mssql/hive-schema-3.0.0.mssql.sql b/metastore/scripts/upgrade/mssql/hive-schema-3.0.0.mssql.sql index fa8fc6ea10..158fa5a0f0 100644 --- a/metastore/scripts/upgrade/mssql/hive-schema-3.0.0.mssql.sql +++ b/metastore/scripts/upgrade/mssql/hive-schema-3.0.0.mssql.sql @@ -231,6 +231,7 @@ CREATE TABLE TAB_COL_STATS LONG_LOW_VALUE bigint NULL, MAX_COL_LEN bigint NULL, NUM_DISTINCTS bigint NULL, + BIT_VECTOR varbinary(max) NULL, NUM_FALSES bigint NULL, NUM_NULLS bigint NOT NULL, NUM_TRUES bigint NULL, diff --git a/metastore/scripts/upgrade/mysql/044-HIVE-16997.mysql.sql b/metastore/scripts/upgrade/mysql/044-HIVE-16997.mysql.sql index 4954b2e63c..bef13640fd 100644 --- a/metastore/scripts/upgrade/mysql/044-HIVE-16997.mysql.sql +++ b/metastore/scripts/upgrade/mysql/044-HIVE-16997.mysql.sql @@ -1 +1,2 @@ ALTER TABLE PART_COL_STATS ADD COLUMN BIT_VECTOR BLOB; +ALTER TABLE TAB_COL_STATS ADD COLUMN BIT_VECTOR BLOB; diff --git a/metastore/scripts/upgrade/mysql/hive-schema-3.0.0.mysql.sql b/metastore/scripts/upgrade/mysql/hive-schema-3.0.0.mysql.sql index 31963d0309..98c107754b 100644 --- a/metastore/scripts/upgrade/mysql/hive-schema-3.0.0.mysql.sql +++ b/metastore/scripts/upgrade/mysql/hive-schema-3.0.0.mysql.sql @@ -662,6 +662,7 @@ CREATE TABLE IF NOT EXISTS `TAB_COL_STATS` ( `BIG_DECIMAL_HIGH_VALUE` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin, `NUM_NULLS` bigint(20) NOT NULL, `NUM_DISTINCTS` bigint(20), + `BIT_VECTOR` blob, `AVG_COL_LEN` double(53,4), `MAX_COL_LEN` bigint(20), `NUM_TRUES` bigint(20), diff --git a/metastore/scripts/upgrade/oracle/044-HIVE-16997.oracle.sql b/metastore/scripts/upgrade/oracle/044-HIVE-16997.oracle.sql index 44e5fa35ba..387b1f6e62 100644 --- a/metastore/scripts/upgrade/oracle/044-HIVE-16997.oracle.sql +++ b/metastore/scripts/upgrade/oracle/044-HIVE-16997.oracle.sql @@ -1 +1,2 @@ ALTER TABLE PART_COL_STATS ADD BIT_VECTOR BLOB NULL; +ALTER TABLE TAB_COL_STATS ADD BIT_VECTOR BLOB NULL; diff --git a/metastore/scripts/upgrade/oracle/hive-schema-3.0.0.oracle.sql b/metastore/scripts/upgrade/oracle/hive-schema-3.0.0.oracle.sql index 81e4208ed9..7b32d782eb 100644 --- a/metastore/scripts/upgrade/oracle/hive-schema-3.0.0.oracle.sql +++ b/metastore/scripts/upgrade/oracle/hive-schema-3.0.0.oracle.sql @@ -479,6 +479,7 @@ CREATE TABLE TAB_COL_STATS ( BIG_DECIMAL_HIGH_VALUE VARCHAR2(4000), NUM_NULLS NUMBER NOT NULL, NUM_DISTINCTS NUMBER, + BIT_VECTOR BLOB, AVG_COL_LEN NUMBER, MAX_COL_LEN NUMBER, NUM_TRUES NUMBER, diff --git a/metastore/scripts/upgrade/postgres/043-HIVE-16997.postgres.sql b/metastore/scripts/upgrade/postgres/043-HIVE-16997.postgres.sql index bee8c4438a..0d62202cb5 100644 --- a/metastore/scripts/upgrade/postgres/043-HIVE-16997.postgres.sql +++ b/metastore/scripts/upgrade/postgres/043-HIVE-16997.postgres.sql @@ -1 +1,2 @@ ALTER TABLE "PART_COL_STATS" ADD COLUMN "BIT_VECTOR" BYTEA; +ALTER TABLE "TAB_COL_STATS" ADD COLUMN "BIT_VECTOR" BYTEA; diff --git a/metastore/scripts/upgrade/postgres/hive-schema-3.0.0.postgres.sql b/metastore/scripts/upgrade/postgres/hive-schema-3.0.0.postgres.sql index 5cb5cb0f14..d2dfdd857e 100644 --- a/metastore/scripts/upgrade/postgres/hive-schema-3.0.0.postgres.sql +++ b/metastore/scripts/upgrade/postgres/hive-schema-3.0.0.postgres.sql @@ -498,6 +498,7 @@ CREATE TABLE "TAB_COL_STATS" ( "BIG_DECIMAL_HIGH_VALUE" character varying(4000) DEFAULT NULL::character varying, "NUM_NULLS" bigint NOT NULL, "NUM_DISTINCTS" bigint, + "BIT_VECTOR" bytea, "AVG_COL_LEN" double precision, "MAX_COL_LEN" bigint, "NUM_TRUES" bigint, -- 2.11.0 (Apple Git-81)