From c7985c29ff11d36853d7866533a06e6fa2d27dc0 Mon Sep 17 00:00:00 2001 From: Ashutosh Chauhan Date: Wed, 18 Oct 2017 17:55:34 -0700 Subject: [PATCH] HIVE-17836 : Persisting nulls in bit vector field fails for postgres backed metastore --- .../apache/hadoop/hive/metastore/ObjectStore.java | 21 +++++++++++++++++---- .../hadoop/hive/metastore/tools/SQLGenerator.java | 6 ++++++ 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index ffb2abdf62..af6a5703cd 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -728,6 +728,7 @@ public boolean commitTransaction() { * @return true if there is an active transaction. If the current transaction * is either committed or rolled back it returns false */ + @Override public boolean isActiveTransaction() { if (currentTransaction == null) { return false; @@ -3254,13 +3255,14 @@ public int getNumPartitionsByFilter(String dbName, String tblName, ? PartFilterExprUtil.getFilterParser(filter).tree : ExpressionTree.EMPTY_TREE; return new GetHelper(dbName, tblName, true, true) { - private SqlFilterForPushdown filter = new SqlFilterForPushdown(); + private final SqlFilterForPushdown filter = new SqlFilterForPushdown(); @Override protected String describeResult() { return "Partition count"; } + @Override protected boolean canUseDirectSql(GetHelper ctx) throws MetaException { return directSql.generateSqlFilterForPushdown(ctx.getTable(), exprTree, filter); } @@ -3285,13 +3287,14 @@ public int getNumPartitionsByExpr(String dbName, String tblName, return new GetHelper(dbName, tblName, true, true) { - private SqlFilterForPushdown filter = new SqlFilterForPushdown(); + private final SqlFilterForPushdown filter = new SqlFilterForPushdown(); @Override protected String describeResult() { return "Partition count"; } + @Override protected boolean canUseDirectSql(GetHelper ctx) throws MetaException { return directSql.generateSqlFilterForPushdown(ctx.getTable(), exprTree, filter); }; @@ -3331,7 +3334,7 @@ protected Integer getJdoResult( final ExpressionTree tree = (filter != null && !filter.isEmpty()) ? PartFilterExprUtil.getFilterParser(filter).tree : ExpressionTree.EMPTY_TREE; return new GetListHelper(dbName, tblName, allowSql, allowJdo) { - private SqlFilterForPushdown filter = new SqlFilterForPushdown(); + private final SqlFilterForPushdown filter = new SqlFilterForPushdown(); @Override protected boolean canUseDirectSql(GetHelper> ctx) throws MetaException { @@ -7280,6 +7283,11 @@ private void writeMTableColumnStatistics(Table table, MTableColumnStatistics mSt if (oldStats != null) { StatObjectConverter.setFieldsIntoOldStats(mStatsObj, oldStats); } else { + if (sqlGenerator.getDbProduct().equals(DatabaseProduct.POSTGRES) && mStatsObj.getBitVector() == null) { + // workaround for DN bug in persisting nulls in pg bytea column + // instead set empty bit vector with header. + mStatsObj.setBitVector(new byte[] {'H','L'}); + } pm.makePersistent(mStatsObj); } } finally { @@ -7316,6 +7324,11 @@ private void writeMPartitionColumnStatistics(Table table, Partition partition, if (oldStats != null) { StatObjectConverter.setFieldsIntoOldStats(mStatsObj, oldStats); } else { + if (sqlGenerator.getDbProduct().equals(DatabaseProduct.POSTGRES) && mStatsObj.getBitVector() == null) { + // workaround for DN bug in persisting nulls in pg bytea column + // instead set empty bit vector with header. + mStatsObj.setBitVector(new byte[] {'H','L'}); + } pm.makePersistent(mStatsObj); } } finally { @@ -8801,7 +8814,7 @@ private static void clearOutPmfClassLoaderCache(PersistenceManagerFactory pmf) { pmCache.setAccessible(true); Set pmSet = (Set)pmCache.get(pmf); for (JDOPersistenceManager pm : pmSet) { - org.datanucleus.ExecutionContext ec = (org.datanucleus.ExecutionContext)pm.getExecutionContext(); + org.datanucleus.ExecutionContext ec = pm.getExecutionContext(); if (ec instanceof org.datanucleus.ExecutionContextThreadedImpl) { ClassLoaderResolver clr = ((org.datanucleus.ExecutionContextThreadedImpl)ec).getClassLoaderResolver(); clearClr(clr); diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SQLGenerator.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SQLGenerator.java index 8268af9559..5b4d4bdf9b 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SQLGenerator.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SQLGenerator.java @@ -38,6 +38,7 @@ Licensed to the Apache Software Foundation (ASF) under one public final class SQLGenerator { static final private Logger LOG = LoggerFactory.getLogger(SQLGenerator.class.getName()); private final DatabaseProduct dbProduct; + private final Configuration conf; public SQLGenerator(DatabaseProduct dbProduct, Configuration conf) { @@ -169,4 +170,9 @@ public String addLimitClause(int numRows, String noSelectsqlQuery) throws MetaEx throw new MetaException(msg); } } + + public DatabaseProduct getDbProduct() { + return dbProduct; + } + } -- 2.13.5 (Apple Git-94)