commit 24b68775043d04327fb3154603addcfab219a299 Author: Janaki Lahorani Date: Mon Apr 2 13:42:39 2018 -0700 HIVE-17824: Enhance MSCK command to support repairing stale partitions in HMS Change-Id: I3657cc03267739eddc6cae7089fdb83807cdce35 diff --git itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java index 024fa1baaef8d642509c5e339f684024f3a11334..52d9668e28392017c82f4918882328c6496d98f6 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java +++ itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java @@ -110,7 +110,8 @@ public String get() { ".*at com\\.jolbox.*", ".*at com\\.zaxxer.*", "org\\.apache\\.hadoop\\.hive\\.metastore\\.model\\.MConstraint@([0-9]|[a-z])*", - "^Repair: Added partition to metastore.*" + "^Repair: Added partition to metastore.*", + "^Repair: Dropped partition from metastore.*" }); public QOutProcessor(FsType fsType) { diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java index fb1efe01dcddb4fd77f05918e46d742922dd313d..9fe866433037e081b91696b63c80b86181632891 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java @@ -21,6 +21,7 @@ import static org.apache.commons.lang.StringUtils.join; import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_CATALOG_NAME; import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_STORAGE; +import static org.apache.hadoop.hive.ql.parse.DDLSemanticAnalyzer.makeBinaryPredicate; import java.io.BufferedWriter; import java.io.DataOutputStream; @@ -205,6 +206,9 @@ import org.apache.hadoop.hive.ql.plan.DropWMMappingDesc; import org.apache.hadoop.hive.ql.plan.DropWMPoolDesc; import org.apache.hadoop.hive.ql.plan.DropWMTriggerDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; import org.apache.hadoop.hive.ql.plan.FileMergeDesc; import org.apache.hadoop.hive.ql.plan.GrantDesc; import org.apache.hadoop.hive.ql.plan.GrantRevokeRoleDDL; @@ -273,6 +277,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; @@ -2133,56 +2138,85 @@ private int msck(Hive db, MsckDesc msckDesc) { String[] names = Utilities.getDbTableName(msckDesc.getTableName()); checker.checkMetastore(names[0], names[1], msckDesc.getPartSpecs(), result); Set partsNotInMs = result.getPartitionsNotInMs(); - if (msckDesc.isRepairPartitions() && !partsNotInMs.isEmpty()) { - AbstractList vals = null; - String settingStr = HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_MSCK_PATH_VALIDATION); - boolean doValidate = !("ignore".equals(settingStr)); - boolean doSkip = doValidate && "skip".equals(settingStr); - // The default setting is "throw"; assume doValidate && !doSkip means throw. - if (doValidate) { - // Validate that we can add partition without escaping. Escaping was originally intended - // to avoid creating invalid HDFS paths; however, if we escape the HDFS path (that we - // deem invalid but HDFS actually supports - it is possible to create HDFS paths with - // unprintable characters like ASCII 7), metastore will create another directory instead - // of the one we are trying to "repair" here. - Iterator iter = partsNotInMs.iterator(); - while (iter.hasNext()) { - CheckResult.PartitionResult part = iter.next(); - try { - vals = Warehouse.makeValsFromName(part.getPartitionName(), vals); - } catch (MetaException ex) { - throw new HiveException(ex); - } - for (String val : vals) { - String escapedPath = FileUtils.escapePathName(val); - assert escapedPath != null; - if (escapedPath.equals(val)) { - continue; + Set partsNotInFs = result.getPartitionsNotOnFs(); + + if (msckDesc.isRepairPartitions()) { + // Repair metadata in HMS + + Table table = db.getTable(msckDesc.getTableName()); + int maxRetries = conf.getIntVar(ConfVars.HIVE_MSCK_REPAIR_BATCH_MAX_RETRIES); + int decayingFactor = 2; + + if (msckDesc.isAddPartitions() && !partsNotInMs.isEmpty()) { + // MSCK called to add missing paritions into metastore and there are + // missing partitions. + + int batchSize = conf.getIntVar(ConfVars.HIVE_MSCK_REPAIR_BATCH_SIZE); + if (batchSize == 0) { + //batching is not enabled. Try to add all the partitions in one call + batchSize = partsNotInMs.size(); + } + + AbstractList vals = null; + String settingStr = HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_MSCK_PATH_VALIDATION); + boolean doValidate = !("ignore".equals(settingStr)); + boolean doSkip = doValidate && "skip".equals(settingStr); + // The default setting is "throw"; assume doValidate && !doSkip means throw. + if (doValidate) { + // Validate that we can add partition without escaping. Escaping was originally intended + // to avoid creating invalid HDFS paths; however, if we escape the HDFS path (that we + // deem invalid but HDFS actually supports - it is possible to create HDFS paths with + // unprintable characters like ASCII 7), metastore will create another directory instead + // of the one we are trying to "repair" here. + Iterator iter = partsNotInMs.iterator(); + while (iter.hasNext()) { + CheckResult.PartitionResult part = iter.next(); + try { + vals = Warehouse.makeValsFromName(part.getPartitionName(), vals); + } catch (MetaException ex) { + throw new HiveException(ex); } - String errorMsg = "Repair: Cannot add partition " + msckDesc.getTableName() - + ':' + part.getPartitionName() + " due to invalid characters in the name"; - if (doSkip) { - repairOutput.add(errorMsg); - iter.remove(); - } else { - throw new HiveException(errorMsg); + for (String val : vals) { + String escapedPath = FileUtils.escapePathName(val); + assert escapedPath != null; + if (escapedPath.equals(val)) { + continue; + } + String errorMsg = "Repair: Cannot add partition " + msckDesc.getTableName() + ':' + + part.getPartitionName() + " due to invalid characters in the name"; + if (doSkip) { + repairOutput.add(errorMsg); + iter.remove(); + } else { + throw new HiveException(errorMsg); + } } } } + try { + createPartitionsInBatches(db, repairOutput, partsNotInMs, table, batchSize, + decayingFactor, maxRetries); + } catch (Exception e) { + throw new HiveException(e); + } } - Table table = db.getTable(msckDesc.getTableName()); - int batchSize = conf.getIntVar(ConfVars.HIVE_MSCK_REPAIR_BATCH_SIZE); - int maxRetries = conf.getIntVar(ConfVars.HIVE_MSCK_REPAIR_BATCH_MAX_RETRIES); - int decayingFactor = 2; - if (batchSize == 0) { - //batching is not enabled. Try to add all the partitions in one call - batchSize = partsNotInMs.size(); - } - try { - createPartitionsInBatches(db, repairOutput, partsNotInMs, table, batchSize, - decayingFactor, maxRetries); - } catch (Exception e) { - throw new HiveException(e); + + if (msckDesc.isDropPartitions() && !partsNotInFs.isEmpty()) { + // MSCK called to drop stale paritions from metastore and there are + // stale partitions. + + int batchSize = conf.getIntVar(ConfVars.HIVE_MSCK_REPAIR_BATCH_SIZE); + if (batchSize == 0) { + //batching is not enabled. Try to drop all the partitions in one call + batchSize = partsNotInFs.size(); + } + + try { + dropPartitionsInBatches(db, repairOutput, partsNotInFs, table, batchSize, + decayingFactor, maxRetries); + } catch (Exception e) { + throw new HiveException(e); + } } } } catch (HiveException e) { @@ -2272,6 +2306,104 @@ public Void execute(int size) throws Exception { }.run(); } + @VisibleForTesting + void dropPartitionsInBatches(Hive db, List repairOutput, + Set partsNotInFs, Table table, int batchSize, int decayingFactor, + int maxRetries) throws Exception { + String dropMsgFormat = + "Repair: Dropped partition from metastore " + table.getTableName() + ":%s"; + // Copy of partitions that will be split into batches + Set batchWork = new HashSet<>(partsNotInFs); + + // Column Types of all partitioned columns. Used for generating partition specification + Map colTypes = new HashMap(); + for (FieldSchema fs : table.getPartitionKeys()) { + colTypes.put(fs.getName().toLowerCase(), fs.getType()); + } + + new RetryUtilities.ExponentiallyDecayingBatchWork(batchSize, decayingFactor, maxRetries) { + @Override + public Void execute(int size) throws Exception { + while (!batchWork.isEmpty()) { + int currentBatchSize = size; + + // to store the partitions that are currently being processed + List lastBatch = new ArrayList<>(currentBatchSize); + + // drop messages for the dropped partitions + List dropMsgs = new ArrayList<>(currentBatchSize); + + // partitions to be dropped in this batch + Map> partSpecs = + new HashMap>(); + + // Key to be used to save the partition to be dropped in partSpecs + int dropKey = 0; + + for (CheckResult.PartitionResult part : batchWork) { + // This batch is full: break out of for loop to execute + if (currentBatchSize == 0) { + break; + } + + // Split by "/" to identify partition parts + String[] parts = part.getPartitionName().split("/"); + + // The expression to identify the partition to be dropped + ExprNodeGenericFuncDesc expr = null; + + // Loop through the partitions and form the expression + for (int i = 0; i < parts.length; i++) { + // Split the partition predicate to identify column and value + String[] partExprParts = parts[i].split("="); + + // Partition Column + String partCol = partExprParts[0]; + + // Column Type + PrimitiveTypeInfo pti = TypeInfoFactory.getPrimitiveTypeInfo(colTypes.get(partCol)); + + // Form the expression node corresponding to column + ExprNodeColumnDesc column = new ExprNodeColumnDesc(pti, partCol, null, true); + + // Build the expression based on the partition predicate + ExprNodeGenericFuncDesc op = + makeBinaryPredicate("=", column, new ExprNodeConstantDesc(pti, partExprParts[1])); + + // the multiple parts to partition predicate are joined using and + expr = (expr == null) ? op : makeBinaryPredicate("and", expr, op); + } + + // Add the expression to partition specification + partSpecs.put(dropKey, Lists.newArrayList(expr)); + + // Increment dropKey to get a new key for hash map + ++dropKey; + + // Add the part to lastBatch to track the parition being dropped + lastBatch.add(part); + + // Update messages + dropMsgs.add(String.format(dropMsgFormat, part.getPartitionName())); + + // Decrement batch size. When this gets to 0, the batch will be executed + currentBatchSize--; + } + + // use drop table descriptor to drop partitions + DropTableDesc dropTblDesc = + new DropTableDesc(table.getFullyQualifiedName(), partSpecs, null, false, null); + dropPartitions(db, table, dropTblDesc); + + // if last batch is successful remove it from partsNotInFs + batchWork.removeAll(lastBatch); + repairOutput.addAll(dropMsgs); + } + return null; + } + }.run(); + } + /** * Write the result of msck to a writer. * diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java index cfe8af8e4c7f8cd630af0010b4e1ca713f072d3b..a8fa043c504d21d2c760d44d6fcae534b4a0d0a7 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java @@ -3644,6 +3644,44 @@ private void analyzeAlterTableArchive(String[] qualified, CommonTree ast, boolea } + /** + * Check if MSCK is called to add partitions. + * + * @param keyWord + * could be ADD, DROP or SYNC. ADD or SYNC will indicate that add partition is on. + * + * @return true if add is on; false otherwise + */ + private static boolean isMsckAddPartition(int keyWord) { + switch (keyWord) { + case HiveParser.KW_DROP: + return false; + case HiveParser.KW_SYNC: + case HiveParser.KW_ADD: + default: + return true; + } + } + + /** + * Check if MSCK is called to drop partitions. + * + * @param keyWord + * could be ADD, DROP or SYNC. DROP or SYNC will indicate that drop partition is on. + * + * @return true if drop is on; false otherwise + */ + private static boolean isMsckDropPartition(int keyWord) { + switch (keyWord) { + case HiveParser.KW_DROP: + case HiveParser.KW_SYNC: + return true; + case HiveParser.KW_ADD: + default: + return false; + } + } + /** * Verify that the information in the metastore matches up with the data on * the fs. @@ -3654,20 +3692,34 @@ private void analyzeAlterTableArchive(String[] qualified, CommonTree ast, boolea */ private void analyzeMetastoreCheck(CommonTree ast) throws SemanticException { String tableName = null; + + boolean addPartitions = true; + boolean dropPartitions = false; + boolean repair = false; if (ast.getChildCount() > 0) { repair = ast.getChild(0).getType() == HiveParser.KW_REPAIR; if (!repair) { tableName = getUnescapedName((ASTNode) ast.getChild(0)); + + if (ast.getChildCount() > 1) { + addPartitions = isMsckAddPartition(ast.getChild(1).getType()); + dropPartitions = isMsckDropPartition(ast.getChild(1).getType()); + } } else if (ast.getChildCount() > 1) { tableName = getUnescapedName((ASTNode) ast.getChild(1)); + + if (ast.getChildCount() > 2) { + addPartitions = isMsckAddPartition(ast.getChild(2).getType()); + dropPartitions = isMsckDropPartition(ast.getChild(2).getType()); + } } } Table tab = getTable(tableName); List> specs = getPartitionSpecs(tab, ast); outputs.add(new WriteEntity(tab, WriteEntity.WriteType.DDL_SHARED)); MsckDesc checkDesc = new MsckDesc(tableName, specs, ctx.getResFile(), - repair); + repair, addPartitions, dropPartitions); rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), checkDesc))); } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g index 3ff6a5e4e30b30c9ba64b8f0a96714cd7e55954c..09a436898444d37b1c04fcd258202d8f296fc4e0 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g +++ ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g @@ -373,6 +373,7 @@ KW_MANAGEMENT: 'MANAGEMENT'; KW_ACTIVE: 'ACTIVE'; KW_UNMANAGED: 'UNMANAGED'; KW_APPLICATION: 'APPLICATION'; +KW_SYNC: 'SYNC'; // Operators // NOTE: if you add a new function/operator, add it to sysFuncNames so that describe function _FUNC_ will work. diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g index 936657b02e23dcc264d8dfb1a0617e2a8e983f80..925783b49980ee8d376815af12b343a4c22496d5 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g +++ ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g @@ -490,6 +490,7 @@ import org.apache.hadoop.hive.conf.HiveConf; xlateMap.put("KW_CLUSTER", "CLUSTER"); xlateMap.put("KW_DISTRIBUTE", "DISTRIBUTE"); xlateMap.put("KW_SORT", "SORT"); + xlateMap.put("KW_SYNC", "SYNC"); xlateMap.put("KW_UNION", "UNION"); xlateMap.put("KW_INTERSECT", "INTERSECT"); xlateMap.put("KW_EXCEPT", "EXCEPT"); @@ -1807,8 +1808,11 @@ withAdminOption metastoreCheck @init { pushMsg("metastore check statement", state); } @after { popMsg(state); } - : KW_MSCK (repair=KW_REPAIR)? (KW_TABLE tableName partitionSpec? (COMMA partitionSpec)*)? - -> ^(TOK_MSCK $repair? (tableName partitionSpec*)?) + : KW_MSCK (repair=KW_REPAIR)? + (KW_TABLE tableName + ((add=KW_ADD | drop=KW_DROP | sync=KW_SYNC) (parts=KW_PARTITIONS))? | + (partitionSpec)?) + -> ^(TOK_MSCK $repair? tableName? $add? $drop? $sync? (partitionSpec*)?) ; resourceList diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/MsckDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/MsckDesc.java index 7e6def40c1d0e5a7f6bea5c498410ee68259ac6f..5d2307c168c48a1366fbf9ba6559288021d16acc 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/MsckDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/MsckDesc.java @@ -35,6 +35,8 @@ private ArrayList> partSpecs; private String resFile; private boolean repairPartitions; + private boolean addPartitions; + private boolean dropPartitions; /** * For serialization use only. @@ -42,20 +44,26 @@ public MsckDesc() { } - /** - * Description of a msck command. - * - * @param tableName - * Table to check, can be null. - * @param partSpecs - * Partition specification, can be null. - * @param resFile - * Where to save the output of the command - * @param repairPartitions - * remove stale / add new partitions found during the check - */ + /** + * Description of a msck command. + * + * @param tableName + * Table to check, can be null. + * @param partSpecs + * Partition specification, can be null. + * @param resFile + * Where to save the output of the command + * @param repairPartitions + * remove stale / add new partitions found during the check + * @param addPartitions + * find partitions that are missing from metastore, and add them when repairPartitions + * is set to true + * @param dropPartitions + * find stale partitions in metastore, and drop them when repairPartitions + * is set to true + */ public MsckDesc(String tableName, List> partSpecs, - Path resFile, boolean repairPartitions) { + Path resFile, boolean repairPartitions, boolean addPartitions, boolean dropPartitions) { super(); this.tableName = tableName; this.partSpecs = new ArrayList>(partSpecs.size()); @@ -64,6 +72,8 @@ public MsckDesc(String tableName, List> partSpecs, } this.resFile = resFile.toString(); this.repairPartitions = repairPartitions; + this.addPartitions = addPartitions; + this.dropPartitions = dropPartitions; } /** @@ -118,6 +128,20 @@ public boolean isRepairPartitions() { return repairPartitions; } + /** + * @return if missing partitions is to be found, and added with repair option + */ + public boolean isAddPartitions() { + return addPartitions; + } + + /** + * @return if stale partitions is to be found, and removed with repair option + */ + public boolean isDropPartitions() { + return dropPartitions; + } + /** * @param repairPartitions * stale / add new partitions found during the check diff --git ql/src/test/queries/clientpositive/msck_repair_drop.q ql/src/test/queries/clientpositive/msck_repair_drop.q new file mode 100644 index 0000000000000000000000000000000000000000..bc14d981729d330613fb9c4a32a4b71757ddd1da --- /dev/null +++ ql/src/test/queries/clientpositive/msck_repair_drop.q @@ -0,0 +1,180 @@ +set hive.mv.files.thread=0; +DROP TABLE IF EXISTS repairtable; + +CREATE TABLE repairtable(col STRING) PARTITIONED BY (p1 STRING, p2 STRING); + +-- repairtable will have partitions created with part keys p1=1, p1=2, p1=3, p1=4 and p1=5 +-- p1=2 will be used to test drop in 3 tests +-- 1. each partition is dropped individually: set hive.msck.repair.batch.size=1; +-- 2. partition are dropped in groups of 3: set hive.msck.repair.batch.size=3; +-- 3. all partitions are dropped in 1 shot: set hive.msck.repair.batch.size=0; +-- the same set of 10 partitions will be created between each drop attempts +-- p1=3, p1=4 and p1=5 will be used to test keywords add, drop and sync + +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=1/p2=11/p3=111; +dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=1/p2=11/p3=111/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=1/p2=12/p3=121; +dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=1/p2=12/p3=121/datafile; + +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=2/p2=21/p3=211; +dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=2/p2=21/p3=211/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=2/p2=22/p3=221; +dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=2/p2=22/p3=221/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=2/p2=23/p3=231; +dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=2/p2=23/p3=231/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=2/p2=24/p3=241; +dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=2/p2=24/p3=241/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=2/p2=25/p3=251; +dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=2/p2=25/p3=251/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=2/p2=26/p3=261; +dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=2/p2=26/p3=261/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=2/p2=27/p3=271; +dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=2/p2=27/p3=271/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=2/p2=28/p3=281; +dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=2/p2=28/p3=281/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=2/p2=29/p3=291; +dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=2/p2=29/p3=291/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=2/p2=210/p3=2101; +dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=2/p2=210/p3=2101/datafile; + +MSCK TABLE default.repairtable; +MSCK REPAIR TABLE default.repairtable; + +-- Now all 12 partitions are in +show partitions default.repairtable; + +-- Remove all p1=2 partitions from file system +dfs -rmr ${system:test.warehouse.dir}/repairtable/p1=2; + +-- test 1: each partition is dropped individually +set hive.msck.repair.batch.size=1; +MSCK TABLE default.repairtable DROP PARTITIONS; +MSCK REPAIR TABLE default.repairtable DROP PARTITIONS; +show partitions default.repairtable; + +-- Recreate p1=2 partitions +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=2/p2=21/p3=211; +dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=2/p2=21/p3=211/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=2/p2=22/p3=221; +dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=2/p2=22/p3=221/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=2/p2=23/p3=231; +dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=2/p2=23/p3=231/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=2/p2=24/p3=241; +dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=2/p2=24/p3=241/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=2/p2=25/p3=251; +dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=2/p2=25/p3=251/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=2/p2=26/p3=261; +dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=2/p2=26/p3=261/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=2/p2=27/p3=271; +dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=2/p2=27/p3=271/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=2/p2=28/p3=281; +dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=2/p2=28/p3=281/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=2/p2=29/p3=291; +dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=2/p2=29/p3=291/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=2/p2=210/p3=2101; +dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=2/p2=210/p3=2101/datafile; + +MSCK TABLE default.repairtable; +MSCK REPAIR TABLE default.repairtable; + +-- Now all 12 partitions are in +show partitions default.repairtable; + +-- Remove all p1=2 partitions from file system +dfs -rmr ${system:test.warehouse.dir}/repairtable/p1=2; + +-- test 2: partition are dropped in groups of 3 +set hive.msck.repair.batch.size=3; +MSCK TABLE default.repairtable DROP PARTITIONS; +MSCK REPAIR TABLE default.repairtable DROP PARTITIONS; +show partitions default.repairtable; + +-- Recreate p1=2 partitions +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=2/p2=21/p3=211; +dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=2/p2=21/p3=211/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=2/p2=22/p3=221; +dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=2/p2=22/p3=221/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=2/p2=23/p3=231; +dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=2/p2=23/p3=231/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=2/p2=24/p3=241; +dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=2/p2=24/p3=241/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=2/p2=25/p3=251; +dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=2/p2=25/p3=251/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=2/p2=26/p3=261; +dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=2/p2=26/p3=261/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=2/p2=27/p3=271; +dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=2/p2=27/p3=271/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=2/p2=28/p3=281; +dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=2/p2=28/p3=281/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=2/p2=29/p3=291; +dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=2/p2=29/p3=291/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=2/p2=210/p3=2101; +dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=2/p2=210/p3=2101/datafile; + +MSCK TABLE default.repairtable; +MSCK REPAIR TABLE default.repairtable; + +-- Now all 12 partitions are in +show partitions default.repairtable; + +-- Remove all p1=2 partitions from file system +dfs -rmr ${system:test.warehouse.dir}/repairtable/p1=2; + +-- test 3. all partitions are dropped in 1 shot +set hive.msck.repair.batch.size=0; +MSCK TABLE default.repairtable DROP PARTITIONS; +MSCK REPAIR TABLE default.repairtable DROP PARTITIONS; +show partitions default.repairtable; + +-- test add parition keyword: begin +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=3/p2=31/p3=311; +dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=3/p2=31/p3=311/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=3/p2=32/p3=321; +dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=3/p2=32/p3=321/datafile; + +MSCK TABLE default.repairtable; +MSCK REPAIR TABLE default.repairtable; +show partitions default.repairtable; + +-- Create p1=4 in filesystem +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=4/p2=41/p3=411; +dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=4/p2=41/p3=411/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=4/p2=42/p3=421; +dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=4/p2=42/p3=421/datafile; + +-- Remove p1=3 from filesystem +dfs -rmr ${system:test.warehouse.dir}/repairtable/p1=3; + +-- Status: p1=3 dropped from filesystem, but exists in metastore +-- p1=4 exists in filesystem but not in metastore +-- test add partition: only brings in p1=4 and doesn't remove p1=3 +MSCK TABLE default.repairtable ADD PARTITIONS; +MSCK REPAIR TABLE default.repairtable ADD PARTITIONS; +show partitions default.repairtable; +-- test add partition keyword: end + +-- test drop partition keyword: begin +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=5/p2=51/p3=511; +dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=5/p2=51/p3=511/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=5/p2=52/p3=521; +dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=5/p2=52/p3=521/datafile; + +-- Status: p1=3 removed from filesystem, but exists in metastore (as part of add test) +-- p1=5 exists in filesystem but not in metastore +-- test drop partition: only removes p1=3 from metastore but doesn't update metadata for p1=5 +MSCK TABLE default.repairtable DROP PARTITIONS; +MSCK REPAIR TABLE default.repairtable DROP PARTITIONS; +show partitions default.repairtable; +-- test drop partition keyword: end + +-- test sync partition keyword: begin +-- Remove p1=4 from filesystem +dfs -rmr ${system:test.warehouse.dir}/repairtable/p1=4; + +-- Status: p1=4 dropped from filesystem, but exists in metastore +-- p1=5 exists in filesystem but not in metastore (as part of drop test) +-- test sync partition: removes p1=4 from metastore and updates metadata for p1=5 +MSCK TABLE default.repairtable SYNC PARTITIONS; +MSCK REPAIR TABLE default.repairtable SYNC PARTITIONS; +show partitions default.repairtable; +-- test sync partition keyword: end diff --git ql/src/test/results/clientpositive/msck_repair_drop.q.out ql/src/test/results/clientpositive/msck_repair_drop.q.out new file mode 100644 index 0000000000000000000000000000000000000000..b105fd7248d53fac19ac1200a9ce83c8de892c4e --- /dev/null +++ ql/src/test/results/clientpositive/msck_repair_drop.q.out @@ -0,0 +1,327 @@ +PREHOOK: query: DROP TABLE IF EXISTS repairtable +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS repairtable +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE repairtable(col STRING) PARTITIONED BY (p1 STRING, p2 STRING) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@repairtable +POSTHOOK: query: CREATE TABLE repairtable(col STRING) PARTITIONED BY (p1 STRING, p2 STRING) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@repairtable +PREHOOK: query: MSCK TABLE default.repairtable +PREHOOK: type: MSCK +PREHOOK: Output: default@repairtable +POSTHOOK: query: MSCK TABLE default.repairtable +POSTHOOK: type: MSCK +POSTHOOK: Output: default@repairtable +Partitions not in metastore: repairtable:p1=1/p2=11 repairtable:p1=1/p2=12 repairtable:p1=2/p2=21 repairtable:p1=2/p2=210 repairtable:p1=2/p2=22 repairtable:p1=2/p2=23 repairtable:p1=2/p2=24 repairtable:p1=2/p2=25 repairtable:p1=2/p2=26 repairtable:p1=2/p2=27 repairtable:p1=2/p2=28 repairtable:p1=2/p2=29 +PREHOOK: query: MSCK REPAIR TABLE default.repairtable +PREHOOK: type: MSCK +PREHOOK: Output: default@repairtable +POSTHOOK: query: MSCK REPAIR TABLE default.repairtable +POSTHOOK: type: MSCK +POSTHOOK: Output: default@repairtable +Partitions not in metastore: repairtable:p1=1/p2=11 repairtable:p1=1/p2=12 repairtable:p1=2/p2=21 repairtable:p1=2/p2=210 repairtable:p1=2/p2=22 repairtable:p1=2/p2=23 repairtable:p1=2/p2=24 repairtable:p1=2/p2=25 repairtable:p1=2/p2=26 repairtable:p1=2/p2=27 repairtable:p1=2/p2=28 repairtable:p1=2/p2=29 +#### A masked pattern was here #### +PREHOOK: query: show partitions default.repairtable +PREHOOK: type: SHOWPARTITIONS +PREHOOK: Input: default@repairtable +POSTHOOK: query: show partitions default.repairtable +POSTHOOK: type: SHOWPARTITIONS +POSTHOOK: Input: default@repairtable +p1=1/p2=11 +p1=1/p2=12 +p1=2/p2=21 +p1=2/p2=210 +p1=2/p2=22 +p1=2/p2=23 +p1=2/p2=24 +p1=2/p2=25 +p1=2/p2=26 +p1=2/p2=27 +p1=2/p2=28 +p1=2/p2=29 +#### A masked pattern was here #### +PREHOOK: query: MSCK TABLE default.repairtable DROP PARTITIONS +PREHOOK: type: MSCK +PREHOOK: Output: default@repairtable +POSTHOOK: query: MSCK TABLE default.repairtable DROP PARTITIONS +POSTHOOK: type: MSCK +POSTHOOK: Output: default@repairtable +Partitions missing from filesystem: repairtable:p1=2/p2=21 repairtable:p1=2/p2=210 repairtable:p1=2/p2=22 repairtable:p1=2/p2=23 repairtable:p1=2/p2=24 repairtable:p1=2/p2=25 repairtable:p1=2/p2=26 repairtable:p1=2/p2=27 repairtable:p1=2/p2=28 repairtable:p1=2/p2=29 +PREHOOK: query: MSCK REPAIR TABLE default.repairtable DROP PARTITIONS +PREHOOK: type: MSCK +PREHOOK: Output: default@repairtable +POSTHOOK: query: MSCK REPAIR TABLE default.repairtable DROP PARTITIONS +POSTHOOK: type: MSCK +POSTHOOK: Output: default@repairtable +POSTHOOK: Output: default@repairtable@p1=2/p2=21 +POSTHOOK: Output: default@repairtable@p1=2/p2=210 +POSTHOOK: Output: default@repairtable@p1=2/p2=22 +POSTHOOK: Output: default@repairtable@p1=2/p2=23 +POSTHOOK: Output: default@repairtable@p1=2/p2=24 +POSTHOOK: Output: default@repairtable@p1=2/p2=25 +POSTHOOK: Output: default@repairtable@p1=2/p2=26 +POSTHOOK: Output: default@repairtable@p1=2/p2=27 +POSTHOOK: Output: default@repairtable@p1=2/p2=28 +POSTHOOK: Output: default@repairtable@p1=2/p2=29 +Partitions missing from filesystem: repairtable:p1=2/p2=21 repairtable:p1=2/p2=210 repairtable:p1=2/p2=22 repairtable:p1=2/p2=23 repairtable:p1=2/p2=24 repairtable:p1=2/p2=25 repairtable:p1=2/p2=26 repairtable:p1=2/p2=27 repairtable:p1=2/p2=28 repairtable:p1=2/p2=29 +#### A masked pattern was here #### +PREHOOK: query: show partitions default.repairtable +PREHOOK: type: SHOWPARTITIONS +PREHOOK: Input: default@repairtable +POSTHOOK: query: show partitions default.repairtable +POSTHOOK: type: SHOWPARTITIONS +POSTHOOK: Input: default@repairtable +p1=1/p2=11 +p1=1/p2=12 +PREHOOK: query: MSCK TABLE default.repairtable +PREHOOK: type: MSCK +PREHOOK: Output: default@repairtable +POSTHOOK: query: MSCK TABLE default.repairtable +POSTHOOK: type: MSCK +POSTHOOK: Output: default@repairtable +Partitions not in metastore: repairtable:p1=2/p2=21 repairtable:p1=2/p2=210 repairtable:p1=2/p2=22 repairtable:p1=2/p2=23 repairtable:p1=2/p2=24 repairtable:p1=2/p2=25 repairtable:p1=2/p2=26 repairtable:p1=2/p2=27 repairtable:p1=2/p2=28 repairtable:p1=2/p2=29 +PREHOOK: query: MSCK REPAIR TABLE default.repairtable +PREHOOK: type: MSCK +PREHOOK: Output: default@repairtable +POSTHOOK: query: MSCK REPAIR TABLE default.repairtable +POSTHOOK: type: MSCK +POSTHOOK: Output: default@repairtable +Partitions not in metastore: repairtable:p1=2/p2=21 repairtable:p1=2/p2=210 repairtable:p1=2/p2=22 repairtable:p1=2/p2=23 repairtable:p1=2/p2=24 repairtable:p1=2/p2=25 repairtable:p1=2/p2=26 repairtable:p1=2/p2=27 repairtable:p1=2/p2=28 repairtable:p1=2/p2=29 +#### A masked pattern was here #### +PREHOOK: query: show partitions default.repairtable +PREHOOK: type: SHOWPARTITIONS +PREHOOK: Input: default@repairtable +POSTHOOK: query: show partitions default.repairtable +POSTHOOK: type: SHOWPARTITIONS +POSTHOOK: Input: default@repairtable +p1=1/p2=11 +p1=1/p2=12 +p1=2/p2=21 +p1=2/p2=210 +p1=2/p2=22 +p1=2/p2=23 +p1=2/p2=24 +p1=2/p2=25 +p1=2/p2=26 +p1=2/p2=27 +p1=2/p2=28 +p1=2/p2=29 +#### A masked pattern was here #### +PREHOOK: query: MSCK TABLE default.repairtable DROP PARTITIONS +PREHOOK: type: MSCK +PREHOOK: Output: default@repairtable +POSTHOOK: query: MSCK TABLE default.repairtable DROP PARTITIONS +POSTHOOK: type: MSCK +POSTHOOK: Output: default@repairtable +Partitions missing from filesystem: repairtable:p1=2/p2=21 repairtable:p1=2/p2=210 repairtable:p1=2/p2=22 repairtable:p1=2/p2=23 repairtable:p1=2/p2=24 repairtable:p1=2/p2=25 repairtable:p1=2/p2=26 repairtable:p1=2/p2=27 repairtable:p1=2/p2=28 repairtable:p1=2/p2=29 +PREHOOK: query: MSCK REPAIR TABLE default.repairtable DROP PARTITIONS +PREHOOK: type: MSCK +PREHOOK: Output: default@repairtable +POSTHOOK: query: MSCK REPAIR TABLE default.repairtable DROP PARTITIONS +POSTHOOK: type: MSCK +POSTHOOK: Output: default@repairtable +POSTHOOK: Output: default@repairtable@p1=2/p2=21 +POSTHOOK: Output: default@repairtable@p1=2/p2=210 +POSTHOOK: Output: default@repairtable@p1=2/p2=22 +POSTHOOK: Output: default@repairtable@p1=2/p2=23 +POSTHOOK: Output: default@repairtable@p1=2/p2=24 +POSTHOOK: Output: default@repairtable@p1=2/p2=25 +POSTHOOK: Output: default@repairtable@p1=2/p2=26 +POSTHOOK: Output: default@repairtable@p1=2/p2=27 +POSTHOOK: Output: default@repairtable@p1=2/p2=28 +POSTHOOK: Output: default@repairtable@p1=2/p2=29 +Partitions missing from filesystem: repairtable:p1=2/p2=21 repairtable:p1=2/p2=210 repairtable:p1=2/p2=22 repairtable:p1=2/p2=23 repairtable:p1=2/p2=24 repairtable:p1=2/p2=25 repairtable:p1=2/p2=26 repairtable:p1=2/p2=27 repairtable:p1=2/p2=28 repairtable:p1=2/p2=29 +#### A masked pattern was here #### +PREHOOK: query: show partitions default.repairtable +PREHOOK: type: SHOWPARTITIONS +PREHOOK: Input: default@repairtable +POSTHOOK: query: show partitions default.repairtable +POSTHOOK: type: SHOWPARTITIONS +POSTHOOK: Input: default@repairtable +p1=1/p2=11 +p1=1/p2=12 +PREHOOK: query: MSCK TABLE default.repairtable +PREHOOK: type: MSCK +PREHOOK: Output: default@repairtable +POSTHOOK: query: MSCK TABLE default.repairtable +POSTHOOK: type: MSCK +POSTHOOK: Output: default@repairtable +Partitions not in metastore: repairtable:p1=2/p2=21 repairtable:p1=2/p2=210 repairtable:p1=2/p2=22 repairtable:p1=2/p2=23 repairtable:p1=2/p2=24 repairtable:p1=2/p2=25 repairtable:p1=2/p2=26 repairtable:p1=2/p2=27 repairtable:p1=2/p2=28 repairtable:p1=2/p2=29 +PREHOOK: query: MSCK REPAIR TABLE default.repairtable +PREHOOK: type: MSCK +PREHOOK: Output: default@repairtable +POSTHOOK: query: MSCK REPAIR TABLE default.repairtable +POSTHOOK: type: MSCK +POSTHOOK: Output: default@repairtable +Partitions not in metastore: repairtable:p1=2/p2=21 repairtable:p1=2/p2=210 repairtable:p1=2/p2=22 repairtable:p1=2/p2=23 repairtable:p1=2/p2=24 repairtable:p1=2/p2=25 repairtable:p1=2/p2=26 repairtable:p1=2/p2=27 repairtable:p1=2/p2=28 repairtable:p1=2/p2=29 +#### A masked pattern was here #### +PREHOOK: query: show partitions default.repairtable +PREHOOK: type: SHOWPARTITIONS +PREHOOK: Input: default@repairtable +POSTHOOK: query: show partitions default.repairtable +POSTHOOK: type: SHOWPARTITIONS +POSTHOOK: Input: default@repairtable +p1=1/p2=11 +p1=1/p2=12 +p1=2/p2=21 +p1=2/p2=210 +p1=2/p2=22 +p1=2/p2=23 +p1=2/p2=24 +p1=2/p2=25 +p1=2/p2=26 +p1=2/p2=27 +p1=2/p2=28 +p1=2/p2=29 +#### A masked pattern was here #### +PREHOOK: query: MSCK TABLE default.repairtable DROP PARTITIONS +PREHOOK: type: MSCK +PREHOOK: Output: default@repairtable +POSTHOOK: query: MSCK TABLE default.repairtable DROP PARTITIONS +POSTHOOK: type: MSCK +POSTHOOK: Output: default@repairtable +Partitions missing from filesystem: repairtable:p1=2/p2=21 repairtable:p1=2/p2=210 repairtable:p1=2/p2=22 repairtable:p1=2/p2=23 repairtable:p1=2/p2=24 repairtable:p1=2/p2=25 repairtable:p1=2/p2=26 repairtable:p1=2/p2=27 repairtable:p1=2/p2=28 repairtable:p1=2/p2=29 +PREHOOK: query: MSCK REPAIR TABLE default.repairtable DROP PARTITIONS +PREHOOK: type: MSCK +PREHOOK: Output: default@repairtable +POSTHOOK: query: MSCK REPAIR TABLE default.repairtable DROP PARTITIONS +POSTHOOK: type: MSCK +POSTHOOK: Output: default@repairtable +POSTHOOK: Output: default@repairtable@p1=2/p2=21 +POSTHOOK: Output: default@repairtable@p1=2/p2=210 +POSTHOOK: Output: default@repairtable@p1=2/p2=22 +POSTHOOK: Output: default@repairtable@p1=2/p2=23 +POSTHOOK: Output: default@repairtable@p1=2/p2=24 +POSTHOOK: Output: default@repairtable@p1=2/p2=25 +POSTHOOK: Output: default@repairtable@p1=2/p2=26 +POSTHOOK: Output: default@repairtable@p1=2/p2=27 +POSTHOOK: Output: default@repairtable@p1=2/p2=28 +POSTHOOK: Output: default@repairtable@p1=2/p2=29 +Partitions missing from filesystem: repairtable:p1=2/p2=21 repairtable:p1=2/p2=210 repairtable:p1=2/p2=22 repairtable:p1=2/p2=23 repairtable:p1=2/p2=24 repairtable:p1=2/p2=25 repairtable:p1=2/p2=26 repairtable:p1=2/p2=27 repairtable:p1=2/p2=28 repairtable:p1=2/p2=29 +#### A masked pattern was here #### +PREHOOK: query: show partitions default.repairtable +PREHOOK: type: SHOWPARTITIONS +PREHOOK: Input: default@repairtable +POSTHOOK: query: show partitions default.repairtable +POSTHOOK: type: SHOWPARTITIONS +POSTHOOK: Input: default@repairtable +p1=1/p2=11 +p1=1/p2=12 +PREHOOK: query: MSCK TABLE default.repairtable +PREHOOK: type: MSCK +PREHOOK: Output: default@repairtable +POSTHOOK: query: MSCK TABLE default.repairtable +POSTHOOK: type: MSCK +POSTHOOK: Output: default@repairtable +Partitions not in metastore: repairtable:p1=3/p2=31 repairtable:p1=3/p2=32 +PREHOOK: query: MSCK REPAIR TABLE default.repairtable +PREHOOK: type: MSCK +PREHOOK: Output: default@repairtable +POSTHOOK: query: MSCK REPAIR TABLE default.repairtable +POSTHOOK: type: MSCK +POSTHOOK: Output: default@repairtable +Partitions not in metastore: repairtable:p1=3/p2=31 repairtable:p1=3/p2=32 +#### A masked pattern was here #### +PREHOOK: query: show partitions default.repairtable +PREHOOK: type: SHOWPARTITIONS +PREHOOK: Input: default@repairtable +POSTHOOK: query: show partitions default.repairtable +POSTHOOK: type: SHOWPARTITIONS +POSTHOOK: Input: default@repairtable +p1=1/p2=11 +p1=1/p2=12 +p1=3/p2=31 +p1=3/p2=32 +#### A masked pattern was here #### +PREHOOK: query: MSCK TABLE default.repairtable ADD PARTITIONS +PREHOOK: type: MSCK +PREHOOK: Output: default@repairtable +POSTHOOK: query: MSCK TABLE default.repairtable ADD PARTITIONS +POSTHOOK: type: MSCK +POSTHOOK: Output: default@repairtable +Partitions not in metastore: repairtable:p1=4/p2=41 repairtable:p1=4/p2=42 +Partitions missing from filesystem: repairtable:p1=3/p2=31 repairtable:p1=3/p2=32 +PREHOOK: query: MSCK REPAIR TABLE default.repairtable ADD PARTITIONS +PREHOOK: type: MSCK +PREHOOK: Output: default@repairtable +POSTHOOK: query: MSCK REPAIR TABLE default.repairtable ADD PARTITIONS +POSTHOOK: type: MSCK +POSTHOOK: Output: default@repairtable +Partitions not in metastore: repairtable:p1=4/p2=41 repairtable:p1=4/p2=42 +Partitions missing from filesystem: repairtable:p1=3/p2=31 repairtable:p1=3/p2=32 +#### A masked pattern was here #### +PREHOOK: query: show partitions default.repairtable +PREHOOK: type: SHOWPARTITIONS +PREHOOK: Input: default@repairtable +POSTHOOK: query: show partitions default.repairtable +POSTHOOK: type: SHOWPARTITIONS +POSTHOOK: Input: default@repairtable +p1=1/p2=11 +p1=1/p2=12 +p1=3/p2=31 +p1=3/p2=32 +p1=4/p2=41 +p1=4/p2=42 +PREHOOK: query: MSCK TABLE default.repairtable DROP PARTITIONS +PREHOOK: type: MSCK +PREHOOK: Output: default@repairtable +POSTHOOK: query: MSCK TABLE default.repairtable DROP PARTITIONS +POSTHOOK: type: MSCK +POSTHOOK: Output: default@repairtable +Partitions not in metastore: repairtable:p1=5/p2=51 repairtable:p1=5/p2=52 +Partitions missing from filesystem: repairtable:p1=3/p2=31 repairtable:p1=3/p2=32 +PREHOOK: query: MSCK REPAIR TABLE default.repairtable DROP PARTITIONS +PREHOOK: type: MSCK +PREHOOK: Output: default@repairtable +POSTHOOK: query: MSCK REPAIR TABLE default.repairtable DROP PARTITIONS +POSTHOOK: type: MSCK +POSTHOOK: Output: default@repairtable +POSTHOOK: Output: default@repairtable@p1=3/p2=31 +POSTHOOK: Output: default@repairtable@p1=3/p2=32 +Partitions not in metastore: repairtable:p1=5/p2=51 repairtable:p1=5/p2=52 +Partitions missing from filesystem: repairtable:p1=3/p2=31 repairtable:p1=3/p2=32 +#### A masked pattern was here #### +PREHOOK: query: show partitions default.repairtable +PREHOOK: type: SHOWPARTITIONS +PREHOOK: Input: default@repairtable +POSTHOOK: query: show partitions default.repairtable +POSTHOOK: type: SHOWPARTITIONS +POSTHOOK: Input: default@repairtable +p1=1/p2=11 +p1=1/p2=12 +p1=4/p2=41 +p1=4/p2=42 +#### A masked pattern was here #### +PREHOOK: query: MSCK TABLE default.repairtable SYNC PARTITIONS +PREHOOK: type: MSCK +PREHOOK: Output: default@repairtable +POSTHOOK: query: MSCK TABLE default.repairtable SYNC PARTITIONS +POSTHOOK: type: MSCK +POSTHOOK: Output: default@repairtable +Partitions not in metastore: repairtable:p1=5/p2=51 repairtable:p1=5/p2=52 +Partitions missing from filesystem: repairtable:p1=4/p2=41 repairtable:p1=4/p2=42 +PREHOOK: query: MSCK REPAIR TABLE default.repairtable SYNC PARTITIONS +PREHOOK: type: MSCK +PREHOOK: Output: default@repairtable +POSTHOOK: query: MSCK REPAIR TABLE default.repairtable SYNC PARTITIONS +POSTHOOK: type: MSCK +POSTHOOK: Output: default@repairtable +POSTHOOK: Output: default@repairtable@p1=4/p2=41 +POSTHOOK: Output: default@repairtable@p1=4/p2=42 +Partitions not in metastore: repairtable:p1=5/p2=51 repairtable:p1=5/p2=52 +Partitions missing from filesystem: repairtable:p1=4/p2=41 repairtable:p1=4/p2=42 +#### A masked pattern was here #### +PREHOOK: query: show partitions default.repairtable +PREHOOK: type: SHOWPARTITIONS +PREHOOK: Input: default@repairtable +POSTHOOK: query: show partitions default.repairtable +POSTHOOK: type: SHOWPARTITIONS +POSTHOOK: Input: default@repairtable +p1=1/p2=11 +p1=1/p2=12 +p1=5/p2=51 +p1=5/p2=52