diff --git a/src/java/org/apache/hadoop/hbase/HStoreKey.java b/src/java/org/apache/hadoop/hbase/HStoreKey.java index e02234d..eba016c 100644 --- a/src/java/org/apache/hadoop/hbase/HStoreKey.java +++ b/src/java/org/apache/hadoop/hbase/HStoreKey.java @@ -455,7 +455,7 @@ public class HStoreKey implements WritableComparable, HeapSize { return getDelimiter(b, 0, b.length, COLUMN_FAMILY_DELIMITER); } - private static int getRequiredDelimiterInReverse(final byte [] b, + public static int getRequiredDelimiterInReverse(final byte [] b, final int offset, final int length, final int delimiter) { int index = getDelimiterInReverse(b, offset, length, delimiter); if (index < 0) { @@ -470,7 +470,7 @@ public class HStoreKey implements WritableComparable, HeapSize { * @return Index of delimiter having started from end of b moving * leftward. */ - private static int getDelimiter(final byte [] b, int offset, final int length, + public static int getDelimiter(final byte [] b, int offset, final int length, final int delimiter) { if (b == null) { throw new NullPointerException(); diff --git a/src/java/org/apache/hadoop/hbase/HbaseFsck.java b/src/java/org/apache/hadoop/hbase/HbaseFsck.java index 6e8ae66..02c51fd 100644 --- a/src/java/org/apache/hadoop/hbase/HbaseFsck.java +++ b/src/java/org/apache/hadoop/hbase/HbaseFsck.java @@ -1,11 +1,318 @@ +/* + * Copyright 2009 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.hadoop.hbase; -/** - * Created by IntelliJ IDEA. - * User: ryan - * Date: Mar 28, 2009 - * Time: 12:49:58 AM - * To change this template use File | Settings | File Templates. - */ +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Writables; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Scanner; +import org.apache.hadoop.hbase.io.RowResult; +import org.apache.hadoop.hbase.filter.RowFilterInterface; +import org.apache.hadoop.hbase.filter.WhileMatchRowFilter; +import org.apache.hadoop.hbase.filter.PrefixRowFilter; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileStatus; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Map; +import java.util.TreeMap; +import java.util.List; +import java.util.SortedMap; +import java.util.Collections; +import java.util.Comparator; + public class HbaseFsck { + private FileSystem fs; + private HBaseConfiguration conf; + private String tableName; + private byte[] fsckTable; + private byte[][] columns; + private HTable meta; + private ArrayList regions; + private String hbaseRootDir; + private boolean fixIt; + + + public static void main(String []args) throws IOException { + if (args.length < 2) { + System.out.println("Usage: HbaseFsck table_name [fix|no]"); + return; + } + boolean fix = false; + if (args[1].equals("fix")) { + fix = true; + System.out.println("Fix flag turned on"); + } + HbaseFsck fsck = new HbaseFsck(args[0], fix); + fsck.fsck(); + } + + public HbaseFsck(String tableName, boolean fixIt) throws IOException { + this.tableName = tableName; + this.fixIt = fixIt; + + conf = new HBaseConfiguration(); + fs = FileSystem.get(conf); + + fsckTable = Bytes.toBytes(tableName + ",,"); + columns = new byte[][] { + Bytes.toBytes("info:regioninfo"), + Bytes.toBytes("historian:split") + }; + meta = new HTable(conf, ".META."); + meta.setScannerCaching(500); + RowFilterInterface filter = + new WhileMatchRowFilter( + new PrefixRowFilter(Bytes.toBytes(tableName + ","))); + Scanner s = meta.getScanner(columns, fsckTable, filter); + regions = new ArrayList(); + + RowResult res; + while ((res = s.next()) != null) { + regions.add(res); + } + s.close(); + + + hbaseRootDir = conf.get(HConstants.HBASE_DIR); + } + + + public boolean fsck() throws IOException { + Path tablePath = new Path(hbaseRootDir + "/" + tableName); + FileStatus tablePathStatus = fs.getFileStatus(tablePath); + if (!tablePathStatus.isDir()) { + System.out.println("Ahh, there's your problem, '"+ tablePath + "' isn't a directory!"); + return false; + } + + // We have what the .META. thinks is the list of regions. Now look at the files on disk: + //List regionDirs = new ArrayList(); + FileStatus [] regionDirs = fs.listStatus( + tablePath); + + if (regions.isEmpty()) { + System.out.println("No region directories in path: " + tablePath); + return false; + } + + System.out.println("Raw region count: " + regions.size()); + + + // check dups of the regions... + SortedMap> dupes = getDupes(regions); + + compareDiskAndMeta(dupes, regionDirs); + + return true; + } + + private void compareDiskAndMeta(SortedMap> dupes, + FileStatus[] rds) throws IOException { + Map regionDirs = new TreeMap(); + for ( FileStatus fs : rds ) { + String name = fs.getPath().getName(); + if (!name.equals("compaction.dir")) { + regionDirs.put(fs.getPath().getName(), fs); + } + } + + System.out.println("Disk region dir count: " + regionDirs.size()); + System.out.println(".META. same-start-key region count: " + dupes.size()); + + // a map between directory and the list of regions with that start key, + // the first one in the list is the matching one. the other ones are + // bogus. + SortedMap> matched = + new TreeMap>(); + List rms = new ArrayList(); + boolean containsExtraMetas = false; + + for( Map.Entry> e : dupes.entrySet()) { + // byte[] is the start key. + for(RowResult reg : e.getValue()) { + // ok find match + String encodedName = Integer.toString(HRegionInfo.encodeRegionName(reg.getRow())); + // find it in the regionDirs. + if (regionDirs.containsKey(encodedName)) { + // foundddd it. + System.out.println("Found encoded name: " + encodedName + ", others: " + + (e.getValue().size()-1)); + regionDirs.remove(encodedName); + rms.add(e.getKey()); + + // do some junk: + List rs = new ArrayList(e.getValue()); + rs.remove(reg); + rs.add(0, reg); + + matched.put(encodedName, rs); + + if (rs.size() > 1) { + containsExtraMetas = true; + } + // should break here. + } + } + } + // make a copy, dont ruin the caller's copy. + SortedMap> dupes2 = + new TreeMap>(dupes); + for( byte[] rm : rms ) { + dupes2.remove(rm); + } + // print some data: + if (!regionDirs.isEmpty()) { + System.out.println("Some dirs exist but don't seem to have .META. entries for:"); + for(Map.Entry e : regionDirs.entrySet()) { + System.out.println(e.getKey()); + } + // Would need to recover the .META. entries from the store files. + } + if (!dupes2.isEmpty()) { + System.out.println("Some META entries dont seem to have region dirs on disk:"); + for( Map.Entry> e : dupes2.entrySet() ) { + System.out.println("start key: " + Bytes.toString(e.getKey())); + for( RowResult r : e.getValue()) { + System.out.println("- META: " + HRegionInfo.encodeRegionName(r.getRow()) + ":" + + Bytes.toString(r.getRow())); + } + } + // A Meta region without an entry on disk = data loss, + // only recovery is to nuke the .META. entry and suck it up. + } + if (regionDirs.isEmpty() && dupes2.isEmpty() && containsExtraMetas) { + System.out.println("looks like you have extra regions in META, delete them and " + + "all might be well"); + // Trivial fix case - delete bogus entries in .META. + + checkAndProposeFix(matched); + + } + } + + private void checkAndProposeFix(SortedMap> matched) throws IOException { + // Build the list of RowResult & HRegionInfos. + // list of HRegionInfo... the new region list. + List regions = new ArrayList(); + // sorted order still. good good. + for( Map.Entry> e: matched.entrySet()) { + HRegionInfo info = (HRegionInfo) Writables.getWritable( + e.getValue().get(0).get("info:regioninfo").getValue(), + new HRegionInfo()); + // compare some things: + if (!e.getKey().equals(Integer.toString(info.getEncodedName()))) { + System.out.println("Encoded name in HRI doesnt match: " + info); + } + regions.add(info); + } + Comparator c = new Comparator() { + public int compare(HRegionInfo hRegionInfo, HRegionInfo hRegionInfo1) { + return Bytes.compareTo(hRegionInfo.getStartKey(), + hRegionInfo1.getStartKey()); + } + }; + Collections.sort(regions, c); + // having created all the regions, compare them 1 at a time... + byte [] prevKey = new byte[0]; + HRegionInfo prevInfo = null; + boolean lastRegionSighted = false; + System.out.println("Reconcilliating region count: " + regions.size()); + for ( HRegionInfo region : regions ) { + // start key should match end key. + if (region.getStartKey().length == 0) { + // first... + prevKey = region.getEndKey(); + continue; + } + if (region.getEndKey().length == 0) { + // at the end... + lastRegionSighted = true; + continue; + } + if (lastRegionSighted) { + System.out.println("Discrepency - more regions when the last region " + + "was already sighted at region: " + region); + System.out.println("Previous region was: " + prevInfo); + return; + } + if (Bytes.compareTo(prevKey, region.getStartKey()) != 0) { + // omgeee + System.out.println("Discrepency between start/end key at end region: " + + region); + System.out.println("Previous region was: " + prevInfo); + return; + } + prevKey = region.getEndKey(); + prevInfo = region; + } + // we can nuke all other rows except for the ones in 'regions'. + if (!fixIt) { + System.out.println("Not fixing"); + return; + } + if (fixIt) { + System.out.println("Fixing your .META. by deleting extra entries:"); + for( List e : matched.values()) { + // skip first, delete others: + for (int i = 1; i < e.size(); i++) { + System.out.println("Deleting entry: " + Bytes.toString(e.get(i).getRow())); + meta.deleteAll(e.get(i).getRow()); + } + } + } + } + + /** + * This clusters the RowResults into clusters that all have the + * same start-key. + */ + private static SortedMap> + getDupes(ArrayList regions) { + // the partition of duplicate regions. + SortedMap> dupes = + new TreeMap>(Bytes.BYTES_COMPARATOR); + + byte [] prev; + for(RowResult region : regions) { + // extract start key part... + byte [] key = region.getRow(); + int start = HStoreKey.getDelimiter(key, 0, key.length, + HRegionInfo.DELIMITER); + int end = HStoreKey.getRequiredDelimiterInReverse(key, + start, key.length - start, HRegionInfo.DELIMITER); + if (end < start) { + System.out.println("problem with region key: " + Bytes.toString(key)); + continue; + } + byte [] startKey = new byte[end-start-1]; + System.arraycopy(key, start+1, startKey, 0, startKey.length); + if (!dupes.containsKey(startKey)) { + dupes.put(startKey, new ArrayList()); + } + dupes.get(startKey).add(region); + } + + return dupes; + } }