diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java index fb91da4..0d93ca0 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java @@ -23,6 +23,7 @@ import static org.apache.hadoop.util.StringUtils.stringifyException; import java.io.BufferedWriter; import java.io.DataOutput; +import java.io.DataOutputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.OutputStreamWriter; @@ -91,8 +92,11 @@ import org.apache.hadoop.hive.ql.metadata.HiveMetaStoreChecker; import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler; import org.apache.hadoop.hive.ql.metadata.HiveUtils; import org.apache.hadoop.hive.ql.metadata.InvalidTableException; +import org.apache.hadoop.hive.ql.metadata.JsonMetaDataFormatter; import org.apache.hadoop.hive.ql.metadata.MetaDataFormatUtils; +import org.apache.hadoop.hive.ql.metadata.MetaDataFormatter; import org.apache.hadoop.hive.ql.metadata.Partition; +import org.apache.hadoop.hive.ql.metadata.TextMetaDataFormatter; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.parse.AlterTablePartMergeFilesDesc; import org.apache.hadoop.hive.ql.plan.AddPartitionDesc; @@ -166,6 +170,8 @@ public class DDLTask extends Task implements Serializable { private static String INTERMEDIATE_ORIGINAL_DIR_SUFFIX; private static String INTERMEDIATE_EXTRACTED_DIR_SUFFIX; + private MetaDataFormatter formatter; + @Override public boolean requireLock() { return this.work != null && this.work.getNeedLock(); @@ -180,6 +186,13 @@ public class DDLTask extends Task implements Serializable { super.initialize(conf, queryPlan, ctx); this.conf = conf; + // Pick the formatter to use to display the results. Either the + // normal human readable output or a json object. + if ("json".equals(conf.get("hive.format"))) + formatter = new JsonMetaDataFormatter(); + else + formatter = new TextMetaDataFormatter(); + INTERMEDIATE_ARCHIVED_DIR_SUFFIX = HiveConf.getVar(conf, ConfVars.METASTORE_INT_ARCHIVED); INTERMEDIATE_ORIGINAL_DIR_SUFFIX = @@ -1812,18 +1825,14 @@ public class DDLTask extends Task implements Serializable { } // write the results in the file - DataOutput outStream = null; + DataOutputStream outStream = null; try { Path resFile = new Path(showParts.getResFile()); FileSystem fs = resFile.getFileSystem(conf); outStream = fs.create(resFile); - Iterator iterParts = parts.iterator(); - while (iterParts.hasNext()) { - // create a row per partition name - outStream.writeBytes(iterParts.next()); - outStream.write(terminator); - } + formatter.showTablePartitons(outStream, parts); + ((FSDataOutputStream) outStream).close(); outStream = null; } catch (FileNotFoundException e) { @@ -2405,88 +2414,14 @@ public class DDLTask extends Task implements Serializable { } // write the results in the file - DataOutput outStream = null; + DataOutputStream outStream = null; try { Path resFile = new Path(showTblStatus.getResFile()); FileSystem fs = resFile.getFileSystem(conf); outStream = fs.create(resFile); - Iterator iterTables = tbls.iterator(); - while (iterTables.hasNext()) { - // create a row per table name - Table tbl = iterTables.next(); - String tableName = tbl.getTableName(); - String tblLoc = null; - String inputFormattCls = null; - String outputFormattCls = null; - if (part != null) { - if (par != null) { - if (par.getLocation() != null) { - tblLoc = par.getDataLocation().toString(); - } - inputFormattCls = par.getInputFormatClass().getName(); - outputFormattCls = par.getOutputFormatClass().getName(); - } - } else { - if (tbl.getPath() != null) { - tblLoc = tbl.getDataLocation().toString(); - } - inputFormattCls = tbl.getInputFormatClass().getName(); - outputFormattCls = tbl.getOutputFormatClass().getName(); - } - - String owner = tbl.getOwner(); - List cols = tbl.getCols(); - String ddlCols = MetaStoreUtils.getDDLFromFieldSchema("columns", cols); - boolean isPartitioned = tbl.isPartitioned(); - String partitionCols = ""; - if (isPartitioned) { - partitionCols = MetaStoreUtils.getDDLFromFieldSchema( - "partition_columns", tbl.getPartCols()); - } - - outStream.writeBytes("tableName:" + tableName); - outStream.write(terminator); - outStream.writeBytes("owner:" + owner); - outStream.write(terminator); - outStream.writeBytes("location:" + tblLoc); - outStream.write(terminator); - outStream.writeBytes("inputformat:" + inputFormattCls); - outStream.write(terminator); - outStream.writeBytes("outputformat:" + outputFormattCls); - outStream.write(terminator); - outStream.writeBytes("columns:" + ddlCols); - outStream.write(terminator); - outStream.writeBytes("partitioned:" + isPartitioned); - outStream.write(terminator); - outStream.writeBytes("partitionColumns:" + partitionCols); - outStream.write(terminator); - // output file system information - Path tablLoc = tbl.getPath(); - List locations = new ArrayList(); - if (isPartitioned) { - if (par == null) { - for (Partition curPart : db.getPartitions(tbl)) { - if (curPart.getLocation() != null) { - locations.add(new Path(curPart.getLocation())); - } - } - } else { - if (par.getLocation() != null) { - locations.add(new Path(par.getLocation())); - } - } - } else { - if (tablLoc != null) { - locations.add(tablLoc); - } - } - if (!locations.isEmpty()) { - writeFileSystemStats(outStream, locations, tablLoc, false, 0); - } + formatter.showTableStatus(outStream, db, conf, tbls, part, par); - outStream.write(terminator); - } ((FSDataOutputStream) outStream).close(); outStream = null; } catch (FileNotFoundException e) { @@ -2522,14 +2457,14 @@ public class DDLTask extends Task implements Serializable { // describe the table - populate the output stream Table tbl = db.getTable(tableName, false); Partition part = null; - DataOutput outStream = null; + DataOutputStream outStream = null; try { Path resFile = new Path(descTbl.getResFile()); if (tbl == null) { FileSystem fs = resFile.getFileSystem(conf); outStream = fs.create(resFile); String errMsg = "Table " + tableName + " does not exist"; - outStream.write(errMsg.getBytes("UTF-8")); + formatter.error(outStream, errMsg); ((FSDataOutputStream) outStream).close(); outStream = null; return 0; @@ -2541,7 +2476,7 @@ public class DDLTask extends Task implements Serializable { outStream = fs.create(resFile); String errMsg = "Partition " + descTbl.getPartSpec() + " for table " + tableName + " does not exist"; - outStream.write(errMsg.getBytes("UTF-8")); + formatter.error(outStream, errMsg); ((FSDataOutputStream) outStream).close(); outStream = null; return 0; @@ -2559,67 +2494,26 @@ public class DDLTask extends Task implements Serializable { } try { - LOG.info("DDLTask: got data for " + tbl.getTableName()); - Path resFile = new Path(descTbl.getResFile()); FileSystem fs = resFile.getFileSystem(conf); outStream = fs.create(resFile); + List cols = null; if (colPath.equals(tableName)) { - List cols = (part == null) ? tbl.getCols() : part.getCols(); + cols = (part == null) ? tbl.getCols() : part.getCols(); if (!descTbl.isFormatted()) { if (tableName.equals(colPath)) { cols.addAll(tbl.getPartCols()); } - outStream.writeBytes(MetaDataFormatUtils.displayColsUnformatted(cols)); - } else { - outStream.writeBytes( - MetaDataFormatUtils.getAllColumnsInformation(cols, - tbl.isPartitioned() ? tbl.getPartCols() : null)); } } else { - List cols = Hive.getFieldsFromDeserializer(colPath, tbl.getDeserializer()); - if (descTbl.isFormatted()) { - outStream.writeBytes(MetaDataFormatUtils.getAllColumnsInformation(cols)); - } else { - outStream.writeBytes(MetaDataFormatUtils.displayColsUnformatted(cols)); - } + cols = Hive.getFieldsFromDeserializer(colPath, tbl.getDeserializer()); } - if (tableName.equals(colPath)) { - - if (descTbl.isFormatted()) { - if (part != null) { - outStream.writeBytes(MetaDataFormatUtils.getPartitionInformation(part)); - } else { - outStream.writeBytes(MetaDataFormatUtils.getTableInformation(tbl)); - } - } - - // if extended desc table then show the complete details of the table - if (descTbl.isExt()) { - // add empty line - outStream.write(terminator); - if (part != null) { - // show partition information - outStream.writeBytes("Detailed Partition Information"); - outStream.write(separator); - outStream.writeBytes(part.getTPartition().toString()); - outStream.write(separator); - // comment column is empty - outStream.write(terminator); - } else { - // show table information - outStream.writeBytes("Detailed Table Information"); - outStream.write(separator); - outStream.writeBytes(tbl.getTTable().toString()); - outStream.write(separator); - outStream.write(terminator); - } - } - } + formatter.describeTable(outStream, colPath, tableName, tbl, part, cols, + descTbl.isFormatted(), descTbl.isExt()); LOG.info("DDLTask: written data for " + tbl.getTableName()); ((FSDataOutputStream) outStream).close(); @@ -2681,128 +2575,6 @@ public class DDLTask extends Task implements Serializable { outStream.write(separator); } - private void writeFileSystemStats(DataOutput outStream, List locations, - Path tabLoc, boolean partSpecified, int indent) throws IOException { - long totalFileSize = 0; - long maxFileSize = 0; - long minFileSize = Long.MAX_VALUE; - long lastAccessTime = 0; - long lastUpdateTime = 0; - int numOfFiles = 0; - - boolean unknown = false; - FileSystem fs = tabLoc.getFileSystem(conf); - // in case all files in locations do not exist - try { - FileStatus tmpStatus = fs.getFileStatus(tabLoc); - lastAccessTime = ShimLoader.getHadoopShims().getAccessTime(tmpStatus); - lastUpdateTime = tmpStatus.getModificationTime(); - if (partSpecified) { - // check whether the part exists or not in fs - tmpStatus = fs.getFileStatus(locations.get(0)); - } - } catch (IOException e) { - LOG.warn( - "Cannot access File System. File System status will be unknown: ", e); - unknown = true; - } - - if (!unknown) { - for (Path loc : locations) { - try { - FileStatus status = fs.getFileStatus(tabLoc); - FileStatus[] files = fs.listStatus(loc); - long accessTime = ShimLoader.getHadoopShims().getAccessTime(status); - long updateTime = status.getModificationTime(); - // no matter loc is the table location or part location, it must be a - // directory. - if (!status.isDir()) { - continue; - } - if (accessTime > lastAccessTime) { - lastAccessTime = accessTime; - } - if (updateTime > lastUpdateTime) { - lastUpdateTime = updateTime; - } - for (FileStatus currentStatus : files) { - if (currentStatus.isDir()) { - continue; - } - numOfFiles++; - long fileLen = currentStatus.getLen(); - totalFileSize += fileLen; - if (fileLen > maxFileSize) { - maxFileSize = fileLen; - } - if (fileLen < minFileSize) { - minFileSize = fileLen; - } - accessTime = ShimLoader.getHadoopShims().getAccessTime( - currentStatus); - updateTime = currentStatus.getModificationTime(); - if (accessTime > lastAccessTime) { - lastAccessTime = accessTime; - } - if (updateTime > lastUpdateTime) { - lastUpdateTime = updateTime; - } - } - } catch (IOException e) { - // ignore - } - } - } - String unknownString = "unknown"; - - for (int k = 0; k < indent; k++) { - outStream.writeBytes(Utilities.INDENT); - } - outStream.writeBytes("totalNumberFiles:"); - outStream.writeBytes(unknown ? unknownString : "" + numOfFiles); - outStream.write(terminator); - - for (int k = 0; k < indent; k++) { - outStream.writeBytes(Utilities.INDENT); - } - outStream.writeBytes("totalFileSize:"); - outStream.writeBytes(unknown ? unknownString : "" + totalFileSize); - outStream.write(terminator); - - for (int k = 0; k < indent; k++) { - outStream.writeBytes(Utilities.INDENT); - } - outStream.writeBytes("maxFileSize:"); - outStream.writeBytes(unknown ? unknownString : "" + maxFileSize); - outStream.write(terminator); - - for (int k = 0; k < indent; k++) { - outStream.writeBytes(Utilities.INDENT); - } - outStream.writeBytes("minFileSize:"); - if (numOfFiles > 0) { - outStream.writeBytes(unknown ? unknownString : "" + minFileSize); - } else { - outStream.writeBytes(unknown ? unknownString : "" + 0); - } - outStream.write(terminator); - - for (int k = 0; k < indent; k++) { - outStream.writeBytes(Utilities.INDENT); - } - outStream.writeBytes("lastAccessTime:"); - outStream.writeBytes((unknown || lastAccessTime < 0) ? unknownString : "" - + lastAccessTime); - outStream.write(terminator); - - for (int k = 0; k < indent; k++) { - outStream.writeBytes(Utilities.INDENT); - } - outStream.writeBytes("lastUpdateTime:"); - outStream.writeBytes(unknown ? unknownString : "" + lastUpdateTime); - outStream.write(terminator); - } - /** * Alter a given table. * diff --git ql/src/java/org/apache/hadoop/hive/ql/metadata/JsonMetaDataFormatter.java ql/src/java/org/apache/hadoop/hive/ql/metadata/JsonMetaDataFormatter.java new file mode 100644 index 0000000..ad5fbaf --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/JsonMetaDataFormatter.java @@ -0,0 +1,355 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.metadata; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.URLDecoder; +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.ql.metadata.Partition; +import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.shims.ShimLoader; +import org.codehaus.jackson.map.ObjectMapper; + +/** + * Format table and index information for machine readability using + * json. + */ +public class JsonMetaDataFormatter implements MetaDataFormatter { + private static final Log LOG = LogFactory.getLog("hive.ql.exec.DDLTask"); + + /** + * Convert the map to a JSON string. + */ + public void asJson(DataOutputStream out, Map data) + throws HiveException + { + try { + new ObjectMapper().writeValue(out, data); + } catch (IOException e) { + throw new HiveException("Unable to convert to json", e); + } + } + + /** + * Write error message. + */ + public void error(DataOutputStream out, String msg) + throws HiveException + { + asJson(out, + MapBuilder.create() + .put("error", msg) + .build()); + } + + /** + * Describe table. + */ + public void describeTable(DataOutputStream out, + String colPath, String tableName, + Table tbl, Partition part, List cols, + boolean isFormatted, boolean isExt) + throws HiveException + { + MapBuilder builder = MapBuilder.create(); + + builder.put("columns", makeColsUnformatted(cols)); + + if (isExt) { + if (part != null) + builder.put("partition", part.getTPartition()); + else + builder.put("table", tbl.getTTable()); + } + + asJson(out, builder.build()); + } + + private List makeColsUnformatted(List cols) { + ArrayList res = new ArrayList(); + for (FieldSchema col : cols) + res.add(makeOneColUnformatted(col)); + return res; + } + + private Map makeOneColUnformatted(FieldSchema col) { + return MapBuilder.create() + .put("name", col.getName()) + .put("type", col.getType()) + .put("comment", col.getComment()) + .build(); + } + + public void showTableStatus(DataOutputStream out, + Hive db, + HiveConf conf, + List
tbls, + Map part, + Partition par) + throws HiveException + { + asJson(out, MapBuilder + .create() + .put("tables", makeAllTableStatus(db, conf, + tbls, part, par)) + .build()); + } + + private List makeAllTableStatus(Hive db, + HiveConf conf, + List
tbls, + Map part, + Partition par) + throws HiveException + { + try { + ArrayList res = new ArrayList(); + for (Table tbl : tbls) + res.add(makeOneTableStatus(tbl, db, conf, part, par)); + return res; + } catch(IOException e) { + throw new HiveException(e); + } + } + + private Map makeOneTableStatus(Table tbl, + Hive db, + HiveConf conf, + Map part, + Partition par) + throws HiveException, IOException + { + String tblLoc = null; + String inputFormattCls = null; + String outputFormattCls = null; + if (part != null) { + if (par != null) { + if (par.getLocation() != null) { + tblLoc = par.getDataLocation().toString(); + } + inputFormattCls = par.getInputFormatClass().getName(); + outputFormattCls = par.getOutputFormatClass().getName(); + } + } else { + if (tbl.getPath() != null) { + tblLoc = tbl.getDataLocation().toString(); + } + inputFormattCls = tbl.getInputFormatClass().getName(); + outputFormattCls = tbl.getOutputFormatClass().getName(); + } + + MapBuilder builder = MapBuilder.create(); + + builder.put("tableName", tbl.getTableName()); + builder.put("owner", tbl.getOwner()); + builder.put("location", tblLoc); + builder.put("inputformat", inputFormattCls); + builder.put("outputformat", outputFormattCls); + builder.put("columns", makeColsUnformatted(tbl.getCols())); + + builder.put("partitioned", tbl.isPartitioned()); + if (tbl.isPartitioned()) + builder.put("partitionColumns", makeColsUnformatted(tbl.getPartCols())); + + putFileSystemsStats(builder, makeTableStatusLocations(tbl, db, par), + conf, tbl.getPath()); + + return builder.build(); + } + + private List makeTableStatusLocations(Table tbl, Hive db, Partition par) + throws HiveException + { + // output file system information + Path tblPath = tbl.getPath(); + List locations = new ArrayList(); + if (tbl.isPartitioned()) { + if (par == null) { + for (Partition curPart : db.getPartitions(tbl)) { + if (curPart.getLocation() != null) { + locations.add(new Path(curPart.getLocation())); + } + } + } else { + if (par.getLocation() != null) { + locations.add(new Path(par.getLocation())); + } + } + } else { + if (tblPath != null) { + locations.add(tblPath); + } + } + + return locations; + } + + // Duplicates logic in TextMetaDataFormatter + private void putFileSystemsStats(MapBuilder builder, List locations, + HiveConf conf, Path tblPath) + throws IOException + { + long totalFileSize = 0; + long maxFileSize = 0; + long minFileSize = Long.MAX_VALUE; + long lastAccessTime = 0; + long lastUpdateTime = 0; + int numOfFiles = 0; + + boolean unknown = false; + FileSystem fs = tblPath.getFileSystem(conf); + // in case all files in locations do not exist + try { + FileStatus tmpStatus = fs.getFileStatus(tblPath); + lastAccessTime = ShimLoader.getHadoopShims().getAccessTime(tmpStatus); + lastUpdateTime = tmpStatus.getModificationTime(); + } catch (IOException e) { + LOG.warn( + "Cannot access File System. File System status will be unknown: ", e); + unknown = true; + } + + if (!unknown) { + for (Path loc : locations) { + try { + FileStatus status = fs.getFileStatus(tblPath); + FileStatus[] files = fs.listStatus(loc); + long accessTime = ShimLoader.getHadoopShims().getAccessTime(status); + long updateTime = status.getModificationTime(); + // no matter loc is the table location or part location, it must be a + // directory. + if (!status.isDir()) { + continue; + } + if (accessTime > lastAccessTime) { + lastAccessTime = accessTime; + } + if (updateTime > lastUpdateTime) { + lastUpdateTime = updateTime; + } + for (FileStatus currentStatus : files) { + if (currentStatus.isDir()) { + continue; + } + numOfFiles++; + long fileLen = currentStatus.getLen(); + totalFileSize += fileLen; + if (fileLen > maxFileSize) { + maxFileSize = fileLen; + } + if (fileLen < minFileSize) { + minFileSize = fileLen; + } + accessTime = ShimLoader.getHadoopShims().getAccessTime( + currentStatus); + updateTime = currentStatus.getModificationTime(); + if (accessTime > lastAccessTime) { + lastAccessTime = accessTime; + } + if (updateTime > lastUpdateTime) { + lastUpdateTime = updateTime; + } + } + } catch (IOException e) { + // ignore + } + } + } + + builder + .put("totalNumberFiles", numOfFiles, ! unknown) + .put("totalFileSize", totalFileSize, ! unknown) + .put("maxFileSize", maxFileSize, ! unknown) + .put("minFileSize", numOfFiles > 0 ? minFileSize : 0, ! unknown) + .put("lastAccessTime", lastAccessTime, ! (unknown || lastAccessTime < 0)) + .put("lastUpdateTime", lastUpdateTime, ! unknown); + } + + /** + * Show the table partitions. + */ + public void showTablePartitons(DataOutputStream out, List parts) + throws HiveException + { + asJson(out, + MapBuilder.create() + .put("partitions", makeTablePartions(parts)) + .build()); + } + + private List makeTablePartions(List parts) + throws HiveException + { + try { + ArrayList res = new ArrayList(); + for (String part : parts) + res.add(makeOneTablePartition(part)); + return res; + } catch (UnsupportedEncodingException e) { + throw new HiveException(e); + } + } + + // This seems like a very wrong implementation. + private Map makeOneTablePartition(String partIdent) + throws UnsupportedEncodingException + { + ArrayList res = new ArrayList(); + + ArrayList names = new ArrayList(); + for (String part : StringUtils.split(partIdent, "/")) { + String name = part; + String val = null; + String[] kv = StringUtils.split(part, "=", 2); + if (kv != null) { + name = kv[0]; + if (kv.length > 1) + val = URLDecoder.decode(kv[1], "UTF-8"); + } + if (val != null) + names.add(name + "='" + val + "'"); + else + names.add(name); + + res.add(MapBuilder.create() + .put("columnName", name) + .put("columnValue", val) + .build()); + } + + return MapBuilder.create() + .put("name", StringUtils.join(names, ",")) + .put("values", res) + .build(); + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/metadata/MapBuilder.java ql/src/java/org/apache/hadoop/hive/ql/metadata/MapBuilder.java new file mode 100644 index 0000000..61a9a97 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/MapBuilder.java @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.metadata; + +import java.util.HashMap; +import java.util.Map; + +/** + * Helper class to build Maps consumed by the JSON formatter. Only + * add non-null entries to the Map. + */ +public class MapBuilder { + private Map map = new HashMap(); + + private MapBuilder() {} + + public static MapBuilder create() { + return new MapBuilder(); + } + + public MapBuilder put(String name, Object val) { + if (val != null) + map.put(name, val); + return this; + } + + public MapBuilder put(String name, boolean val) { + map.put(name, Boolean.valueOf(val)); + return this; + } + + public MapBuilder put(String name, int val) { + map.put(name, Integer.valueOf(val)); + return this; + } + + public MapBuilder put(String name, long val) { + map.put(name, Long.valueOf(val)); + return this; + } + + public MapBuilder put(String name, T val, boolean use) { + if (use) + put(name, val); + return this; + } + + public Map build() { + return map; + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/metadata/MetaDataFormatter.java ql/src/java/org/apache/hadoop/hive/ql/metadata/MetaDataFormatter.java new file mode 100644 index 0000000..f565789 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/MetaDataFormatter.java @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.metadata; + +import java.io.DataOutputStream; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.ql.metadata.Partition; +import org.apache.hadoop.hive.ql.metadata.Table; + +/** + * Interface to format table and index information. We can format it + * for human readability (lines of text) or for machine readability + * (json). + */ +public interface MetaDataFormatter { + /** + * Write error message. + */ + public void error(DataOutputStream out, String msg) + throws HiveException; + + /** + * Describe table. + */ + public void describeTable(DataOutputStream out, + String colPath, String tableName, + Table tbl, Partition part, List cols, + boolean isFormatted, boolean isExt) + throws HiveException; + + /** + * Show the table status. + */ + public void showTableStatus(DataOutputStream out, + Hive db, + HiveConf conf, + List
tbls, + Map part, + Partition par) + throws HiveException; + + /** + * Show the table partitions. + */ + public void showTablePartitons(DataOutputStream out, + List parts) + throws HiveException; +} + diff --git ql/src/java/org/apache/hadoop/hive/ql/metadata/TextMetaDataFormatter.java ql/src/java/org/apache/hadoop/hive/ql/metadata/TextMetaDataFormatter.java new file mode 100644 index 0000000..5bdc4c4 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/TextMetaDataFormatter.java @@ -0,0 +1,372 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.metadata; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.MetaStoreUtils; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.ql.exec.Utilities; +import org.apache.hadoop.hive.ql.metadata.Partition; +import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.shims.ShimLoader; + +/** + * Format table and index information for human readability using + * simple lines of text. + */ +public class TextMetaDataFormatter implements MetaDataFormatter { + private static final Log LOG = LogFactory.getLog("hive.ql.exec.DDLTask"); + + private static final int separator = Utilities.tabCode; + private static final int terminator = Utilities.newLineCode; + + public void error(DataOutputStream out, String msg) + throws HiveException + { + try { + out.write(msg.getBytes("UTF-8")); + } catch (Exception e) { + throw new HiveException(e); + } + } + + public void describeTable(DataOutputStream outStream, + String colPath, String tableName, + Table tbl, Partition part, List cols, + boolean isFormatted, boolean isExt) + throws HiveException + { + try { + auxDescribeTable(outStream, colPath, tableName, tbl, part, cols, + isFormatted, isExt); + } catch (IOException e) { + throw new HiveException(e); + } + } + + private void auxDescribeTable(DataOutputStream outStream, + String colPath, String tableName, + Table tbl, Partition part, List cols, + boolean isFormatted, boolean isExt) + throws HiveException, IOException + { + if (colPath.equals(tableName)) { + if (!isFormatted) { + outStream.writeBytes(MetaDataFormatUtils.displayColsUnformatted(cols)); + } else { + outStream.writeBytes( + MetaDataFormatUtils.getAllColumnsInformation(cols, + tbl.isPartitioned() ? tbl.getPartCols() : null)); + } + } else { + if (isFormatted) { + outStream.writeBytes(MetaDataFormatUtils.getAllColumnsInformation(cols)); + } else { + outStream.writeBytes(MetaDataFormatUtils.displayColsUnformatted(cols)); + } + } + + if (tableName.equals(colPath)) { + + if (isFormatted) { + if (part != null) { + outStream.writeBytes(MetaDataFormatUtils.getPartitionInformation(part)); + } else { + outStream.writeBytes(MetaDataFormatUtils.getTableInformation(tbl)); + } + } + + // if extended desc table then show the complete details of the table + if (isExt) { + // add empty line + outStream.write(terminator); + if (part != null) { + // show partition information + outStream.writeBytes("Detailed Partition Information"); + outStream.write(separator); + outStream.writeBytes(part.getTPartition().toString()); + outStream.write(separator); + // comment column is empty + outStream.write(terminator); + } else { + // show table information + outStream.writeBytes("Detailed Table Information"); + outStream.write(separator); + outStream.writeBytes(tbl.getTTable().toString()); + outStream.write(separator); + outStream.write(terminator); + } + } + } + } + + public void showTableStatus(DataOutputStream outStream, + Hive db, + HiveConf conf, + List
tbls, + Map part, + Partition par) + throws HiveException + { + try { + auxShowTableStatus(outStream, db, conf, tbls, part, par); + } catch (IOException e) { + throw new HiveException(e); + } + } + + private void auxShowTableStatus(DataOutputStream outStream, + Hive db, + HiveConf conf, + List
tbls, + Map part, + Partition par) + throws HiveException, IOException + { + Iterator
iterTables = tbls.iterator(); + while (iterTables.hasNext()) { + // create a row per table name + Table tbl = iterTables.next(); + String tableName = tbl.getTableName(); + String tblLoc = null; + String inputFormattCls = null; + String outputFormattCls = null; + if (part != null) { + if (par != null) { + if (par.getLocation() != null) { + tblLoc = par.getDataLocation().toString(); + } + inputFormattCls = par.getInputFormatClass().getName(); + outputFormattCls = par.getOutputFormatClass().getName(); + } + } else { + if (tbl.getPath() != null) { + tblLoc = tbl.getDataLocation().toString(); + } + inputFormattCls = tbl.getInputFormatClass().getName(); + outputFormattCls = tbl.getOutputFormatClass().getName(); + } + + String owner = tbl.getOwner(); + List cols = tbl.getCols(); + String ddlCols = MetaStoreUtils.getDDLFromFieldSchema("columns", cols); + boolean isPartitioned = tbl.isPartitioned(); + String partitionCols = ""; + if (isPartitioned) { + partitionCols = MetaStoreUtils.getDDLFromFieldSchema( + "partition_columns", tbl.getPartCols()); + } + + outStream.writeBytes("tableName:" + tableName); + outStream.write(terminator); + outStream.writeBytes("owner:" + owner); + outStream.write(terminator); + outStream.writeBytes("location:" + tblLoc); + outStream.write(terminator); + outStream.writeBytes("inputformat:" + inputFormattCls); + outStream.write(terminator); + outStream.writeBytes("outputformat:" + outputFormattCls); + outStream.write(terminator); + outStream.writeBytes("columns:" + ddlCols); + outStream.write(terminator); + outStream.writeBytes("partitioned:" + isPartitioned); + outStream.write(terminator); + outStream.writeBytes("partitionColumns:" + partitionCols); + outStream.write(terminator); + // output file system information + Path tblPath = tbl.getPath(); + List locations = new ArrayList(); + if (isPartitioned) { + if (par == null) { + for (Partition curPart : db.getPartitions(tbl)) { + if (curPart.getLocation() != null) { + locations.add(new Path(curPart.getLocation())); + } + } + } else { + if (par.getLocation() != null) { + locations.add(new Path(par.getLocation())); + } + } + } else { + if (tblPath != null) { + locations.add(tblPath); + } + } + if (!locations.isEmpty()) { + writeFileSystemStats(outStream, conf, locations, tblPath, false, 0); + } + + outStream.write(terminator); + } + } + + private void writeFileSystemStats(DataOutputStream outStream, + HiveConf conf, + List locations, + Path tblPath, boolean partSpecified, int indent) + throws IOException + { + long totalFileSize = 0; + long maxFileSize = 0; + long minFileSize = Long.MAX_VALUE; + long lastAccessTime = 0; + long lastUpdateTime = 0; + int numOfFiles = 0; + + boolean unknown = false; + FileSystem fs = tblPath.getFileSystem(conf); + // in case all files in locations do not exist + try { + FileStatus tmpStatus = fs.getFileStatus(tblPath); + lastAccessTime = ShimLoader.getHadoopShims().getAccessTime(tmpStatus); + lastUpdateTime = tmpStatus.getModificationTime(); + if (partSpecified) { + // check whether the part exists or not in fs + tmpStatus = fs.getFileStatus(locations.get(0)); + } + } catch (IOException e) { + LOG.warn( + "Cannot access File System. File System status will be unknown: ", e); + unknown = true; + } + + if (!unknown) { + for (Path loc : locations) { + try { + FileStatus status = fs.getFileStatus(tblPath); + FileStatus[] files = fs.listStatus(loc); + long accessTime = ShimLoader.getHadoopShims().getAccessTime(status); + long updateTime = status.getModificationTime(); + // no matter loc is the table location or part location, it must be a + // directory. + if (!status.isDir()) { + continue; + } + if (accessTime > lastAccessTime) { + lastAccessTime = accessTime; + } + if (updateTime > lastUpdateTime) { + lastUpdateTime = updateTime; + } + for (FileStatus currentStatus : files) { + if (currentStatus.isDir()) { + continue; + } + numOfFiles++; + long fileLen = currentStatus.getLen(); + totalFileSize += fileLen; + if (fileLen > maxFileSize) { + maxFileSize = fileLen; + } + if (fileLen < minFileSize) { + minFileSize = fileLen; + } + accessTime = ShimLoader.getHadoopShims().getAccessTime( + currentStatus); + updateTime = currentStatus.getModificationTime(); + if (accessTime > lastAccessTime) { + lastAccessTime = accessTime; + } + if (updateTime > lastUpdateTime) { + lastUpdateTime = updateTime; + } + } + } catch (IOException e) { + // ignore + } + } + } + String unknownString = "unknown"; + + for (int k = 0; k < indent; k++) { + outStream.writeBytes(Utilities.INDENT); + } + outStream.writeBytes("totalNumberFiles:"); + outStream.writeBytes(unknown ? unknownString : "" + numOfFiles); + outStream.write(terminator); + + for (int k = 0; k < indent; k++) { + outStream.writeBytes(Utilities.INDENT); + } + outStream.writeBytes("totalFileSize:"); + outStream.writeBytes(unknown ? unknownString : "" + totalFileSize); + outStream.write(terminator); + + for (int k = 0; k < indent; k++) { + outStream.writeBytes(Utilities.INDENT); + } + outStream.writeBytes("maxFileSize:"); + outStream.writeBytes(unknown ? unknownString : "" + maxFileSize); + outStream.write(terminator); + + for (int k = 0; k < indent; k++) { + outStream.writeBytes(Utilities.INDENT); + } + outStream.writeBytes("minFileSize:"); + if (numOfFiles > 0) { + outStream.writeBytes(unknown ? unknownString : "" + minFileSize); + } else { + outStream.writeBytes(unknown ? unknownString : "" + 0); + } + outStream.write(terminator); + + for (int k = 0; k < indent; k++) { + outStream.writeBytes(Utilities.INDENT); + } + outStream.writeBytes("lastAccessTime:"); + outStream.writeBytes((unknown || lastAccessTime < 0) ? unknownString : "" + + lastAccessTime); + outStream.write(terminator); + + for (int k = 0; k < indent; k++) { + outStream.writeBytes(Utilities.INDENT); + } + outStream.writeBytes("lastUpdateTime:"); + outStream.writeBytes(unknown ? unknownString : "" + lastUpdateTime); + outStream.write(terminator); + } + + /** + * Show the table partitions. + */ + public void showTablePartitons(DataOutputStream outStream, List parts) + throws HiveException + { + try { + for (String part : parts) { + outStream.writeBytes(part); + outStream.write(terminator); + } + } catch (IOException e) { + throw new HiveException(e); + } + } +}