diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java index fb91da4..6fd6d7d 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java @@ -23,6 +23,7 @@ import static org.apache.hadoop.util.StringUtils.stringifyException; import java.io.BufferedWriter; import java.io.DataOutput; +import java.io.DataOutputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.OutputStreamWriter; @@ -91,8 +92,11 @@ import org.apache.hadoop.hive.ql.metadata.HiveMetaStoreChecker; import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler; import org.apache.hadoop.hive.ql.metadata.HiveUtils; import org.apache.hadoop.hive.ql.metadata.InvalidTableException; +import org.apache.hadoop.hive.ql.metadata.JsonMetaDataFormatter; import org.apache.hadoop.hive.ql.metadata.MetaDataFormatUtils; +import org.apache.hadoop.hive.ql.metadata.MetaDataFormatter; import org.apache.hadoop.hive.ql.metadata.Partition; +import org.apache.hadoop.hive.ql.metadata.TextMetaDataFormatter; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.parse.AlterTablePartMergeFilesDesc; import org.apache.hadoop.hive.ql.plan.AddPartitionDesc; @@ -166,6 +170,8 @@ public class DDLTask extends Task implements Serializable { private static String INTERMEDIATE_ORIGINAL_DIR_SUFFIX; private static String INTERMEDIATE_EXTRACTED_DIR_SUFFIX; + private MetaDataFormatter formatter; + @Override public boolean requireLock() { return this.work != null && this.work.getNeedLock(); @@ -180,6 +186,13 @@ public class DDLTask extends Task implements Serializable { super.initialize(conf, queryPlan, ctx); this.conf = conf; + // Pick the formatter to use to display the results. Either the + // normal human readable output or a json object. + if ("json".equals(conf.get("hive.format"))) + formatter = new JsonMetaDataFormatter(); + else + formatter = new TextMetaDataFormatter(); + INTERMEDIATE_ARCHIVED_DIR_SUFFIX = HiveConf.getVar(conf, ConfVars.METASTORE_INT_ARCHIVED); INTERMEDIATE_ORIGINAL_DIR_SUFFIX = @@ -375,17 +388,31 @@ public class DDLTask extends Task implements Serializable { } } catch (InvalidTableException e) { - console.printError("Table " + e.getTableName() + " does not exist"); + formatter.consoleError(console, "Table " + e.getTableName() + " does not exist", + formatter.MISSING); LOG.debug(stringifyException(e)); return 1; + } catch (AlreadyExistsException e) { + formatter.consoleError(console, e.getMessage(), + "\n" + stringifyException(e), + formatter.CONFLICT); + return 1; + } catch (NoSuchObjectException e) { + formatter.consoleError(console, e.getMessage(), + "\n" + stringifyException(e), + formatter.MISSING); + return 1; } catch (HiveException e) { - console.printError("FAILED: Error in metadata: " + e.getMessage(), "\n" - + stringifyException(e)); + formatter.consoleError(console, + "FAILED: Error in metadata: " + e.getMessage(), + "\n" + stringifyException(e), + formatter.ERROR); LOG.debug(stringifyException(e)); return 1; } catch (Exception e) { - console.printError("Failed with exception " + e.getMessage(), "\n" - + stringifyException(e)); + formatter.consoleError(console, "Failed with exception " + e.getMessage(), + "\n" + stringifyException(e), + formatter.ERROR); return (1); } assert false; @@ -1801,7 +1828,9 @@ public class DDLTask extends Task implements Serializable { tbl = db.getTable(tabName); if (!tbl.isPartitioned()) { - console.printError("Table " + tabName + " is not a partitioned table"); + formatter.consoleError(console, + "Table " + tabName + " is not a partitioned table", + formatter.ERROR); return 1; } if (showParts.getPartSpec() != null) { @@ -1812,18 +1841,14 @@ public class DDLTask extends Task implements Serializable { } // write the results in the file - DataOutput outStream = null; + DataOutputStream outStream = null; try { Path resFile = new Path(showParts.getResFile()); FileSystem fs = resFile.getFileSystem(conf); outStream = fs.create(resFile); - Iterator iterParts = parts.iterator(); - while (iterParts.hasNext()) { - // create a row per partition name - outStream.writeBytes(iterParts.next()); - outStream.write(terminator); - } + formatter.showTablePartitons(outStream, parts); + ((FSDataOutputStream) outStream).close(); outStream = null; } catch (FileNotFoundException e) { @@ -1920,24 +1945,22 @@ public class DDLTask extends Task implements Serializable { LOG.info("results : " + databases.size()); // write the results in the file - DataOutput outStream = null; + DataOutputStream outStream = null; try { Path resFile = new Path(showDatabasesDesc.getResFile()); FileSystem fs = resFile.getFileSystem(conf); outStream = fs.create(resFile); - for (String database : databases) { - // create a row per database name - outStream.writeBytes(database); - outStream.write(terminator); - } + formatter.showDatabases(outStream, databases); ((FSDataOutputStream) outStream).close(); outStream = null; } catch (FileNotFoundException e) { - LOG.warn("show databases: " + stringifyException(e)); + formatter.logWarn(outStream, "show databases: " + stringifyException(e), + formatter.ERROR); return 1; } catch (IOException e) { - LOG.warn("show databases: " + stringifyException(e)); + formatter.logWarn(outStream, "show databases: " + stringifyException(e), + formatter.ERROR); return 1; } catch (Exception e) { throw new HiveException(e.toString()); @@ -1976,26 +1999,23 @@ public class DDLTask extends Task implements Serializable { } // write the results in the file - DataOutput outStream = null; + DataOutputStream outStream = null; try { Path resFile = new Path(showTbls.getResFile()); FileSystem fs = resFile.getFileSystem(conf); outStream = fs.create(resFile); - SortedSet sortedTbls = new TreeSet(tbls); - Iterator iterTbls = sortedTbls.iterator(); - while (iterTbls.hasNext()) { - // create a row per table name - outStream.writeBytes(iterTbls.next()); - outStream.write(terminator); - } + SortedSet sortedTbls = new TreeSet(tbls); + formatter.showTables(outStream, sortedTbls); ((FSDataOutputStream) outStream).close(); outStream = null; } catch (FileNotFoundException e) { - LOG.warn("show table: " + stringifyException(e)); + formatter.logWarn(outStream, "show table: " + stringifyException(e), + formatter.ERROR); return 1; } catch (IOException e) { - LOG.warn("show table: " + stringifyException(e)); + formatter.logWarn(outStream, "show table: " + stringifyException(e), + formatter.ERROR); return 1; } catch (Exception e) { throw new HiveException(e.toString()); @@ -2319,7 +2339,7 @@ public class DDLTask extends Task implements Serializable { } private int descDatabase(DescDatabaseDesc descDatabase) throws HiveException { - DataOutput outStream = null; + DataOutputStream outStream = null; try { Path resFile = new Path(descDatabase.getResFile()); FileSystem fs = resFile.getFileSystem(conf); @@ -2327,37 +2347,32 @@ public class DDLTask extends Task implements Serializable { Database database = db.getDatabase(descDatabase.getDatabaseName()); - if (database != null) { - outStream.writeBytes(database.getName()); - outStream.write(separator); - if (database.getDescription() != null) { - outStream.writeBytes(database.getDescription()); - } - outStream.write(separator); - if (database.getLocationUri() != null) { - outStream.writeBytes(database.getLocationUri()); - } - - outStream.write(separator); - if (descDatabase.isExt() && database.getParametersSize() > 0) { - Map params = database.getParameters(); - outStream.writeBytes(params.toString()); - } - + if (database == null) { + formatter.error(outStream, + "No such database: " + descDatabase.getDatabaseName(), + formatter.MISSING); } else { - outStream.writeBytes("No such database: " + descDatabase.getDatabaseName()); - } - - outStream.write(terminator); + Map params = null; + if(descDatabase.isExt()) + params = database.getParameters(); + formatter.showDatabaseDescription(outStream, + database.getName(), + database.getDescription(), + database.getLocationUri(), + params); + } ((FSDataOutputStream) outStream).close(); outStream = null; - } catch (FileNotFoundException e) { - LOG.warn("describe database: " + stringifyException(e)); + formatter.logWarn(outStream, + "describe database: " + stringifyException(e), + formatter.ERROR); return 1; } catch (IOException e) { - LOG.warn("describe database: " + stringifyException(e)); + formatter.logWarn(outStream, + "describe database: " + stringifyException(e), + formatter.ERROR); return 1; } catch (Exception e) { throw new HiveException(e.toString()); @@ -2405,95 +2420,23 @@ public class DDLTask extends Task implements Serializable { } // write the results in the file - DataOutput outStream = null; + DataOutputStream outStream = null; try { Path resFile = new Path(showTblStatus.getResFile()); FileSystem fs = resFile.getFileSystem(conf); outStream = fs.create(resFile); - Iterator iterTables = tbls.iterator(); - while (iterTables.hasNext()) { - // create a row per table name - Table tbl = iterTables.next(); - String tableName = tbl.getTableName(); - String tblLoc = null; - String inputFormattCls = null; - String outputFormattCls = null; - if (part != null) { - if (par != null) { - if (par.getLocation() != null) { - tblLoc = par.getDataLocation().toString(); - } - inputFormattCls = par.getInputFormatClass().getName(); - outputFormattCls = par.getOutputFormatClass().getName(); - } - } else { - if (tbl.getPath() != null) { - tblLoc = tbl.getDataLocation().toString(); - } - inputFormattCls = tbl.getInputFormatClass().getName(); - outputFormattCls = tbl.getOutputFormatClass().getName(); - } - - String owner = tbl.getOwner(); - List cols = tbl.getCols(); - String ddlCols = MetaStoreUtils.getDDLFromFieldSchema("columns", cols); - boolean isPartitioned = tbl.isPartitioned(); - String partitionCols = ""; - if (isPartitioned) { - partitionCols = MetaStoreUtils.getDDLFromFieldSchema( - "partition_columns", tbl.getPartCols()); - } + formatter.showTableStatus(outStream, db, conf, tbls, part, par); - outStream.writeBytes("tableName:" + tableName); - outStream.write(terminator); - outStream.writeBytes("owner:" + owner); - outStream.write(terminator); - outStream.writeBytes("location:" + tblLoc); - outStream.write(terminator); - outStream.writeBytes("inputformat:" + inputFormattCls); - outStream.write(terminator); - outStream.writeBytes("outputformat:" + outputFormattCls); - outStream.write(terminator); - outStream.writeBytes("columns:" + ddlCols); - outStream.write(terminator); - outStream.writeBytes("partitioned:" + isPartitioned); - outStream.write(terminator); - outStream.writeBytes("partitionColumns:" + partitionCols); - outStream.write(terminator); - // output file system information - Path tablLoc = tbl.getPath(); - List locations = new ArrayList(); - if (isPartitioned) { - if (par == null) { - for (Partition curPart : db.getPartitions(tbl)) { - if (curPart.getLocation() != null) { - locations.add(new Path(curPart.getLocation())); - } - } - } else { - if (par.getLocation() != null) { - locations.add(new Path(par.getLocation())); - } - } - } else { - if (tablLoc != null) { - locations.add(tablLoc); - } - } - if (!locations.isEmpty()) { - writeFileSystemStats(outStream, locations, tablLoc, false, 0); - } - - outStream.write(terminator); - } ((FSDataOutputStream) outStream).close(); outStream = null; } catch (FileNotFoundException e) { - LOG.info("show table status: " + stringifyException(e)); + formatter.logInfo(outStream, "show table status: " + stringifyException(e), + formatter.ERROR); return 1; } catch (IOException e) { - LOG.info("show table status: " + stringifyException(e)); + formatter.logInfo(outStream, "show table status: " + stringifyException(e), + formatter.ERROR); return 1; } catch (Exception e) { throw new HiveException(e); @@ -2522,14 +2465,14 @@ public class DDLTask extends Task implements Serializable { // describe the table - populate the output stream Table tbl = db.getTable(tableName, false); Partition part = null; - DataOutput outStream = null; + DataOutputStream outStream = null; try { Path resFile = new Path(descTbl.getResFile()); if (tbl == null) { FileSystem fs = resFile.getFileSystem(conf); outStream = fs.create(resFile); String errMsg = "Table " + tableName + " does not exist"; - outStream.write(errMsg.getBytes("UTF-8")); + formatter.error(outStream, errMsg, formatter.MISSING); ((FSDataOutputStream) outStream).close(); outStream = null; return 0; @@ -2541,7 +2484,7 @@ public class DDLTask extends Task implements Serializable { outStream = fs.create(resFile); String errMsg = "Partition " + descTbl.getPartSpec() + " for table " + tableName + " does not exist"; - outStream.write(errMsg.getBytes("UTF-8")); + formatter.error(outStream, errMsg, formatter.MISSING); ((FSDataOutputStream) outStream).close(); outStream = null; return 0; @@ -2549,87 +2492,50 @@ public class DDLTask extends Task implements Serializable { tbl = part.getTable(); } } catch (FileNotFoundException e) { - LOG.info("describe table: " + stringifyException(e)); + formatter.logInfo(outStream, "describe table: " + stringifyException(e), + formatter.ERROR); return 1; } catch (IOException e) { - LOG.info("describe table: " + stringifyException(e)); + formatter.logInfo(outStream, "describe table: " + stringifyException(e), + formatter.ERROR); return 1; } finally { IOUtils.closeStream((FSDataOutputStream) outStream); } try { - LOG.info("DDLTask: got data for " + tbl.getTableName()); - Path resFile = new Path(descTbl.getResFile()); FileSystem fs = resFile.getFileSystem(conf); outStream = fs.create(resFile); + List cols = null; if (colPath.equals(tableName)) { - List cols = (part == null) ? tbl.getCols() : part.getCols(); + cols = (part == null) ? tbl.getCols() : part.getCols(); if (!descTbl.isFormatted()) { if (tableName.equals(colPath)) { cols.addAll(tbl.getPartCols()); } - outStream.writeBytes(MetaDataFormatUtils.displayColsUnformatted(cols)); - } else { - outStream.writeBytes( - MetaDataFormatUtils.getAllColumnsInformation(cols, - tbl.isPartitioned() ? tbl.getPartCols() : null)); } } else { - List cols = Hive.getFieldsFromDeserializer(colPath, tbl.getDeserializer()); - if (descTbl.isFormatted()) { - outStream.writeBytes(MetaDataFormatUtils.getAllColumnsInformation(cols)); - } else { - outStream.writeBytes(MetaDataFormatUtils.displayColsUnformatted(cols)); - } + cols = Hive.getFieldsFromDeserializer(colPath, tbl.getDeserializer()); } - if (tableName.equals(colPath)) { - - if (descTbl.isFormatted()) { - if (part != null) { - outStream.writeBytes(MetaDataFormatUtils.getPartitionInformation(part)); - } else { - outStream.writeBytes(MetaDataFormatUtils.getTableInformation(tbl)); - } - } - - // if extended desc table then show the complete details of the table - if (descTbl.isExt()) { - // add empty line - outStream.write(terminator); - if (part != null) { - // show partition information - outStream.writeBytes("Detailed Partition Information"); - outStream.write(separator); - outStream.writeBytes(part.getTPartition().toString()); - outStream.write(separator); - // comment column is empty - outStream.write(terminator); - } else { - // show table information - outStream.writeBytes("Detailed Table Information"); - outStream.write(separator); - outStream.writeBytes(tbl.getTTable().toString()); - outStream.write(separator); - outStream.write(terminator); - } - } - } + formatter.describeTable(outStream, colPath, tableName, tbl, part, cols, + descTbl.isFormatted(), descTbl.isExt()); LOG.info("DDLTask: written data for " + tbl.getTableName()); ((FSDataOutputStream) outStream).close(); outStream = null; } catch (FileNotFoundException e) { - LOG.info("describe table: " + stringifyException(e)); + formatter.logInfo(outStream, "describe table: " + stringifyException(e), + formatter.ERROR); return 1; } catch (IOException e) { - LOG.info("describe table: " + stringifyException(e)); + formatter.logInfo(outStream, "describe table: " + stringifyException(e), + formatter.ERROR); return 1; } catch (Exception e) { throw new HiveException(e); @@ -2681,128 +2587,6 @@ public class DDLTask extends Task implements Serializable { outStream.write(separator); } - private void writeFileSystemStats(DataOutput outStream, List locations, - Path tabLoc, boolean partSpecified, int indent) throws IOException { - long totalFileSize = 0; - long maxFileSize = 0; - long minFileSize = Long.MAX_VALUE; - long lastAccessTime = 0; - long lastUpdateTime = 0; - int numOfFiles = 0; - - boolean unknown = false; - FileSystem fs = tabLoc.getFileSystem(conf); - // in case all files in locations do not exist - try { - FileStatus tmpStatus = fs.getFileStatus(tabLoc); - lastAccessTime = ShimLoader.getHadoopShims().getAccessTime(tmpStatus); - lastUpdateTime = tmpStatus.getModificationTime(); - if (partSpecified) { - // check whether the part exists or not in fs - tmpStatus = fs.getFileStatus(locations.get(0)); - } - } catch (IOException e) { - LOG.warn( - "Cannot access File System. File System status will be unknown: ", e); - unknown = true; - } - - if (!unknown) { - for (Path loc : locations) { - try { - FileStatus status = fs.getFileStatus(tabLoc); - FileStatus[] files = fs.listStatus(loc); - long accessTime = ShimLoader.getHadoopShims().getAccessTime(status); - long updateTime = status.getModificationTime(); - // no matter loc is the table location or part location, it must be a - // directory. - if (!status.isDir()) { - continue; - } - if (accessTime > lastAccessTime) { - lastAccessTime = accessTime; - } - if (updateTime > lastUpdateTime) { - lastUpdateTime = updateTime; - } - for (FileStatus currentStatus : files) { - if (currentStatus.isDir()) { - continue; - } - numOfFiles++; - long fileLen = currentStatus.getLen(); - totalFileSize += fileLen; - if (fileLen > maxFileSize) { - maxFileSize = fileLen; - } - if (fileLen < minFileSize) { - minFileSize = fileLen; - } - accessTime = ShimLoader.getHadoopShims().getAccessTime( - currentStatus); - updateTime = currentStatus.getModificationTime(); - if (accessTime > lastAccessTime) { - lastAccessTime = accessTime; - } - if (updateTime > lastUpdateTime) { - lastUpdateTime = updateTime; - } - } - } catch (IOException e) { - // ignore - } - } - } - String unknownString = "unknown"; - - for (int k = 0; k < indent; k++) { - outStream.writeBytes(Utilities.INDENT); - } - outStream.writeBytes("totalNumberFiles:"); - outStream.writeBytes(unknown ? unknownString : "" + numOfFiles); - outStream.write(terminator); - - for (int k = 0; k < indent; k++) { - outStream.writeBytes(Utilities.INDENT); - } - outStream.writeBytes("totalFileSize:"); - outStream.writeBytes(unknown ? unknownString : "" + totalFileSize); - outStream.write(terminator); - - for (int k = 0; k < indent; k++) { - outStream.writeBytes(Utilities.INDENT); - } - outStream.writeBytes("maxFileSize:"); - outStream.writeBytes(unknown ? unknownString : "" + maxFileSize); - outStream.write(terminator); - - for (int k = 0; k < indent; k++) { - outStream.writeBytes(Utilities.INDENT); - } - outStream.writeBytes("minFileSize:"); - if (numOfFiles > 0) { - outStream.writeBytes(unknown ? unknownString : "" + minFileSize); - } else { - outStream.writeBytes(unknown ? unknownString : "" + 0); - } - outStream.write(terminator); - - for (int k = 0; k < indent; k++) { - outStream.writeBytes(Utilities.INDENT); - } - outStream.writeBytes("lastAccessTime:"); - outStream.writeBytes((unknown || lastAccessTime < 0) ? unknownString : "" - + lastAccessTime); - outStream.write(terminator); - - for (int k = 0; k < indent; k++) { - outStream.writeBytes(Utilities.INDENT); - } - outStream.writeBytes("lastUpdateTime:"); - outStream.writeBytes(unknown ? unknownString : "" + lastUpdateTime); - outStream.write(terminator); - } - /** * Alter a given table. * @@ -2822,8 +2606,10 @@ public class DDLTask extends Task implements Serializable { if(alterTbl.getPartSpec() != null) { part = db.getPartition(tbl, alterTbl.getPartSpec(), false); if(part == null) { - console.printError("Partition : " + alterTbl.getPartSpec().toString() - + " does not exist."); + formatter.consoleError(console, + "Partition : " + alterTbl.getPartSpec().toString() + + " does not exist.", + formatter.MISSING); return 1; } } @@ -2853,7 +2639,9 @@ public class DDLTask extends Task implements Serializable { while (iterOldCols.hasNext()) { String oldColName = iterOldCols.next().getName(); if (oldColName.equalsIgnoreCase(newColName)) { - console.printError("Column '" + newColName + "' exists"); + formatter.consoleError(console, + "Column '" + newColName + "' exists", + formatter.CONFLICT); return 1; } } @@ -2885,7 +2673,9 @@ public class DDLTask extends Task implements Serializable { String oldColName = col.getName(); if (oldColName.equalsIgnoreCase(newName) && !oldColName.equalsIgnoreCase(oldName)) { - console.printError("Column '" + newName + "' exists"); + formatter.consoleError(console, + "Column '" + newName + "' exists", + formatter.CONFLICT); return 1; } else if (oldColName.equalsIgnoreCase(oldName)) { col.setName(newName); @@ -2913,12 +2703,16 @@ public class DDLTask extends Task implements Serializable { // did not find the column if (!found) { - console.printError("Column '" + oldName + "' does not exist"); + formatter.consoleError(console, + "Column '" + oldName + "' does not exists", + formatter.MISSING); return 1; } // after column is not null, but we did not find it. if ((afterCol != null && !afterCol.trim().equals("")) && position < 0) { - console.printError("Column '" + afterCol + "' does not exist"); + formatter.consoleError(console, + "Column '" + afterCol + "' does not exists", + formatter.MISSING); return 1; } @@ -2939,8 +2733,10 @@ public class DDLTask extends Task implements Serializable { && !tbl.getSerializationLib().equals(LazySimpleSerDe.class.getName()) && !tbl.getSerializationLib().equals(ColumnarSerDe.class.getName()) && !tbl.getSerializationLib().equals(DynamicSerDe.class.getName())) { - console.printError("Replace columns is not supported for this table. " - + "SerDe may be incompatible."); + formatter.consoleError(console, + "Replace columns is not supported for this table. " + + "SerDe may be incompatible.", + formatter.ERROR); return 1; } tbl.getTTable().getSd().setCols(alterTbl.getNewCols()); @@ -3071,7 +2867,9 @@ public class DDLTask extends Task implements Serializable { throw new HiveException(e); } } else { - console.printError("Unsupported Alter commnad"); + formatter.consoleError(console, + "Unsupported Alter commnad", + formatter.ERROR); return 1; } @@ -3082,8 +2880,9 @@ public class DDLTask extends Task implements Serializable { try { tbl.checkValidity(); } catch (HiveException e) { - console.printError("Invalid table columns : " + e.getMessage(), - stringifyException(e)); + formatter.consoleError(console, + "Invalid table columns : " + e.getMessage(), + formatter.ERROR); return 1; } } else { @@ -3249,8 +3048,10 @@ public class DDLTask extends Task implements Serializable { try { user = conf.getUser(); } catch (IOException e) { - console.printError("Unable to get current user: " + e.getMessage(), - stringifyException(e)); + formatter.consoleError(console, + "Unable to get current user: " + e.getMessage(), + stringifyException(e), + formatter.ERROR); return false; } @@ -3672,8 +3473,10 @@ public class DDLTask extends Task implements Serializable { try { tbl.setOwner(conf.getUser()); } catch (IOException e) { - console.printError("Unable to get current user: " + e.getMessage(), - stringifyException(e)); + formatter.consoleError(console, + "Unable to get current user: " + e.getMessage(), + stringifyException(e), + formatter.ERROR); return 1; } // set create time diff --git ql/src/java/org/apache/hadoop/hive/ql/metadata/JsonMetaDataFormatter.java ql/src/java/org/apache/hadoop/hive/ql/metadata/JsonMetaDataFormatter.java new file mode 100644 index 0000000..4e2eaa1 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/JsonMetaDataFormatter.java @@ -0,0 +1,460 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.metadata; + +import java.io.DataOutputStream; +import java.io.OutputStream; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.URLDecoder; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.ql.metadata.Partition; +import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.ql.session.SessionState.LogHelper; +import org.apache.hadoop.hive.shims.ShimLoader; +import org.codehaus.jackson.map.ObjectMapper; + +/** + * Format table and index information for machine readability using + * json. + */ +public class JsonMetaDataFormatter implements MetaDataFormatter { + private static final Log LOG = LogFactory.getLog("hive.ql.exec.DDLTask"); + + /** + * Convert the map to a JSON string. + */ + public void asJson(OutputStream out, Map data) + throws HiveException + { + try { + new ObjectMapper().writeValue(out, data); + } catch (IOException e) { + throw new HiveException("Unable to convert to json", e); + } + } + + /** + * Write an error message. + */ + public void error(OutputStream out, String msg, int errorCode) + throws HiveException + { + asJson(out, + MapBuilder.create() + .put("error", msg) + .put("errorCode", errorCode) + .build()); + } + + /** + * Write a log warn message. + */ + public void logWarn(OutputStream out, String msg, int errorCode) + throws HiveException + { + LOG.warn(msg); + error(out, msg, errorCode); + } + + /** + * Write a log info message. + */ + public void logInfo(OutputStream out, String msg, int errorCode) + throws HiveException + { + LOG.info(msg); + error(out, msg, errorCode); + } + + /** + * Write a console error message. + */ + public void consoleError(LogHelper console, String msg, int errorCode) { + try { + console.printError(msg); + error(console.getOutStream(), msg, errorCode); + } catch (HiveException e) { + console.printError("unable to create json: " + e); + } + } + + /** + * Write a console error message. + */ + public void consoleError(LogHelper console, String msg, String detail, + int errorCode) + { + try { + console.printError(msg, detail); + asJson(console.getOutStream(), + MapBuilder.create() + .put("error", msg) + .put("errorDetail", detail) + .put("errorCode", errorCode) + .build()); + } catch (HiveException e) { + console.printError("unable to create json: " + e); + } + } + + /** + * Show a list of tables. + */ + public void showTables(DataOutputStream out, Set tables) + throws HiveException + { + asJson(out, + MapBuilder.create() + .put("tables", tables) + .build()); + } + + /** + * Describe table. + */ + public void describeTable(DataOutputStream out, + String colPath, String tableName, + Table tbl, Partition part, List cols, + boolean isFormatted, boolean isExt) + throws HiveException + { + MapBuilder builder = MapBuilder.create(); + + builder.put("columns", makeColsUnformatted(cols)); + + if (isExt) { + if (part != null) + builder.put("partitionInfo", part.getTPartition()); + else + builder.put("tableInfo", tbl.getTTable()); + } + + asJson(out, builder.build()); + } + + private List makeColsUnformatted(List cols) { + ArrayList res = new ArrayList(); + for (FieldSchema col : cols) + res.add(makeOneColUnformatted(col)); + return res; + } + + private Map makeOneColUnformatted(FieldSchema col) { + return MapBuilder.create() + .put("name", col.getName()) + .put("type", col.getType()) + .put("comment", col.getComment()) + .build(); + } + + public void showTableStatus(DataOutputStream out, + Hive db, + HiveConf conf, + List
tbls, + Map part, + Partition par) + throws HiveException + { + asJson(out, MapBuilder + .create() + .put("tables", makeAllTableStatus(db, conf, + tbls, part, par)) + .build()); + } + + private List makeAllTableStatus(Hive db, + HiveConf conf, + List
tbls, + Map part, + Partition par) + throws HiveException + { + try { + ArrayList res = new ArrayList(); + for (Table tbl : tbls) + res.add(makeOneTableStatus(tbl, db, conf, part, par)); + return res; + } catch(IOException e) { + throw new HiveException(e); + } + } + + private Map makeOneTableStatus(Table tbl, + Hive db, + HiveConf conf, + Map part, + Partition par) + throws HiveException, IOException + { + String tblLoc = null; + String inputFormattCls = null; + String outputFormattCls = null; + if (part != null) { + if (par != null) { + if (par.getLocation() != null) { + tblLoc = par.getDataLocation().toString(); + } + inputFormattCls = par.getInputFormatClass().getName(); + outputFormattCls = par.getOutputFormatClass().getName(); + } + } else { + if (tbl.getPath() != null) { + tblLoc = tbl.getDataLocation().toString(); + } + inputFormattCls = tbl.getInputFormatClass().getName(); + outputFormattCls = tbl.getOutputFormatClass().getName(); + } + + MapBuilder builder = MapBuilder.create(); + + builder.put("tableName", tbl.getTableName()); + builder.put("owner", tbl.getOwner()); + builder.put("location", tblLoc); + builder.put("inputFormat", inputFormattCls); + builder.put("outputFormat", outputFormattCls); + builder.put("columns", makeColsUnformatted(tbl.getCols())); + + builder.put("partitioned", tbl.isPartitioned()); + if (tbl.isPartitioned()) + builder.put("partitionColumns", makeColsUnformatted(tbl.getPartCols())); + + putFileSystemsStats(builder, makeTableStatusLocations(tbl, db, par), + conf, tbl.getPath()); + + return builder.build(); + } + + private List makeTableStatusLocations(Table tbl, Hive db, Partition par) + throws HiveException + { + // output file system information + Path tblPath = tbl.getPath(); + List locations = new ArrayList(); + if (tbl.isPartitioned()) { + if (par == null) { + for (Partition curPart : db.getPartitions(tbl)) { + if (curPart.getLocation() != null) { + locations.add(new Path(curPart.getLocation())); + } + } + } else { + if (par.getLocation() != null) { + locations.add(new Path(par.getLocation())); + } + } + } else { + if (tblPath != null) { + locations.add(tblPath); + } + } + + return locations; + } + + // Duplicates logic in TextMetaDataFormatter + private void putFileSystemsStats(MapBuilder builder, List locations, + HiveConf conf, Path tblPath) + throws IOException + { + long totalFileSize = 0; + long maxFileSize = 0; + long minFileSize = Long.MAX_VALUE; + long lastAccessTime = 0; + long lastUpdateTime = 0; + int numOfFiles = 0; + + boolean unknown = false; + FileSystem fs = tblPath.getFileSystem(conf); + // in case all files in locations do not exist + try { + FileStatus tmpStatus = fs.getFileStatus(tblPath); + lastAccessTime = ShimLoader.getHadoopShims().getAccessTime(tmpStatus); + lastUpdateTime = tmpStatus.getModificationTime(); + } catch (IOException e) { + LOG.warn( + "Cannot access File System. File System status will be unknown: ", e); + unknown = true; + } + + if (!unknown) { + for (Path loc : locations) { + try { + FileStatus status = fs.getFileStatus(tblPath); + FileStatus[] files = fs.listStatus(loc); + long accessTime = ShimLoader.getHadoopShims().getAccessTime(status); + long updateTime = status.getModificationTime(); + // no matter loc is the table location or part location, it must be a + // directory. + if (!status.isDir()) { + continue; + } + if (accessTime > lastAccessTime) { + lastAccessTime = accessTime; + } + if (updateTime > lastUpdateTime) { + lastUpdateTime = updateTime; + } + for (FileStatus currentStatus : files) { + if (currentStatus.isDir()) { + continue; + } + numOfFiles++; + long fileLen = currentStatus.getLen(); + totalFileSize += fileLen; + if (fileLen > maxFileSize) { + maxFileSize = fileLen; + } + if (fileLen < minFileSize) { + minFileSize = fileLen; + } + accessTime = ShimLoader.getHadoopShims().getAccessTime( + currentStatus); + updateTime = currentStatus.getModificationTime(); + if (accessTime > lastAccessTime) { + lastAccessTime = accessTime; + } + if (updateTime > lastUpdateTime) { + lastUpdateTime = updateTime; + } + } + } catch (IOException e) { + // ignore + } + } + } + + builder + .put("totalNumberFiles", numOfFiles, ! unknown) + .put("totalFileSize", totalFileSize, ! unknown) + .put("maxFileSize", maxFileSize, ! unknown) + .put("minFileSize", numOfFiles > 0 ? minFileSize : 0, ! unknown) + .put("lastAccessTime", lastAccessTime, ! (unknown || lastAccessTime < 0)) + .put("lastUpdateTime", lastUpdateTime, ! unknown); + } + + /** + * Show the table partitions. + */ + public void showTablePartitons(DataOutputStream out, List parts) + throws HiveException + { + asJson(out, + MapBuilder.create() + .put("partitions", makeTablePartions(parts)) + .build()); + } + + private List makeTablePartions(List parts) + throws HiveException + { + try { + ArrayList res = new ArrayList(); + for (String part : parts) + res.add(makeOneTablePartition(part)); + return res; + } catch (UnsupportedEncodingException e) { + throw new HiveException(e); + } + } + + // This seems like a very wrong implementation. + private Map makeOneTablePartition(String partIdent) + throws UnsupportedEncodingException + { + ArrayList res = new ArrayList(); + + ArrayList names = new ArrayList(); + for (String part : StringUtils.split(partIdent, "/")) { + String name = part; + String val = null; + String[] kv = StringUtils.split(part, "=", 2); + if (kv != null) { + name = kv[0]; + if (kv.length > 1) + val = URLDecoder.decode(kv[1], "UTF-8"); + } + if (val != null) + names.add(name + "='" + val + "'"); + else + names.add(name); + + res.add(MapBuilder.create() + .put("columnName", name) + .put("columnValue", val) + .build()); + } + + return MapBuilder.create() + .put("name", StringUtils.join(names, ",")) + .put("values", res) + .build(); + } + + /** + * Show a list of databases + */ + public void showDatabases(DataOutputStream out, List databases) + throws HiveException + { + asJson(out, + MapBuilder.create() + .put("databases", databases) + .build()); + } + + /** + * Show the description of a database + */ + public void showDatabaseDescription(DataOutputStream out, + String database, + String comment, + String location, + Map params) + throws HiveException + { + if (params == null || params.isEmpty()) { + asJson(out, MapBuilder + .create() + .put("database", database) + .put("comment", comment) + .put("location", location) + .build()); + } else { + asJson(out, MapBuilder + .create() + .put("database", database) + .put("comment", comment) + .put("location", location) + .put("params", params) + .build()); + } + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/metadata/MapBuilder.java ql/src/java/org/apache/hadoop/hive/ql/metadata/MapBuilder.java new file mode 100644 index 0000000..61a9a97 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/MapBuilder.java @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.metadata; + +import java.util.HashMap; +import java.util.Map; + +/** + * Helper class to build Maps consumed by the JSON formatter. Only + * add non-null entries to the Map. + */ +public class MapBuilder { + private Map map = new HashMap(); + + private MapBuilder() {} + + public static MapBuilder create() { + return new MapBuilder(); + } + + public MapBuilder put(String name, Object val) { + if (val != null) + map.put(name, val); + return this; + } + + public MapBuilder put(String name, boolean val) { + map.put(name, Boolean.valueOf(val)); + return this; + } + + public MapBuilder put(String name, int val) { + map.put(name, Integer.valueOf(val)); + return this; + } + + public MapBuilder put(String name, long val) { + map.put(name, Long.valueOf(val)); + return this; + } + + public MapBuilder put(String name, T val, boolean use) { + if (use) + put(name, val); + return this; + } + + public Map build() { + return map; + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/metadata/MetaDataFormatter.java ql/src/java/org/apache/hadoop/hive/ql/metadata/MetaDataFormatter.java new file mode 100644 index 0000000..a015cb4 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/MetaDataFormatter.java @@ -0,0 +1,133 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.metadata; + +import java.io.DataOutputStream; +import java.io.OutputStream; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.ql.metadata.Partition; +import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.ql.session.SessionState.LogHelper; + +/** + * Interface to format table and index information. We can format it + * for human readability (lines of text) or for machine readability + * (json). + */ +public interface MetaDataFormatter { + /** + * Generic error code. This and the other error codes are + * designed to match the HTTP status codes. + */ + static final int ERROR = 500; + + /** + * Missing error code. + */ + static final int MISSING = 404; + + /** + * Conflict error code. + */ + static final int CONFLICT = 409; + + /** + * Write an error message. + */ + public void error(OutputStream out, String msg, int errorCode) + throws HiveException; + + /** + * Write a log warn message. + */ + public void logWarn(OutputStream out, String msg, int errorCode) + throws HiveException; + + /** + * Write a log info message. + */ + public void logInfo(OutputStream out, String msg, int errorCode) + throws HiveException; + + /** + * Write a console error message. + */ + public void consoleError(LogHelper console, String msg, int errorCode); + + /** + * Write a console error message. + */ + public void consoleError(LogHelper console, String msg, String detail, + int errorCode); + + /** + * Show a list of tables. + */ + public void showTables(DataOutputStream out, Set tables) + throws HiveException; + + /** + * Describe table. + */ + public void describeTable(DataOutputStream out, + String colPath, String tableName, + Table tbl, Partition part, List cols, + boolean isFormatted, boolean isExt) + throws HiveException; + + /** + * Show the table status. + */ + public void showTableStatus(DataOutputStream out, + Hive db, + HiveConf conf, + List
tbls, + Map part, + Partition par) + throws HiveException; + + /** + * Show the table partitions. + */ + public void showTablePartitons(DataOutputStream out, + List parts) + throws HiveException; + + /** + * Show the databases + */ + public void showDatabases(DataOutputStream out, List databases) + throws HiveException; + + /** + * Describe a database. + */ + public void showDatabaseDescription(DataOutputStream out, + String database, + String comment, + String location, + Map params) + throws HiveException; +} + diff --git ql/src/java/org/apache/hadoop/hive/ql/metadata/TextMetaDataFormatter.java ql/src/java/org/apache/hadoop/hive/ql/metadata/TextMetaDataFormatter.java new file mode 100644 index 0000000..6c44bd1 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/TextMetaDataFormatter.java @@ -0,0 +1,477 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.metadata; + +import java.io.DataOutputStream; +import java.io.OutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.MetaStoreUtils; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.ql.exec.Utilities; +import org.apache.hadoop.hive.ql.metadata.Partition; +import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.ql.session.SessionState.LogHelper; +import org.apache.hadoop.hive.shims.ShimLoader; + +/** + * Format table and index information for human readability using + * simple lines of text. + */ +public class TextMetaDataFormatter implements MetaDataFormatter { + private static final Log LOG = LogFactory.getLog("hive.ql.exec.DDLTask"); + + private static final int separator = Utilities.tabCode; + private static final int terminator = Utilities.newLineCode; + + /** + * Write an error message. + */ + public void error(OutputStream out, String msg, int errorCode) + throws HiveException + { + try { + out.write(msg.getBytes("UTF-8")); + out.write(terminator); + } catch (Exception e) { + throw new HiveException(e); + } + } + + /** + * Write a log warn message. + */ + public void logWarn(OutputStream out, String msg, int errorCode) + throws HiveException + { + LOG.warn(msg); + } + + /** + * Write a log info message. + */ + public void logInfo(OutputStream out, String msg, int errorCode) + throws HiveException + { + LOG.info(msg); + } + + /** + * Write a console error message. + */ + public void consoleError(LogHelper console, String msg, int errorCode) { + console.printError(msg); + } + + /** + * Write a console error message. + */ + public void consoleError(LogHelper console, String msg, String detail, + int errorCode) + { + console.printError(msg, detail); + } + + /** + * Show a list of tables. + */ + public void showTables(DataOutputStream out, Set tables) + throws HiveException + { + Iterator iterTbls = tables.iterator(); + + try { + while (iterTbls.hasNext()) { + // create a row per table name + out.writeBytes(iterTbls.next()); + out.write(terminator); + } + } catch (IOException e) { + throw new HiveException(e); + } + } + + public void describeTable(DataOutputStream outStream, + String colPath, String tableName, + Table tbl, Partition part, List cols, + boolean isFormatted, boolean isExt) + throws HiveException + { + try { + auxDescribeTable(outStream, colPath, tableName, tbl, part, cols, + isFormatted, isExt); + } catch (IOException e) { + throw new HiveException(e); + } + } + + private void auxDescribeTable(DataOutputStream outStream, + String colPath, String tableName, + Table tbl, Partition part, List cols, + boolean isFormatted, boolean isExt) + throws HiveException, IOException + { + if (colPath.equals(tableName)) { + if (!isFormatted) { + outStream.writeBytes(MetaDataFormatUtils.displayColsUnformatted(cols)); + } else { + outStream.writeBytes( + MetaDataFormatUtils.getAllColumnsInformation(cols, + tbl.isPartitioned() ? tbl.getPartCols() : null)); + } + } else { + if (isFormatted) { + outStream.writeBytes(MetaDataFormatUtils.getAllColumnsInformation(cols)); + } else { + outStream.writeBytes(MetaDataFormatUtils.displayColsUnformatted(cols)); + } + } + + if (tableName.equals(colPath)) { + + if (isFormatted) { + if (part != null) { + outStream.writeBytes(MetaDataFormatUtils.getPartitionInformation(part)); + } else { + outStream.writeBytes(MetaDataFormatUtils.getTableInformation(tbl)); + } + } + + // if extended desc table then show the complete details of the table + if (isExt) { + // add empty line + outStream.write(terminator); + if (part != null) { + // show partition information + outStream.writeBytes("Detailed Partition Information"); + outStream.write(separator); + outStream.writeBytes(part.getTPartition().toString()); + outStream.write(separator); + // comment column is empty + outStream.write(terminator); + } else { + // show table information + outStream.writeBytes("Detailed Table Information"); + outStream.write(separator); + outStream.writeBytes(tbl.getTTable().toString()); + outStream.write(separator); + outStream.write(terminator); + } + } + } + } + + public void showTableStatus(DataOutputStream outStream, + Hive db, + HiveConf conf, + List
tbls, + Map part, + Partition par) + throws HiveException + { + try { + auxShowTableStatus(outStream, db, conf, tbls, part, par); + } catch (IOException e) { + throw new HiveException(e); + } + } + + private void auxShowTableStatus(DataOutputStream outStream, + Hive db, + HiveConf conf, + List
tbls, + Map part, + Partition par) + throws HiveException, IOException + { + Iterator
iterTables = tbls.iterator(); + while (iterTables.hasNext()) { + // create a row per table name + Table tbl = iterTables.next(); + String tableName = tbl.getTableName(); + String tblLoc = null; + String inputFormattCls = null; + String outputFormattCls = null; + if (part != null) { + if (par != null) { + if (par.getLocation() != null) { + tblLoc = par.getDataLocation().toString(); + } + inputFormattCls = par.getInputFormatClass().getName(); + outputFormattCls = par.getOutputFormatClass().getName(); + } + } else { + if (tbl.getPath() != null) { + tblLoc = tbl.getDataLocation().toString(); + } + inputFormattCls = tbl.getInputFormatClass().getName(); + outputFormattCls = tbl.getOutputFormatClass().getName(); + } + + String owner = tbl.getOwner(); + List cols = tbl.getCols(); + String ddlCols = MetaStoreUtils.getDDLFromFieldSchema("columns", cols); + boolean isPartitioned = tbl.isPartitioned(); + String partitionCols = ""; + if (isPartitioned) { + partitionCols = MetaStoreUtils.getDDLFromFieldSchema( + "partition_columns", tbl.getPartCols()); + } + + outStream.writeBytes("tableName:" + tableName); + outStream.write(terminator); + outStream.writeBytes("owner:" + owner); + outStream.write(terminator); + outStream.writeBytes("location:" + tblLoc); + outStream.write(terminator); + outStream.writeBytes("inputformat:" + inputFormattCls); + outStream.write(terminator); + outStream.writeBytes("outputformat:" + outputFormattCls); + outStream.write(terminator); + outStream.writeBytes("columns:" + ddlCols); + outStream.write(terminator); + outStream.writeBytes("partitioned:" + isPartitioned); + outStream.write(terminator); + outStream.writeBytes("partitionColumns:" + partitionCols); + outStream.write(terminator); + // output file system information + Path tblPath = tbl.getPath(); + List locations = new ArrayList(); + if (isPartitioned) { + if (par == null) { + for (Partition curPart : db.getPartitions(tbl)) { + if (curPart.getLocation() != null) { + locations.add(new Path(curPart.getLocation())); + } + } + } else { + if (par.getLocation() != null) { + locations.add(new Path(par.getLocation())); + } + } + } else { + if (tblPath != null) { + locations.add(tblPath); + } + } + if (!locations.isEmpty()) { + writeFileSystemStats(outStream, conf, locations, tblPath, false, 0); + } + + outStream.write(terminator); + } + } + + private void writeFileSystemStats(DataOutputStream outStream, + HiveConf conf, + List locations, + Path tblPath, boolean partSpecified, int indent) + throws IOException + { + long totalFileSize = 0; + long maxFileSize = 0; + long minFileSize = Long.MAX_VALUE; + long lastAccessTime = 0; + long lastUpdateTime = 0; + int numOfFiles = 0; + + boolean unknown = false; + FileSystem fs = tblPath.getFileSystem(conf); + // in case all files in locations do not exist + try { + FileStatus tmpStatus = fs.getFileStatus(tblPath); + lastAccessTime = ShimLoader.getHadoopShims().getAccessTime(tmpStatus); + lastUpdateTime = tmpStatus.getModificationTime(); + if (partSpecified) { + // check whether the part exists or not in fs + tmpStatus = fs.getFileStatus(locations.get(0)); + } + } catch (IOException e) { + LOG.warn( + "Cannot access File System. File System status will be unknown: ", e); + unknown = true; + } + + if (!unknown) { + for (Path loc : locations) { + try { + FileStatus status = fs.getFileStatus(tblPath); + FileStatus[] files = fs.listStatus(loc); + long accessTime = ShimLoader.getHadoopShims().getAccessTime(status); + long updateTime = status.getModificationTime(); + // no matter loc is the table location or part location, it must be a + // directory. + if (!status.isDir()) { + continue; + } + if (accessTime > lastAccessTime) { + lastAccessTime = accessTime; + } + if (updateTime > lastUpdateTime) { + lastUpdateTime = updateTime; + } + for (FileStatus currentStatus : files) { + if (currentStatus.isDir()) { + continue; + } + numOfFiles++; + long fileLen = currentStatus.getLen(); + totalFileSize += fileLen; + if (fileLen > maxFileSize) { + maxFileSize = fileLen; + } + if (fileLen < minFileSize) { + minFileSize = fileLen; + } + accessTime = ShimLoader.getHadoopShims().getAccessTime( + currentStatus); + updateTime = currentStatus.getModificationTime(); + if (accessTime > lastAccessTime) { + lastAccessTime = accessTime; + } + if (updateTime > lastUpdateTime) { + lastUpdateTime = updateTime; + } + } + } catch (IOException e) { + // ignore + } + } + } + String unknownString = "unknown"; + + for (int k = 0; k < indent; k++) { + outStream.writeBytes(Utilities.INDENT); + } + outStream.writeBytes("totalNumberFiles:"); + outStream.writeBytes(unknown ? unknownString : "" + numOfFiles); + outStream.write(terminator); + + for (int k = 0; k < indent; k++) { + outStream.writeBytes(Utilities.INDENT); + } + outStream.writeBytes("totalFileSize:"); + outStream.writeBytes(unknown ? unknownString : "" + totalFileSize); + outStream.write(terminator); + + for (int k = 0; k < indent; k++) { + outStream.writeBytes(Utilities.INDENT); + } + outStream.writeBytes("maxFileSize:"); + outStream.writeBytes(unknown ? unknownString : "" + maxFileSize); + outStream.write(terminator); + + for (int k = 0; k < indent; k++) { + outStream.writeBytes(Utilities.INDENT); + } + outStream.writeBytes("minFileSize:"); + if (numOfFiles > 0) { + outStream.writeBytes(unknown ? unknownString : "" + minFileSize); + } else { + outStream.writeBytes(unknown ? unknownString : "" + 0); + } + outStream.write(terminator); + + for (int k = 0; k < indent; k++) { + outStream.writeBytes(Utilities.INDENT); + } + outStream.writeBytes("lastAccessTime:"); + outStream.writeBytes((unknown || lastAccessTime < 0) ? unknownString : "" + + lastAccessTime); + outStream.write(terminator); + + for (int k = 0; k < indent; k++) { + outStream.writeBytes(Utilities.INDENT); + } + outStream.writeBytes("lastUpdateTime:"); + outStream.writeBytes(unknown ? unknownString : "" + lastUpdateTime); + outStream.write(terminator); + } + + /** + * Show the table partitions. + */ + public void showTablePartitons(DataOutputStream outStream, List parts) + throws HiveException + { + try { + for (String part : parts) { + outStream.writeBytes(part); + outStream.write(terminator); + } + } catch (IOException e) { + throw new HiveException(e); + } + } + + /** + * Show the list of databases + */ + public void showDatabases(DataOutputStream outStream, List databases) + throws HiveException + { + try { + for (String database : databases) { + // create a row per database name + outStream.writeBytes(database); + outStream.write(terminator); + } + } catch (IOException e) { + throw new HiveException(e); + } + } + + /** + * Describe a database + */ + public void showDatabaseDescription(DataOutputStream outStream, + String database, + String comment, + String location, + Map params) + throws HiveException + { + try { + outStream.writeBytes(database); + outStream.write(separator); + if (comment != null) + outStream.writeBytes(comment); + outStream.write(separator); + if (location != null) + outStream.writeBytes(location); + outStream.write(separator); + if (params != null && !params.isEmpty()) { + outStream.writeBytes(params.toString()); + } + outStream.write(terminator); + } catch (IOException e) { + throw new HiveException(e); + } + } +} diff --git ql/src/test/queries/clientpositive/describe_database_json.q ql/src/test/queries/clientpositive/describe_database_json.q new file mode 100644 index 0000000..6dbbda0 --- /dev/null +++ ql/src/test/queries/clientpositive/describe_database_json.q @@ -0,0 +1,23 @@ +set hive.format=json; + +CREATE DATABASE IF NOT EXISTS jsondb1 COMMENT 'Test database' LOCATION '${hiveconf:hive.metastore.warehouse.dir}/jsondb1' WITH DBPROPERTIES ('id' = 'jsondb1'); + +DESCRIBE DATABASE jsondb1; + +DESCRIBE DATABASE EXTENDED jsondb1; + +SHOW DATABASES; + +SHOW DATABASES LIKE 'json*'; + +DROP DATABASE jsondb1; + +CREATE DATABASE jsondb1; + +DESCRIBE DATABASE jsondb1; + +DESCRIBE DATABASE EXTENDED jsondb1; + +DROP DATABASE jsondb1; + +set hive.format=text; diff --git ql/src/test/queries/clientpositive/describe_table_json.q ql/src/test/queries/clientpositive/describe_table_json.q new file mode 100644 index 0000000..78f8113 --- /dev/null +++ ql/src/test/queries/clientpositive/describe_table_json.q @@ -0,0 +1,19 @@ +set hive.format=json; + +CREATE TABLE IF NOT EXISTS jsontable (key INT, value STRING) COMMENT 'json table' STORED AS TEXTFILE; + +SHOW TABLES; + +SHOW TABLES LIKE 'json*'; + +SHOW TABLE EXTENDED LIKE 'json*'; + +ALTER TABLE jsontable SET TBLPROPERTIES ('id' = 'jsontable'); + +DESCRIBE jsontable; + +DESCRIBE extended jsontable; + +DROP TABLE jsontable; + +set hive.format=text; diff --git ql/src/test/queries/clientpositive/misc_json.q ql/src/test/queries/clientpositive/misc_json.q new file mode 100644 index 0000000..cc746aa --- /dev/null +++ ql/src/test/queries/clientpositive/misc_json.q @@ -0,0 +1,13 @@ +set hive.format=json; + +CREATE TABLE IF NOT EXISTS jsontable (key INT, value STRING) COMMENT 'json table' STORED AS TEXTFILE; + +ALTER TABLE jsontable ADD COLUMNS (name STRING COMMENT 'a new column'); + +ALTER TABLE jsontable RENAME TO jsontable2; + +SHOW TABLE EXTENDED LIKE jsontable2; + +DROP TABLE jsontable2; + +set hive.format=text; diff --git ql/src/test/queries/clientpositive/partitions_json.q ql/src/test/queries/clientpositive/partitions_json.q new file mode 100644 index 0000000..c7f3f2a --- /dev/null +++ ql/src/test/queries/clientpositive/partitions_json.q @@ -0,0 +1,19 @@ +set hive.format=json; + +CREATE TABLE add_part_test (key STRING, value STRING) PARTITIONED BY (ds STRING); +SHOW PARTITIONS add_part_test; + +ALTER TABLE add_part_test ADD PARTITION (ds='2010-01-01'); +SHOW PARTITIONS add_part_test; + +ALTER TABLE add_part_test ADD IF NOT EXISTS PARTITION (ds='2010-01-01'); +SHOW PARTITIONS add_part_test; + +ALTER TABLE add_part_test ADD IF NOT EXISTS PARTITION (ds='2010-01-02'); +SHOW PARTITIONS add_part_test; + +SHOW TABLE EXTENDED LIKE add_part_test PARTITION (ds='2010-01-02'); + +ALTER TABLE add_part_test DROP PARTITION (ds='2010-01-02'); + +DROP TABLE add_part_test; diff --git ql/src/test/results/clientpositive/describe_database_json.q.out ql/src/test/results/clientpositive/describe_database_json.q.out new file mode 100644 index 0000000..5414469 --- /dev/null +++ ql/src/test/results/clientpositive/describe_database_json.q.out @@ -0,0 +1,46 @@ +#### A masked pattern was here #### +PREHOOK: type: CREATEDATABASE +#### A masked pattern was here #### +POSTHOOK: type: CREATEDATABASE +PREHOOK: query: DESCRIBE DATABASE jsondb1 +PREHOOK: type: DESCDATABASE +POSTHOOK: query: DESCRIBE DATABASE jsondb1 +POSTHOOK: type: DESCDATABASE +#### A masked pattern was here #### +PREHOOK: query: DESCRIBE DATABASE EXTENDED jsondb1 +PREHOOK: type: DESCDATABASE +POSTHOOK: query: DESCRIBE DATABASE EXTENDED jsondb1 +POSTHOOK: type: DESCDATABASE +#### A masked pattern was here #### +PREHOOK: query: SHOW DATABASES +PREHOOK: type: SHOWDATABASES +POSTHOOK: query: SHOW DATABASES +POSTHOOK: type: SHOWDATABASES +{"databases":["default","jsondb1"]} +PREHOOK: query: SHOW DATABASES LIKE 'json*' +PREHOOK: type: SHOWDATABASES +POSTHOOK: query: SHOW DATABASES LIKE 'json*' +POSTHOOK: type: SHOWDATABASES +{"databases":["jsondb1"]} +PREHOOK: query: DROP DATABASE jsondb1 +PREHOOK: type: DROPDATABASE +POSTHOOK: query: DROP DATABASE jsondb1 +POSTHOOK: type: DROPDATABASE +PREHOOK: query: CREATE DATABASE jsondb1 +PREHOOK: type: CREATEDATABASE +POSTHOOK: query: CREATE DATABASE jsondb1 +POSTHOOK: type: CREATEDATABASE +PREHOOK: query: DESCRIBE DATABASE jsondb1 +PREHOOK: type: DESCDATABASE +POSTHOOK: query: DESCRIBE DATABASE jsondb1 +POSTHOOK: type: DESCDATABASE +#### A masked pattern was here #### +PREHOOK: query: DESCRIBE DATABASE EXTENDED jsondb1 +PREHOOK: type: DESCDATABASE +POSTHOOK: query: DESCRIBE DATABASE EXTENDED jsondb1 +POSTHOOK: type: DESCDATABASE +#### A masked pattern was here #### +PREHOOK: query: DROP DATABASE jsondb1 +PREHOOK: type: DROPDATABASE +POSTHOOK: query: DROP DATABASE jsondb1 +POSTHOOK: type: DROPDATABASE diff --git ql/src/test/results/clientpositive/describe_table_json.q.out ql/src/test/results/clientpositive/describe_table_json.q.out new file mode 100644 index 0000000..d4f497f --- /dev/null +++ ql/src/test/results/clientpositive/describe_table_json.q.out @@ -0,0 +1,46 @@ +PREHOOK: query: CREATE TABLE IF NOT EXISTS jsontable (key INT, value STRING) COMMENT 'json table' STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +POSTHOOK: query: CREATE TABLE IF NOT EXISTS jsontable (key INT, value STRING) COMMENT 'json table' STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@jsontable +PREHOOK: query: SHOW TABLES +PREHOOK: type: SHOWTABLES +POSTHOOK: query: SHOW TABLES +POSTHOOK: type: SHOWTABLES +{"tables":["jsontable","src","src1","src_json","src_sequencefile","src_thrift","srcbucket","srcbucket2","srcpart"]} +PREHOOK: query: SHOW TABLES LIKE 'json*' +PREHOOK: type: SHOWTABLES +POSTHOOK: query: SHOW TABLES LIKE 'json*' +POSTHOOK: type: SHOWTABLES +{"tables":["jsontable"]} +PREHOOK: query: SHOW TABLE EXTENDED LIKE 'json*' +PREHOOK: type: SHOW_TABLESTATUS +POSTHOOK: query: SHOW TABLE EXTENDED LIKE 'json*' +POSTHOOK: type: SHOW_TABLESTATUS +{"tables":[]} +PREHOOK: query: ALTER TABLE jsontable SET TBLPROPERTIES ('id' = 'jsontable') +PREHOOK: type: ALTERTABLE_PROPERTIES +PREHOOK: Input: default@jsontable +PREHOOK: Output: default@jsontable +POSTHOOK: query: ALTER TABLE jsontable SET TBLPROPERTIES ('id' = 'jsontable') +POSTHOOK: type: ALTERTABLE_PROPERTIES +POSTHOOK: Input: default@jsontable +POSTHOOK: Output: default@jsontable +PREHOOK: query: DESCRIBE jsontable +PREHOOK: type: DESCTABLE +POSTHOOK: query: DESCRIBE jsontable +POSTHOOK: type: DESCTABLE +{"columns":[{"name":"key","type":"int"},{"name":"value","type":"string"}]} +PREHOOK: query: DESCRIBE extended jsontable +PREHOOK: type: DESCTABLE +POSTHOOK: query: DESCRIBE extended jsontable +POSTHOOK: type: DESCTABLE +#### A masked pattern was here #### +PREHOOK: query: DROP TABLE jsontable +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@jsontable +PREHOOK: Output: default@jsontable +POSTHOOK: query: DROP TABLE jsontable +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@jsontable +POSTHOOK: Output: default@jsontable diff --git ql/src/test/results/clientpositive/misc_json.q.out ql/src/test/results/clientpositive/misc_json.q.out new file mode 100644 index 0000000..42d3ace --- /dev/null +++ ql/src/test/results/clientpositive/misc_json.q.out @@ -0,0 +1,35 @@ +PREHOOK: query: CREATE TABLE IF NOT EXISTS jsontable (key INT, value STRING) COMMENT 'json table' STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +POSTHOOK: query: CREATE TABLE IF NOT EXISTS jsontable (key INT, value STRING) COMMENT 'json table' STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@jsontable +PREHOOK: query: ALTER TABLE jsontable ADD COLUMNS (name STRING COMMENT 'a new column') +PREHOOK: type: ALTERTABLE_ADDCOLS +PREHOOK: Input: default@jsontable +PREHOOK: Output: default@jsontable +POSTHOOK: query: ALTER TABLE jsontable ADD COLUMNS (name STRING COMMENT 'a new column') +POSTHOOK: type: ALTERTABLE_ADDCOLS +POSTHOOK: Input: default@jsontable +POSTHOOK: Output: default@jsontable +PREHOOK: query: ALTER TABLE jsontable RENAME TO jsontable2 +PREHOOK: type: ALTERTABLE_RENAME +PREHOOK: Input: default@jsontable +PREHOOK: Output: default@jsontable +POSTHOOK: query: ALTER TABLE jsontable RENAME TO jsontable2 +POSTHOOK: type: ALTERTABLE_RENAME +POSTHOOK: Input: default@jsontable +POSTHOOK: Output: default@jsontable +POSTHOOK: Output: default@jsontable2 +PREHOOK: query: SHOW TABLE EXTENDED LIKE jsontable2 +PREHOOK: type: SHOW_TABLESTATUS +POSTHOOK: query: SHOW TABLE EXTENDED LIKE jsontable2 +POSTHOOK: type: SHOW_TABLESTATUS +#### A masked pattern was here #### +PREHOOK: query: DROP TABLE jsontable2 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@jsontable2 +PREHOOK: Output: default@jsontable2 +POSTHOOK: query: DROP TABLE jsontable2 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@jsontable2 +POSTHOOK: Output: default@jsontable2 diff --git ql/src/test/results/clientpositive/partitions_json.q.out ql/src/test/results/clientpositive/partitions_json.q.out new file mode 100644 index 0000000..deb7a1f --- /dev/null +++ ql/src/test/results/clientpositive/partitions_json.q.out @@ -0,0 +1,68 @@ +PREHOOK: query: CREATE TABLE add_part_test (key STRING, value STRING) PARTITIONED BY (ds STRING) +PREHOOK: type: CREATETABLE +POSTHOOK: query: CREATE TABLE add_part_test (key STRING, value STRING) PARTITIONED BY (ds STRING) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@add_part_test +PREHOOK: query: SHOW PARTITIONS add_part_test +PREHOOK: type: SHOWPARTITIONS +POSTHOOK: query: SHOW PARTITIONS add_part_test +POSTHOOK: type: SHOWPARTITIONS +{"partitions":[]} +PREHOOK: query: ALTER TABLE add_part_test ADD PARTITION (ds='2010-01-01') +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Input: default@add_part_test +POSTHOOK: query: ALTER TABLE add_part_test ADD PARTITION (ds='2010-01-01') +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Input: default@add_part_test +POSTHOOK: Output: default@add_part_test@ds=2010-01-01 +PREHOOK: query: SHOW PARTITIONS add_part_test +PREHOOK: type: SHOWPARTITIONS +POSTHOOK: query: SHOW PARTITIONS add_part_test +POSTHOOK: type: SHOWPARTITIONS +{"partitions":[{"values":[{"columnName":"ds","columnValue":"2010-01-01"}],"name":"ds='2010-01-01'"}]} +PREHOOK: query: ALTER TABLE add_part_test ADD IF NOT EXISTS PARTITION (ds='2010-01-01') +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Input: default@add_part_test +PREHOOK: Output: default@add_part_test@ds=2010-01-01 +POSTHOOK: query: ALTER TABLE add_part_test ADD IF NOT EXISTS PARTITION (ds='2010-01-01') +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Input: default@add_part_test +POSTHOOK: Output: default@add_part_test@ds=2010-01-01 +PREHOOK: query: SHOW PARTITIONS add_part_test +PREHOOK: type: SHOWPARTITIONS +POSTHOOK: query: SHOW PARTITIONS add_part_test +POSTHOOK: type: SHOWPARTITIONS +{"partitions":[{"values":[{"columnName":"ds","columnValue":"2010-01-01"}],"name":"ds='2010-01-01'"}]} +PREHOOK: query: ALTER TABLE add_part_test ADD IF NOT EXISTS PARTITION (ds='2010-01-02') +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Input: default@add_part_test +POSTHOOK: query: ALTER TABLE add_part_test ADD IF NOT EXISTS PARTITION (ds='2010-01-02') +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Input: default@add_part_test +POSTHOOK: Output: default@add_part_test@ds=2010-01-02 +PREHOOK: query: SHOW PARTITIONS add_part_test +PREHOOK: type: SHOWPARTITIONS +POSTHOOK: query: SHOW PARTITIONS add_part_test +POSTHOOK: type: SHOWPARTITIONS +{"partitions":[{"values":[{"columnName":"ds","columnValue":"2010-01-01"}],"name":"ds='2010-01-01'"},{"values":[{"columnName":"ds","columnValue":"2010-01-02"}],"name":"ds='2010-01-02'"}]} +PREHOOK: query: SHOW TABLE EXTENDED LIKE add_part_test PARTITION (ds='2010-01-02') +PREHOOK: type: SHOW_TABLESTATUS +POSTHOOK: query: SHOW TABLE EXTENDED LIKE add_part_test PARTITION (ds='2010-01-02') +POSTHOOK: type: SHOW_TABLESTATUS +#### A masked pattern was here #### +PREHOOK: query: ALTER TABLE add_part_test DROP PARTITION (ds='2010-01-02') +PREHOOK: type: ALTERTABLE_DROPPARTS +PREHOOK: Input: default@add_part_test +PREHOOK: Output: default@add_part_test@ds=2010-01-02 +POSTHOOK: query: ALTER TABLE add_part_test DROP PARTITION (ds='2010-01-02') +POSTHOOK: type: ALTERTABLE_DROPPARTS +POSTHOOK: Input: default@add_part_test +POSTHOOK: Output: default@add_part_test@ds=2010-01-02 +PREHOOK: query: DROP TABLE add_part_test +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@add_part_test +PREHOOK: Output: default@add_part_test +POSTHOOK: query: DROP TABLE add_part_test +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@add_part_test +POSTHOOK: Output: default@add_part_test