commit 73031c661f292ae61697dba344a6ceb762877be5 Author: Alex Behm Date: Wed Dec 14 10:17:47 2016 -0800 IMPALA-3641: Adding additional logging. Change-Id: I0c6ed6e95c88834630532d9f15265e0d3f275f9f diff --git a/fe/src/main/java/org/apache/impala/analysis/Analyzer.java b/fe/src/main/java/org/apache/impala/analysis/Analyzer.java index 8bea3aa..7d386ce 100644 --- a/fe/src/main/java/org/apache/impala/analysis/Analyzer.java +++ b/fe/src/main/java/org/apache/impala/analysis/Analyzer.java @@ -2467,7 +2467,10 @@ public class Analyzer { if (db == null) { throw new DatabaseNotFoundException("Database not found: " + dbName); } - return db.containsTable(tableName); + Table table = db.getTable(tableName); + if (table == null) return false; + LOG.debug("IMPALA-3641 Tbl exists: " + table.debugImpala3641()); + return true; } catch (DatabaseNotFoundException e) { throw new AnalysisException(DB_DOES_NOT_EXIST_ERROR_MSG + dbName); } diff --git a/fe/src/main/java/org/apache/impala/analysis/CreateTableStmt.java b/fe/src/main/java/org/apache/impala/analysis/CreateTableStmt.java index 1139005..b0a4bc6 100644 --- a/fe/src/main/java/org/apache/impala/analysis/CreateTableStmt.java +++ b/fe/src/main/java/org/apache/impala/analysis/CreateTableStmt.java @@ -34,6 +34,8 @@ import org.apache.impala.util.AvroSchemaConverter; import org.apache.impala.util.AvroSchemaParser; import org.apache.impala.util.AvroSchemaUtils; import org.apache.impala.util.KuduUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; @@ -46,6 +48,8 @@ import com.google.common.primitives.Ints; */ public class CreateTableStmt extends StatementBase { + private final static Logger LOG = LoggerFactory.getLogger(CreateTableStmt.class); + @VisibleForTesting final static String KUDU_STORAGE_HANDLER_ERROR_MESSAGE = "Kudu tables must be" + " specified using 'STORED AS KUDU' without using the storage handler table" @@ -160,7 +164,14 @@ public class CreateTableStmt extends StatementBase { public void analyze(Analyzer analyzer) throws AnalysisException { super.analyze(analyzer); owner_ = analyzer.getUser().getName(); - tableDef_.analyze(analyzer); + try { + tableDef_.analyze(analyzer); + } catch (AnalysisException e) { + if (e.getMessage().startsWith(Analyzer.TBL_ALREADY_EXISTS_ERROR_MSG)) { + LOG.debug("IMPALA-3641 Tbl exists: " + analyzer.getCatalog().debugImpala3641()); + } + throw e; + } analyzeKuduFormat(analyzer); // Avro tables can have empty column defs because they can infer them from the Avro // schema. Likewise for external Kudu tables, the schema can be read from Kudu. diff --git a/fe/src/main/java/org/apache/impala/analysis/TableDef.java b/fe/src/main/java/org/apache/impala/analysis/TableDef.java index ae314c3..a8f73f2 100644 --- a/fe/src/main/java/org/apache/impala/analysis/TableDef.java +++ b/fe/src/main/java/org/apache/impala/analysis/TableDef.java @@ -21,6 +21,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import org.apache.hadoop.fs.permission.FsAction; import org.apache.impala.authorization.Privilege; import org.apache.impala.catalog.HdfsStorageDescriptor; import org.apache.impala.catalog.RowFormat; @@ -36,8 +37,6 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; -import org.apache.hadoop.fs.permission.FsAction; - /** * Represents the table parameters in a CREATE TABLE statement. These parameters * correspond to the following clauses in a CREATE TABLE statement: diff --git a/fe/src/main/java/org/apache/impala/catalog/CatalogDeltaLog.java b/fe/src/main/java/org/apache/impala/catalog/CatalogDeltaLog.java index 27839b3..751d9e1 100644 --- a/fe/src/main/java/org/apache/impala/catalog/CatalogDeltaLog.java +++ b/fe/src/main/java/org/apache/impala/catalog/CatalogDeltaLog.java @@ -23,6 +23,8 @@ import java.util.TreeMap; import org.apache.impala.thrift.TCatalogObject; import org.apache.impala.thrift.TTable; +import org.apache.log4j.Logger; + import com.google.common.base.Preconditions; /** @@ -44,11 +46,17 @@ import com.google.common.base.Preconditions; * "invalidate metadata" is run concurrently with async catalog operations. */ public class CatalogDeltaLog { + private static final Logger LOG = Logger.getLogger(CatalogDeltaLog.class); + // Map of the catalog version an object was removed from the catalog // to the catalog object, ordered by catalog version. private SortedMap removedCatalogObjects_ = new TreeMap(); + public synchronized String debugImpala3641() { + return removedCatalogObjects_.toString(); + } + /** * Adds a new item to the map of removed catalog objects. */ @@ -58,6 +66,7 @@ public class CatalogDeltaLog { } /** + * TODO(Alex): IMPALA-3641, versions should be >=? * Given the current catalog version, removes all items with catalogVersion < * currectCatalogVersion. Such objects do not need to be tracked in the delta * log anymore because they are consistent with the state store's view of the @@ -67,9 +76,12 @@ public class CatalogDeltaLog { // Nothing will be garbage collected so avoid creating a new object. if (!removedCatalogObjects_.isEmpty() && removedCatalogObjects_.firstKey() < currentCatalogVersion) { + LOG.debug("IMPALA-3641 Garbage collecting: " + + removedCatalogObjects_.firstKey() + " " + currentCatalogVersion); removedCatalogObjects_ = new TreeMap( removedCatalogObjects_.tailMap(currentCatalogVersion)); } + LOG.debug("IMPALA-3641 " + debugImpala3641()); } /** diff --git a/fe/src/main/java/org/apache/impala/catalog/ImpaladCatalog.java b/fe/src/main/java/org/apache/impala/catalog/ImpaladCatalog.java index 521a844..3e87f0a 100644 --- a/fe/src/main/java/org/apache/impala/catalog/ImpaladCatalog.java +++ b/fe/src/main/java/org/apache/impala/catalog/ImpaladCatalog.java @@ -22,9 +22,6 @@ import java.util.concurrent.atomic.AtomicBoolean; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; -import org.apache.log4j.Logger; -import org.apache.thrift.TException; - import org.apache.impala.catalog.MetaStoreClientPool.MetaStoreClient; import org.apache.impala.common.ImpalaException; import org.apache.impala.thrift.TCatalogObject; @@ -38,6 +35,8 @@ import org.apache.impala.thrift.TTable; import org.apache.impala.thrift.TUniqueId; import org.apache.impala.thrift.TUpdateCatalogCacheRequest; import org.apache.impala.thrift.TUpdateCatalogCacheResponse; +import org.apache.log4j.Logger; +import org.apache.thrift.TException; /** * Thread safe Catalog for an Impalad. The Impalad catalog can be updated either via @@ -91,6 +90,11 @@ public class ImpaladCatalog extends Catalog { // Used during table creation. private final String defaultKuduMasterHosts_; + public synchronized String debugImpala3641() { + return "catalog@" + lastSyncedCatalogVersion_ + + " DeltaLog: " + catalogDeltaLog_.debugImpala3641(); + } + /** * C'tor used by tests that need to validate the ImpaladCatalog outside of the * CatalogServer. @@ -130,6 +134,8 @@ public class ImpaladCatalog extends Catalog { } } + LOG.debug("IMPALA-3641 " + lastSyncedCatalogVersion_ + " " + req.toString()); + // First process all updates long newCatalogVersion = lastSyncedCatalogVersion_; for (TCatalogObject catalogObject: req.getUpdated_objects()) { @@ -337,6 +343,7 @@ public class ImpaladCatalog extends Catalog { "Unexpected TCatalogObjectType: " + catalogObject.getType()); } + LOG.debug("IMPALA-3641 Removing obj " + catalogObject.getCatalog_version() + " " + lastSyncedCatalogVersion_); if (catalogObject.getCatalog_version() > lastSyncedCatalogVersion_) { catalogDeltaLog_.addRemovedObject(catalogObject); } diff --git a/fe/src/main/java/org/apache/impala/catalog/Table.java b/fe/src/main/java/org/apache/impala/catalog/Table.java index 9919dbf..38aef10 100644 --- a/fe/src/main/java/org/apache/impala/catalog/Table.java +++ b/fe/src/main/java/org/apache/impala/catalog/Table.java @@ -28,10 +28,7 @@ import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.metastore.api.FieldSchema; -import org.apache.log4j.Logger; - import org.apache.impala.analysis.TableName; -import org.apache.impala.common.AnalysisException; import org.apache.impala.common.ImpalaRuntimeException; import org.apache.impala.common.Pair; import org.apache.impala.thrift.TAccessLevel; @@ -43,6 +40,8 @@ import org.apache.impala.thrift.TTable; import org.apache.impala.thrift.TTableDescriptor; import org.apache.impala.thrift.TTableStats; import org.apache.impala.util.HdfsCachingUtil; +import org.apache.log4j.Logger; + import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -488,4 +487,9 @@ public abstract class Table implements CatalogObject { } return new Pair(cachePoolName, cacheReplication); } + + public String debugImpala3641() { + return getTableName().toString() + "@" + catalogVersion_ + " " + + getClass().getSimpleName() + " " + isLoaded(); + } } diff --git a/fe/src/main/java/org/apache/impala/service/Frontend.java b/fe/src/main/java/org/apache/impala/service/Frontend.java index d236cb1..fcceb99 100644 --- a/fe/src/main/java/org/apache/impala/service/Frontend.java +++ b/fe/src/main/java/org/apache/impala/service/Frontend.java @@ -904,12 +904,25 @@ public class Frontend { // Only re-throw the AnalysisException if there were no missing tables. if (missingTbls.isEmpty()) throw e; + String lcStmt = queryCtx.client_request.stmt.toLowerCase(); + if (lcStmt.startsWith("drop table") || lcStmt.startsWith("create table")) { + LOG.debug("IMPALA-3641 Tbl load for stmt: " + queryCtx.client_request.stmt + + " " + missingTbls.toString()); + } + // Some tables/views were missing, request and wait for them to load. if (!requestTblLoadAndWait(missingTbls, MISSING_TBL_LOAD_WAIT_TIMEOUT_MS)) { if (LOG.isTraceEnabled()) { LOG.trace(String.format("Missing tables were not received in %dms. Load " + "request will be retried.", MISSING_TBL_LOAD_WAIT_TIMEOUT_MS)); } + if (lcStmt.startsWith("drop table") || lcStmt.startsWith("create table")) { + LOG.debug("IMPALA-3641 Tbl load timed out: " + queryCtx.client_request.stmt ); + } + } + if (lcStmt.startsWith("drop table") || lcStmt.startsWith("create table")) { + LOG.debug("IMPALA-3641 Tbl load request succeeded: " + + queryCtx.client_request.stmt ); } } }