diff --git a/contrib/src/java/org/apache/hadoop/hive/contrib/udtf/example/GenericUDTFCount2.java b/contrib/src/java/org/apache/hadoop/hive/contrib/udtf/example/GenericUDTFCount2.java index 0bd788e..032322a 100644 --- a/contrib/src/java/org/apache/hadoop/hive/contrib/udtf/example/GenericUDTFCount2.java +++ b/contrib/src/java/org/apache/hadoop/hive/contrib/udtf/example/GenericUDTFCount2.java @@ -36,8 +36,8 @@ */ public class GenericUDTFCount2 extends GenericUDTF { - Integer count = Integer.valueOf(0); - Object forwardObj[] = new Object[1]; + private transient Integer count = Integer.valueOf(0); + private transient Object forwardObj[] = new Object[1]; @Override public void close() throws HiveException { diff --git a/contrib/src/java/org/apache/hadoop/hive/contrib/udtf/example/GenericUDTFExplode2.java b/contrib/src/java/org/apache/hadoop/hive/contrib/udtf/example/GenericUDTFExplode2.java index da5ae9c..a4dd152 100644 --- a/contrib/src/java/org/apache/hadoop/hive/contrib/udtf/example/GenericUDTFExplode2.java +++ b/contrib/src/java/org/apache/hadoop/hive/contrib/udtf/example/GenericUDTFExplode2.java @@ -67,7 +67,7 @@ public StructObjectInspector initialize(ObjectInspector[] args) fieldOIs); } - Object forwardObj[] = new Object[2]; + private transient Object forwardObj[] = new Object[2]; @Override public void process(Object[] o) throws HiveException { diff --git a/hcatalog/pom.xml b/hcatalog/pom.xml index 9f48a41..7521d0e 100644 --- a/hcatalog/pom.xml +++ b/hcatalog/pom.xml @@ -215,7 +215,16 @@ false - + + sonatype-snapshots + https://oss.sonatype.org/content/repositories/snapshots/ + + false + + + true + + diff --git a/ivy/ivysettings.xml b/ivy/ivysettings.xml index d230f2c..51ce5ac 100644 --- a/ivy/ivysettings.xml +++ b/ivy/ivysettings.xml @@ -47,6 +47,10 @@ checkmodified="${ivy.checkmodified}" changingPattern="${ivy.changingPattern}"/> + + @@ -68,6 +72,7 @@ + @@ -77,11 +82,13 @@ + + diff --git a/ivy/libraries.properties b/ivy/libraries.properties index c306440..aa8e742 100644 --- a/ivy/libraries.properties +++ b/ivy/libraries.properties @@ -51,6 +51,7 @@ jetty.version=6.1.26 jline.version=0.9.94 json.version=20090211 junit.version=4.10 +kryo.version=2.22-SNAPSHOT libfb303.version=0.9.0 libthrift.version=0.9.0 log4j.version=1.2.16 diff --git a/ql/build.xml b/ql/build.xml index 64e7b59..ac2b7b0 100644 --- a/ql/build.xml +++ b/ql/build.xml @@ -250,6 +250,15 @@ + + + + + + + @@ -298,7 +307,9 @@ - + + + diff --git a/ql/ivy.xml b/ql/ivy.xml index 08a8d6f..df07f4e 100644 --- a/ql/ivy.xml +++ b/ql/ivy.xml @@ -46,6 +46,8 @@ rev="${protobuf.version}" transitive="false"/> + diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java index e27222f..710102f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java @@ -466,7 +466,7 @@ public int compile(String command, boolean resetTaskIds) { // deserialize the queryPlan FileInputStream fis = new FileInputStream(queryPlanFileName); - QueryPlan newPlan = Utilities.deserializeObject(fis); + QueryPlan newPlan = Utilities.deserializeObject(fis, QueryPlan.class); fis.close(); // Use the deserialized plan diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnInfo.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnInfo.java index ee44051..5ee2054 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnInfo.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnInfo.java @@ -54,7 +54,7 @@ */ private boolean isVirtualCol; - private transient ObjectInspector objectInspector; + private ObjectInspector objectInspector; private boolean isHiddenVirtualCol; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeColumnEvaluator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeColumnEvaluator.java index 3f1e001..24c8281 100755 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeColumnEvaluator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeColumnEvaluator.java @@ -31,12 +31,12 @@ */ public class ExprNodeColumnEvaluator extends ExprNodeEvaluator { - transient boolean simpleCase; - transient StructObjectInspector inspector; - transient StructField field; - transient StructObjectInspector[] inspectors; - transient StructField[] fields; - transient boolean[] unionField; + private transient boolean simpleCase; + private transient StructObjectInspector inspector; + private transient StructField field; + private transient StructObjectInspector[] inspectors; + private transient StructField[] fields; + private transient boolean[] unionField; public ExprNodeColumnEvaluator(ExprNodeColumnDesc expr) { super(expr); @@ -61,18 +61,17 @@ public ObjectInspector initialize(ObjectInspector rowInspector) throws HiveExcep fields = new StructField[names.length]; unionField = new boolean[names.length]; int unionIndex = -1; - for (int i = 0; i < names.length; i++) { if (i == 0) { inspectors[0] = (StructObjectInspector) rowInspector; } else { - if (unionIndex != -1) { + if (unionIndex == -1) { + inspectors[i] = (StructObjectInspector) fields[i - 1] + .getFieldObjectInspector(); + } else { inspectors[i] = (StructObjectInspector) ( (UnionObjectInspector)fields[i-1].getFieldObjectInspector()). getObjectInspectors().get(unionIndex); - } else { - inspectors[i] = (StructObjectInspector) fields[i - 1] - .getFieldObjectInspector(); } } // to support names like _colx:1._coly diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableDummyOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableDummyOperator.java index 70cbff8..e00b7d3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableDummyOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableDummyOperator.java @@ -38,7 +38,7 @@ protected void initializeOp(Configuration hconf) throws HiveException { this.outputObjInspector = serde.getObjectInspector(); initializeChildren(hconf); } catch (Exception e) { - LOG.error("Generating output obj inspector from dummy object error"); + LOG.error("Generating output obj inspector from dummy object error", e); e.printStackTrace(); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java index 50580f4..fa3ad8f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java @@ -58,7 +58,7 @@ private static final long serialVersionUID = 1L; - private Configuration configuration; + private transient Configuration configuration; protected List> childOperators; protected List> parentOperators; protected String operatorId; @@ -196,7 +196,7 @@ public void setDone(boolean done) { } // non-bean fields needed during compilation - private transient RowSchema rowSchema; + private RowSchema rowSchema; public void setSchema(RowSchema rowSchema) { this.rowSchema = rowSchema; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/RowSchema.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/RowSchema.java index 234bbcd..1bfcee6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/RowSchema.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/RowSchema.java @@ -51,8 +51,10 @@ public void setSignature(ArrayList signature) { @Override public String toString() { StringBuilder sb = new StringBuilder('('); - for (ColumnInfo col: signature) { - sb.append(col.toString()); + if (signature != null) { + for (ColumnInfo col : signature) { + sb.append(col.toString()); + } } sb.append(')'); return sb.toString(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java index edb55fa..cbbf98e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java @@ -27,6 +27,7 @@ import java.beans.XMLDecoder; import java.beans.XMLEncoder; import java.io.BufferedReader; +import java.io.BufferedWriter; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.DataInput; @@ -34,6 +35,7 @@ import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; +import java.io.FileWriter; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; @@ -76,9 +78,11 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.antlr.runtime.CommonToken; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.WordUtils; import org.apache.commons.logging.Log; @@ -112,6 +116,9 @@ import org.apache.hadoop.hive.ql.io.OneNullRowInputFormat; import org.apache.hadoop.hive.ql.io.RCFile; import org.apache.hadoop.hive.ql.io.ReworkMapredInputFormat; +import org.apache.hadoop.hive.ql.io.rcfile.merge.MergeWork; +import org.apache.hadoop.hive.ql.io.rcfile.stats.PartialScanWork; +import org.apache.hadoop.hive.ql.io.rcfile.truncate.ColumnTruncateWork; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Partition; import org.apache.hadoop.hive.ql.metadata.Table; @@ -165,6 +172,11 @@ import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.Shell; +import com.esotericsoftware.kryo.Kryo; +import com.esotericsoftware.kryo.io.Input; +import com.esotericsoftware.kryo.io.Output; + + /** * Utilities. * @@ -268,8 +280,19 @@ public static BaseWork getBaseWork(Configuration conf, String name) { localPath = new Path(name); } InputStream in = new FileInputStream(localPath.toUri().getPath()); - BaseWork ret = deserializeObject(in); - gWork = ret; + if(MAP_PLAN_NAME.equals(name)){ + if("org.apache.hadoop.hive.ql.io.rcfile.merge.RCFileMergeMapper".equals(conf.get("mapred.mapper.class"))) { + gWork = deserializeObject(in, MergeWork.class); + } else if("org.apache.hadoop.hive.ql.io.rcfile.truncate.ColumnTruncateMapper".equals(conf.get("mapred.mapper.class"))) { + gWork = deserializeObject(in, ColumnTruncateWork.class); + } else if("org.apache.hadoop.hive.ql.io.rcfile.stats.PartialScanMapper".equals(conf.get("mapred.mapper.class"))) { + gWork = deserializeObject(in, PartialScanWork.class); + } else { + gWork = deserializeObject(in, MapWork.class); + } + } else { + gWork = deserializeObject(in, ReduceWork.class); + } gWorkMap.put(path, gWork); } return gWork; @@ -586,11 +609,61 @@ protected void initialize(Class type, Object oldInstance, Object newInstance, En } } - /** - * Serialize the object. This helper function mainly makes sure that enums, - * counters, etc are handled properly. - */ - public static void serializeObject(Object plan, OutputStream out) { + private static class CommonTokenSerializer extends com.esotericsoftware.kryo.Serializer { + + @Override + public CommonToken read(Kryo kryo, Input input, Class clazz) { + return new CommonToken(input.readInt(), input.readString()); + } + + @Override + public void write(Kryo kryo, Output output, CommonToken token) { + output.writeInt(token.getType()); + output.writeString(token.getText()); + } + + } + + private static class SqlDateSerializer extends com.esotericsoftware.kryo.Serializer { + + @Override + public java.sql.Date read(Kryo kryo, Input input, Class clazz) { + return new java.sql.Date(input.readLong()); + } + + @Override + public void write(Kryo kryo, Output output, java.sql.Date sqlDate) { + output.writeLong(sqlDate.getTime()); + } + + } + + /* + * This helper function is ONLY FOR DEBUGGING. + * It writes the plan in a file. It could be used to isolate any Kryo bug. + * Need to be removed before final review. + */ + private static void createPlanXml(Object plan) { + try { + File fl = new File("/tmp/myplan.xml-" + counter.incrementAndGet()); + // if (fl.exists() == false) { + LOG.info("Plan written to file : " + fl.getAbsolutePath() + " plan class :" + plan.getClass()); + BufferedWriter bw = new BufferedWriter(new FileWriter(fl)); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + createPlanXml(plan, baos); + bw.write(baos.toString()); + bw.close(); + // } else { + // LOG.info("File already exists "+ fl.getAbsolutePath()); + // } + } catch (IOException e1) { + // TODO Auto-generated catch block + e1.printStackTrace(); + } + + } + + private static void createPlanXml(Object plan, OutputStream out) { XMLEncoder e = new XMLEncoder(out); e.setExceptionListener(new ExceptionListener() { public void exceptionThrown(Exception e) { @@ -613,22 +686,69 @@ public void exceptionThrown(Exception e) { } /** + * Serialize the object. This helper function mainly makes sure that enums, + * counters, etc are handled properly. + * @throws IOException + */ + public static void serializeObject(Object plan, OutputStream out) throws IOException { + createPlanXml(plan); //For DEBUGGING ONLY.Comment before review. + com.esotericsoftware.minlog.Log.trace("START SERIALIZE"); + Kryo kryo = getKryo(); + /*Kryo kryo = new Kryo(); + //kryo.register(CommonToken.class, new CommonTokenSerializer()); + kryo.register(java.sql.Date.class, new SqlDateSerializer());*/ + Output output = new Output(out); + kryo.writeObject(output, plan); + output.close(); + com.esotericsoftware.minlog.Log.trace("END SERIALIZE"); + } + + /** * De-serialize an object. This helper function mainly makes sure that enums, * counters, etc are handled properly. */ - @SuppressWarnings("unchecked") - public static T deserializeObject(InputStream in) { - XMLDecoder d = null; + public static T deserializeObject(InputStream in, Class clazz ) { + com.esotericsoftware.minlog.Log.trace("START DESERIALIZE"); + Kryo kryo = getKryo(); + /*Kryo kryo = new Kryo(); + kryo.setClassLoader(Thread.currentThread().getContextClassLoader()); + //kryo.register(CommonToken.class, new CommonTokenSerializer()); + kryo.register(java.sql.Date.class, new SqlDateSerializer());*/ + Input inp = new Input(in); + T t = kryo.readObject(inp,clazz); + inp.close(); + com.esotericsoftware.minlog.Log.trace("END DESERIALIZE"); + return t; + } + + //TODO: Ensure to reuse the same instance of Kryo. There is overhead to create a Kryo object. + + protected static Kryo getKryo() { + Kryo kryo = new Kryo(); + kryo.setClassLoader(Thread.currentThread().getContextClassLoader()); + //kryo.register(CommonToken.class, new CommonTokenSerializer()); + kryo.register(java.sql.Date.class, new SqlDateSerializer()); + return kryo; + } + + public static void main(String[] args) throws Exception { try { - d = new XMLDecoder(in, null, null); - return (T) d.readObject(); - } finally { - if (null != d) { - d.close(); - } + com.esotericsoftware.minlog.Log.TRACE(); + deserializeObject(new FileInputStream("/tmp/kryo-mapredwork-2"), MapredWork.class); + } catch (Exception e) { + e.printStackTrace(System.out); } } + private static final AtomicInteger counter = new AtomicInteger(0); + + public static MapredWork copyMRWork(MapredWork work) { + //com.esotericsoftware.minlog.Log.TRACE(); + ByteArrayOutputStream baos = new ByteArrayOutputStream(4096); + createPlanXml(work, baos); + return (MapredWork)(new XMLDecoder(new ByteArrayInputStream(baos.toByteArray()),null,null).readObject()); + } + public static TableDesc defaultTd; static { // by default we expect ^A separated strings diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java index eb9205b..2cc70f3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java @@ -716,12 +716,12 @@ public static void main(String[] args) throws IOException, HiveException { int ret; if (localtask) { memoryMXBean = ManagementFactory.getMemoryMXBean(); - MapredLocalWork plan = (MapredLocalWork) Utilities.deserializeObject(pathData); + MapredLocalWork plan = Utilities.deserializeObject(pathData, MapredLocalWork.class); MapredLocalTask ed = new MapredLocalTask(plan, conf, isSilent); ret = ed.executeFromChildJVM(new DriverContext()); } else { - MapredWork plan = (MapredWork) Utilities.deserializeObject(pathData); + MapredWork plan = Utilities.deserializeObject(pathData, MapredWork.class); ExecDriver ed = new ExecDriver(plan, conf, isSilent); ret = ed.execute(new DriverContext()); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HadoopJobExecHelper.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HadoopJobExecHelper.java index cd872b2..b0e6ef3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HadoopJobExecHelper.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HadoopJobExecHelper.java @@ -35,10 +35,10 @@ import org.apache.hadoop.hive.common.JavaUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.MapRedStats; +import org.apache.hadoop.hive.ql.exec.Operator.ProgressCounter; import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.exec.TaskHandle; import org.apache.hadoop.hive.ql.exec.Utilities; -import org.apache.hadoop.hive.ql.exec.Operator.ProgressCounter; import org.apache.hadoop.hive.ql.history.HiveHistory.Keys; import org.apache.hadoop.hive.ql.plan.ReducerTimeStatsPerJob; import org.apache.hadoop.hive.ql.session.SessionState; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java index 49a0ee3..ff5f3d7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java @@ -39,14 +39,11 @@ import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.exec.Utilities.StreamPrinter; -import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.ql.plan.MapWork; import org.apache.hadoop.hive.ql.plan.MapredWork; import org.apache.hadoop.hive.ql.plan.OperatorDesc; import org.apache.hadoop.hive.ql.plan.ReduceWork; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.shims.ShimLoader; -import org.apache.hadoop.mapred.JobConf; /** * Extension of ExecDriver: * - can optionally spawn a map-reduce task from a separate jvm @@ -69,7 +66,6 @@ private transient ContentSummary inputSummary = null; private transient boolean runningViaChild = false; - private transient boolean inputSizeEstimated = false; private transient long totalInputFileSize; private transient long totalInputNumFiles; @@ -79,10 +75,6 @@ public MapRedTask() { super(); } - public MapRedTask(MapredWork plan, JobConf job, boolean isSilent) throws HiveException { - throw new RuntimeException("Illegal Constructor call"); - } - @Override public int execute(DriverContext driverContext) { @@ -408,7 +400,7 @@ private void setNumberOfReducers() throws IOException { if (inputSummary == null) { inputSummary = Utilities.getInputSummary(driverContext.getCtx(), work.getMapWork(), null); } - int reducers = Utilities.estimateNumberOfReducers(conf, inputSummary, work.getMapWork(), + int reducers = Utilities.estimateNumberOfReducers(conf, inputSummary, work.getMapWork(), work.isFinalMapRed()); rWork.setNumReduceTasks(reducers); console diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/BucketingSortingCtx.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/BucketingSortingCtx.java index c8354ff..296fecb 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/BucketingSortingCtx.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/BucketingSortingCtx.java @@ -36,19 +36,19 @@ */ public class BucketingSortingCtx implements NodeProcessorCtx { - boolean disableBucketing; + private boolean disableBucketing; // A mapping from an operator to the columns by which it's output is bucketed - Map, List> bucketedColsByOp; + private Map, List> bucketedColsByOp; // A mapping from a directory which a FileSinkOperator writes into to the columns by which that // output is bucketed - Map> bucketedColsByDirectory; + private Map> bucketedColsByDirectory; // A mapping from an operator to the columns by which it's output is sorted - Map, List> sortedColsByOp; + private Map, List> sortedColsByOp; // A mapping from a directory which a FileSinkOperator writes into to the columns by which that // output is sorted - Map> sortedColsByDirectory; + private Map> sortedColsByDirectory; public BucketingSortingCtx(boolean disableBucketing) { this.disableBucketing = disableBucketing; @@ -189,13 +189,18 @@ public String toString() { * to be constant for all equivalent columns. */ public static final class SortCol implements BucketSortCol, Serializable { + + public SortCol() { + super(); + } + private static final long serialVersionUID = 1L; // Equivalent aliases for the column - private final List names = new ArrayList(); + private List names = new ArrayList(); // Indexes of those equivalent columns - private final List indexes = new ArrayList(); + private List indexes = new ArrayList(); // Sort order (+|-) - private final char sortOrder; + private char sortOrder; public SortCol(String name, int index, char sortOrder) { this(sortOrder); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CommonJoinTaskDispatcher.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CommonJoinTaskDispatcher.java index 14a4b11..7af2a9e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CommonJoinTaskDispatcher.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CommonJoinTaskDispatcher.java @@ -17,8 +17,6 @@ */ package org.apache.hadoop.hive.ql.optimizer.physical; -import java.io.ByteArrayInputStream; -import java.io.InputStream; import java.io.Serializable; import java.io.UnsupportedEncodingException; import java.util.ArrayList; @@ -502,9 +500,10 @@ public static boolean cannotConvert(String bigTableAlias, } // deep copy a new mapred work from xml // Once HIVE-4396 is in, it would be faster to use a cheaper method to clone the plan - String xml = currTask.getWork().toXML(); - InputStream in = new ByteArrayInputStream(xml.getBytes("UTF-8")); - MapredWork newWork = Utilities.deserializeObject(in); + + MapredWork newWork = Utilities.copyMRWork(currTask.getWork()); + //MapredWork newWork = kryo.copy(currTask.getWork()); + // create map join task and set big table as i ObjectPair newTaskAlias = convertTaskToMapJoinTask(newWork, i); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenMRSkewJoinProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenMRSkewJoinProcessor.java index ee4d4d1..b6bbe5f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenMRSkewJoinProcessor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenMRSkewJoinProcessor.java @@ -18,9 +18,7 @@ package org.apache.hadoop.hive.ql.optimizer.physical; -import java.io.ByteArrayInputStream; import java.io.Serializable; -import java.io.UnsupportedEncodingException; import java.util.ArrayList; import java.util.HashMap; import java.util.LinkedHashMap; @@ -241,16 +239,7 @@ public static void processSkewJoin(JoinOperator joinOp, HiveConf.ConfVars.HIVE_MAPPER_CANNOT_SPAN_MULTIPLE_PARTITIONS); newPlan.setMapperCannotSpanPartns(mapperCannotSpanPartns); - MapredWork clonePlan = null; - try { - String xmlPlan = currPlan.toXML(); - StringBuilder sb = new StringBuilder(xmlPlan); - ByteArrayInputStream bis; - bis = new ByteArrayInputStream(sb.toString().getBytes("UTF-8")); - clonePlan = Utilities.deserializeObject(bis); - } catch (UnsupportedEncodingException e) { - throw new SemanticException(e); - } + MapredWork clonePlan = Utilities.copyMRWork(currPlan); Operator[] parentOps = new TableScanOperator[tags.length]; for (int k = 0; k < tags.length; k++) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LocalMapJoinProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LocalMapJoinProcFactory.java index 85a2bdf..c3077ca 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LocalMapJoinProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LocalMapJoinProcFactory.java @@ -163,7 +163,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx ctx, Object.. if (parent instanceof TableScanOperator) { tbl = ((TableScanOperator) parent).getTableDesc(); } else { - throw new SemanticException(); + throw new SemanticException("Parent not TableScanOperator as expected. Please report."); } } else { // get parent schema diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SortMergeJoinTaskDispatcher.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SortMergeJoinTaskDispatcher.java index 35122c2..0265bbe 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SortMergeJoinTaskDispatcher.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SortMergeJoinTaskDispatcher.java @@ -17,8 +17,6 @@ */ package org.apache.hadoop.hive.ql.optimizer.physical; -import java.io.ByteArrayInputStream; -import java.io.InputStream; import java.io.Serializable; import java.io.UnsupportedEncodingException; import java.util.ArrayList; @@ -148,11 +146,8 @@ private void genSMBJoinWork(MapWork currWork, SMBMapJoinOperator smbJoinOp) { private MapredWork convertSMBWorkToJoinWork(MapredWork currWork, SMBMapJoinOperator oldSMBJoinOp) throws SemanticException { try { - String xml = currWork.toXML(); - // deep copy a new mapred work - InputStream in = new ByteArrayInputStream(xml.getBytes("UTF-8")); - MapredWork currJoinWork = Utilities.deserializeObject(in); + MapredWork currJoinWork = Utilities.copyMRWork(currWork); SMBMapJoinOperator newSMBJoinOp = getSMBMapJoinOp(currJoinWork); // Add the row resolver for the new operator @@ -169,14 +164,11 @@ private MapredWork convertSMBWorkToJoinWork(MapredWork currWork, SMBMapJoinOpera } // create map join task and set big table as bigTablePosition - private ObjectPair convertSMBTaskToMapJoinTask(String xml, + private ObjectPair convertSMBTaskToMapJoinTask(MapredWork newWork, int bigTablePosition, SMBMapJoinOperator smbJoinOp, QBJoinTree joinTree) throws UnsupportedEncodingException, SemanticException { - // deep copy a new mapred work from xml - InputStream in = new ByteArrayInputStream(xml.getBytes("UTF-8")); - MapredWork newWork = Utilities.deserializeObject(in); // create a mapred task for this work MapRedTask newTask = (MapRedTask) TaskFactory.get(newWork, physicalContext .getParseContext().getConf()); @@ -240,7 +232,6 @@ private boolean isEligibleForOptimization(SMBMapJoinOperator originalSMBJoinOp) public Task processCurrentTask(MapRedTask currTask, ConditionalTask conditionalTask, Context context) throws SemanticException { - // whether it contains a sort merge join operator MapredWork currWork = currTask.getWork(); SMBMapJoinOperator originalSMBJoinOp = getSMBMapJoinOp(currWork); @@ -290,7 +281,6 @@ private boolean isEligibleForOptimization(SMBMapJoinOperator originalSMBJoinOp) long aliasTotalKnownInputSize = getTotalKnownInputSize(context, currJoinWork.getMapWork(), pathToAliases, aliasToSize); - String xml = currJoinWork.toXML(); long ThresholdOfSmallTblSizeSum = HiveConf.getLongVar(conf, HiveConf.ConfVars.HIVESMALLTABLESFILESIZE); @@ -302,7 +292,7 @@ private boolean isEligibleForOptimization(SMBMapJoinOperator originalSMBJoinOp) // create map join task for the given big table position ObjectPair newTaskAlias = - convertSMBTaskToMapJoinTask(xml, bigTablePosition, newSMBJoinOp, joinTree); + convertSMBTaskToMapJoinTask(Utilities.copyMRWork(currJoinWork), bigTablePosition, newSMBJoinOp, joinTree); MapRedTask newTask = newTaskAlias.getFirst(); String bigTableAlias = newTaskAlias.getSecond(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/QBJoinTree.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/QBJoinTree.java index 0aaf47e..9834e73 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/QBJoinTree.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/QBJoinTree.java @@ -51,22 +51,22 @@ // keeps track of the right-hand-side table name of the left-semi-join, and // its list of join keys - private final HashMap> rhsSemijoin; + private transient final HashMap> rhsSemijoin; // join conditions - private ArrayList> expressions; + private transient ArrayList> expressions; // key index to nullsafe join flag private ArrayList nullsafes; // filters - private ArrayList> filters; + private transient ArrayList> filters; // outerjoin-pos = other-pos:filter-len, other-pos:filter-len, ... private int[][] filterMap; // filters for pushing - private ArrayList> filtersForPushing; + private transient ArrayList> filtersForPushing; // user asked for map-side join private boolean mapSideJoin; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/RowResolver.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/RowResolver.java index 908546e..3b94bfb 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/RowResolver.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/RowResolver.java @@ -264,7 +264,11 @@ public boolean getIsExprResolver() { @Override public String toString() { StringBuilder sb = new StringBuilder(); - + // XXX this class is messed up on read and toString() fails which is called by kryo + // thus to debug the issue I have removed this call + /*if(true) { + return "RowResolver"; + }*/ for (Map.Entry> e : rslvMap .entrySet()) { String tab = e.getKey(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java index ff90bc8..b2c3b3f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java @@ -52,13 +52,13 @@ * distributed on the cluster. The ExecMapper will ultimately deserialize this * class on the data nodes and setup it's operator pipeline accordingly. * - * This class is also used in the explain command any property with the + * This class is also used in the explain command any property with the * appropriate annotation will be displayed in the explain output. */ @SuppressWarnings({"serial", "deprecation"}) public class MapWork extends BaseWork { - private static transient final Log LOG = LogFactory.getLog(MapWork.class); + private static final Log LOG = LogFactory.getLog(MapWork.class); private boolean hadoopSupportsSplittable; @@ -102,15 +102,15 @@ public static final int SAMPLING_ON_START = 2; // sampling on task running // the following two are used for join processing - private QBJoinTree joinTree; - private LinkedHashMap, OpParseContext> opParseCtxMap; + private transient QBJoinTree joinTree; + private transient LinkedHashMap, OpParseContext> opParseCtxMap; private boolean mapperCannotSpanPartns; // used to indicate the input is sorted, and so a BinarySearchRecordReader shoudl be used private boolean inputFormatSorted = false; - private transient boolean useBucketizedHiveInputFormat; + private boolean useBucketizedHiveInputFormat; public MapWork() { } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/MapredWork.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/MapredWork.java index 7298cff..f3203bf 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/MapredWork.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/MapredWork.java @@ -18,12 +18,10 @@ package org.apache.hadoop.hive.ql.plan; -import java.io.ByteArrayOutputStream; import java.util.ArrayList; import java.util.List; import org.apache.hadoop.hive.ql.exec.Operator; -import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.mapred.JobConf; @@ -82,10 +80,4 @@ public void configureJobConf(JobConf job) { return ops; } - - public String toXML() { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - Utilities.serializeObject(this, baos); - return (baos.toString()); - } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDesc.java index da03755..aaf9ff9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDesc.java @@ -52,7 +52,7 @@ */ boolean isMapSide = false; - HiveConf cfg; + transient HiveConf cfg; static{ PTFUtils.makeTransient(PTFDesc.class, "llInfo"); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java index 5f983db..644f0c2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java @@ -47,7 +47,7 @@ private java.util.Properties properties; private String serdeClassName; - private transient String baseFileName; + private String baseFileName; public void setBaseFileName(String baseFileName) { this.baseFileName = baseFileName; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFnGrams.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFnGrams.java index 4f28628..4a7caab 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFnGrams.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFnGrams.java @@ -160,15 +160,15 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) throws SemanticE */ public static class GenericUDAFnGramEvaluator extends GenericUDAFEvaluator { // For PARTIAL1 and COMPLETE: ObjectInspectors for original data - private StandardListObjectInspector outerInputOI; - private StandardListObjectInspector innerInputOI; - private PrimitiveObjectInspector inputOI; - private PrimitiveObjectInspector nOI; - private PrimitiveObjectInspector kOI; - private PrimitiveObjectInspector pOI; + private transient StandardListObjectInspector outerInputOI; + private transient StandardListObjectInspector innerInputOI; + private transient PrimitiveObjectInspector inputOI; + private transient PrimitiveObjectInspector nOI; + private transient PrimitiveObjectInspector kOI; + private transient PrimitiveObjectInspector pOI; // For PARTIAL2 and FINAL: ObjectInspectors for partial aggregations - private StandardListObjectInspector loi; + private transient StandardListObjectInspector loi; @Override public ObjectInspector init(Mode m, ObjectInspector[] parameters) throws HiveException { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFArray.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFArray.java index 4176d3f..c429367 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFArray.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFArray.java @@ -38,8 +38,8 @@ @Description(name = "array", value = "_FUNC_(n0, n1...) - Creates an array with the given elements ") public class GenericUDFArray extends GenericUDF { - private Converter[] converters; - private ArrayList ret = new ArrayList(); + private transient Converter[] converters; + private transient ArrayList ret = new ArrayList(); @Override public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBridge.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBridge.java index a9feba4..3e01364 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBridge.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBridge.java @@ -57,7 +57,7 @@ /** * The underlying UDF class. */ - Class udfClass; + private Class udfClass; /** * The underlying UDF class Name. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFormatNumber.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFormatNumber.java index f23193e..a9a0176 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFormatNumber.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFormatNumber.java @@ -54,10 +54,10 @@ + " '12,332.1235'") public class GenericUDFFormatNumber extends GenericUDF { private transient ObjectInspector[] argumentOIs; - private final Text resultText = new Text(); - private final StringBuilder pattern = new StringBuilder(""); - private final DecimalFormat numberFormat = new DecimalFormat(""); - private int lastDValue = -1; + private transient final Text resultText = new Text(); + private transient final StringBuilder pattern = new StringBuilder(""); + private transient final DecimalFormat numberFormat = new DecimalFormat(""); + private transient int lastDValue = -1; @Override public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFIndex.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFIndex.java index 0659dfa..5911f2c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFIndex.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFIndex.java @@ -36,11 +36,11 @@ */ @Description(name = "index", value = "_FUNC_(a, n) - Returns the n-th element of a ") public class GenericUDFIndex extends GenericUDF { - private MapObjectInspector mapOI; + private transient MapObjectInspector mapOI; private boolean mapKeyPreferWritable; - private ListObjectInspector listOI; - private PrimitiveObjectInspector indexOI; - private ObjectInspector returnOI; + private transient ListObjectInspector listOI; + private transient PrimitiveObjectInspector indexOI; + private transient ObjectInspector returnOI; @Override public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFNamedStruct.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFNamedStruct.java index 2c18d93..7b18d82 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFNamedStruct.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFNamedStruct.java @@ -35,7 +35,7 @@ value = "_FUNC_(name1, val1, name2, val2, ...) - Creates a struct with the given " + "field names and values") public class GenericUDFNamedStruct extends GenericUDF { - Object[] ret; + private transient Object[] ret; @Override public ObjectInspector initialize(ObjectInspector[] arguments) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFStruct.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFStruct.java index 258c0bf..7c8dfcf 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFStruct.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFStruct.java @@ -31,7 +31,7 @@ @Description(name = "struct", value = "_FUNC_(col1, col2, col3, ...) - Creates a struct with the given field values") public class GenericUDFStruct extends GenericUDF { - Object[] ret; + private transient Object[] ret; @Override public ObjectInspector initialize(ObjectInspector[] arguments) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToUnixTimeStamp.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToUnixTimeStamp.java index b0a755e..c1b2607 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToUnixTimeStamp.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToUnixTimeStamp.java @@ -47,7 +47,7 @@ private transient TimestampObjectInspector inputTimestampOI; private transient StringObjectInspector patternOI; - private String lasPattern = "yyyy-MM-dd HH:mm:ss"; + private transient String lasPattern = "yyyy-MM-dd HH:mm:ss"; private transient final SimpleDateFormat formatter = new SimpleDateFormat(lasPattern); @Override diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFExplode.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFExplode.java index de2dfb1..9aec825 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFExplode.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFExplode.java @@ -77,8 +77,8 @@ public StructObjectInspector initialize(ObjectInspector[] args) throws UDFArgume fieldOIs); } - private final Object[] forwardListObj = new Object[1]; - private final Object[] forwardMapObj = new Object[2]; + private transient final Object[] forwardListObj = new Object[1]; + private transient final Object[] forwardMapObj = new Object[2]; @Override public void process(Object[] o) throws HiveException { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFJSONTuple.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFJSONTuple.java index 490198a..521f4e6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFJSONTuple.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFJSONTuple.java @@ -63,9 +63,9 @@ int numCols; // number of output columns String[] paths; // array of path expressions, each of which corresponds to a column - Text[] retCols; // array of returned column values - Text[] cols; // object pool of non-null Text, avoid creating objects all the time - Object[] nullCols; // array of null column values + private transient Text[] retCols; // array of returned column values + private transient Text[] cols; // object pool of non-null Text, avoid creating objects all the time + private transient Object[] nullCols; // array of null column values private transient ObjectInspector[] inputOIs; // input ObjectInspectors boolean pathParsed = false; boolean seenErrors = false; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFParseUrlTuple.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFParseUrlTuple.java index 96ec55f..dff9ba6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFParseUrlTuple.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFParseUrlTuple.java @@ -18,11 +18,11 @@ package org.apache.hadoop.hive.ql.udf.generic; -import java.net.URL; import java.net.MalformedURLException; +import java.net.URL; import java.util.ArrayList; -import java.util.regex.Pattern; import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -65,13 +65,13 @@ PARTNAME[] partnames; // mapping from pathnames to enum PARTNAME Text[] retCols; // array of returned column values Text[] cols; // object pool of non-null Text, avoid creating objects all the time - Object[] nullCols; // array of null column values - ObjectInspector[] inputOIs; // input ObjectInspectors + private transient Object[] nullCols; // array of null column values + private transient ObjectInspector[] inputOIs; // input ObjectInspectors boolean pathParsed = false; boolean seenErrors = false; - URL url = null; - Pattern p = null; - String lastKey = null; + private transient URL url = null; + private transient Pattern p = null; + private transient String lastKey = null; @Override public void close() throws HiveException { @@ -122,7 +122,7 @@ public StructObjectInspector initialize(ObjectInspector[] args) // all returned type will be Text fieldOIs.add(PrimitiveObjectInspectorFactory.writableStringObjectInspector); } - + return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs); } @@ -137,7 +137,7 @@ public void process(Object[] o) throws HiveException { if (!pathParsed) { for (int i = 0;i < numCols; ++i) { paths[i] = ((StringObjectInspector) inputOIs[i+1]).getPrimitiveJavaObject(o[i+1]); - + if (paths[i] == null) { partnames[i] = PARTNAME.NULLNAME; } else if (paths[i].equals("HOST")) { @@ -158,11 +158,11 @@ public void process(Object[] o) throws HiveException { partnames[i] = PARTNAME.USERINFO; } else if (paths[i].startsWith("QUERY:")) { partnames[i] = PARTNAME.QUERY_WITH_KEY; - paths[i] = paths[i].substring(6); // update paths[i], e.g., from "QUERY:id" to "id" + paths[i] = paths[i].substring(6); // update paths[i], e.g., from "QUERY:id" to "id" } else { partnames[i] = PARTNAME.NULLNAME; - } - } + } + } pathParsed = true; } @@ -171,9 +171,9 @@ public void process(Object[] o) throws HiveException { forward(nullCols); return; } - + try { - String ret = null; + String ret = null; url = new URL(urlStr); for (int i = 0; i < numCols; ++i) { ret = evaluate(url, i); @@ -188,7 +188,7 @@ public void process(Object[] o) throws HiveException { } forward(retCols); - return; + return; } catch (MalformedURLException e) { // parsing error, invalid url string if (!seenErrors) { @@ -204,10 +204,11 @@ public void process(Object[] o) throws HiveException { public String toString() { return "parse_url_tuple"; } - + private String evaluate(URL url, int index) { - if (url == null || index < 0 || index >= partnames.length) + if (url == null || index < 0 || index >= partnames.length) { return null; + } switch (partnames[index]) { case HOST : return url.getHost(); @@ -239,5 +240,5 @@ private String evaluateQuery(String query, String key) { return m.group(2); } return null; - } + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFStack.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFStack.java index 2c423fa..1a744e0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFStack.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFStack.java @@ -48,7 +48,7 @@ public void close() throws HiveException { } private transient List argOIs = new ArrayList(); - Object[] forwardObj = null; + private transient Object[] forwardObj = null; private transient ArrayList returnOIResolvers = new ArrayList(); IntWritable numRows = null; diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardConstantListObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardConstantListObjectInspector.java index 5aad51f..90b13e0 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardConstantListObjectInspector.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardConstantListObjectInspector.java @@ -31,7 +31,9 @@ implements ConstantObjectInspector { private List value; - + private StandardConstantListObjectInspector() { + super(null); + } /** * Call ObjectInspectorFactory.getStandardListObjectInspector instead. */ diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardConstantMapObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardConstantMapObjectInspector.java index ff190af..5c081fa 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardConstantMapObjectInspector.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardConstantMapObjectInspector.java @@ -32,6 +32,9 @@ private Map value; + private StandardConstantMapObjectInspector() { + super(null, null); + } /** * Call ObjectInspectorFactory.getStandardMapObjectInspector instead. */ diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardListObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardListObjectInspector.java index af14663..c78b559 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardListObjectInspector.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardListObjectInspector.java @@ -30,7 +30,7 @@ */ public class StandardListObjectInspector implements SettableListObjectInspector { - ObjectInspector listElementObjectInspector; + private ObjectInspector listElementObjectInspector; /** * Call ObjectInspectorFactory.getStandardListObjectInspector instead. diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardMapObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardMapObjectInspector.java index 2f5c1d9..a02bd44 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardMapObjectInspector.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardMapObjectInspector.java @@ -36,9 +36,12 @@ */ public class StandardMapObjectInspector implements SettableMapObjectInspector { - ObjectInspector mapKeyObjectInspector; - ObjectInspector mapValueObjectInspector; + private ObjectInspector mapKeyObjectInspector; + private ObjectInspector mapValueObjectInspector; + private StandardMapObjectInspector() { + this(null, null); + } /** * Call ObjectInspectorFactory.getStandardMapObjectInspector instead. */ diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantBinaryObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantBinaryObjectInspector.java index fe91a82..48b280f 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantBinaryObjectInspector.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantBinaryObjectInspector.java @@ -28,8 +28,12 @@ WritableBinaryObjectInspector implements ConstantObjectInspector { - private final BytesWritable value; + private BytesWritable value; + private WritableConstantBinaryObjectInspector() { + super(); + } + public WritableConstantBinaryObjectInspector(BytesWritable value) { super(); this.value = value; diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantBooleanObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantBooleanObjectInspector.java index 03a92e8..7dea428 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantBooleanObjectInspector.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantBooleanObjectInspector.java @@ -31,6 +31,9 @@ private BooleanWritable value; + private WritableConstantBooleanObjectInspector() { + super(); + } WritableConstantBooleanObjectInspector(BooleanWritable value) { super(); this.value = value; diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantByteObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantByteObjectInspector.java index 808f8e4..c648eea 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantByteObjectInspector.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantByteObjectInspector.java @@ -31,6 +31,9 @@ private ByteWritable value; + private WritableConstantByteObjectInspector() { + super(); + } WritableConstantByteObjectInspector(ByteWritable value) { super(); this.value = value; diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantDateObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantDateObjectInspector.java index 3bd87e9..2c492f5 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantDateObjectInspector.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantDateObjectInspector.java @@ -31,6 +31,9 @@ private DateWritable value; + private WritableConstantDateObjectInspector() { + super(); + } WritableConstantDateObjectInspector(DateWritable value) { super(); this.value = value; diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantDoubleObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantDoubleObjectInspector.java index 4fe1639..8dec8d6 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantDoubleObjectInspector.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantDoubleObjectInspector.java @@ -31,6 +31,9 @@ private DoubleWritable value; + private WritableConstantDoubleObjectInspector() { + super(); + } WritableConstantDoubleObjectInspector(DoubleWritable value) { super(); this.value = value; diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantFloatObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantFloatObjectInspector.java index d40bdc7..1703554 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantFloatObjectInspector.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantFloatObjectInspector.java @@ -31,6 +31,9 @@ private FloatWritable value; + private WritableConstantFloatObjectInspector() { + super(); + } WritableConstantFloatObjectInspector(FloatWritable value) { super(); this.value = value; diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantHiveDecimalObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantHiveDecimalObjectInspector.java index c0413e2..9f9357c 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantHiveDecimalObjectInspector.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantHiveDecimalObjectInspector.java @@ -27,8 +27,11 @@ public class WritableConstantHiveDecimalObjectInspector extends WritableHiveDecimalObjectInspector implements ConstantObjectInspector { - private final HiveDecimalWritable value; + private HiveDecimalWritable value; + private WritableConstantHiveDecimalObjectInspector() { + super(); + } WritableConstantHiveDecimalObjectInspector(HiveDecimalWritable value) { this.value = value; } diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantIntObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantIntObjectInspector.java index 8cb8c1c..f9958e0 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantIntObjectInspector.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantIntObjectInspector.java @@ -31,6 +31,9 @@ private IntWritable value; + private WritableConstantIntObjectInspector() { + super(); + } WritableConstantIntObjectInspector(IntWritable value) { super(); this.value = value; diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantLongObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantLongObjectInspector.java index 42f9eed..16174b2 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantLongObjectInspector.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantLongObjectInspector.java @@ -31,6 +31,9 @@ private LongWritable value; + private WritableConstantLongObjectInspector() { + super(); + } WritableConstantLongObjectInspector(LongWritable value) { super(); this.value = value; diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantShortObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantShortObjectInspector.java index a476fa0..3e95729 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantShortObjectInspector.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantShortObjectInspector.java @@ -31,6 +31,9 @@ private ShortWritable value; + private WritableConstantShortObjectInspector() { + super(); + } WritableConstantShortObjectInspector(ShortWritable value) { super(); this.value = value; diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantStringObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantStringObjectInspector.java index 802357b..22805bb 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantStringObjectInspector.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantStringObjectInspector.java @@ -31,6 +31,9 @@ private Text value; + private WritableConstantStringObjectInspector() { + super(); + } WritableConstantStringObjectInspector(Text value) { super(); this.value = value; diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantTimestampObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantTimestampObjectInspector.java index 2ddd3c3..4ce58e0 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantTimestampObjectInspector.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantTimestampObjectInspector.java @@ -31,6 +31,9 @@ private TimestampWritable value; + private WritableConstantTimestampObjectInspector() { + super(); + } WritableConstantTimestampObjectInspector(TimestampWritable value) { super(); this.value = value;