Index: cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java =================================================================== --- cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java (revision 986564) +++ cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java (working copy) @@ -59,9 +59,9 @@ public static final String prompt = "hive"; public static final String prompt2 = " "; // when ';' is not yet seen - + public static final String HIVERCFILE = ".hiverc"; - + private final LogHelper console; private final Configuration conf; @@ -261,7 +261,7 @@ } ss.setIsSilent(saveSilent); } - + public static void main(String[] args) throws Exception { OptionsProcessor oproc = new OptionsProcessor(); @@ -275,9 +275,10 @@ CliSessionState ss = new CliSessionState(new HiveConf(SessionState.class)); ss.in = System.in; + String encoding = HiveConf.getVar(ss.getConf(), HiveConf.ConfVars.HIVECLIENCODING); try { - ss.out = new PrintStream(System.out, true, "UTF-8"); - ss.err = new PrintStream(System.err, true, "UTF-8"); + ss.out = new PrintStream(System.out, true, encoding); + ss.err = new PrintStream(System.err, true, encoding); } catch (UnsupportedEncodingException e) { System.exit(3); } @@ -311,7 +312,7 @@ // Execute -i init files (always in silent mode) cli.processInitFiles(ss); - + if (ss.execString != null) { System.exit(cli.processLine(ss.execString)); } Index: common/src/java/org/apache/hadoop/hive/conf/HiveConf.java =================================================================== --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (revision 986564) +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (working copy) @@ -174,6 +174,9 @@ HIVEJAR("hive.jar.path", ""), HIVEAUXJARS("hive.aux.jars.path", ""), + // hive cli encoding + HIVECLIENCODING("hive.cli.encoding", "UTF-8"), + // hive added files and jars HIVEADDEDFILES("hive.added.files.path", ""), HIVEADDEDJARS("hive.added.jars.path", ""), Index: ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java (revision 986564) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java (working copy) @@ -28,6 +28,7 @@ import org.apache.hadoop.hive.ql.DriverContext; import org.apache.hadoop.hive.ql.QueryPlan; import org.apache.hadoop.hive.ql.plan.FetchWork; +import org.apache.hadoop.hive.ql.plan.PartitionDesc; import org.apache.hadoop.hive.ql.plan.TableDesc; import org.apache.hadoop.hive.ql.plan.api.StageType; import org.apache.hadoop.hive.serde.Constants; @@ -47,6 +48,7 @@ private FetchOperator ftOp; private LazySimpleSerDe mSerde; private int totalRows; + private String encodingString; public FetchTask() { super(); @@ -63,11 +65,23 @@ mSerde = new LazySimpleSerDe(); Properties mSerdeProp = new Properties(); mSerdeProp.put(Constants.SERIALIZATION_FORMAT, "" + Utilities.tabCode); - mSerdeProp.put(Constants.SERIALIZATION_NULL_FORMAT, (work) + mSerdeProp.put(Constants.SERIALIZATION_NULL_FORMAT, work .getSerializationNullFormat()); mSerde.initialize(job, mSerdeProp); mSerde.setUseJSONSerialize(true); + TableDesc desc = getTblDesc(); + if (desc == null && getPartDesc() != null) { + if (getPartDesc().size() > 0) { + desc = getPartDesc().get(0).getTableDesc(); + } + } + + if (desc != null) { + encodingString = desc.getProperties().getProperty(Constants.SERIALIZATION_ENCODING, "UTF-8"); + } + LOG.info("Serialization encoding:" + encodingString); + ftOp = new FetchOperator(work, job); } catch (Exception e) { // Bail out ungracefully - we should never hit @@ -84,13 +98,20 @@ } /** - * Return the tableDesc of the fetchWork. + * Return the TableDesc of the fetchWork. */ public TableDesc getTblDesc() { return work.getTblDesc(); } /** + * Return the PartitionDesc of the fetchWork. + */ + public ArrayList getPartDesc() { + return work.getPartDesc(); + } + + /** * Return the maximum number of rows returned by fetch. */ public int getMaxRows() { @@ -127,7 +148,8 @@ return true; } - res.add(((Text) mSerde.serialize(io.o, io.oi)).toString()); + Text row = (Text)mSerde.serialize(io.o, io.oi); + res.add(new String(row.getBytes(), 0, row.getLength(), encodingString)); numRows++; } totalRows += numRows; @@ -152,11 +174,13 @@ @Override protected void localizeMRTmpFilesImpl(Context ctx) { String s = work.getTblDir(); - if ((s != null) && ctx.isMRTmpFileURI(s)) + if ((s != null) && ctx.isMRTmpFileURI(s)) { work.setTblDir(ctx.localizeMRTmpFileURI(s)); + } ArrayList ls = work.getPartDir(); - if (ls != null) + if (ls != null) { ctx.localizePaths(ls); + } } } Index: ql/src/java/org/apache/hadoop/hive/ql/plan/FetchWork.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/plan/FetchWork.java (revision 986567) +++ ql/src/java/org/apache/hadoop/hive/ql/plan/FetchWork.java (working copy) @@ -45,6 +45,8 @@ */ private String serializationNullFormat = "NULL"; + private String serializationEncoding = "UTF-8"; + public FetchWork() { } @@ -76,6 +78,14 @@ serializationNullFormat = format; } + public void setSerializationEncoding(String serializationEncoding) { + this.serializationEncoding = serializationEncoding; + } + + public String getSerializationEncoding() { + return serializationEncoding; + } + /** * @return the tblDir */ Index: serde/src/gen-java/org/apache/hadoop/hive/serde/Constants.java =================================================================== --- serde/src/gen-java/org/apache/hadoop/hive/serde/Constants.java (revision 986564) +++ serde/src/gen-java/org/apache/hadoop/hive/serde/Constants.java (working copy) @@ -5,14 +5,8 @@ */ package org.apache.hadoop.hive.serde; -import java.util.List; -import java.util.ArrayList; -import java.util.Map; -import java.util.HashMap; -import java.util.Set; import java.util.HashSet; -import java.util.Collections; -import org.apache.log4j.Logger; +import java.util.Set; public class Constants { @@ -22,6 +16,8 @@ public static final String SERIALIZATION_FORMAT = "serialization.format"; + public static final String SERIALIZATION_ENCODING = "serialization.encoding"; + public static final String SERIALIZATION_DDL = "serialization.ddl"; public static final String SERIALIZATION_NULL_FORMAT = "serialization.null.format";