diff --git ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java index 3805f9d..01a00d8 100644 --- ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java +++ ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java @@ -108,6 +108,8 @@ public boolean accept(File filePath) { } private List templatePaths = new ArrayList(); + + private String hiveRootDirectory; private String outputDirectory; @@ -185,6 +187,18 @@ public String getTemplate() { return template; } + public void setHiveRootDirectory(File hiveRootDirectory) { + try { + this.hiveRootDirectory = hiveRootDirectory.getCanonicalPath(); + } catch (IOException ioe) { + throw new BuildException(ioe); + } + } + + public String getHiveRootDirectory() { + return hiveRootDirectory; + } + public void setTemplatePath(String templatePath) throws Exception { templatePaths.clear(); for (String relativePath : templatePath.split(",")) { @@ -302,14 +316,15 @@ public void execute() throws BuildException { List qFiles = new ArrayList(); HashMap qFilesMap = new HashMap(); + File hiveRootDir = null; + File queryDir = null; File outDir = null; File resultsDir = null; File logDir = null; try { - File inpDir = null; if (queryDirectory != null) { - inpDir = new File(queryDirectory); + queryDir = new File(queryDirectory); } if (queryFile != null && !queryFile.equals("")) { @@ -318,31 +333,37 @@ public void execute() throws BuildException { if (includeOnly != null && !includeOnly.contains(qFile)) { continue; } - if (null != inpDir) { - qFiles.add(new File(inpDir, qFile)); + if (null != queryDir) { + qFiles.add(new File(queryDir, qFile)); } else { qFiles.add(new File(qFile)); } } } else if (queryFileRegex != null && !queryFileRegex.equals("")) { - qFiles.addAll(Arrays.asList(inpDir.listFiles( + qFiles.addAll(Arrays.asList(queryDir.listFiles( new QFileRegexFilter(queryFileRegex, includeOnly)))); } else if (runDisabled != null && runDisabled.equals("true")) { - qFiles.addAll(Arrays.asList(inpDir.listFiles(new DisabledQFileFilter(includeOnly)))); + qFiles.addAll(Arrays.asList(queryDir.listFiles(new DisabledQFileFilter(includeOnly)))); } else { - qFiles.addAll(Arrays.asList(inpDir.listFiles(new QFileFilter(includeOnly)))); + qFiles.addAll(Arrays.asList(queryDir.listFiles(new QFileFilter(includeOnly)))); } if (excludeQueryFile != null && !excludeQueryFile.equals("")) { // Exclude specified query files, comma separated for (String qFile : excludeQueryFile.split(",")) { - if (null != inpDir) { - qFiles.remove(new File(inpDir, qFile)); + if (null != queryDir) { + qFiles.remove(new File(queryDir, qFile)); } else { qFiles.remove(new File(qFile)); } } } + + hiveRootDir = new File(hiveRootDirectory); + if (!hiveRootDir.exists()) { + throw new BuildException("Hive Root Directory " + + hiveRootDir.getCanonicalPath() + " does not exist"); + } Collections.sort(qFiles); for (File qFile : qFiles) { @@ -397,6 +418,8 @@ public void execute() throws BuildException { // For each of the qFiles generate the test VelocityContext ctx = new VelocityContext(); ctx.put("className", className); + ctx.put("hiveRootDir", getEscapedCanonicalPath(hiveRootDir)); + ctx.put("queryDir", getEscapedCanonicalPath(queryDir)); ctx.put("qfiles", qFiles); ctx.put("qfilesMap", qFilesMap); ctx.put("resultsDir", getEscapedCanonicalPath(resultsDir)); diff --git bin/ext/beeline.sh bin/ext/beeline.sh index 4195d3d..6c0435d 100644 --- bin/ext/beeline.sh +++ bin/ext/beeline.sh @@ -18,13 +18,12 @@ THISSERVICE=beeline export SERVICE_LIST="${SERVICE_LIST}${THISSERVICE} " beeline () { - - CLASS=org.apache.hive.jdbc.beeline.HiveBeeline; + CLASS=org.apache.hive.beeline.BeeLine; execHiveCmd $CLASS "$@" } beeline_help () { - CLASS=org.apache.hive.jdbc.beeline.HiveBeeline; + CLASS=org.apache.hive.beeline.BeeLine; execHiveCmd $CLASS "--help" } diff --git build-common.xml build-common.xml index e68ecea..e565e2e 100644 --- build-common.xml +++ build-common.xml @@ -38,7 +38,9 @@ + + @@ -77,6 +79,7 @@ + @@ -97,6 +100,7 @@ + + + @@ -226,6 +232,7 @@ + @@ -280,7 +287,7 @@ + + + + + + @@ -413,8 +426,10 @@ + + @@ -422,6 +437,7 @@ + @@ -440,9 +456,11 @@ + + @@ -486,6 +504,23 @@ + + + + + + + + + + + + + + + + + diff --git build.properties build.properties index 2959558..b004409 100644 --- build.properties +++ build.properties @@ -72,8 +72,8 @@ jsp.test.jar=${hadoop.root}/lib/jetty-ext/jsp-api.jar common.jar=${hadoop.root}/lib/commons-httpclient-3.0.1.jar # module names needed for build process -iterate.hive.all=ant,shims,common,serde,metastore,ql,contrib,service,cli,jdbc,hwi,hbase-handler,pdk,builtins -iterate.hive.modules=shims,common,serde,metastore,ql,contrib,service,cli,jdbc,hwi,hbase-handler,pdk,builtins +iterate.hive.all=ant,shims,common,serde,metastore,ql,contrib,service,cli,jdbc,beeline,hwi,hbase-handler,pdk,builtins +iterate.hive.modules=shims,common,serde,metastore,ql,contrib,service,cli,jdbc,beeline,hwi,hbase-handler,pdk,builtins iterate.hive.tests=ql,contrib,hbase-handler,hwi,jdbc,metastore,odbc,serde,service iterate.hive.thrift=ql,service,metastore,serde iterate.hive.cpp=odbc @@ -91,7 +91,16 @@ test.junit.timeout=43200000 # Use this property to selectively disable tests from the command line: # ant test -Dtest.junit.exclude="**/TestCliDriver.class" # ant test -Dtest.junit.exclude="**/Test*CliDriver.class,**/TestPartitions.class" -test.junit.exclude= +test.junit.exclude="**/TestBeeLineDriver.class, **/TestHiveServer2Concurrency.class" +test.continue.on.failure=false + +test.submodule.exclude= +test.junit.maxmemory=512m + +test.concurrency.num.threads=1 +#test.beelinepositive.exclude=add_part_exist.q,alter1.q,alter2.q,alter4.q,alter5.q,alter_rename_partition.q,alter_rename_partition_authorization.q,archive.q,archive_corrupt.q,archive_multi.q,archive_mr_1806.q,archive_multi_mr_1806.q,authorization_1.q,authorization_2.q,authorization_4.q,authorization_5.q,authorization_6.q,authorization_7.q,ba_table1.q,ba_table2.q,ba_table3.q,ba_table_udfs.q,binary_table_bincolserde.q,binary_table_colserde.q,cluster.q,columnarserde_create_shortcut.q,combine2.q,constant_prop.q,create_nested_type.q,create_or_replace_view.q,create_struct_table.q,create_union_table.q,database.q,database_location.q,database_properties.q,ddltime.q,describe_database_json.q,drop_database_removes_partition_dirs.q,escape1.q,escape2.q,exim_00_nonpart_empty.q,exim_01_nonpart.q,exim_02_00_part_empty.q,exim_02_part.q,exim_03_nonpart_over_compat.q,exim_04_all_part.q,exim_04_evolved_parts.q,exim_05_some_part.q,exim_06_one_part.q,exim_07_all_part_over_nonoverlap.q,exim_08_nonpart_rename.q,exim_09_part_spec_nonoverlap.q,exim_10_external_managed.q,exim_11_managed_external.q,exim_12_external_location.q,exim_13_managed_location.q,exim_14_managed_location_over_existing.q,exim_15_external_part.q,exim_16_part_external.q,exim_17_part_managed.q,exim_18_part_external.q,exim_19_00_part_external_location.q,exim_19_part_external_location.q,exim_20_part_managed_location.q,exim_21_export_authsuccess.q,exim_22_import_exist_authsuccess.q,exim_23_import_part_authsuccess.q,exim_24_import_nonexist_authsuccess.q,global_limit.q,groupby_complex_types.q,groupby_complex_types_multi_single_reducer.q,index_auth.q,index_auto.q,index_auto_empty.q,index_bitmap.q,index_bitmap1.q,index_bitmap2.q,index_bitmap3.q,index_bitmap_auto.q,index_bitmap_rc.q,index_compact.q,index_compact_1.q,index_compact_2.q,index_compact_3.q,index_stale_partitioned.q,init_file.q,input16.q,input16_cc.q,input46.q,input_columnarserde.q,input_dynamicserde.q,input_lazyserde.q,input_testxpath3.q,input_testxpath4.q,insert2_overwrite_partitions.q,insertexternal1.q,join_thrift.q,lateral_view.q,load_binary_data.q,load_exist_part_authsuccess.q,load_nonpart_authsuccess.q,load_part_authsuccess.q,loadpart_err.q,lock1.q,lock2.q,lock3.q,lock4.q,merge_dynamic_partition.q,multi_insert.q,multi_insert_move_tasks_share_dependencies.q,null_column.q,ppd_clusterby.q,query_with_semi.q,rename_column.q,sample6.q,sample_islocalmode_hook.q,set_processor_namespaces.q,show_tables.q,source.q,split_sample.q,str_to_map.q,transform1.q,udaf_collect_set.q,udaf_context_ngrams.q,udaf_histogram_numeric.q,udaf_ngrams.q,udaf_percentile_approx.q,udf_array.q,udf_bitmap_and.q,udf_bitmap_or.q,udf_explode.q,udf_format_number.q,udf_map.q,udf_map_keys.q,udf_map_values.q,udf_max.q,udf_min.q,udf_named_struct.q,udf_percentile.q,udf_printf.q,udf_sentences.q,udf_sort_array.q,udf_split.q,udf_struct.q,udf_substr.q,udf_translate.q,udf_union.q,udf_xpath.q,udtf_stack.q,view.q,virtual_column.q + + # # Ivy Properties @@ -107,7 +116,7 @@ ivy.changingPattern=.*SNAPSHOT ivy.publish.pattern=[artifact]-[revision].[ext] ivy.artifact.retrieve.pattern=[conf]/[artifact]-[revision](-[classifier]).[ext] ivysettings.xml=${ivy.conf.dir}/ivysettings.xml -ivyresolvelog=download-only +ivyresolvelog=default ivy.mvn.repo=http://repo2.maven.org/maven2 ivy_repo_url=${ivy.mvn.repo}/org/apache/ivy/ivy/${ivy.version}/ivy-${ivy.version}.jar hive.ivy.org=org.apache.hive diff --git build.xml build.xml index 8f25ee2..eb2bff8 100644 --- build.xml +++ build.xml @@ -138,6 +138,7 @@ + @@ -386,16 +387,18 @@ + + + + + + - - - - @@ -442,6 +445,9 @@ + + + @@ -455,14 +461,15 @@ + + - @@ -655,6 +662,7 @@ + @@ -879,89 +887,91 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + description="Resolve, Retrieve Ivy-managed artifacts for docs configuration"> + + + - - + + + - - + + @@ -1164,6 +1180,12 @@ + + + + + + @@ -1182,18 +1204,15 @@ - - - - - - + + + @@ -1235,6 +1254,14 @@ output.file="${mvn.pom.dir}/hive-anttasks-${version}.pom.asc" gpg.passphrase="${gpg.passphrase}"/> + + @@ -1259,6 +1286,14 @@ output.file="${mvn.pom.dir}/hive-contrib-${version}.pom.asc" gpg.passphrase="${gpg.passphrase}"/> + + @@ -1291,14 +1326,6 @@ output.file="${mvn.pom.dir}/hive-metastore-${version}.pom.asc" gpg.passphrase="${gpg.passphrase}"/> - - @@ -1307,14 +1334,6 @@ output.file="${mvn.pom.dir}/hive-pdk-${version}.pom.asc" gpg.passphrase="${gpg.passphrase}"/> - - diff --git cli/build.xml cli/build.xml index 6e70d5f..092a68b 100755 --- cli/build.xml +++ cli/build.xml @@ -40,6 +40,11 @@ to call at top-level: ant deploy-contrib compile-core-test + + + + + diff --git cli/ivy.xml cli/ivy.xml index 0d1c64a..4bf543e 100644 --- cli/ivy.xml +++ cli/ivy.xml @@ -30,6 +30,7 @@ + diff --git common/build.xml common/build.xml index 24ad8f5..731f26e 100755 --- common/build.xml +++ common/build.xml @@ -29,6 +29,11 @@ to call at top-level: ant deploy-contrib compile-core-test + + + + + vars = new HashMap(); + private final List restrictList = new ArrayList(); static { ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); @@ -146,6 +149,16 @@ }; /** + * The conf variables that depends on current user + */ + public static final HiveConf.ConfVars[] userVars = { + HiveConf.ConfVars.SCRATCHDIR, + HiveConf.ConfVars.LOCALSCRATCHDIR, + HiveConf.ConfVars.DOWNLOADED_RESOURCES_DIR, + HiveConf.ConfVars.HIVEHISTORYFILELOC + }; + + /** * ConfVars. * * These are the default configuration properties for Hive. Each HiveConf @@ -688,6 +701,24 @@ HIVE_DDL_OUTPUT_FORMAT("hive.ddl.output.format", null), HIVE_ENTITY_SEPARATOR("hive.entity.separator", "@"), + HIVE_SERVER2_THRIFT_MIN_WORKER_THREADS("hive.server2.thrift.min.worker.threads", 5), + HIVE_SERVER2_THRIFT_MAX_WORKER_THREADS("hive.server2.thrift.max.worker.threads", 100), + + HIVE_SERVER2_THRIFT_PORT("hive.server2.thrift.port", 10000), + HIVE_SERVER2_THRIFT_BIND_HOST("hive.server2.thrift.bind.host", ""), + + + // HiveServer2 auth configuration + HIVE_SERVER2_AUTHENTICATION("hive.server2.authentication", "NONE"), + HIVE_SERVER2_KERBEROS_KEYTAB("hive.server2.authentication.kerberos.keytab", ""), + HIVE_SERVER2_KERBEROS_PRINCIPAL("hive.server2.authentication.kerberos.principal", ""), + HIVE_SERVER2_PLAIN_LDAP_URL("hive.server2.authentication.ldap.url", null), + HIVE_SERVER2_PLAIN_LDAP_BASEDN("hive.server2.authentication.ldap.baseDN", null), + HIVE_SERVER2_KERBEROS_IMPERSONATION("hive.server2.enable.impersonation", false), + HIVE_SERVER2_CUSTOM_AUTHENTICATION_CLASS("hive.server2.custom.authentication.class", null), + + HIVE_CONF_RESTRICTED_LIST("hive.conf.restricted.list", null), + // If this is set all move tasks at the end of a multi-insert query will only begin once all // outputs are ready HIVE_MULTI_INSERT_MOVE_TASKS_SHARE_DEPENDENCIES( @@ -870,6 +901,13 @@ private static synchronized InputStream getConfVarInputStream() { return new LoopingByteArrayInputStream(confVarByteArray); } + public void verifyAndSet(String name, String value) throws IllegalArgumentException { + if (restrictList.contains(name)) { + throw new IllegalArgumentException("Cann't modify " + name + " at runtime"); + } + set(name, value); + } + public static int getIntVar(Configuration conf, ConfVars var) { assert (var.valClass == Integer.class); return conf.getInt(var.varname, var.defaultIntVal); @@ -1057,8 +1095,18 @@ private void initialize(Class cls) { if (auxJars == null) { auxJars = this.get(ConfVars.HIVEAUXJARS.varname); } + + // setup list of conf vars that are not allowed to change runtime + String restrictListStr = this.get(ConfVars.HIVE_CONF_RESTRICTED_LIST.toString()); + if (restrictListStr != null) { + for (String entry : restrictListStr.split(",")) { + restrictList.add(entry); + } + } + restrictList.add(ConfVars.HIVE_CONF_RESTRICTED_LIST.toString()); } + /** * Apply system properties to this object if the property name is defined in ConfVars * and the value is non-null and not an empty string. diff --git conf/hive-default.xml.template conf/hive-default.xml.template index abbab99..2aab282 100644 --- conf/hive-default.xml.template +++ conf/hive-default.xml.template @@ -1120,7 +1120,7 @@ hive.profiler.retries.wait 3000 - The base waiting window (in milliseconds) before the next retry. The actual wait time is calculated by baseWindow * failues + baseWindow * (failure + 1) * (random number between [0.0,1.0]). + The base waiting window (in milliseconds) before the next retry. The actual wait time is calculated by baseWindow * failues baseWindow * (failure 1) * (random number between [0.0,1.0]). @@ -1174,7 +1174,7 @@ hive.stats.retries.wait 3000 - The base waiting window (in milliseconds) before the next retry. The actual wait time is calculated by baseWindow * failues + baseWindow * (failure + 1) * (random number between [0.0,1.0]). + The base waiting window (in milliseconds) before the next retry. The actual wait time is calculated by baseWindow * failues baseWindow * (failure 1) * (random number between [0.0,1.0]). @@ -1676,7 +1676,7 @@ Currently the query should be single sourced not having any subquery and should not have any aggregations or distincts (which incurrs RS), lateral views and joins. 1. minimal : SELECT STAR, FILTER on partition columns, LIMIT only - 2. more : SELECT, FILTER, LIMIT only (+TABLESAMPLE, virtual columns) + 2. more : SELECT, FILTER, LIMIT only (TABLESAMPLE, virtual columns) @@ -1692,7 +1692,6 @@ The number of miliseconds between HMSHandler retry attempts - hive.server.read.socket.timeout 10 @@ -1765,5 +1764,92 @@ false Whether to enable using Column Position Alias in Group By or Order By + + + hive.server2.thrift.min.worker.threads + 5 + Minimum number of Thrift worker threads + + + + hive.server2.thrift.max.worker.threads + 100 + Maximum number of Thrift worker threads + + + + hive.server2.thrift.port + 10000 + Port number of HiveServer2 Thrift interface. + Can be overridden by setting $HIVE_SERVER2_THRIFT_PORT + + + + hive.server2.thrift.bind.host + localhost + Bind host on which to run the HiveServer2 Thrift interface. + Can be overridden by setting $HIVE_SERVER2_THRIFT_BIND_HOST + + + + hive.server2.authentication + NONE + + Client authentication types. + NONE: no authentication check + LDAP: LDAP/AD based authentication + KERBEROS: Kerberos/GSSAPI authentication + CUSTOM: Custom authentication provider + (Use with property hive.server2.custom.authentication.class) + + + + + hive.server2.custom.authentication.class + + + Custom authentication class. Used when property + 'hive.server2.authentication' is set to 'CUSTOM'. Provided class + must be a proper implementation of the interface + org.apache.hive.service.auth.PasswdAuthenticationProvider. HiveServer2 + will call its Authenticate(user, passed) method to authenticate requests. + The implementation may optionally extend the Hadoop's + org.apache.hadoop.conf.Configured class to grab Hive's Configuration object. + + + + + >hive.server2.authentication.kerberos.principal + + + Kerberos server principal + + + + + >hive.server2.authentication.kerberos.keytab + + + Kerberos keytab file for server principal + + + + + hive.server2.authentication.ldap.url + + + LDAP connection URL + + + + + + hive.server2.authentication.ldap.baseDN + + + LDAP base DN + + + diff --git contrib/build.xml contrib/build.xml index 277c985..5d33d83 100644 --- contrib/build.xml +++ contrib/build.xml @@ -49,7 +49,8 @@ - --> - - + diff --git hbase-handler/build.xml hbase-handler/build.xml index 8676ca3..8e23a09 100644 --- hbase-handler/build.xml +++ hbase-handler/build.xml @@ -47,7 +47,8 @@ - - - + diff --git ivy/ivysettings.xml ivy/ivysettings.xml index 28c481e..d230f2c 100644 --- ivy/ivysettings.xml +++ ivy/ivysettings.xml @@ -61,7 +61,7 @@ diff --git ivy/libraries.properties ivy/libraries.properties index a4f35f9..16a0f70 100644 --- ivy/libraries.properties +++ ivy/libraries.properties @@ -37,6 +37,7 @@ commons-compress.version=1.4.1 commons-configuration.version=1.6 commons-dbcp.version=1.4 commons-httpclient.version=3.0.1 +commons-io.version=2.4 commons-lang.version=2.4 commons-logging.version=1.0.4 commons-logging-api.version=1.0.4 @@ -51,8 +52,6 @@ jdo-api.version=2.3-ec jdom.version=1.1 jetty.version=6.1.26 jline.version=0.9.94 -sqlline.version=1_0_2 -sqlline.branch=1.0.2 json.version=20090211 junit.version=4.10 libfb303.version=0.9.0 @@ -63,6 +62,7 @@ mockito-all.version=1.8.2 rat.version=0.8 slf4j-api.version=1.6.1 slf4j-log4j12.version=1.6.1 +tempus-fugit.version=1.1 velocity.version=1.5 zookeeper.version=3.4.3 javolution.version=5.5.1 diff --git jdbc/ivy.xml jdbc/ivy.xml index 29777a3..9269bd1 100644 --- jdbc/ivy.xml +++ jdbc/ivy.xml @@ -33,8 +33,6 @@ transitive="false"/> - diff --git metastore/build.xml metastore/build.xml index 9e60b66..0e94611 100755 --- metastore/build.xml +++ metastore/build.xml @@ -22,15 +22,6 @@ - - - You must set the 'thrift.home' property! - Executing ${thrift.home}/bin/thrift on ${ant.project.name}/if/hive_metastore.thrift - - - - - diff --git ql/build.xml ql/build.xml index 7098d16..05956e2 100644 --- ql/build.xml +++ ql/build.xml @@ -27,26 +27,22 @@ + + - - - You must set the 'thrift.home' property! - Executing ${thrift.home}/bin/thrift on ${ant.project.name}/if/queryplan.thrift - - - - - + + - - - + hadoopVersion="${hadoopVersion}"/> + + - - - + diff --git ql/src/java/org/apache/hadoop/hive/ql/Driver.java ql/src/java/org/apache/hadoop/hive/ql/Driver.java index a373c8b..5ce28c9 100644 --- ql/src/java/org/apache/hadoop/hive/ql/Driver.java +++ ql/src/java/org/apache/hadoop/hive/ql/Driver.java @@ -112,6 +112,8 @@ static final private Log LOG = LogFactory.getLog(Driver.class.getName()); static final private LogHelper console = new LogHelper(LOG); + private static final Object compileMonitor = new Object(); + private int maxRows = 100; ByteStream.Output bos = new ByteStream.Output(); @@ -895,7 +897,10 @@ public CommandProcessorResponse run(String command) throws CommandNeedRetryExcep perfLogger.PerfLogBegin(LOG, PerfLogger.DRIVER_RUN); perfLogger.PerfLogBegin(LOG, PerfLogger.TIME_TO_SUBMIT); - int ret = compile(command); + int ret; + synchronized (compileMonitor) { + ret = compile(command); + } if (ret != 0) { releaseLocks(ctx.getHiveLocks()); return new CommandProcessorResponse(ret, errorMessage, SQLState); diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/CopyTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/CopyTask.java index 9391acd..23bebc5 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/CopyTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/CopyTask.java @@ -72,13 +72,12 @@ public int execute(DriverContext driverContext) { } if (!dstFs.mkdirs(toPath)) { - console - .printError("Cannot make target directory: " + toPath.toString()); + console.printError("Cannot make target directory: " + toPath.toString()); return 2; } for (FileStatus oneSrc : srcs) { - System.out.println("Copying file: " + oneSrc.getPath().toString()); + console.printInfo("Copying file: " + oneSrc.getPath().toString()); LOG.debug("Copying file: " + oneSrc.getPath().toString()); if (!FileUtil.copy(srcFs, oneSrc.getPath(), dstFs, toPath, false, // delete // source diff --git ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index 7442792..5befddc 100644 --- ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -182,6 +182,10 @@ public static Hive get() throws HiveException { return db; } + public static void set(Hive hive) { + hiveDB.set(hive); + } + public static void closeCurrent() { hiveDB.remove(); } @@ -2409,6 +2413,26 @@ public Table newTable(String tableName) throws HiveException { } } + public String getDelegationToken(String owner, String renewer) + throws HiveException{ + try { + return getMSC().getDelegationToken(owner, renewer); + } catch(Exception e) { + LOG.error(StringUtils.stringifyException(e)); + throw new HiveException(e); + } + } + + public void cancelDelegationToken(String tokenStrForm) + throws HiveException { + try { + getMSC().cancelDelegationToken(tokenStrForm); + } catch(Exception e) { + LOG.error(StringUtils.stringifyException(e)); + throw new HiveException(e); + } + } + private static String[] getQualifiedNames(String qualifiedName) { return qualifiedName.split("\\."); } diff --git ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java index 64e2c13..912c4ad 100644 --- ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java +++ ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java @@ -116,15 +116,27 @@ private CommandProcessorResponse setVariable(String varname, String varvalue){ return new CommandProcessorResponse(0); } else if (varname.startsWith(SetProcessor.HIVECONF_PREFIX)){ String propName = varname.substring(SetProcessor.HIVECONF_PREFIX.length()); - String error = setConf(varname, propName, varvalue, false); - return new CommandProcessorResponse(error == null ? 0 : 1, error, null); + try { + ss.getConf().verifyAndSet(propName, new VariableSubstitution().substitute(ss.getConf(),varvalue)); + return new CommandProcessorResponse(0); + } catch (IllegalArgumentException e) { + ss.out.println(e.getMessage()); + return new CommandProcessorResponse(-1, e.getMessage(), "42000"); + } } else if (varname.startsWith(SetProcessor.HIVEVAR_PREFIX)) { String propName = varname.substring(SetProcessor.HIVEVAR_PREFIX.length()); ss.getHiveVariables().put(propName, new VariableSubstitution().substitute(ss.getConf(),varvalue)); return new CommandProcessorResponse(0); } else { - String error = setConf(varname, varname, varvalue, true); - return new CommandProcessorResponse(error == null ? 0 : 1, error, null); + String substitutedValue = new VariableSubstitution().substitute(ss.getConf(),varvalue); + try { + ss.getConf().verifyAndSet(varname, substitutedValue ); + ss.getOverriddenConfigurations().put(varname, substitutedValue); + return new CommandProcessorResponse(0); + } catch (IllegalArgumentException e) { + ss.out.println(e.getMessage()); + return new CommandProcessorResponse(-1, e.getMessage(), "42000"); + } } } @@ -209,7 +221,7 @@ private CommandProcessorResponse getVariable(String varname){ } } else { dumpOption(varname); - return new CommandProcessorResponse(0); + return new CommandProcessorResponse(0, null, null, getSchema()); } } diff --git ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java index f167fad..f9da449 100644 --- ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java +++ ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java @@ -172,17 +172,14 @@ public void normalizeNames(File path) throws Exception { normalizeNames(file); } } else { - // System.out.println("Trying to match: " + path.getPath()); Matcher m = reduceTok.matcher(path.getName()); if (m.matches()) { String name = m.group(1) + "reduce" + m.group(3); - // System.out.println("Matched new name: " + name); path.renameTo(new File(path.getParent(), name)); } else { m = mapTok.matcher(path.getName()); if (m.matches()) { String name = m.group(1) + "map_" + m.group(3); - // System.out.println("Matched new name: " + name); path.renameTo(new File(path.getParent(), name)); } } @@ -193,6 +190,14 @@ public QTestUtil(String outDir, String logDir) throws Exception { this(outDir, logDir, false, "0.20"); } + public String getOutputDirectory() { + return outDir; + } + + public String getLogDirectory() { + return logDir; + } + private String getHadoopMainVersion(String input) { if (input == null) { return null; diff --git serde/build.xml serde/build.xml index 5f11529..a2c23d1 100644 --- serde/build.xml +++ serde/build.xml @@ -51,27 +51,6 @@ - - - You must set the 'thrift.home' property! - Executing ${thrift.home}/bin/thrift to build java serde Constants... - - - - Executing ${thrift.home}/bin/thrift to build complex.thrift test classes... - - - - Executing ${thrift.home}/bin/thrift to build testthrift.thrift classes... - - - - Executing ${thrift.home}/bin/thrift to build megastruct.thrift classes... - - - - - Generating data/files/complex.seq... diff --git service/build.xml service/build.xml index 19bdb9f..cb0beb4 100644 --- service/build.xml +++ service/build.xml @@ -22,15 +22,6 @@ - - - You must set the 'thrift.home' property! - Executing ${thrift.home}/bin/thrift on ${ant.project.name}/if/hive_service.thrift - - - - - pvea) throws + public void setTokenStr(UserGroupInformation ugi, String tokenStr, String tokenService) + throws IOException { + throw new UnsupportedOperationException("Tokens are not supported in current hadoop version"); + } + + @Override + public T doAs(UserGroupInformation ugi, PrivilegedExceptionAction pvea) throws IOException, InterruptedException { try { - Subject.doAs(SecurityUtil.getSubject(ugi),pvea); + return Subject.doAs(SecurityUtil.getSubject(ugi),pvea); } catch (PrivilegedActionException e) { throw new IOException(e); } @@ -555,6 +561,21 @@ public UserGroupInformation createRemoteUser(String userName, List group } @Override + public void loginUserFromKeytab(String principal, String keytabFile) throws IOException { + throw new UnsupportedOperationException("Kerberos login is not supported in current hadoop version"); + } + + @Override + public UserGroupInformation createProxyUser(String userName) throws IOException { + return createRemoteUser(userName, null); + } + + @Override + public boolean isSecurityEnabled() { + return false; + } + + @Override public String getTaskAttemptLogUrl(JobConf conf, String taskTrackerHttpAddress, String taskAttemptId) throws MalformedURLException { diff --git shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java index 2d32b07..1d1bde6 100644 --- shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java +++ shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java @@ -39,6 +39,7 @@ import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hive.io.HiveIOExceptionHandlerUtil; +import org.apache.hadoop.hive.thrift.DelegationTokenIdentifier; import org.apache.hadoop.hive.thrift.DelegationTokenSelector; import org.apache.hadoop.http.HtmlQuoting; import org.apache.hadoop.io.Text; @@ -59,6 +60,7 @@ import org.apache.hadoop.mapred.lib.CombineFileInputFormat; import org.apache.hadoop.mapred.lib.CombineFileSplit; import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; @@ -463,19 +465,19 @@ public int createHadoopArchive(Configuration conf, Path sourceDir, Path destDir, return ToolRunner.run(har, args.toArray(new String[0])); } - + /* * This particular instance is for Hadoop 1.0 which creates an archive * with only the relative path of the archived directory stored within * the archive as compared to the full path in case of earlier versions. * See this api in Hadoop20Shims for comparison. */ - public URI getHarUri(URI original, URI base, URI originalBase) + public URI getHarUri(URI original, URI base, URI originalBase) throws URISyntaxException { URI relative = originalBase.relativize(original); if (relative.isAbsolute()) { throw new URISyntaxException("Couldn't create URI for location.", - "Relative: " + relative + " Base: " + "Relative: " + relative + " Base: " + base + " OriginalBase: " + originalBase); } @@ -538,8 +540,27 @@ public String getTokenStrForm(String tokenSignature) throws IOException { } @Override - public void doAs(UserGroupInformation ugi, PrivilegedExceptionAction pvea) throws IOException, InterruptedException { - ugi.doAs(pvea); + public void setTokenStr(UserGroupInformation ugi, String tokenStr, String tokenService) throws IOException { + Token delegationToken = new Token(); + delegationToken.decodeFromUrlString(tokenStr); + delegationToken.setService(new Text(tokenService)); + ugi.addToken(delegationToken); + } + + @Override + public T doAs(UserGroupInformation ugi, PrivilegedExceptionAction pvea) throws IOException, InterruptedException { + return ugi.doAs(pvea); + } + + @Override + public UserGroupInformation createProxyUser(String userName) throws IOException { + return UserGroupInformation.createProxyUser( + userName, UserGroupInformation.getLoginUser()); + } + + @Override + public boolean isSecurityEnabled() { + return UserGroupInformation.isSecurityEnabled(); } @Override @@ -557,6 +578,12 @@ public void closeAllForUGI(UserGroupInformation ugi) { } @Override + public void loginUserFromKeytab(String principal, String keytabFile) throws IOException { + String hostPrincipal = SecurityUtil.getServerPrincipal(principal, "0.0.0.0"); + UserGroupInformation.loginUserFromKeytab(hostPrincipal, keytabFile); + } + + @Override abstract public JobTrackerState getJobTrackerState(ClusterStatus clusterStatus) throws Exception; @Override diff --git shims/src/common-secure/java/org/apache/hadoop/hive/thrift/HadoopThriftAuthBridge20S.java shims/src/common-secure/java/org/apache/hadoop/hive/thrift/HadoopThriftAuthBridge20S.java index ff45021..777226f 100644 --- shims/src/common-secure/java/org/apache/hadoop/hive/thrift/HadoopThriftAuthBridge20S.java +++ shims/src/common-secure/java/org/apache/hadoop/hive/thrift/HadoopThriftAuthBridge20S.java @@ -40,6 +40,8 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge.Client; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hive.thrift.client.TUGIAssumingTransport; import org.apache.hadoop.security.SaslRpcServer; import org.apache.hadoop.security.SaslRpcServer.AuthMethod; @@ -62,6 +64,7 @@ import org.apache.thrift.transport.TTransportException; import org.apache.thrift.transport.TTransportFactory; +import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION; /** * Functions that bridge Thrift's SASL transports to Hadoop's @@ -76,6 +79,14 @@ public Client createClient() { } @Override + public Client createClientWithConf(String authType) { + Configuration conf = new Configuration(); + conf.set(HADOOP_SECURITY_AUTHENTICATION, authType); + UserGroupInformation.setConfiguration(conf); + return new Client(); + } + + @Override public Server createServer(String keytabFile, String principalConf) throws TTransportException { return new Server(keytabFile, principalConf); } @@ -233,7 +244,7 @@ public Server() throws TTransportException { /** * Create a server with a kerberos keytab/principal. */ - private Server(String keytabFile, String principalConf) + protected Server(String keytabFile, String principalConf) throws TTransportException { if (keytabFile == null || keytabFile.isEmpty()) { throw new TTransportException("No keytab specified"); @@ -293,7 +304,15 @@ public TTransportFactory createTransportFactory() throws TTransportException */ @Override public TProcessor wrapProcessor(TProcessor processor) { - return new TUGIAssumingProcessor(processor, secretManager); + return new TUGIAssumingProcessor(processor, secretManager, true); + } + + /** + * Wrap a TProcessor to capture the client information like connecting userid, ip etc + */ + @Override + public TProcessor wrapNonAssumingProcessor(TProcessor processor) { + return new TUGIAssumingProcessor(processor, secretManager, false); } protected DelegationTokenStore getTokenStore(Configuration conf) @@ -398,6 +417,18 @@ protected synchronized AuthenticationMethod initialValue() { } }; + private static ThreadLocal remoteUser = new ThreadLocal () { + @Override + protected synchronized String initialValue() { + return null; + } + }; + + @Override + public String getRemoteUser() { + return remoteUser.get(); + } + /** CallbackHandler for SASL DIGEST-MD5 mechanism */ // This code is pretty much completely based on Hadoop's // SaslRpcServer.SaslDigestCallbackHandler - the only reason we could not @@ -479,12 +510,15 @@ public void handle(Callback[] callbacks) throws InvalidToken, * * This is used on the server side to set the UGI for each specific call. */ - private class TUGIAssumingProcessor implements TProcessor { + protected class TUGIAssumingProcessor implements TProcessor { final TProcessor wrapped; DelegationTokenSecretManager secretManager; - TUGIAssumingProcessor(TProcessor wrapped, DelegationTokenSecretManager secretManager) { + boolean useProxy; + TUGIAssumingProcessor(TProcessor wrapped, DelegationTokenSecretManager secretManager, + boolean useProxy) { this.wrapped = wrapped; this.secretManager = secretManager; + this.useProxy = useProxy; } public boolean process(final TProtocol inProt, final TProtocol outProt) throws TException { @@ -513,17 +547,23 @@ public boolean process(final TProtocol inProt, final TProtocol outProt) throws T remoteAddress.set(socket.getInetAddress()); UserGroupInformation clientUgi = null; try { - clientUgi = UserGroupInformation.createProxyUser( - endUser, UserGroupInformation.getLoginUser()); - return clientUgi.doAs(new PrivilegedExceptionAction() { - public Boolean run() { - try { - return wrapped.process(inProt, outProt); - } catch (TException te) { - throw new RuntimeException(te); + if (useProxy) { + clientUgi = UserGroupInformation.createProxyUser( + endUser, UserGroupInformation.getLoginUser()); + remoteUser.set(clientUgi.getShortUserName()); + return clientUgi.doAs(new PrivilegedExceptionAction() { + public Boolean run() { + try { + return wrapped.process(inProt, outProt); + } catch (TException te) { + throw new RuntimeException(te); + } } - } - }); + }); + } else { + remoteUser.set(endUser); + return wrapped.process(inProt, outProt); + } } catch (RuntimeException rte) { if (rte.getCause() instanceof TException) { throw (TException)rte.getCause(); diff --git shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java index bdb2500..946e1c1 100644 --- shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java +++ shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java @@ -192,15 +192,15 @@ public URI getHarUri(URI original, URI base, URI originalBase) public void closeAllForUGI(UserGroupInformation ugi); public UserGroupInformation getUGIForConf(Configuration conf) throws LoginException, IOException; - /** * Used by metastore server to perform requested rpc in client context. + * @param * @param ugi * @param pvea * @throws IOException * @throws InterruptedException */ - public void doAs(UserGroupInformation ugi, PrivilegedExceptionAction pvea) throws + public T doAs(UserGroupInformation ugi, PrivilegedExceptionAction pvea) throws IOException, InterruptedException; /** @@ -226,6 +226,12 @@ public void doAs(UserGroupInformation ugi, PrivilegedExceptionAction pvea) public boolean isSecureShimImpl(); /** + * Return true if the hadoop configuration has security enabled + * @return + */ + public boolean isSecurityEnabled(); + + /** * Get the string form of the token given a token signature. * The signature is used as the value of the "service" field in the token for lookup. * Ref: AbstractDelegationTokenSelector in Hadoop. If there exists such a token @@ -242,6 +248,16 @@ public void doAs(UserGroupInformation ugi, PrivilegedExceptionAction pvea) */ String getTokenStrForm(String tokenSignature) throws IOException; + /** + * Add a delegation token to the given ugi + * @param ugi + * @param tokenStr + * @param tokenService + * @throws IOException + */ + void setTokenStr(UserGroupInformation ugi, String tokenStr, String tokenService) + throws IOException; + enum JobTrackerState { INITIALIZING, RUNNING }; @@ -290,6 +306,12 @@ public void doAs(UserGroupInformation ugi, PrivilegedExceptionAction pvea) public String getJobLauncherHttpAddress(Configuration conf); + /** + * Perform kerberos login using the given principal and keytab + * @throws IOException + */ + public void loginUserFromKeytab(String principal, String keytabFile) throws IOException; + /** * Move the directory/file to trash. In case of the symlinks or mount points, the file is * moved to the trashbin in the actual volume of the path p being deleted @@ -321,6 +343,13 @@ public boolean moveToAppropriateTrash(FileSystem fs, Path path, Configuration co public short getDefaultReplication(FileSystem fs, Path path); /** + * Create the proxy ugi for the given userid + * @param userName + * @return + */ + UserGroupInformation createProxyUser(String userName) throws IOException; + + /** * InputSplitShim. * */ @@ -380,4 +409,5 @@ public boolean moveToAppropriateTrash(FileSystem fs, Path path, Configuration co RecordReader getRecordReader(JobConf job, InputSplitShim split, Reporter reporter, Class> rrClass) throws IOException; } + } diff --git shims/src/common/java/org/apache/hadoop/hive/thrift/HadoopThriftAuthBridge.java shims/src/common/java/org/apache/hadoop/hive/thrift/HadoopThriftAuthBridge.java index ecaa2d7..9b0ec0a 100644 --- shims/src/common/java/org/apache/hadoop/hive/thrift/HadoopThriftAuthBridge.java +++ shims/src/common/java/org/apache/hadoop/hive/thrift/HadoopThriftAuthBridge.java @@ -37,6 +37,11 @@ public Client createClient() { "The current version of Hadoop does not support Authentication"); } + public Client createClientWithConf(String authType) { + throw new UnsupportedOperationException( + "The current version of Hadoop does not support Authentication"); + } + public Server createServer(String keytabFile, String principalConf) throws TTransportException { throw new UnsupportedOperationException( @@ -67,7 +72,9 @@ public abstract TTransport createClientTransport( public static abstract class Server { public abstract TTransportFactory createTransportFactory() throws TTransportException; public abstract TProcessor wrapProcessor(TProcessor processor); + public abstract TProcessor wrapNonAssumingProcessor(TProcessor processor); public abstract InetAddress getRemoteAddress(); + public abstract String getRemoteUser(); public abstract void startDelegationTokenSecretManager(Configuration conf) throws IOException; public abstract String getDelegationToken(String owner, String renewer) throws IOException, InterruptedException;