Index: ant/findbugs.xml =================================================================== --- ant/findbugs.xml (revision 1360995) +++ ant/findbugs.xml (working copy) @@ -45,6 +45,7 @@ + - + + + + + + + + + + + + + + Index: conf/templeton-log4j.properties =================================================================== --- conf/templeton-log4j.properties (revision 0) +++ conf/templeton-log4j.properties (revision 0) @@ -0,0 +1,31 @@ +# +# templeton-log4j - configure the Templeton logging system + +# Define some default values that can be overridden by system properties +templeton.root.logger = DEBUG, standard +templeton.log.dir = . +templeton.log.file = templeton.log + +log4j.rootLogger = ${templeton.root.logger} + +# Logging Threshold +log4j.threshhold = DEBUG + +log4j.appender.standard = org.apache.log4j.DailyRollingFileAppender +log4j.appender.standard.File = ${templeton.log.dir}/${templeton.log.file} + +# Rollver at midnight +log4j.appender.DRFA.DatePattern = .yyyy-MM-dd + +log4j.appender.DRFA.layout = org.apache.log4j.PatternLayout + +log4j.appender.standard.layout = org.apache.log4j.PatternLayout +log4j.appender.standard.layout.conversionPattern = %-5p | %d{DATE} | %c | %m%n + +# Class logging settings +log4j.logger.com.sun.jersey = DEBUG +log4j.logger.com.sun.jersey.spi.container.servlet.WebComponent = ERROR +log4j.logger.org.apache.hadoop = INFO +log4j.logger.org.apache.hadoop.conf = WARN +log4j.logger.org.apache.zookeeper = WARN +log4j.logger.org.eclipse.jetty = INFO Index: conf/templeton-default.xml =================================================================== --- conf/templeton-default.xml (revision 0) +++ conf/templeton-default.xml (revision 0) @@ -0,0 +1,211 @@ + + + + + + + + + + + templeton.port + 50111 + The HTTP port for the main server. + + + + templeton.hadoop.conf.dir + ${env.HADOOP_CONF_DIR} + The path to the Hadoop configuration. + + + + templeton.jar + ${env.TEMPLETON_HOME}/templeton-0.1.0-dev.jar + The path to the Templeton jar file. + + + + templeton.libjars + ${env.TEMPLETON_HOME}/lib/zookeeper-3.3.4.jar + Jars to add to the classpath. + + + + templeton.override.jars + hdfs:///user/templeton/ugi.jar + + Jars to add the the HADOOP_CLASSPATH for all Map Reduce jobs. + This is a list of jars that must exist on hdfs that are added + to the Distributed Cache. + + + + + templeton.override.enabled + true + + Enable the override path in templeton.override.jars + + + + + templeton.streaming.jar + hdfs:///user/templeton/hadoop-streaming.jar + The hdfs path to the Hadoop streaming jar file. + + + + templeton.hadoop + ${env.HADOOP_PREFIX}/bin/hadoop + The path to the Hadoop executable. + + + + templeton.pig.archive + hdfs:///user/templeton/pig-0.9.2.tar.gz + The path to the Pig archive. + + + + templeton.pig.path + pig-0.9.2.tar.gz/pig-0.9.2/bin/pig + The path to the Pig executable. + + + + templeton.hcat + ${env.HCAT_PREFIX}/bin/hcat + The path to the hcatalog executable. + + + + templeton.hive.archive + hdfs:///user/templeton/hcatalog-0.3.0.tar.gz + The path to the Hive archive. + + + + templeton.hive.path + hcatalog-0.3.0.tar.gz/hcatalog-0.3.0/bin/hive + The path to the Hive executable. + + + + templeton.hive.properties + hive.metastore.local=false,hive.metastore.uris=thrift://localhost:9933,hive.metastore.sasl.enabled=false + Properties to set when running hive. + + + + templeton.exec.encoding + UTF-8 + The encoding of the stdout and stderr data. + + + + templeton.exec.timeout + 10000 + + How long in milliseconds a program is allowed to run on the + Templeton box. + + + + + templeton.exec.max-procs + 16 + The maximum number of processes allowed to run at once. + + + + templeton.exec.max-output-bytes + 1048576 + + The maximum number of bytes from stdout or stderr stored in ram. + + + + + templeton.exec.envs + HADOOP_PREFIX,HADOOP_HOME,JAVA_HOME + The environment variables passed through to exec. + + + + templeton.zookeeper.hosts + 127.0.0.1:2181 + ZooKeeper servers, as comma separated host:port pairs + + + + templeton.zookeeper.session-timeout + 30000 + ZooKeeper session timeout in milliseconds + + + + templeton.callback.retry.interval + 10000 + How long to wait between callback retry attempts in milliseconds + + + + templeton.callback.retry.attempts + 5 + How many times to retry the callback + + + + templeton.storage.class + org.apache.hcatalog.templeton.tool.HDFSStorage + The class to use as storage + + + + templeton.storage.root + /templeton-hadoop + The path to the directory to use for storage + + + + templeton.hdfs.cleanup.interval + 43200000 + The maximum delay between a thread's cleanup checks + + + + templeton.hdfs.cleanup.maxage + 604800000 + The maximum age of a templeton job + + + + templeton.zookeeper.cleanup.interval + 43200000 + The maximum delay between a thread's cleanup checks + + + + templeton.zookeeper.cleanup.maxage + 604800000 + The maximum age of a templeton job + + + Index: src/test/e2e/templeton/tests/ddl.conf =================================================================== --- src/test/e2e/templeton/tests/ddl.conf (revision 0) +++ src/test/e2e/templeton/tests/ddl.conf (revision 0) @@ -0,0 +1,1110 @@ +############################################################################### +# curl command tests for templeton +# +# + +#use Yahoo::Miners::Test::PigSetup; + +#PigSetup::setup(); + +#my $me = `whoami`; +#chomp $me; + +$cfg = +{ + 'driver' => 'Curl', + + 'groups' => + [ +##============================================================================================================= + { + 'name' => 'Hcat_table', + 'tests' => + [ + { + #drop table if exists + 'num' => 1, + 'method' => 'POST', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl', + 'status_code' => 200, + 'post_options' => ['user.name=:UNAME:','exec=drop table if exists templetontest_tab2'], + 'json_field_substr_match' => {'stderr' => 'OK'}, + + }, + { + #create table + 'num' => 2, + 'method' => 'POST', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl', + 'status_code' => 200, + 'post_options' => ['user.name=:UNAME:','exec=create table templetontest_tab2(i int, j bigint) STORED AS rcfile;'], + 'json_field_substr_match' => {'stderr' => 'OK', 'exitcode' => '^0$'} + }, + { + #show tables + 'num' => 3, + 'method' => 'POST', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl', + 'status_code' => 200, + 'post_options' => ['user.name=:UNAME:','exec=show tables;'], + 'json_field_substr_match' => {'stderr' => 'OK', 'stdout' => 'templetontest_tab2', 'exitcode' => '^0$'} + }, + { + #create table again, should fail + 'num' => 4, + 'method' => 'POST', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl', + 'status_code' => 200, + 'post_options' => ['user.name=:UNAME:','exec=create table templetontest_tab2(i int, j bigint) STORED AS rcfile;'], + 'json_field_substr_match' => { 'exitcode' => '^9$'} + }, + { + #describe table + 'num' => 5, + 'method' => 'POST', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl', + 'status_code' => 200, + 'post_options' => ['user.name=:UNAME:','exec=describe templetontest_tab2;'], + 'json_field_substr_match' => { 'stdout' => '.*i\s+int.*\n.*j.*bigint.*', 'exitcode' => '^0$', 'stderr' => 'OK'} + }, + { + #alter table - file format + 'num' => 6, + 'method' => 'POST', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl', + 'status_code' => 200, + 'post_options' => ['user.name=:UNAME:','exec=alter table templetontest_tab2 SET FILEFORMAT rcfile;'], + 'json_field_substr_match' => { 'exitcode' => '^0$', 'stderr' => 'OK'} + }, + { + #alter table - add column + 'num' => 7, + 'method' => 'POST', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl', + 'status_code' => 200, + 'post_options' => ['user.name=:UNAME:','exec=alter table templetontest_tab2 add columns(newcolx int);'], + 'json_field_substr_match' => { 'exitcode' => '^0$', 'stderr' => 'OK'} + }, + { + #describe table + 'num' => 8, + 'method' => 'POST', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl', + 'status_code' => 200, + 'post_options' => ['user.name=:UNAME:','exec=describe templetontest_tab2;'], + 'json_field_substr_match' => { 'stdout' => '.*newcolx\s+int.*', 'exitcode' => '^0$', 'stderr' => 'OK'} + }, + ] + }, +##============================================================================================================= + { + 'name' => 'REST_DDL_DB', + 'tests' => + [ + { + #cleanup + #drop database if exists + 'num' => 1, + 'method' => 'POST', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl', + 'status_code' => 200, + 'post_options' => ['user.name=:UNAME:','exec=drop database if exists templeton_testempty_db'], + 'json_field_substr_match' => {'stderr' => 'OK'}, + }, + { + #create db + 'num' => 2, + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/templeton_testempty_db?user.name=:UNAME:', + 'status_code' => 200, + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{"comment":"Hello there", "location":"", "properties":{"a":"b"}}'] + }, + { + #negative test - create same db again + 'ignore' => 'does not return corect status code', + 'num' => 3, + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/templeton_testempty_db?user.name=:UNAME:', + 'status_code' => 409, + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{"comment":"Hello there", "location":"", "properties":{"a":"b"}}'] + }, + { #show databases + 'num' => 4, + 'method' => 'GET', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/?user.name=:UNAME:', + 'status_code' => 200, + 'format_header' => 'Content-Type: application/json', + 'json_field_substr_match' => {'databases' => 'templeton_testempty_db'}, + }, + { #show databases w like + 'num' => 5, + 'method' => 'GET', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/?like=*empty*&user.name=:UNAME:', + 'status_code' => 200, + 'json_field_substr_match' => {'databases' => 'templeton_testempty_db'}, + }, + + { #desc db + 'num' => 6, + 'method' => 'GET', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/templeton_testempty_db?user.name=:UNAME:', + 'status_code' => 200, + 'json_field_substr_match' => {'database' => 'templeton_testempty_db', 'location' => 'templeton_testempty_db'}, + }, + { #drop db + 'num' => 7, + 'method' => 'DELETE', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/templeton_testempty_db?user.name=:UNAME:', + 'status_code' => 200, + }, + { + #-ve test , describe on non existent db + 'num' => 8, + 'ignore' => 'ignore waiting for bugfix', + 'method' => 'GET', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/test_no_such_db?user.name=:UNAME:', + 'status_code' => 404, + 'json_field_substr_match' => {'error' => 'No such database'}, + }, + ] + }, +##============================================================================================================= + { + 'name' => 'REST_DDL_TABLE_BASIC', + 'tests' => + [ + { + #setup + #create db if not exists + 'num' => 1, + 'method' => 'POST', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl', + 'status_code' => 200, + 'post_options' => ['user.name=:UNAME:','exec=create database if not exists templeton_testdb1;'], + 'json_field_substr_match' => {'stderr' => 'OK'}, + }, + + { + #cleanup + #drop table if exists + 'num' => 2, + 'method' => 'DELETE', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/templeton_testdb1/table/templeton_testtab1?user.name=:UNAME:&ifExists=true', + 'status_code' => 200, + 'json_field_substr_match' => {'database' => 'templeton_testdb1', 'table' => 'templeton_testtab1'}, + }, + { + #setup + #create table if not exists + 'ignore' => 'rest api now available', + 'num' => 3, + 'method' => 'POST', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl', + 'status_code' => 200, + 'post_options' => ['user.name=:UNAME:','exec=use templeton_testdb1; create table if not exists templeton_testtab1 (i int, j bigint) STORED AS rcfile;'], + 'json_field_substr_match' => {'stderr' => 'OK'}, + }, + + { + #create table + 'num' => 4, + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/templeton_testdb1/table/templeton_testtab1?user.name=:UNAME:', + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{ + "external" : "true", "ifNotExists": "true", + "columns": [ + { "name" : "i", "type" : "int" }, + { "name" : "j", "type" : "bigint", "comment" : "some comment" } + ], + "location" : "/tmp/tab1", + "format" : { "storedAs" : "rcfile"} + }'], + 'status_code' => 200, + 'json_field_substr_match' => {'database' => 'templeton_testdb1', 'table' => 'templeton_testtab1'}, + + }, + { + #negative test - create same table again + 'ignore' => 'currently returns success, needs to be fixed', + 'num' => 5, + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/templeton_testdb1/table/templeton_testtab1?user.name=:UNAME:', + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{ + "external" : "true", "ifNotExists": "true", + "columns": [ + { "name" : "i", "type" : "int" }, + { "name" : "j", "type" : "bigint", "comment" : "some comment" } + ], + "location" : "/tmp/tab1", + "format" : { "storedAs" : "rcfile"} + }'], + + 'status_code' => 409, + }, + { #show tables + 'num' => 6, + 'method' => 'GET', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/templeton_testdb1/table/?user.name=:UNAME:', + 'status_code' => 200, + 'json_field_substr_match' => {'tables' => 'templeton_testtab1'}, + }, + { #show tables like '%table1%' + 'num' => 7, + 'method' => 'GET', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/templeton_testdb1/table/?user.name=:UNAME:&like=*testtab1*', + 'status_code' => 200, + 'json_field_substr_match' => {'tables' => 'templeton_testtab1'}, + }, + { #desc table + 'num' => 8, + 'method' => 'GET', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/templeton_testdb1/table/templeton_testtab1?user.name=:UNAME:', + 'status_code' => 200, + # 'json_field_substr_match' => {'table-name' => 'templeton_testtab1'}, + 'json_field_match_object' => { 'columns' => '[ + { "name" : "i", "type" : "int"}, + { "name" : "j", "type" : "bigint", "comment" : "some comment" } + ]' }, + }, + { #drop table + 'num' => 9, + 'method' => 'DELETE', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/templeton_testdb1/table/templeton_testtab1?user.name=:UNAME:', + 'status_code' => 200, + }, + { + #-ve test , describe on non existent table + 'num' => 10, + 'method' => 'GET', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/templeton_testdb1/table/templeton_testtab1?user.name=:UNAME:', + 'status_code' => 404, + 'json_field_substr_match' => {'error' => 'templeton_testtab1 does not exist'}, + }, + { + #-ve test , describe on non existent table + 'num' => 11, + 'method' => 'GET', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/templeton_testdb1/table/templeton_testtab1?user.name=:UNAME:&format=extended:', + 'status_code' => 404, + 'json_field_substr_match' => {'error' => 'templeton_testtab1 does not exist'}, + }, + + ] + }, + +##============================================================================================================= + { + 'name' => 'REST_DDL_TABLE_EXTENDED', + 'tests' => + [ + + { + #cleanup + #drop table if exists + 'num' => 1, + 'method' => 'DELETE', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/templetontest_tabe?user.name=:UNAME:&ifExists=true', + 'status_code' => 200, + 'json_field_substr_match' => {'database' => 'default', 'table' => 'templetontest_tabe'}, + + }, + + + + { + #setup + #create table if not exists + 'num' => 2, + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/templetontest_tabe?user.name=:UNAME:', + 'status_code' => 200, + 'format_header' => 'Content-Type: application/json', + 'post_options' => [ + + '{"ifNotExists": "false", + "columns": [ + { "name" : "i", "type" : "int" }, + { "name" : "j", "type" : "bigint" }, + { "name" : "ip", "type" : "string", "comment" : "IP Address of the User" } + ], + "format" : { + "storedAs" : "rcfile", + "rowFormat" : { + "fieldsTerminatedBy" : "\u0001", + "collectionItemsTerminatedBy" : "\u0002", + "mapKeysTerminatedBy" : "\u0003", + "linesTerminatedBy" : "\n", + + "serde" : { + "name" : "org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe", + "properties" : { + "key" : "value" + } + } + } + }, + "comment" : "This is the page view table", + "partitionedBy" : [ { "name" : "dt", "type" : "string"}, { "name" : "country", "type" : "string"} ], + "tableProperties" : {"prop1" : "val1" , "prop2" : "val2" }, + "clusteredBy" : {"columnNames": ["i", "j"], "sortedBy": [ {"columnName": "i", "order": "ASC"}, {"columnName": "j", "order": "ASC"}], "numberOfBuckets": 10 } + }', + ], + 'json_field_substr_match' => {'database' => 'default', 'table' => 'templetontest_tabe'}, + + + }, + + { #desc table + 'num' => 3, + 'method' => 'GET', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/templetontest_tabe?user.name=:UNAME:&format=extended', + 'status_code' => 200, + #TODO: check for clustered section + 'json_field_substr_match' => + { + 'location' => "templetontest_tabe", + 'inputFormat' => "org.apache.hadoop.hive.ql.io.RCFileInputFormat", + 'lastAccessTime' => 0, + 'lastUpdateTime' => '\d{13}', + 'maxFileSize' => 0, + 'minFileSize' => 0, + 'outputFormat' => "org.apache.hadoop.hive.ql.io.RCFileOutputFormat", + 'partitioned' => "true", + 'table' => "templetontest_tabe", + + + }, + 'json_field_match_object' => + {'columns' => '[ + { + "name" : "i", + "type" : "int" + }, + { + "name" : "j", + "type" : "bigint" + }, + { + "comment" : "IP Address of the User", + "name" : "ip", + "type" : "string" + } + ]', + "partitionColumns" => '[ + { + "name" : "dt", + "type" : "string" + }, + { + "name" : "country", + "type" : "string" + } + ]', + }, + }, + + ] + + }, +##============================================================================================================= + { + 'name' => 'REST_DDL_TABLE_RENAME', + 'tests' => + [ + { + #cleanup table 1 + #drop table if exists + 'num' => 1, + 'method' => 'POST', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl', + 'status_code' => 200, + 'post_options' => ['user.name=:UNAME:','exec=drop table if exists templeton_testtab_rename1'], + 'json_field_substr_match' => {'stderr' => 'OK'}, + }, + { + #cleanup table 2 + #drop table if exists + 'num' => 2, + 'method' => 'POST', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl', + 'status_code' => 200, + 'post_options' => ['user.name=:UNAME:','exec=drop table if exists templeton_testtab_rename2'], + 'json_field_substr_match' => {'stderr' => 'OK'}, + }, + { + #setup + #create table if not exists + 'num' => 3, + 'method' => 'POST', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl', + 'status_code' => 200, + 'post_options' => ['user.name=:UNAME:','exec=create table if not exists templeton_testtab_rename1 (i int, j bigint) STORED AS rcfile;'], + 'json_field_substr_match' => {'stderr' => 'OK'}, + }, + { + #rename table + 'num' => 4, + 'method' => 'POST', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/templeton_testtab_rename1', + 'status_code' => 200, + 'post_options' => ['user.name=:UNAME:','rename=templeton_testtab_rename2'], + 'json_field_substr_match' => {'table' => 'templeton_testtab_rename2'}, + }, + + { + #desc table + 'num' => 5, + 'method' => 'GET', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/templeton_testtab_rename2?user.name=:UNAME:', + 'status_code' => 200, + # 'json_field_substr_match' => {'table-name' => 'templeton_testtab1'}, + 'json_field_match_object' => { 'columns' => '[ + { "name" : "i", "type" : "int" }, + { "name" : "j", "type" : "bigint" } + ]' }, + }, + + + ] + + }, + + +##============================================================================================================= + { + 'name' => 'REST_DDL_PARTITIONS', + 'tests' => + [ + + { + #cleanup + #drop table if exists + 'num' => 1, + 'method' => 'POST', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl', + 'status_code' => 200, + 'post_options' => ['user.name=:UNAME:','exec=drop table if exists templetontest_parts'], + 'json_field_substr_match' => {'stderr' => 'OK'}, + }, + + + { + #setup + #create table if not exists + 'num' => 2, + 'method' => 'POST', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl', + 'status_code' => 200, + 'post_options' => ['user.name=:UNAME:', + 'exec=create table if not exists templetontest_parts (i int, j bigint, ip STRING COMMENT "IP Address of the User") +COMMENT "This is the page view table" + PARTITIONED BY(dt STRING, country STRING) +ROW FORMAT DELIMITED + FIELDS TERMINATED BY "\001" + COLLECTION ITEMS TERMINATED BY "\002" + MAP KEYS TERMINATED BY "\003" +STORED AS rcfile +--LOCATION "table1_location" '], + 'json_field_substr_match' => {'stderr' => 'OK'}, + }, + { + #create partition 1 + 'num' => 3, + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/templetontest_parts/partition/dt=%2720120101%27,country=%27US%27?user.name=:UNAME:', + 'status_code' => 200, + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{"location" : "loc1"}' ], + 'json_field_substr_match' => {'database' => 'default', + 'table' => 'templetontest_parts', + 'partition' => "dt='20120101',country='US'" + }, + }, + { + #create partition 2 + 'num' => 4, + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/templetontest_parts/partition/dt=%2720120101%27,country=%27IN%27?user.name=:UNAME:', + 'status_code' => 200, + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{"location" : "loc2"}' ], + 'json_field_substr_match' => {'database' => 'default', + 'table' => 'templetontest_parts', + 'partition' => "dt='20120101',country='IN'" + }, + }, + { + #-ve test. create partition 2 -with same name + 'num' => 5, + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/templetontest_parts/partition/dt=%2720120101%27,country=%27IN%27?user.name=:UNAME:', + 'status_code' => 409, + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{"location" : "loc2"}' ], + 'json_field_substr_match' => {'error' => 'Partition already exists', + }, + }, + + + { + #lookup partitions + 'num' => 6, + 'method' => 'GET', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/templetontest_parts/partition/?user.name=:UNAME:', + 'status_code' => 200, + 'json_field_match_object' => + { + "partitions" => + '[ + { + "name" : "dt=\'20120101\',country=\'IN\'", + "values" : [ + { + "columnName" : "dt", + "columnValue" : "20120101" + }, + { + "columnName" : "country", + "columnValue" : "IN" + } + ] + }, + { + "name" : "dt=\'20120101\',country=\'US\'", + "values" : [ + { + "columnName" : "dt", + "columnValue" : "20120101" + }, + { + "columnName" : "country", + "columnValue" : "US" + } + ] + } + ]' + + }, + }, + + { + #describe a partition + 'num' => 7, + 'method' => 'GET', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/templetontest_parts/partition/dt=\'20120101\',country=\'IN\'?user.name=:UNAME:', + 'status_code' => 200, + 'json_field_substr_match' => + { + "database" => "default", + "table" => "templetontest_parts", + "partition" => "dt=\'20120101\',country=\'IN\'", + + }, + 'json_field_match_object' => + { + "columns" => '[ + { + "name" : "i", + "type" : "int" + }, + { + "name" : "j", + "type" : "bigint" + }, + { + "comment" : "IP Address of the User", + "name" : "ip", + "type" : "string" + } + ]', + + "partitionColumns" => '[ + { + "name" : "dt", + "type" : "string" + }, + { + "name" : "country", + "type" : "string" + } + ]', + }, + }, + + + { + #describe a partition - extended + 'num' => 8, + 'method' => 'GET', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/templetontest_parts/partition/dt=\'20120101\',country=\'IN\'?user.name=:UNAME:&format=extended', + 'status_code' => 200, + 'json_field_substr_match' => + { + + "outputFormat" => "org.apache.hadoop.hive.ql.io.RCFileOutputFormat", + "inputFormat" => "org.apache.hadoop.hive.ql.io.RCFileInputFormat", + "lastAccessTime" => 0, + "maxFileSize" => 0, + "location" => "loc2", + "totalNumberFiles" => 0, + "lastUpdateTime" => '\d{13}', + "minFileSize" => 0, + "partitioned" => 'true', + "totalFileSize" => 0, + "table" => "templetontest_parts" + }, + 'json_field_match_object' => + { + "columns" => '[ + { + "name" : "i", + "type" : "int" + }, + { + "name" : "j", + "type" : "bigint" + }, + { + "comment" : "IP Address of the User", + "name" : "ip", + "type" : "string" + } + ]', + + "partitionColumns" => '[ + { + "name" : "dt", + "type" : "string" + }, + { + "name" : "country", + "type" : "string" + } + ]', + }, + }, + + { + #delete partition 2 + 'num' => 9, + 'method' => 'DELETE', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/templetontest_parts/partition/dt=%2720120101%27,country=%27IN%27?user.name=:UNAME:', + 'status_code' => 200, + 'json_field_substr_match' => {'database' => 'default', + 'table' => 'templetontest_parts', + }, + }, + + { + #delete partition 2 again + 'num' => 10, + 'method' => 'DELETE', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/templetontest_parts/partition/dt=%2720120101%27,country=%27IN%27?user.name=:UNAME:', + 'status_code' => 200, #should return 404 when bug is fixed + 'json_field_substr_match' => + { 'database' => 'default', + 'table' => 'templetontest_parts', + }, + }, + { + #lookup deleted partition 2 + 'num' => 11, + 'method' => 'GET', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/templetontest_parts/partition/dt=%2720120101%27,country=%27IN%27?user.name=:UNAME:', + 'status_code' => 500, #should return 404 when bug is fixed + 'json_field_substr_match' => + { + 'error' => 'Partition.*for table templetontest_parts does not exist' + }, + }, + + ] + + }, + +##============================================================================================================= + + { + 'name' => 'REST_DDL_COLUMN', + 'tests' => + [ + { + #cleanup table + #drop table if exists + 'num' => 1, + 'method' => 'DELETE', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/templeton_testcol_tab?user.name=:UNAME:', + 'status_code' => 200, + }, + { + #setup + #create table if not exists + 'num' => 2, + 'method' => 'POST', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl?user.name=:UNAME:', + 'status_code' => 200, + 'post_options' => ['user.name=:UNAME:','exec=create table if not exists templeton_testcol_tab (i int comment "column with comment", j bigint) STORED AS rcfile;'], + 'json_field_substr_match' => {'stderr' => 'OK'}, + }, + { + #get column + 'num' => 3, + 'method' => 'GET', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/templeton_testcol_tab/column/?user.name=:UNAME:', + 'status_code' => 200, + 'json_field_substr_match' => + { + 'database' => 'default', + 'table' => 'templeton_testcol_tab', + + }, + 'json_field_match_object' => + { + 'columns' => '[ + { "name" : "i", "type" : "int", "comment" : "column with comment"}, + { "name" : "j", "type" : "bigint" } + ]' + }, + }, + + { + #get specific column + 'num' => 4, + 'method' => 'GET', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/templeton_testcol_tab/column/i?user.name=:UNAME:', + 'status_code' => 200, + 'json_field_substr_match' => + { + 'database' => 'default', + 'table' => 'templeton_testcol_tab', + + }, + 'json_field_match_object' => + { + 'column' => '{ "name" : "i", "type" : "int", "comment" : "column with comment" }' + }, + }, + + { + #create new column + 'num' => 5, + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/templeton_testcol_tab/column/k?user.name=:UNAME:', + 'status_code' => 200, + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{"type": "string", "comment": "The k column"}' ], + + 'json_field_substr_match' => + { + 'database' => 'default', + 'table' => 'templeton_testcol_tab', + 'column' => 'k', + }, + }, + { + #create duplicate column + 'num' => 6, + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/templeton_testcol_tab/column/k?user.name=:UNAME:', + 'status_code' => 409, + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{"type": "string", "comment": "The k column"}' ], + + }, + + + + + ] + + }, +##============================================================================================================= + + { + 'name' => 'REST_DDL_TABLE_PROPS', + 'tests' => + [ + + { + #cleanup + #drop table if exists + 'num' => 1, + 'method' => 'DELETE', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/templetontest_tprop?user.name=:UNAME:&ifExists=true', + 'status_code' => 200, + 'json_field_substr_match' => {'database' => 'default', 'table' => 'templetontest_tprop'}, + + }, + + + + { + #setup + #create table if not exists + 'num' => 2, + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/templetontest_tprop?user.name=:UNAME:', + 'status_code' => 200, + 'format_header' => 'Content-Type: application/json', + 'post_options' => [ + + '{"ifNotExists": "true", + "columns": [ + { "name" : "i", "type" : "int" }, + { "name" : "j", "type" : "bigint" }, + { "name" : "ip", "type" : "string", "comment" : "IP Address of the User" } + ], + "format" : { "storedAs" : "rcfile"}, + "comment" : "This is the page view table", + "partitionedBy" : [ { "name" : "dt", "type" : "string"}, { "name" : "country", "type" : "string"} ], + "tableProperties" : {"prop1" : "val1" , "prop2" : "val2" } + }', + ], + 'json_field_substr_match' => {'database' => 'default', 'table' => 'templetontest_tprop'}, + + + }, + { + + 'num' => 3, + 'method' => 'GET', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/templetontest_tprop/property?user.name=:UNAME:', + 'status_code' => 200, + #using the substring match instead of object match for properties because object contains a property with timestamp value + 'json_field_substr_match' => + {'database' => 'default', + 'table' => 'templetontest_tprop', + 'properties' => '(.*prop1.*val1|.*prop2.*val2)|(.*prop1.*val1|.*prop2.*val2)' + }, + }, + { + #add new property + 'num' => 4, + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/templetontest_tprop/property/prop3?user.name=:UNAME:', + 'status_code' => 200, + 'json_field_substr_match' => + {'database' => 'default', 'table' => 'templetontest_tprop', "property" => "prop3"}, + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{"value" : "val3"}' ], + + }, + { + #get new property + 'num' => 5, + 'method' => 'GET', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/templetontest_tprop/property/prop3?user.name=:UNAME:', + 'status_code' => 200, + 'json_field_substr_match' => + {'database' => 'default', 'table' => 'templetontest_tprop'}, + 'json_field_match_object' => + { 'property' => '{"prop3" : "val3"}'}, + 'format_header' => 'Content-Type: application/json', + + }, + { + #add new property + 'num' => 6, + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/templetontest_tprop/property/prop3?user.name=:UNAME:', + 'status_code' => 200, + 'json_field_substr_match' => + {'database' => 'default', 'table' => 'templetontest_tprop', "property" => "prop3"}, + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{"value" : "newval3"}' ], + + }, + { + #get new property + 'num' => 7, + 'method' => 'GET', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/templetontest_tprop/property/prop3?user.name=:UNAME:', + 'status_code' => 200, + 'json_field_substr_match' => + {'database' => 'default', 'table' => 'templetontest_tprop'}, + 'json_field_match_object' => + { 'property' => '{"prop3" : "newval3"}'}, + 'format_header' => 'Content-Type: application/json', + + }, + + + { + #cleanup + #drop table if exists + 'num' => 10, + 'method' => 'DELETE', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/templetontest_tprop?user.name=:UNAME:&ifExists=true', + 'status_code' => 200, + 'json_field_substr_match' => {'database' => 'default', 'table' => 'templetontest_tprop'}, + + }, + ] +}, + +##============================================================================================================= + { + 'name' => 'REST_DDL_TABLE_GROUP_PERMS', #test the group and permission flags + 'ignore' => 'these features are tested as part of hcat auth tests. current test cases dont work with auth enabled', + 'tests' => + [ + + { + #cleanup + #drop table if exists + 'num' => 1, + 'method' => 'DELETE', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/templetontest_tgrouperm?user.name=:UNAME:&ifExists=true', + 'status_code' => 200, + 'json_field_substr_match' => {'database' => 'default', 'table' => 'templetontest_tgrouperm'}, + + }, + + + + { + #create table -ve test - no such group + 'num' => 2, + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/templetontest_tgrouperm?user.name=:UNAME:', + 'status_code' => 500, + 'format_header' => 'Content-Type: application/json', + 'post_options' => [ + + '{"ifNotExists": "true", + "group" : "nosuchgroupAAAABBB", + "columns": [ + { "name" : "i", "type" : "int" }, + { "name" : "j", "type" : "bigint" }, + { "name" : "ip", "type" : "string", "comment" : "IP Address of the User" } + ], + "format" : { "storedAs" : "rcfile"}, + "comment" : "This is the page view table", + "partitionedBy" : [ { "name" : "dt", "type" : "string"}, { "name" : "country", "type" : "string"} ], + "tableProperties" : {"prop1" : "val1" , "prop2" : "val2" } + }', + ], +# 'json_field_substr_match' => {'database' => 'default', 'table' => 'templetontest_tgrouperm'}, +#TODO - add checks after error message is standardised + + }, + { + #create table with no permissions + 'num' => 3, + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/templetontest_tgrouperm?user.name=:UNAME:', + 'status_code' => 200, + 'format_header' => 'Content-Type: application/json', + 'post_options' => [ + + '{"ifNotExists": "true", + "permissions" : "---------", + "columns": [ + { "name" : "i", "type" : "int" }, + { "name" : "j", "type" : "bigint" }, + { "name" : "ip", "type" : "string", "comment" : "IP Address of the User" } + ], + "format" : { "storedAs" : "rcfile"}, + "comment" : "This is the page view table", + "partitionedBy" : [ { "name" : "dt", "type" : "string"}, { "name" : "country", "type" : "string"} ], + "tableProperties" : {"prop1" : "val1" , "prop2" : "val2" } + }', + ], + 'json_field_substr_match' => {'database' => 'default', 'table' => 'templetontest_tgrouperm'}, + + + }, + + { + #create partition in table that does not have permissions + 'num' => 4, + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/templetontest_tgrouperm/partition/dt=%2720120101%27,country=%27US%27?user.name=:UNAME:', + 'status_code' => 500, + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{"location" : "loc1"}' ], + 'json_field_substr_match' => {'database' => 'default', + 'table' => 'templetontest_tgrouperm', + 'error' => 'Permission denied', + 'partition' => "dt='20120101',country='US'" + }, + }, + + + ] + +}, + +##============================================================================================================= + { + 'name' => 'HCAT_GROUP_PERMS', + 'ignore' => 'these features are tested as part of hcat auth tests. current test cases dont work with auth enabled', + 'tests' => + [ + { + #drop table if exists + 'num' => 1, + 'method' => 'POST', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl', + 'status_code' => 200, + 'post_options' => ['user.name=:UNAME:','exec=drop table if exists templetontest_hcatgp'], + 'json_field_substr_match' => {'stderr' => 'OK'}, + + }, + { + #create table . -ve test + 'num' => 2, + 'method' => 'POST', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl', + 'status_code' => 200, + 'post_options' => ['user.name=:UNAME:', + 'group=no_such_group_AA', + 'exec=create table templetontest_hcatgp(i int, j bigint) STORED AS rcfile;'], + 'json_field_substr_match' => {'stderr' => 'User does not belong to no_such_group_AA', 'exitcode' => '^1$'} + }, + { + #create table with no permissions + 'num' => 3, + 'method' => 'POST', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl', + 'status_code' => 200, + 'post_options' => ['user.name=:UNAME:', + 'permissions=---------', + 'exec=create table templetontest_hcatgp(i int, j bigint) + PARTITIONED BY(dt STRING, country STRING) + STORED AS rcfile;' + ], + 'json_field_substr_match' => {'stderr' => 'OK', 'exitcode' => '^0$'} + }, + + { + #create partition in table that does not have permissions + 'num' => 4, + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/templetontest_hcatgp/partition/dt=%2720120101%27,country=%27US%27?user.name=:UNAME:', + 'status_code' => 500, + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{"location" : "loc1"}' ], + 'json_field_substr_match' => {'database' => 'default', + 'table' => 'templetontest_hcatgp', + 'error' => 'Permission denied', + 'partition' => "dt='20120101',country='US'" + }, + }, + + ] + +} + + + + + + ] +}, + ; + Index: src/test/e2e/templeton/tests/hcatperms.conf =================================================================== --- src/test/e2e/templeton/tests/hcatperms.conf (revision 0) +++ src/test/e2e/templeton/tests/hcatperms.conf (revision 0) @@ -0,0 +1,1478 @@ +############################################################################### +# curl command tests for templeton +# +# + +#use Yahoo::Miners::Test::PigSetup; + +#PigSetup::setup(); + +#my $me = `whoami`; +#chomp $me; + +$cfg = +{ + 'driver' => 'Curl', + + 'groups' => + [ + +##============================================================================================================= + { + 'name' => 'DB_OPS', + 'tests' => + [ + { + #check if db created by UNAME_OTHER is visible for user UNAME + 'num' => 1, + + 'setup' => [ + { + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_a', + 'user_name' => ':UNAME_OTHER:', + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{"comment":"Hello there", "properties":{"a":"b"}}'], + } + ], + 'method' => 'GET', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/', + 'user_name' => ':UNAME:', + + 'format_header' => 'Content-Type: application/json', + 'json_field_substr_match' => {'databases' => 'hcatperms_a'}, + + 'status_code' => 200, + }, + + + { + + #check if group id is set as per spec + 'num' => 2, + 'setup' => [ + { + 'method' => 'DELETE', + 'format_header' => 'Content-Type: application/json', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_b', + 'user_name' => ':UNAME:', + }, + { + 'method' => 'PUT', + 'post_options' => [ + '{"permissions" : "rwxrwxrwx", + "group" : ":UGROUP:" + }' + ], + 'format_header' => 'Content-Type: application/json', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_b', + 'user_name' => ':UNAME:', + }, + + ], + + + 'method' => 'GET', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_b', + 'user_name' => ':UNAME:', + 'format_header' => 'Content-Type: application/json', + 'json_field_substr_match' => {'location_perms' => 'rwxrwxrwx', 'location_group' => ':UGROUP:'}, + 'status_code' => 200, + }, + { + + #check if group id is set as per spec + 'num' => 3, + 'setup' => [ + { + 'method' => 'DELETE', + 'format_header' => 'Content-Type: application/json', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_c', + 'user_name' => ':UNAME:', + }, + { + 'method' => 'PUT', + 'post_options' => [ + '{"permissions" : "rwxr-x---", + "group" : ":UGROUP:" + }' + ], + 'format_header' => 'Content-Type: application/json', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_c', + 'user_name' => ':UNAME:', + }, + + ], + + + 'method' => 'GET', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_c', + 'user_name' => ':UNAME:', + 'format_header' => 'Content-Type: application/json', + 'json_field_substr_match' => {'location_perms' => 'rwxr-x---', 'location_group' => ':UGROUP:'}, + 'status_code' => 200, + }, + + { + + #check if user belonging to same group is able to access + 'num' => 4, + + 'setup' => [ + { + 'method' => 'DELETE', + 'format_header' => 'Content-Type: application/json', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_d', + 'user_name' => ':UNAME:', + }, + + { + 'method' => 'PUT', + 'post_options' => [ + '{ + "permissions" : "rwxr-x---", + "group" : ":UGROUP:" + }' + ], + + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_d', + 'user_name' => ':UNAME:', + 'format_header' => 'Content-Type: application/json', + } + ], + + + + 'method' => 'GET', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_d', + 'user_name' => ':UNAME_GROUP:', + 'format_header' => 'Content-Type: application/json', + 'json_field_substr_match' => {'location_perms' => 'rwxr-x---', 'location_group' => ':UGROUP:'}, + 'status_code' => 200, + }, + + { + + #check default db permissions + #the default group is currently the group of parent dir + + 'num' => 5, + + 'setup' => [ + { + 'method' => 'DELETE', + 'format_header' => 'Content-Type: application/json', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_e', + 'user_name' => ':UNAME:', + }, + + { + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_e', + 'user_name' => ':UNAME:', + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{}'] + + } + ], + + + + 'method' => 'GET', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_e', + 'user_name' => ':UNAME:', + 'format_header' => 'Content-Type: application/json', + 'json_field_substr_match' => {'location_perms' => 'rwx------'}, + 'status_code' => 200, + }, + + { + + #check if user can delete db + 'num' => 6, + + 'setup' => [ + { + 'method' => 'DELETE', + 'format_header' => 'Content-Type: application/json', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_f', + 'user_name' => ':UNAME:', + }, + + { + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_f', + 'user_name' => ':UNAME:', + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{}'] + + } + ], + + + + 'method' => 'DELETE', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_f', + 'user_name' => ':UNAME:', + 'format_header' => 'Content-Type: application/json', + 'status_code' => 200, + }, + + { + + #check if user belonging to group can delete table + 'num' => 7, + + 'setup' => [ + { + 'method' => 'DELETE', + 'format_header' => 'Content-Type: application/json', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_:TNUM:', + 'user_name' => ':UNAME:', + }, + + { + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_:TNUM:', + 'user_name' => ':UNAME:', + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{ "permissions" : "rwxrwx---", "group" : ":UGROUP:" }'] + + } + ], + + + + 'method' => 'DELETE', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_:TNUM:', + 'user_name' => ':UNAME_GROUP:', + 'format_header' => 'Content-Type: application/json', + 'status_code' => 200, + }, + { + + #check if user belonging to group is unable to delete table + 'num' => 8, + 'setup' => [ + { + 'method' => 'DELETE', + 'format_header' => 'Content-Type: application/json', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_:TNUM:', + 'user_name' => ':UNAME:', + }, + + { + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_:TNUM:', + 'user_name' => ':UNAME:', + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{ "permissions" : "rwx------", "group" : ":UGROUP:" }'] + + } + ], + + + 'method' => 'DELETE', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_:TNUM:', + 'user_name' => ':UNAME_GROUP:', + 'format_header' => 'Content-Type: application/json', + 'status_code' => 500, + }, + + { + + #check if user belonging to 'other' is unable to delete table + 'num' => 10, + 'setup' => [ + { + 'method' => 'DELETE', + 'format_header' => 'Content-Type: application/json', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_:TNUM:', + 'user_name' => ':UNAME:', + }, + + { + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_:TNUM:', + 'user_name' => ':UNAME:', + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{ "permissions" : "rwx------", "group" : ":UGROUP:" }'] + + } + ], + + + 'method' => 'DELETE', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_:TNUM:', + 'user_name' => ':UNAME_OTHER:', + 'format_header' => 'Content-Type: application/json', + 'status_code' => 500, + }, + + + { + + #check if user is able to create table + 'num' => 11, + 'setup' => [ + { + 'method' => 'DELETE', + 'format_header' => 'Content-Type: application/json', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_:TNUM:?ifExists=true&option=cascade', + 'user_name' => ':UNAME:', + 'status_code' => 200, + }, + + { + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_:TNUM:', + 'user_name' => ':UNAME:', + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{ "permissions" : "rwx------" }'] + + }, + + ], + + + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_:TNUM:/table/permstable_:TNUM:', + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{ + "columns": [ + { "name" : "i", "type" : "int" }, + { "name" : "j", "type" : "bigint", "comment" : "some comment" } + ], + "format" : { "storedAs" : "rcfile"} + }'], + 'user_name' => ':UNAME:', + 'status_code' => 200, + + }, + + { + + #check if group user is able to create table + 'num' => 12, + 'setup' => [ + { + 'method' => 'DELETE', + 'format_header' => 'Content-Type: application/json', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_:TNUM:?ifExists=true&option=cascade', + 'user_name' => ':UNAME:', + 'status_code' => 200, + }, + + { + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_:TNUM:', + 'user_name' => ':UNAME:', + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{ "permissions" : "rwxrwx---" , "group" : ":UGROUP:"} '] + + }, + + ], + + + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_:TNUM:/table/permstable_:TNUM:', + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{ + "columns": [ + { "name" : "i", "type" : "int" }, + { "name" : "j", "type" : "bigint", "comment" : "some comment" } + ], + "format" : { "storedAs" : "rcfile"} + }'], + 'user_name' => ':UNAME_GROUP:', + 'status_code' => 200, + + }, + + { + + #check if group user is unable to create table without permissions + 'num' => 13, + 'setup' => [ + { + 'method' => 'DELETE', + 'format_header' => 'Content-Type: application/json', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_:TNUM:?ifExists=true&option=cascade', + 'user_name' => ':UNAME:', + 'status_code' => 200, + }, + + { + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_:TNUM:', + 'user_name' => ':UNAME:', + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{ "permissions" : "rwx------", "group" : ":UGROUP:" }'] + + }, + + ], + + + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_:TNUM:/table/permstable_:TNUM:', + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{ + "columns": [ + { "name" : "i", "type" : "int" }, + { "name" : "j", "type" : "bigint", "comment" : "some comment" } + ], + "format" : { "storedAs" : "rcfile"} + }'], + 'user_name' => ':UNAME_GROUP:', + 'status_code' => 500, + + }, + + { + + #check if other user is unable to create table without permissions + 'num' => 14, + 'setup' => [ + { + 'method' => 'DELETE', + 'format_header' => 'Content-Type: application/json', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_:TNUM:?ifExists=true&option=cascade', + 'user_name' => ':UNAME:', + 'status_code' => 200, + }, + + { + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_:TNUM:', + 'user_name' => ':UNAME:', + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{ "permissions" : "rwxrwx---", "group" : ":UGROUP:" }'], + 'status_code' => 200, + }, + + ], + + + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_:TNUM:/table/permstable_:TNUM:', + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{ + "columns": [ + { "name" : "i", "type" : "int" }, + { "name" : "j", "type" : "bigint", "comment" : "some comment" } + ], + "format" : { "storedAs" : "rcfile"} + }'], + 'user_name' => ':UNAME_OTHER:', + 'status_code' => 500, + + }, + + { + + #show table allowed for user + 'num' => 15, + 'setup' => [ + { + 'method' => 'DELETE', + 'format_header' => 'Content-Type: application/json', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_:TNUM:?ifExists=true&option=cascade', + 'user_name' => ':UNAME:', + 'status_code' => 200, + }, + + { + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_:TNUM:', + 'user_name' => ':UNAME:', + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{ "permissions" : "rwxrwx---", "group" : ":UGROUP:" }'], + 'status_code' => 200, + }, + { + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_:TNUM:/table/permstable_:TNUM:', + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{ + "columns": [ + { "name" : "i", "type" : "int" } + ], + "format" : { "storedAs" : "rcfile"} + }'], + 'user_name' => ':UNAME:', + 'status_code' => 200, + } + + ], + + + 'method' => 'GET', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_:TNUM:/table/', + 'format_header' => 'Content-Type: application/json', + 'user_name' => ':UNAME:', + 'status_code' => 200, + 'json_field_substr_match' => {'tables' => 'permstable_:TNUM:'}, + + + }, + + + { + + #show table allowed for group + 'num' => 16, + 'setup' => [ + { + 'method' => 'DELETE', + 'format_header' => 'Content-Type: application/json', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_:TNUM:?ifExists=true&option=cascade', + 'user_name' => ':UNAME:', + 'status_code' => 200, + }, + + { + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_:TNUM:', + 'user_name' => ':UNAME:', + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{ "permissions" : "rwxrwx---", "group" : ":UGROUP:" }'], + 'status_code' => 200, + }, + { + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_:TNUM:/table/permstable_:TNUM:', + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{ + "columns": [ + { "name" : "i", "type" : "int" } + ], + "format" : { "storedAs" : "rcfile"} + }'], + 'user_name' => ':UNAME:', + 'status_code' => 200, + } + + ], + + + 'method' => 'GET', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_:TNUM:/table/', + 'format_header' => 'Content-Type: application/json', + 'user_name' => ':UNAME_GROUP:', + 'status_code' => 200, + 'json_field_substr_match' => {'tables' => 'permstable_:TNUM:'}, + + + }, + + { + + #show table not allowed for group user without permissions + 'num' => 17, + 'setup' => [ + { #drop db cascade + 'method' => 'DELETE', + 'format_header' => 'Content-Type: application/json', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_:TNUM:?ifExists=true&option=cascade', + 'user_name' => ':UNAME:', + 'status_code' => 200, + }, + + { + #create db + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_:TNUM:', + 'user_name' => ':UNAME:', + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{ "permissions" : "rwx------", "group" : ":UGROUP:" }'], + 'status_code' => 200, + }, + { + #create table + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_:TNUM:/table/permstable_:TNUM:', + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{ + "columns": [ + { "name" : "i", "type" : "int" } + ], + "format" : { "storedAs" : "rcfile"} + }'], + 'user_name' => ':UNAME:', + 'status_code' => 200, + } + + ], + + + 'method' => 'GET', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_:TNUM:/table/', + 'format_header' => 'Content-Type: application/json', + 'user_name' => ':UNAME_GROUP:', + 'status_code' => 500, + + + }, + + { + #show table not allowed for other user without permissions + 'num' => 18, + 'setup' => [ + { #drop db cascade + 'method' => 'DELETE', + 'format_header' => 'Content-Type: application/json', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_:TNUM:?ifExists=true&option=cascade', + 'user_name' => ':UNAME:', + 'status_code' => 200, + }, + + { + #create db + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_:TNUM:', + 'user_name' => ':UNAME:', + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{ "permissions" : "rwx------", "group" : ":UGROUP:" }'], + 'status_code' => 200, + }, + { + #create table + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_:TNUM:/table/permstable_:TNUM:', + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{ + "columns": [ + { "name" : "i", "type" : "int" } + ], + "format" : { "storedAs" : "rcfile"} + }'], + 'user_name' => ':UNAME:', + 'status_code' => 200, + } + + ], + + + 'method' => 'GET', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_:TNUM:/table/', + 'format_header' => 'Content-Type: application/json', + 'user_name' => ':UNAME_OTHER:', + 'status_code' => 500, + 'json_field_substr_match' => {'error' => 'unable to show tables'}, + + + }, + + + { + + #describe db - by user + 'num' => 19, + 'setup' => [ + { #drop db cascade + 'method' => 'DELETE', + 'format_header' => 'Content-Type: application/json', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_:TNUM:?ifExists=true&option=cascade', + 'user_name' => ':UNAME:', + 'status_code' => 200, + }, + + { + #create db + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_:TNUM:', + 'user_name' => ':UNAME:', + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{ "permissions" : "rwxrwx---" }'], + 'status_code' => 200, + }, + + ], + + + 'method' => 'GET', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_:TNUM:', + 'format_header' => 'Content-Type: application/json', + 'user_name' => ':UNAME:', + 'status_code' => 200, + 'json_field_substr_match' => {'database' => 'hcatperms_:TNUM:', + 'location' => 'hcatperms_:TNUM:'}, + + + }, + + { + + #describe db - by group user + 'num' => 20, + 'setup' => [ + { #drop db cascade + 'method' => 'DELETE', + 'format_header' => 'Content-Type: application/json', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_:TNUM:?ifExists=true&option=cascade', + 'user_name' => ':UNAME:', + 'status_code' => 200, + }, + + { + #create db + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_:TNUM:', + 'user_name' => ':UNAME:', + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{ "permissions" : "rwxrwx---", "group" : ":UGROUP:" }'], + 'status_code' => 200, + }, + + ], + + + 'method' => 'GET', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_:TNUM:', + 'format_header' => 'Content-Type: application/json', + 'user_name' => ':UNAME_GROUP:', + 'status_code' => 200, + 'json_field_substr_match' => {'database' => 'hcatperms_:TNUM:', + 'location' => 'hcatperms_:TNUM:'}, + + + }, + + { + + #describe db - by group user without permissions + 'num' => 21, + 'setup' => [ + { #drop db cascade + 'method' => 'DELETE', + 'format_header' => 'Content-Type: application/json', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_:TNUM:?ifExists=true&option=cascade', + 'user_name' => ':UNAME:', + 'status_code' => 200, + }, + + { + #create db + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_:TNUM:', + 'user_name' => ':UNAME:', + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{ "permissions" : "rwx------", "group" : ":UGROUP:" }'], + 'status_code' => 200, + }, + + ], + + + 'method' => 'GET', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_:TNUM:', + 'format_header' => 'Content-Type: application/json', + 'user_name' => ':UNAME_GROUP:', + 'status_code' => 500, + + }, + + { + + #describe db - by other user without permissions + 'num' => 22, + 'setup' => [ + { #drop db cascade + 'method' => 'DELETE', + 'format_header' => 'Content-Type: application/json', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_:TNUM:?ifExists=true&option=cascade', + 'user_name' => ':UNAME:', + 'status_code' => 200, + }, + + { + #create db + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_:TNUM:', + 'user_name' => ':UNAME:', + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{ "permissions" : "rwx------", "group" : ":UGROUP:" }'], + 'status_code' => 200, + }, + + ], + + + 'method' => 'GET', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_:TNUM:', + 'format_header' => 'Content-Type: application/json', + 'user_name' => ':UNAME_OTHER:', + 'status_code' => 500, + + }, + + ] + }, + +##============================================================================================================= + + { + 'name' => 'TABLE_OPS', + 'tests' => + [ + { + #test if the permissions and group for table are set correctly + 'num' => 1, + 'setup' => [ + { #drop table + 'method' => 'DELETE', + 'format_header' => 'Content-Type: application/json', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/permstable_:TNUM:?ifExists=true', + 'user_name' => ':UNAME:', + 'status_code' => 200, + }, + { #create table + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/permstable_:TNUM:', + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{ "columns": [ { "name" : "i", "type" : "int" } ],"format" : { "storedAs" : "rcfile"}, + "permissions" : "rwxr-xr-x", "group" : ":UGROUP:" }'], + 'user_name' => ':UNAME:', + 'status_code' => 200, + } + + ], + + + 'method' => 'GET', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/permstable_:TNUM:?format=extended', + 'format_header' => 'Content-Type: application/json', + 'user_name' => ':UNAME:', + 'status_code' => 200, + 'json_field_substr_match' => {'location_perms' => 'rwxr-xr-x', 'location_group' => ':UGROUP:'}, + + }, + + { + #check default permissions + #the default group is currently the group of parent dir + 'num' => 2, + 'setup' => [ + { #drop table + 'method' => 'DELETE', + 'format_header' => 'Content-Type: application/json', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/permstable_:TNUM:?ifExists=true', + 'user_name' => ':UNAME:', + 'status_code' => 200, + }, + { #create table + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/permstable_:TNUM:', + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{ "columns": [ { "name" : "i", "type" : "int" } ],"format" : { "storedAs" : "rcfile"} }'], + 'user_name' => ':UNAME:', + 'status_code' => 200, + } + + ], + + + 'method' => 'GET', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/permstable_:TNUM:?format=extended', + 'format_header' => 'Content-Type: application/json', + 'user_name' => ':UNAME:', + 'status_code' => 200, + 'json_field_substr_match' => {'location_perms' => 'rwx------', 'location_owner' => ':UNAME:'}, + + }, + + { + #alter table permissions for user + 'num' => 3, + 'setup' => [ + { #drop table + 'method' => 'DELETE', + 'format_header' => 'Content-Type: application/json', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/permstable_:TNUM:?ifExists=true', + 'user_name' => ':UNAME:', + 'status_code' => 200, + }, + { #create table + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/permstable_:TNUM:', + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{ "columns": [ { "name" : "i", "type" : "int" } ],"format" : { "storedAs" : "rcfile"} }'], + 'user_name' => ':UNAME:', + 'status_code' => 200, + } + + ], + + #alter table add column j + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/permstable_:TNUM:/column/j', + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{"type": "string", "comment": "The 2nd col"}'], + 'user_name' => ':UNAME:', + 'status_code' => 200, + }, + + { + #alter table for group user - with permissions + 'num' => 4, + 'setup' => [ + { #drop table + 'method' => 'DELETE', + 'format_header' => 'Content-Type: application/json', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/permstable_:TNUM:?ifExists=true', + 'user_name' => ':UNAME:', + 'status_code' => 200, + }, + { #create table + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/permstable_:TNUM:', + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{ "columns": [ { "name" : "i", "type" : "int" } ], + "format" : { "storedAs" : "rcfile"}, + "permissions" : "rwxrwxr-x", "group" : ":UGROUP:" }' ], + 'user_name' => ':UNAME:', + 'status_code' => 200, + } + + ], + + #alter table add column j + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/permstable_:TNUM:/column/j', + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{"type": "string", "comment": "The 2nd col"}'], + 'user_name' => ':UNAME_GROUP:', + 'status_code' => 200, + }, + + { + #alter table for group user -without permissions + 'num' => 5, + 'setup' => [ + { #drop table + 'method' => 'DELETE', + 'format_header' => 'Content-Type: application/json', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/permstable_:TNUM:?ifExists=true', + 'user_name' => ':UNAME:', + 'status_code' => 200, + }, + { #create table + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/permstable_:TNUM:', + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{ "columns": [ { "name" : "i", "type" : "int" } ], + "format" : { "storedAs" : "rcfile"}, + "permissions" : "rwxr-xr-x", "group" : ":UGROUP:" }' ], + 'user_name' => ':UNAME:', + 'status_code' => 200, + } + + ], + + #alter table add column j + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/permstable_:TNUM:/column/j', + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{"type": "string", "comment": "The 2nd col"}'], + 'user_name' => ':UNAME_GROUP:', + 'status_code' => 500, + }, + + { + #alter table for other user -without permissions + 'num' => 6, + 'setup' => [ + { #drop table + 'method' => 'DELETE', + 'format_header' => 'Content-Type: application/json', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/permstable_:TNUM:?ifExists=true', + 'user_name' => ':UNAME:', + 'status_code' => 200, + }, + { #create table + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/permstable_:TNUM:', + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{ "columns": [ { "name" : "i", "type" : "int" } ], + "format" : { "storedAs" : "rcfile"}, + "permissions" : "rwxr-xr-x", "group" : ":UGROUP:" }' ], + 'user_name' => ':UNAME:', + 'status_code' => 200, + } + + ], + + #alter table add column j + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/permstable_:TNUM:/column/j', + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{"type": "string", "comment": "The 2nd col"}'], + 'user_name' => ':UNAME_OTHER:', + 'status_code' => 500, + }, + + { + #drop table by user + 'num' => 7, + 'setup' => [ + { #drop table + 'method' => 'DELETE', + 'format_header' => 'Content-Type: application/json', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/permstable_:TNUM:?ifExists=true', + 'user_name' => ':UNAME:', + 'status_code' => 200, + }, + { #create table + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/permstable_:TNUM:', + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{ "columns": [ { "name" : "i", "type" : "int" } ], + "format" : { "storedAs" : "rcfile"}, + "permissions" : "rwxr-xr-x", "group" : ":UGROUP:" }' ], + 'user_name' => ':UNAME:', + 'status_code' => 200, + } + + ], + + #drop table + 'method' => 'DELETE', + 'format_header' => 'Content-Type: application/json', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/permstable_:TNUM:', + 'user_name' => ':UNAME:', + 'status_code' => 200, + }, + + { + #drop table by group user with permissions + 'num' => 8, + 'setup' => [ + { #drop table + 'method' => 'DELETE', + 'format_header' => 'Content-Type: application/json', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/permstable_:TNUM:?ifExists=true', + 'user_name' => ':UNAME:', + 'status_code' => 200, + }, + { #create table + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/permstable_:TNUM:', + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{ "columns": [ { "name" : "i", "type" : "int" } ], + "format" : { "storedAs" : "rcfile"}, + "permissions" : "rwxrwx---", "group" : ":UGROUP:" }' ], + 'user_name' => ':UNAME:', + 'status_code' => 200, + } + + ], + + #drop table + 'method' => 'DELETE', + 'format_header' => 'Content-Type: application/json', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/permstable_:TNUM:', + 'user_name' => ':UNAME_GROUP:', + 'status_code' => 200, + }, + + { + #drop table by group user without permissions + 'num' => 9, + 'setup' => [ + { #drop table + 'method' => 'DELETE', + 'format_header' => 'Content-Type: application/json', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/permstable_:TNUM:?ifExists=true', + 'user_name' => ':UNAME:', + 'status_code' => 200, + }, + { #create table + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/permstable_:TNUM:', + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{ "columns": [ { "name" : "i", "type" : "int" } ], + "format" : { "storedAs" : "rcfile"}, + "permissions" : "rwx------", "group" : ":UGROUP:" }' ], + 'user_name' => ':UNAME:', + 'status_code' => 200, + } + + ], + + #drop table + 'method' => 'DELETE', + 'format_header' => 'Content-Type: application/json', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/permstable_:TNUM:', + 'user_name' => ':UNAME_GROUP:', + 'status_code' => 500, + }, + + { + #drop table by other user without permissions + 'num' => 10, + 'setup' => [ + { #drop table + 'method' => 'DELETE', + 'format_header' => 'Content-Type: application/json', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/permstable_:TNUM:?ifExists=true', + 'user_name' => ':UNAME:', + 'status_code' => 200, + }, + { #create table + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/permstable_:TNUM:', + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{ "columns": [ { "name" : "i", "type" : "int" } ], + "format" : { "storedAs" : "rcfile"}, + "permissions" : "rwx------", "group" : ":UGROUP:" }' ], + 'user_name' => ':UNAME:', + 'status_code' => 200, + } + + ], + + #drop table + 'method' => 'DELETE', + 'format_header' => 'Content-Type: application/json', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/permstable_:TNUM:', + 'user_name' => ':UNAME_OTHER:', + 'status_code' => 500, + }, + + + { + #show table allowed for user + 'num' => 11, + 'setup' => [ + { + 'method' => 'DELETE', + 'format_header' => 'Content-Type: application/json', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/tabpermsdb:TNUM:?ifExists=true&option=cascade', + 'user_name' => ':UNAME:', + 'status_code' => 200, + }, + + { + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/tabpermsdb:TNUM:', + 'user_name' => ':UNAME:', + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{ "permissions" : "rwxrwx---" }'], + 'status_code' => 200, + }, + { + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/tabpermsdb:TNUM:/table/permstable_:TNUM:', + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{ + "columns": [ + { "name" : "i", "type" : "int" } + ], + "format" : { "storedAs" : "rcfile"} + }'], + 'user_name' => ':UNAME:', + 'status_code' => 200, + } + + ], + + + 'method' => 'GET', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/tabpermsdb:TNUM:/table/', + 'format_header' => 'Content-Type: application/json', + 'user_name' => ':UNAME:', + 'status_code' => 200, + 'json_field_substr_match' => {'tables' => 'permstable_:TNUM:'}, + + }, + + { + #show table allowed for group user with permissions + 'num' => 12, + 'setup' => [ + { + 'method' => 'DELETE', + 'format_header' => 'Content-Type: application/json', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/tabpermsdb:TNUM:?ifExists=true&option=cascade', + 'user_name' => ':UNAME:', + 'status_code' => 200, + }, + + { + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/tabpermsdb:TNUM:', + 'user_name' => ':UNAME:', + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{ "permissions" : "rwxrwx---", "group" : ":UGROUP:" }'], + 'status_code' => 200, + }, + { + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/tabpermsdb:TNUM:/table/permstable_:TNUM:', + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{ + "columns": [ { "name" : "i", "type" : "int" }], "permissions" : "rwxrwx---", "group" : ":UGROUP:" } + "format" : { "storedAs" : "rcfile"} + }'], + 'user_name' => ':UNAME:', + 'status_code' => 200, + } + + ], + + + 'method' => 'GET', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/tabpermsdb:TNUM:/table/', + 'format_header' => 'Content-Type: application/json', + 'user_name' => ':UNAME_GROUP:', + 'status_code' => 200, + 'json_field_substr_match' => {'tables' => 'permstable_:TNUM:'}, + + }, + + { + #show table not allowed for group user without permissions + 'num' => 13, + 'ignore' => 'HIVE-3009', + 'setup' => [ + { + 'method' => 'DELETE', + 'format_header' => 'Content-Type: application/json', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/tabpermsdb:TNUM:?ifExists=true&option=cascade', + 'user_name' => ':UNAME:', + 'status_code' => 200, + }, + + { + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/tabpermsdb:TNUM:', + 'user_name' => ':UNAME:', + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{ "permissions" : "rwxrwx---" }'], + 'status_code' => 200, + }, + { + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/tabpermsdb:TNUM:/table/permstable_:TNUM:', + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{ + "columns": [ { "name" : "i", "type" : "int" }], "permissions" : "rwx------", "group" : ":UGROUP:" } + "format" : { "storedAs" : "rcfile"} + }'], + 'user_name' => ':UNAME:', + 'status_code' => 200, + } + + ], + + + 'method' => 'GET', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/tabpermsdb:TNUM:/table/', + 'format_header' => 'Content-Type: application/json', + 'user_name' => ':UNAME_GROUP:', + 'status_code' => 500, + 'json_field_substr_match' => {'error' => 'unable to show tables'}, + + }, + + { + #show table not allowed for other user without permissions + 'num' => 14, + 'ignore' => 'HIVE-3009', + 'setup' => [ + { + 'method' => 'DELETE', + 'format_header' => 'Content-Type: application/json', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/tabpermsdb:TNUM:?ifExists=true&option=cascade', + 'user_name' => ':UNAME:', + 'status_code' => 200, + }, + + { + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/tabpermsdb:TNUM:', + 'user_name' => ':UNAME:', + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{ "permissions" : "rwxrwxrwx" }'], + 'status_code' => 200, + }, + { + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/tabpermsdb:TNUM:/table/permstable_:TNUM:', + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{ + "columns": [ { "name" : "i", "type" : "int" }], "permissions" : "rwx------", "group" : ":UGROUP:" } + "format" : { "storedAs" : "rcfile"} + }'], + 'user_name' => ':UNAME:', + 'status_code' => 200, + } + + ], + + + 'method' => 'GET', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/tabpermsdb:TNUM:/table/', + 'format_header' => 'Content-Type: application/json', + 'user_name' => ':UNAME_OTHER:', + 'status_code' => 500, + 'json_field_substr_match' => {'error' => 'unable to show tables'}, + + }, + + { + #show table not allowed for other user without permissions + 'num' => 15, + 'ignore' => 'see HIVE-3009', + 'setup' => [ + { + 'method' => 'DELETE', + 'format_header' => 'Content-Type: application/json', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/tabpermsdb:TNUM:?ifExists=true&option=cascade', + 'user_name' => ':UNAME:', + 'status_code' => 200, + }, + + { + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/tabpermsdb:TNUM:', + 'user_name' => ':UNAME:', + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{ "permissions" : "rwxrwxrwx" }'], + 'status_code' => 200, + }, + { + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/tabpermsdb:TNUM:/table/permstable_:TNUM:', + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{ + "columns": [ { "name" : "i", "type" : "int" }], "permissions" : "rwx------", "group" : ":UGROUP:" } + "format" : { "storedAs" : "rcfile"} + }'], + 'user_name' => ':UNAME:', + 'status_code' => 200, + } + + ], + + + 'method' => 'GET', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/tabpermsdb:TNUM:/table/', + 'format_header' => 'Content-Type: application/json', + 'user_name' => ':UNAME_OTHER:', + 'status_code' => 500, + 'json_field_substr_match' => {'error' => 'unable to show tables'}, + + }, + + { + #describe table test for user done as part of permissions check test + #describe table by group user + 'num' => 16, + 'setup' => [ + { #drop table + 'method' => 'DELETE', + 'format_header' => 'Content-Type: application/json', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/permstable_:TNUM:?ifExists=true', + 'user_name' => ':UNAME:', + 'status_code' => 200, + }, + { #create table + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/permstable_:TNUM:', + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{ "columns": [ { "name" : "i", "type" : "int" } ],"format" : { "storedAs" : "rcfile"}, + "permissions" : "rwxr-x---", "group" : ":UGROUP:" }'], + 'user_name' => ':UNAME:', + 'status_code' => 200, + } + + ], + + + 'method' => 'GET', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/permstable_:TNUM:?format=extended', + 'format_header' => 'Content-Type: application/json', + 'user_name' => ':UNAME_GROUP:', + 'status_code' => 200, + 'json_field_substr_match' => {'location_perms' => 'rwxr-x---', 'location_group' => ':UGROUP:'}, + + }, + + { + #describe extended, table by group user without permission + 'num' => 17, + 'ignore' => 'see HIVE-3009', + 'setup' => [ + { #drop table + 'method' => 'DELETE', + 'format_header' => 'Content-Type: application/json', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/permstable_:TNUM:?ifExists=true', + 'user_name' => ':UNAME:', + 'status_code' => 200, + }, + { #create table + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/permstable_:TNUM:', + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{ "columns": [ { "name" : "i", "type" : "int" } ],"format" : { "storedAs" : "rcfile"}, + "permissions" : "rwx------", "group" : ":UGROUP:" }'], + 'user_name' => ':UNAME:', + 'status_code' => 200, + } + + ], + + + 'method' => 'GET', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/permstable_:TNUM:?format=extended', + 'format_header' => 'Content-Type: application/json', + 'user_name' => ':UNAME_GROUP:', + 'status_code' => 500, + + }, + + { + #describe extended, table by other user without permission + 'num' => 18, + 'ignore' => 'see HIVE-3009', + 'setup' => [ + { #drop table + 'method' => 'DELETE', + 'format_header' => 'Content-Type: application/json', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/permstable_:TNUM:?ifExists=true', + 'user_name' => ':UNAME:', + 'status_code' => 200, + }, + { #create table + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/permstable_:TNUM:', + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['{ "columns": [ { "name" : "i", "type" : "int" } ],"format" : { "storedAs" : "rcfile"}, + "permissions" : "rwx------", "group" : ":UGROUP:" }'], + 'user_name' => ':UNAME:', + 'status_code' => 200, + } + + ], + + + 'method' => 'GET', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/permstable_:TNUM:?format=extended', + 'format_header' => 'Content-Type: application/json', + 'user_name' => ':UNAME_OTHER:', + 'status_code' => 500, + + }, + + + + ] } + + + ] +}, + ; + Index: src/test/e2e/templeton/tests/jobsubmission.conf =================================================================== --- src/test/e2e/templeton/tests/jobsubmission.conf (revision 0) +++ src/test/e2e/templeton/tests/jobsubmission.conf (revision 0) @@ -0,0 +1,359 @@ +############################################################################### +# curl command tests for templeton +# +# + +#use Yahoo::Miners::Test::PigSetup; + +#PigSetup::setup(); + +#my $me = `whoami`; +#chomp $me; + +$cfg = +{ + 'driver' => 'Curl', + + 'groups' => + [ +##============================================================================================================= + { + 'name' => 'TestStreaming', + 'tests' => + [ + { + 'num' => 1, + 'method' => 'POST', + 'url' => ':TEMPLETON_URL:/templeton/v1/mapreduce/streaming', + 'post_options' => ['user.name=:UNAME:','input=:INPDIR_HDFS:/nums.txt','output=:OUTDIR:/mycounts', + 'mapper=/bin/cat', 'reducer=/usr/bin/wc'], + 'json_field_substr_match' => { 'id' => '\d+'}, + #results + 'status_code' => 200, + 'check_job_created' => 1, + 'check_job_complete' => 'SUCCESS', + 'check_call_back' => 1, + }, + { + #-ve test - no input file + 'num' => 2, + 'ignore' => 'wait for fix in hadoop 1.0.3', + 'method' => 'POST', + 'url' => ':TEMPLETON_URL:/templeton/v1/mapreduce/streaming', + 'post_options' => ['user.name=:UNAME:','input=:INPDIR_HDFS:/nums.txt','output=:OUTDIR:/mycounts', + 'mapper=/bin/ls no_such-file-12e3', 'reducer=/usr/bin/wc'], + 'json_field_substr_match' => { 'id' => '\d+'}, + #results + 'status_code' => 200, + 'check_job_created' => 1, + 'check_job_complete' => 'FAILURE', + 'check_call_back' => 1, + }, + + ] + }, +##============================================================================================================= + { + 'name' => 'TestKillJob', + 'tests' => + [ + { + 'num' => 1, + 'method' => 'POST', + 'url' => ':TEMPLETON_URL:/templeton/v1/mapreduce/streaming', + 'post_options' => ['user.name=:UNAME:','input=:INPDIR_HDFS:/nums.txt','output=:OUTDIR:/mycounts', + 'mapper=/bin/sleep 100', 'reducer=/usr/bin/wc'], + 'json_field_substr_match' => { 'id' => '\d+'}, + #results + 'status_code' => 200, + 'check_job_created' => 1, + 'check_job_complete' => 'KILLED', +# 'check_call_back' => 1, #TODO - enable call back check after fix + 'kill_job_timeout' => 10, + }, + ] + }, +##============================================================================================================= + { + 'name' => 'TestMapReduce', + 'tests' => + [ + { + + 'num' => 1, + 'method' => 'POST', + 'url' => ':TEMPLETON_URL:/templeton/v1/mapreduce/jar', + 'post_options' => ['user.name=:UNAME:','arg=:INPDIR_HDFS:/nums.txt', 'arg= :OUTDIR:/wc.txt', + 'jar=:INPDIR_HDFS:/hexamples.jar', 'class=wordcount', ], + 'json_field_substr_match' => { 'id' => '\d+'}, + #results + 'status_code' => 200, + 'check_job_created' => 1, + 'check_job_complete' => 'SUCCESS', + 'check_call_back' => 1, + }, + ] + }, +##============================================================================================================= + { + 'name' => 'TestPig', + 'tests' => + [ + { + #test syntax error + 'ignore' => 'fails in current version', + 'num' => 1, + 'method' => 'POST', + 'url' => ':TEMPLETON_URL:/templeton/v1/pig', + 'post_options' => ['user.name=:UNAME:','execute=asdf', ], + 'json_field_substr_match' => { 'id' => '\d+'}, + #results + 'status_code' => 200, + 'check_job_created' => 1, + 'check_job_complete' => 'FAILURE', + 'check_call_back' => 1, + }, + { + #valid syntax, hdfs operation through pig + 'num' => 2, + 'method' => 'POST', + 'url' => ':TEMPLETON_URL:/templeton/v1/pig', + 'post_options' => ['user.name=:UNAME:','execute=fs -ls :INPDIR_HDFS:;', ], + 'json_field_substr_match' => { 'id' => '\d+'}, + #results + 'status_code' => 200, + 'check_job_created' => 1, + 'check_job_complete' => 'SUCCESS', + 'check_call_back' => 1, + }, + { + #syntax check - valid syntax + 'num' => 3, + 'method' => 'POST', + 'url' => ':TEMPLETON_URL:/templeton/v1/pig', + 'post_options' => ['user.name=:UNAME:','arg=-check', 'file=:INPDIR_HDFS:/loadstore.pig', 'arg=-p', 'arg=INPDIR=:INPDIR_HDFS:','arg=-p', 'arg=OUTDIR=:OUTDIR:', ], + 'json_field_substr_match' => { 'id' => '\d+'}, + #results + 'status_code' => 200, + 'check_job_created' => 1, + 'check_job_complete' => 'SUCCESS', + 'check_call_back' => 1, + }, + { + #syntax check cmd - valid syntax + 'num' => 4, + 'method' => 'POST', + 'url' => ':TEMPLETON_URL:/templeton/v1/pig', + 'post_options' => ['user.name=:UNAME:', 'arg=-d', 'arg=INFO' , 'execute=fs -ls :INPDIR_HDFS: ', ], + 'json_field_substr_match' => { 'id' => '\d+'}, + #results + 'status_code' => 200, + 'check_job_created' => 1, + 'check_job_complete' => 'SUCCESS', + 'check_call_back' => 1, + }, + + { + #a simple load store script + 'num' => 5, + 'method' => 'POST', + 'url' => ':TEMPLETON_URL:/templeton/v1/pig', + 'post_options' => ['user.name=:UNAME:', 'arg=-p', 'arg=INPDIR=:INPDIR_HDFS:','arg=-p', 'arg=OUTDIR=:OUTDIR:', 'file=:INPDIR_HDFS:/loadstore.pig', ], + 'json_field_substr_match' => { 'id' => '\d+'}, + #results + 'status_code' => 200, + 'check_job_created' => 1, + 'check_job_complete' => 'SUCCESS', + 'check_call_back' => 1, + }, + + + { + #pig query registering jar + 'num' => 6, + 'method' => 'POST', + 'url' => ':TEMPLETON_URL:/templeton/v1/pig', + 'post_options' => ['user.name=:UNAME:', 'arg=-p', 'arg=INPDIR=:INPDIR_HDFS:','arg=-p', 'arg=OUTDIR=:OUTDIR:', 'file=:INPDIR_HDFS:/jarregistered.pig', + 'files=:INPDIR_HDFS:/piggybank.jar' ], + 'json_field_substr_match' => { 'id' => '\d+'}, + #results + 'status_code' => 200, + 'check_job_created' => 1, + 'check_job_complete' => 'SUCCESS', + 'check_call_back' => 1, + }, + + { + #macro + 'num' => 7, + 'method' => 'POST', + 'url' => ':TEMPLETON_URL:/templeton/v1/pig', + 'post_options' => ['user.name=:UNAME:', 'arg=-p', 'arg=INPDIR=:INPDIR_HDFS:','arg=-p', 'arg=OUTDIR=:OUTDIR:', 'file=:INPDIR_HDFS:/rowcount_withmacro.pig', + 'files=:INPDIR_HDFS:/rowcountmacro.pig' ], + 'json_field_substr_match' => { 'id' => '\d+'}, + #results + 'status_code' => 200, + 'check_job_created' => 1, + 'check_job_complete' => 'SUCCESS', + 'check_call_back' => 1, + }, + + { + #no file to be copied, should result in launcher job error + 'num' => 8, + ignore => 'check is disabled for now in templeton', + 'method' => 'POST', + 'url' => ':TEMPLETON_URL:/templeton/v1/pig', + 'post_options' => ['user.name=:UNAME:', 'arg=-p', 'arg=INPDIR=:INPDIR_HDFS:','arg=-p', 'arg= OUTDIR=:OUTDIR:', 'file=:INPDIR_HDFS:/no_such_file.pig', + 'files=:INPDIR_HDFS:/rowcountmacro.pig' ], + 'json_field_substr_match' => { 'error' => 'does not exist'}, + #results + 'status_code' => 200, + 'check_job_complete' => 'FAILURE', + + }, + + + + #test 9 + #TODO jython test + + + + + ] + }, +##============================================================================================================= + { + 'name' => 'TestHive', + 'tests' => + [ + { + #test syntax error + 'ignore' => 'fails in current version', + 'num' => 1, + 'method' => 'POST', + 'url' => ':TEMPLETON_URL:/templeton/v1/hive', + 'post_options' => ['user.name=:UNAME:','execute=asdf', ], + 'json_field_substr_match' => { 'id' => '\d+'}, + #results + 'status_code' => 200, + 'check_job_created' => 1, + 'check_job_complete' => 'FAILURE', + + }, + + { + #test show tables + 'num' => 2, + 'method' => 'POST', + 'url' => ':TEMPLETON_URL:/templeton/v1/hive', + 'post_options' => ['user.name=:UNAME:','execute=show tables', ], + 'json_field_substr_match' => { 'id' => '\d+'}, + #results + 'status_code' => 200, + 'check_job_created' => 1, + 'check_job_complete' => 'SUCCESS', + 'check_call_back' => 1, + }, + + { + #test show tables + 'num' => 3, + 'method' => 'POST', + 'url' => ':TEMPLETON_URL:/templeton/v1/hive', + 'post_options' => ['user.name=:UNAME:','execute=drop table if exists mynums;', ], + 'json_field_substr_match' => { 'id' => '\d+'}, + #results + 'status_code' => 200, + 'check_job_created' => 1, + 'check_job_complete' => 'SUCCESS', + 'check_call_back' => 1, + }, + { + #test show tables + 'num' => 4, + 'method' => 'POST', + 'url' => ':TEMPLETON_URL:/templeton/v1/hive', + 'post_options' => ['user.name=:UNAME:','execute=create external table mynums(a int, b int) location ":INPDIR_HDFS:/numstable/";', ], + 'json_field_substr_match' => { 'id' => '\d+'}, + #results + 'status_code' => 200, + 'check_job_created' => 1, + 'check_job_complete' => 'SUCCESS', + 'check_call_back' => 1, + }, + + { + #test describe + 'num' => 5, + 'method' => 'POST', + 'url' => ':TEMPLETON_URL:/templeton/v1/hive', + 'post_options' => ['user.name=:UNAME:','execute=describe formatted mynums', ], + 'json_field_substr_match' => { 'id' => '\d+'}, + #results + 'status_code' => 200, + 'check_job_created' => 1, + 'check_job_complete' => 'SUCCESS', + 'check_call_back' => 1, + }, + + { + #test select * + 'num' => 6, + 'method' => 'POST', + 'url' => ':TEMPLETON_URL:/templeton/v1/hive', + 'post_options' => ['user.name=:UNAME:','execute=select * from mynums', ], + 'json_field_substr_match' => { 'id' => '\d+'}, + #results + 'status_code' => 200, + 'check_job_created' => 1, + 'check_job_complete' => 'SUCCESS', + 'check_call_back' => 1, + }, + + { + #test select a,b + 'num' => 7, + 'method' => 'POST', + 'url' => ':TEMPLETON_URL:/templeton/v1/hive', + 'post_options' => ['user.name=:UNAME:','execute=select a,b from mynums', ], + 'json_field_substr_match' => { 'id' => '\d+'}, + #results + 'status_code' => 200, + 'check_job_created' => 1, + 'check_job_complete' => 'SUCCESS', + 'check_call_back' => 1, + + }, + + + { + #test udfs : select a,rand(b) + 'num' => 8, + 'method' => 'POST', + 'url' => ':TEMPLETON_URL:/templeton/v1/hive', + 'post_options' => ['user.name=:UNAME:','execute=select a,rand(b) from mynums', ], + 'json_field_substr_match' => { 'id' => '\d+'}, + #results + 'status_code' => 200, + 'check_job_created' => 1, + 'check_job_complete' => 'SUCCESS', + + }, + + + + ] + }, + + + + + + + + ] +}, + ; + Index: src/test/e2e/templeton/tests/serverstatus.conf =================================================================== --- src/test/e2e/templeton/tests/serverstatus.conf (revision 0) +++ src/test/e2e/templeton/tests/serverstatus.conf (revision 0) @@ -0,0 +1,57 @@ +############################################################################### +# curl command tests for templeton +# +# + +#use Yahoo::Miners::Test::PigSetup; + +#PigSetup::setup(); + +#my $me = `whoami`; +#chomp $me; + +$cfg = +{ + 'driver' => 'Curl', + + 'groups' => + [ +##============================================================================================================= + { + 'name' => 'TypesSupported', + 'tests' => + [ + { + #types supported + 'num' => 1, + 'method' => 'GET', + 'url' => ':TEMPLETON_URL:/templeton/v1', + 'status_code' => 200, + }, + ] + }, +##============================================================================================================= + { + 'name' => 'VersionsSupported', + 'tests' => + [ + { + #versions supported + 'num' => 1, + 'method' => 'GET', + 'url' => ':TEMPLETON_URL:/templeton/v1/version', + 'status_code' => 200, + 'json_field_substr_match' => {'version' => 'v1'}, + 'json_field_match_object' => {'supportedVersions' => '["v1"]'} + }, + ] + }, + + + + + + ] +}, + ; + Index: src/test/e2e/templeton/conf/default.conf =================================================================== --- src/test/e2e/templeton/conf/default.conf (revision 0) +++ src/test/e2e/templeton/conf/default.conf (revision 0) @@ -0,0 +1,40 @@ +############################################################################ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +my $me = `whoami`; +chomp $me; + +# The contents of this file can be rewritten to fit your installation. +# Also, you can define the following environment variables and set things up as in the test setup +# TH_ROOT Root directory where test harness is installed +# TH_OUT Root directory where output data will be stored + +$cfg = { + #HDFS + 'outpathbase' => '/tmp/templeton_test_out' + , 'localoutpathbase' => "$ENV{TH_WORKING_DIR}/" + + #TEST + , 'tmpPath' => '/tmp/pigtest' + + , 'userhomePath' => "$ENV{HOME}" + ,'local.bin' => '/usr/bin' + + ,'logDir' => "$ENV{TH_OUT}/log" +# ,'propertiesFile' => "./conf/testpropertiesfile.conf" + ,'harness.console.level' => 'ERROR' + +}; Index: src/test/e2e/templeton/inpdir/nums.txt =================================================================== --- src/test/e2e/templeton/inpdir/nums.txt (revision 0) +++ src/test/e2e/templeton/inpdir/nums.txt (revision 0) @@ -0,0 +1,2 @@ +1 3 +2 4 Index: src/test/e2e/templeton/inpdir/rowcountmacro.pig =================================================================== --- src/test/e2e/templeton/inpdir/rowcountmacro.pig (revision 0) +++ src/test/e2e/templeton/inpdir/rowcountmacro.pig (revision 0) @@ -0,0 +1 @@ +DEFINE row_count(X) RETURNS Z { Y = group $X all; $Z = foreach Y generate COUNT($X); }; Index: src/test/e2e/templeton/inpdir/jaradditionaljars.pig =================================================================== --- src/test/e2e/templeton/inpdir/jaradditionaljars.pig (revision 0) +++ src/test/e2e/templeton/inpdir/jaradditionaljars.pig (revision 0) @@ -0,0 +1,5 @@ +-- no register command +l = load '$INPDIR/nums.txt' as (a : chararray); +f = foreach l generate org.apache.pig.piggybank.evaluation.string.Reverse($0); +store f into '$OUTDIR/reversed.txt'; + Index: src/test/e2e/templeton/inpdir/jarregistered.pig =================================================================== --- src/test/e2e/templeton/inpdir/jarregistered.pig (revision 0) +++ src/test/e2e/templeton/inpdir/jarregistered.pig (revision 0) @@ -0,0 +1,5 @@ +register piggybank.jar +l = load '$INPDIR/nums.txt' as (a : chararray); +f = foreach l generate org.apache.pig.piggybank.evaluation.string.Reverse($0); +store f into '$OUTDIR/reversed.txt'; + Index: src/test/e2e/templeton/inpdir/udfs.py =================================================================== --- src/test/e2e/templeton/inpdir/udfs.py (revision 0) +++ src/test/e2e/templeton/inpdir/udfs.py (revision 0) @@ -0,0 +1,3 @@ +@outputSchema("word:chararray") +def helloworld(): + return 'Hello, World' Index: src/test/e2e/templeton/inpdir/loadstore.pig =================================================================== --- src/test/e2e/templeton/inpdir/loadstore.pig (revision 0) +++ src/test/e2e/templeton/inpdir/loadstore.pig (revision 0) @@ -0,0 +1,2 @@ +l = load '$INPDIR/nums.txt'; +store l into '$OUTDIR/loadstore.out'; Index: src/test/e2e/templeton/inpdir/rowcount_withmacro.pig =================================================================== --- src/test/e2e/templeton/inpdir/rowcount_withmacro.pig (revision 0) +++ src/test/e2e/templeton/inpdir/rowcount_withmacro.pig (revision 0) @@ -0,0 +1,5 @@ +import 'rowcountmacro.pig'; + +l = load '$INPDIR/nums.txt'; +c = row_count(l); +store c into '$OUTDIR/rowcount.out'; Index: src/test/e2e/templeton/inpdir/pythonudf.pig =================================================================== --- src/test/e2e/templeton/inpdir/pythonudf.pig (revision 0) +++ src/test/e2e/templeton/inpdir/pythonudf.pig (revision 0) @@ -0,0 +1,5 @@ +register 'udfs.py' using jython as myfuncs; +l = load '$INPDIR/nums.txt' as (a : chararray); +f = foreach l generate $0, myfuncs.helloworld() ; +store f into '$OUTDIR/pyhello.txt'; + Index: src/test/e2e/templeton/deployAndTest.pl =================================================================== --- src/test/e2e/templeton/deployAndTest.pl (revision 0) +++ src/test/e2e/templeton/deployAndTest.pl (revision 0) @@ -0,0 +1,158 @@ +#!/usr/local/bin/perl -w +use strict; + +my $TESTDIR = '/tmp/templetontest/'; +my $TEST_INP_DIR = '/tmp/test_inpdir/'; #dir on hadoop +my $TEST_USER = 'hortonth'; +my $WEBHDFS_URL = 'http://localhost:50070'; +my $TEMPLETON_URL = 'http://localhost:8080'; +my $CATALINA_HOME = $ENV{'CATALINA_HOME'}; + +#use env variables if they have been set +if(defined $ENV{'TESTDIR'}){ + $TESTDIR = $ENV{'TESTDIR'}; +} +if(defined $ENV{'TEST_INP_DIR'}){ + $TEST_INP_DIR = $ENV{'TEST_INP_DIR'}; +} +if(defined $ENV{'TEST_USER'}){ + $TEST_USER = $ENV{'TEST_USER'}; +} +if(defined $ENV{'WEBHDFS_URL'}){ + $WEBHDFS_URL = $ENV{'WEBHDFS_URL'}; +} +if(defined $ENV{'TEMPLETON_URL'}){ + $TEMPLETON_URL = $ENV{'TEMPLETON_URL'}; +} + +if(! defined $ENV{'HCAT_PREFIX'}){ + $ENV{'HCAT_PREFIX'}='/usr/'; +} + +if(! defined $ENV{'HADOOP_PREFIX'}){ + $ENV{'HADOOP_PREFIX'}='/usr/'; +} + +my $host = `hostname` ; +chomp $host; + +if(! defined $ENV{'ZOOKEEPER_HOST'}){ + $ENV{'ZOOKEEPER_HOST'} = $host . ':2181'; +} + +if(! defined $ENV{'METASTORE_HOST'}){ + $ENV{'METASTORE_HOST'} = $host . ':9933'; +} + +print STDERR "##################################################################\n"; +print STDERR "Using the following settings for environment variables\n" . + " (Set them to override the default values) \n" . + "WEBHDFS_URL : $WEBHDFS_URL \n" . + "TEMPLETON_URL : $TEMPLETON_URL \n" . + 'CATALINA_HOME :' . $ENV{'CATALINA_HOME'} . "\n" . + 'HADOOP_PREFIX :' . $ENV{'HADOOP_PREFIX'} . "\n" . + 'HCAT_PREFIX :' . $ENV{'HCAT_PREFIX'} . "\n" . + 'ZOOKEEPER_HOST :' . $ENV{'ZOOKEEPER_HOST'} . "\n" . + 'METASTORE_HOST :' . $ENV{'METASTORE_HOST'} . "\n" +; +print STDERR "##################################################################\n"; + +system("rm -rf $TESTDIR/"); + +#restart tomcat with updated env variables +my $templeton_src = "$TESTDIR/templeton_src"; +$ENV{'TEMPLETON_HOME'} = "$templeton_src/templeton"; + +system ("$CATALINA_HOME/bin/shutdown.sh") == 0 or die "tomcat shutdown failed" ; +sleep 3; + + + +#get templeton git repo, build and install +system("mkdir -p $templeton_src") == 0 or die "could not create dir $templeton_src: $!"; +chdir "$templeton_src" or die "could not change directory to $templeton_src : $!"; +system ('git clone git@github.com:hortonworks/templeton.git') == 0 or die "could not clone templeton git repo"; +chdir 'templeton' or die 'could not change dir : $!'; + + +#put a templeton-site.xml in $TEMPLETON_HOME with zookeeper hostname +writeTempletonSiteXml(); +system ('ant install-war') == 0 or die "templeton build failed"; + +#tomcat should have shutdown by now, try starting it +system ("$CATALINA_HOME/bin/startup.sh") == 0 or die 'tomcat startup failed'; +sleep 3; + +my $tdir = "$templeton_src/templeton/src/test/e2e/templeton"; +chdir $tdir or die "could not change dir $tdir : $!"; + +#copy input files +system("hadoop fs -rmr $TEST_INP_DIR"); +system("hadoop fs -copyFromLocal $tdir/inpdir $TEST_INP_DIR") == 0 or die "failed to copy input dir : $!"; +system("hadoop fs -chmod -R 777 $TEST_INP_DIR") == 0 or die "failed to set input dir permissions : $!"; + +#start tests +my $cmd = "ant test -Dinpdir.hdfs=$TEST_INP_DIR -Dtest.user.name=$TEST_USER" . + " -Dharness.webhdfs.url=$WEBHDFS_URL -Dharness.templeton.url=$TEMPLETON_URL "; + +system($cmd) == 0 or die "templeton tests failed"; + +############################# +sub writeTempletonSiteXml { + my $conf = $ENV{'TEMPLETON_HOME'} . "/templeton-site.xml"; + open ( CFH, ">$conf" ) or die $!; + + + print CFH ' + + + templeton.zookeeper.hosts + ' . + $ENV{'ZOOKEEPER_HOST'} . +' + ZooKeeper servers, as comma separated host:port pairs + + + + templeton.hive.properties + hive.metastore.local=false,hive.metastore.uris=thrift://' . + $ENV{'METASTORE_HOST'} . + ',hive.metastore.sasl.enabled=false,hive.metastore.execute.setugi=true + Properties to set when running hive. + + + + templeton.hive.archive + hdfs:///user/templeton/hcatalog-0.3.0.tar.gz + The path to the Hive archive. + + + + templeton.hive.path + hcatalog-0.3.0.tar.gz/hcatalog-0.3.0/bin/hive + The path to the Hive executable. + + + + templeton.pig.archive + hdfs:///user/templeton/pig-0.9.2.tar.gz + The path to the Pig archive. + + + + templeton.pig.path + pig-0.9.2.tar.gz/pig-0.9.2/bin/pig + The path to the Pig executable. + + + +'; + close CFH or die $!; + +; + + + + + +} Property changes on: src/test/e2e/templeton/deployAndTest.pl ___________________________________________________________________ Added: svn:executable + * Index: src/test/e2e/templeton/README.txt =================================================================== --- src/test/e2e/templeton/README.txt (revision 0) +++ src/test/e2e/templeton/README.txt (revision 0) @@ -0,0 +1,104 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +End to end tests +--------------- +End to end tests in templeton runs tests against an existing templeton server. +It runs hcat, mapreduce, streaming, hive and pig tests. + +Test cases +---------- +The tests are defined in src/test/e2e/templeton/tests/*.conf + +Test framework +-------------- +The test framework is derived from the one used in pig, there is more documentation here on the framework - +https://cwiki.apache.org/confluence/display/PIG/HowToTest + + +Setup +----- +1. Templeton needs to be installed and setup to be able to run hcat, maprduce, hive and pig commands. + +2. Install perl and following perl modules (cpan -i ) +* IPC::Run +* JSON +* Data::Dump +* Number::Compare +* Text::Glob +* Data::Compare +* File::Find::Rule +* HTTP::Daemon + +Tips: +* Using perlbrew (http://perlbrew.pl) should make installing perl modules easier. +* Use 'yes | cpan -i ' to avoid answering the 100's of questions cpan asks. + + + +3. Copy contents of src/test/e2e/templeton/inpdir to hdfs + + +Running the tests +----------------- +Use the following command to run tests - + +ant test -Dinpdir.hdfs= -Dtest.user.name= \ + -Dsecure.mode= -Dharness.webhdfs.url= -Dharness.templeton.url= + +If you want to run specific test group you can specify the group, for example: -Dtests.to.run='-t TestHive' + +If you want to run specific test in a group group you can specify the test, for example: -Dtests.to.run='-t TestHive_1' + + +Running the hcat authorization tests +------------------------------------ +Hcat authorization tests run commands as different users to test if authorization is done right. + +ant test-hcat-authorization -Dkeytab.dir= + -Dsecure.mode= -Dtest.group.name= -Dinpdir.hdfs= + -Dtest.user.name= -Dtest.group.user.name= + -Dtest.other.user.name= + -Dharness.webhdfs.url= -Dharness.templeton.url= + +The is expected to have keytab filenames of the form - user_name.*keytab . + + +Notes +----- + + + +Enable webhdfs by adding the following to your hadoop hdfs-site.xml : + + + dfs.webhdfs.enabled + true + + + dfs.http.address + 127.0.0.1:8085 + true + + +You can build a server that will measure test coverage by using templeton: +ant clean; ant e2e +This assumes you've got webhdfs at the address above, the inpdir info in /user/templeton, and templeton running on the default port. You can change any of those properties in the build file. + +It's best to set HADOOP_HOME_WARN_SUPPRESS=true everywhere you can. Index: src/test/e2e/templeton/newtests/rowcountmacro.pig =================================================================== --- src/test/e2e/templeton/newtests/rowcountmacro.pig (revision 0) +++ src/test/e2e/templeton/newtests/rowcountmacro.pig (revision 0) @@ -0,0 +1 @@ +DEFINE row_count(X) RETURNS Z { Y = group $X all; $Z = foreach Y generate COUNT($X); }; Index: src/test/e2e/templeton/newtests/jaradditionaljars.pig =================================================================== --- src/test/e2e/templeton/newtests/jaradditionaljars.pig (revision 0) +++ src/test/e2e/templeton/newtests/jaradditionaljars.pig (revision 0) @@ -0,0 +1,5 @@ +-- no register command +l = load '$INPDIR/nums.txt' as (a : chararray); +f = foreach l generate org.apache.pig.piggybank.evaluation.string.Reverse($0); +store f into '$OUTDIR/reversed.txt'; + Index: src/test/e2e/templeton/newtests/jarregistered.pig =================================================================== --- src/test/e2e/templeton/newtests/jarregistered.pig (revision 0) +++ src/test/e2e/templeton/newtests/jarregistered.pig (revision 0) @@ -0,0 +1,5 @@ +register piggybank.jar +l = load '$INPDIR/nums.txt' as (a : chararray); +f = foreach l generate org.apache.pig.piggybank.evaluation.string.Reverse($0); +store f into '$OUTDIR/reversed.txt'; + Index: src/test/e2e/templeton/newtests/udfs.py =================================================================== --- src/test/e2e/templeton/newtests/udfs.py (revision 0) +++ src/test/e2e/templeton/newtests/udfs.py (revision 0) @@ -0,0 +1,3 @@ +@outputSchema("word:chararray") +def helloworld(): + return 'Hello, World' Index: src/test/e2e/templeton/newtests/loadstore.pig =================================================================== --- src/test/e2e/templeton/newtests/loadstore.pig (revision 0) +++ src/test/e2e/templeton/newtests/loadstore.pig (revision 0) @@ -0,0 +1,2 @@ +l = load '$INPDIR/nums.txt'; +store l into '$OUTDIR/loadstore.out'; Index: src/test/e2e/templeton/newtests/rowcount_withmacro.pig =================================================================== --- src/test/e2e/templeton/newtests/rowcount_withmacro.pig (revision 0) +++ src/test/e2e/templeton/newtests/rowcount_withmacro.pig (revision 0) @@ -0,0 +1,5 @@ +import 'rowcountmacro.pig'; + +l = load '$INPDIR/nums.txt'; +c = row_count(l); +store c into '$OUTDIR/rowcount.out'; Index: src/test/e2e/templeton/newtests/pigtest.txt =================================================================== --- src/test/e2e/templeton/newtests/pigtest.txt (revision 0) +++ src/test/e2e/templeton/newtests/pigtest.txt (revision 0) @@ -0,0 +1,49 @@ +checks for all of them - +- lanucher launched ok +- job succeeded (and pig exit status 0)/failed (tests below succeed unless specified) + + +#1 +pig -e 'fs -ls :INPDIR:/' +- stdout after job finish has list of files including some known one (.*nums.txt.*) + +#2 +pig -e 'erroneous syntax' +- job failed +- stderr contains syntax error message + +#3 +pig -check "l = load 'x';" + +#4 +- add loadstore.pig to files to be copied +pig -p INPDIR=:INPDIR: -p OUTDIR=:OUTDIR: loadstore.pig + +#5 +- add piggybank.jar to files to be copied +pig -p INPDIR=:INPDIR: -p OUTDIR=:OUTDIR: jarregistered.pig + +#6 +#jar specified in cmdline option instead of explicit register in script +- add piggybank.jar to files to be copied +pig -Dpig.additional.jars=piggybank.jar -p INPDIR=:INPDIR: -p OUTDIR=:OUTDIR: jaradditionaljars.pig + + +#7 +- add rowcountmacro.pig to files to be copied +pig -p INPDIR=:INPDIR: -p OUTDIR=:OUTDIR: rowcount_withmacro.pig + + +#8 +# test error when input file is not present +pig -p INPDIR=:INPDIR: -p OUTDIR=:OUTDIR: no_such_file.pig +- non zero exit status + +#9 +#i haven't been able to get this test working, it complains of not being able to find PyException class +# would reccomend taking daniel's help to get it working +- add udfs.py to files to be copied +pig -p INPDIR=:INPDIR: -p OUTDIR=:OUTDIR: pythonudf.pig + + + Index: src/test/e2e/templeton/newtests/pythonudf.pig =================================================================== --- src/test/e2e/templeton/newtests/pythonudf.pig (revision 0) +++ src/test/e2e/templeton/newtests/pythonudf.pig (revision 0) @@ -0,0 +1,5 @@ +register 'udfs.py' using jython as myfuncs; +l = load '$INPDIR/nums.txt' as (a : chararray); +f = foreach l generate $0, myfuncs.helloworld() ; +store f into '$OUTDIR/pyhello.txt'; + Index: src/test/e2e/templeton/build.xml =================================================================== --- src/test/e2e/templeton/build.xml (revision 0) +++ src/test/e2e/templeton/build.xml (revision 0) @@ -0,0 +1,95 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Index: src/test/e2e/templeton/drivers/TestDriverCurl.pm =================================================================== --- src/test/e2e/templeton/drivers/TestDriverCurl.pm (revision 0) +++ src/test/e2e/templeton/drivers/TestDriverCurl.pm (revision 0) @@ -0,0 +1,1325 @@ +############################################################################ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package TestDriverCurl; + +########################################################################### +# Class: TestDriver +# A base class for TestDrivers. +# + +use TestDriverFactory; +use TestReport; +use File::Path; +use IPC::Run; +use Data::Dump qw(dump); +use JSON; +use HTTP::Daemon; +use HTTP::Status; +use Data::Compare; +use strict; +use English; +use Storable qw(dclone); +use File::Glob ':glob'; + +my $passedStr = 'passed'; +my $failedStr = 'failed'; +my $abortedStr = 'aborted'; +my $skippedStr = 'skipped'; +my $dependStr = 'failed_dependency'; + + +############################################################################## +# Sub: printResults +# Static function, can be used by test_harness.pl +# Print the results so far, given the testStatuses hash. +# +# Paramaters: +# testStatuses - reference to hash of test status results. +# log - reference to file handle to print results to. +# prefix - A title to prefix to the results +# +# Returns: +# None. +# +sub printResults + { + my ($testStatuses, $log, $prefix) = @_; + + my ($pass, $fail, $abort, $depend, $skipped) = (0, 0, 0, 0, 0); + + foreach (keys(%$testStatuses)) { + ($testStatuses->{$_} eq $passedStr) && $pass++; + ($testStatuses->{$_} eq $failedStr) && $fail++; + ($testStatuses->{$_} eq $abortedStr) && $abort++; + ($testStatuses->{$_} eq $dependStr) && $depend++; + ($testStatuses->{$_} eq $skippedStr) && $skipped++; + } + + my $msg = "$prefix, PASSED: $pass FAILED: $fail SKIPPED: $skipped ABORTED: $abort " + . "FAILED DEPENDENCY: $depend"; + print $log "$msg\n"; + print "$msg\n"; + + } + +############################################################################## +# Sub: printGroupResultsXml +# Print the results for the group using junit xml schema using values from the testStatuses hash. +# +# Paramaters: +# $report - the report object to use to generate the report +# $groupName - the name of the group to report totals for +# $testStatuses - the hash containing the results for the tests run so far +# $totalDuration- The total time it took to run the group of tests +# +# Returns: +# None. +# +sub printGroupResultsXml + { + my ( $report, $groupName, $testStatuses, $totalDuration) = @_; + $totalDuration=0 if ( !$totalDuration ); + + my ($pass, $fail, $abort, $depend) = (0, 0, 0, 0); + + foreach my $key (keys(%$testStatuses)) { + if ( $key =~ /^$groupName/ ) { + ($testStatuses->{$key} eq $passedStr) && $pass++; + ($testStatuses->{$key} eq $failedStr) && $fail++; + ($testStatuses->{$key} eq $abortedStr) && $abort++; + ($testStatuses->{$key} eq $dependStr) && $depend++; + } + } + + my $total= $pass + $fail + $abort; + $report->totals( $groupName, $total, $fail, $abort, $totalDuration ); + + } + +############################################################################## +# Sub: new +# Constructor, should only be called by TestDriverFactory. +# +# Paramaters: +# None +# +# Returns: +# None. +sub new + { + my $proto = shift; + my $class = ref($proto) || $proto; + my $self = {}; + + bless($self, $class); + + $self->{'wrong_execution_mode'} = "_xyz_wrong_execution_mode_zyx_"; + + return $self; + } + +############################################################################## +# Sub: globalSetup +# Set up before any tests are run. This gives each individual driver a chance to do +# setup. This function will only be called once, before all the tests are +# run. A driver need not implement it. It is a virtual function. +# +# Paramaters: +# globalHash - Top level hash from config file (does not have any group +# or test information in it). +# log - log file handle +# +# Returns: +# None +# +sub globalSetup + { + my ($self, $globalHash, $log) = @_; + my $subName = (caller(0))[3]; + + + # Setup the output path + my $me = `whoami`; + chomp $me; + $globalHash->{'runid'} = $me . "." . time; + + # if "-ignore false" was provided on the command line, + # it means do run tests even when marked as 'ignore' + if (defined($globalHash->{'ignore'}) && $globalHash->{'ignore'} eq 'false') { + $self->{'ignore'} = 'false'; + } + + if (! defined $globalHash->{'localpathbase'}) { + $globalHash->{'localpathbase'} = '/tmp'; + } + + $globalHash->{'outpath'} = $globalHash->{'outpathbase'} . "/" . $globalHash->{'runid'} . "/"; + $globalHash->{'localpath'} = $globalHash->{'localpathbase'} . "/" . $globalHash->{'runid'} . "/"; + $globalHash->{'webhdfs_url'} = $ENV{'WEBHDFS_URL'}; + $globalHash->{'templeton_url'} = $ENV{'TEMPLETON_URL'}; + $globalHash->{'current_user'} = $ENV{'USER_NAME'}; + $globalHash->{'current_group_user'} = $ENV{'GROUP_USER_NAME'}; + $globalHash->{'current_other_user'} = $ENV{'OTHER_USER_NAME'}; + $globalHash->{'current_group'} = $ENV{'GROUP_NAME'}; + $globalHash->{'keytab_dir'} = $ENV{'KEYTAB_DIR'}; + + + + $globalHash->{'inpdir_local'} = $ENV{'TH_INPDIR_LOCAL'}; + $globalHash->{'inpdir_hdfs'} = $ENV{'TH_INPDIR_HDFS'}; + + $globalHash->{'is_secure_mode'} = $ENV{'SECURE_MODE'}; + + # add libexec location to the path + if (defined($ENV{'PATH'})) { + $ENV{'PATH'} = $globalHash->{'scriptPath'} . ":" . $ENV{'PATH'}; + } else { + $ENV{'PATH'} = $globalHash->{'scriptPath'}; + } + + IPC::Run::run(['mkdir', '-p', $globalHash->{'localpath'}], \undef, $log, $log) or + die "Cannot create localpath directory " . $globalHash->{'localpath'} . + " " . "$ERRNO\n"; + + # Create the temporary directory + IPC::Run::run(['mkdir', '-p', $globalHash->{'tmpPath'}], \undef, $log, $log) or + die "Cannot create temporary directory " . $globalHash->{'tmpPath'} . + " " . "$ERRNO\n"; + + } + +############################################################################### +# Sub: globalCleanup +# Clean up after all tests have run. This gives each individual driver a chance to do +# cleanup. This function will only be called once, after all the tests are +# run. A driver need not implement it. It is a virtual function. +# +# Paramaters: +# globalHash - Top level hash from config file (does not have any group +# or test information in it). +# log - log file handle +# +# Returns: +# None +sub globalCleanup + { + my ($self, $globalHash, $log) = @_; + + IPC::Run::run(['rm', '-rf', $globalHash->{'tmpPath'}], \undef, $log, $log) or + warn "Cannot remove temporary directory " . $globalHash->{'tmpPath'} . + " " . "$ERRNO\n"; + + } + +############################################################################### +# Sub: runTest +# Run a test. This is a pure virtual function. +# +# Parameters: +# testcmd - reference to hash with meta tags and command to run tests. +# Interpretation of the tags and the command are up to the subclass. +# log - reference to a stream pointer for the logs +# +# Returns: +# @returns reference to hash. Contents of hash are defined by the subclass. +# +sub runTest + { + my ($self, $testCmd, $log) = @_; + my $subName = (caller(0))[3]; + + # Handle the various methods of running used in + # the original TestDrivers + + if ( $testCmd->{'url'} ) { + return $self->runCurlCmd( $testCmd, $log); + } else { + die "$subName FATAL Did not find a testCmd that " . + "I know how to handle : " . $testCmd->{'Curl'}; + } + + + } + +############################################################################### + +sub replaceParameters + { + my ($self, $testCmd, $aPfix, $log) = @_; + + my $url = $testCmd->{$aPfix . 'url'}; + $url =~ s/:WEBHDFS_URL:/$testCmd->{'webhdfs_url'}/g; + $url =~ s/:TEMPLETON_URL:/$testCmd->{'templeton_url'}/g; + $url = $self->replaceParametersInArg($url, $testCmd, $log); + $testCmd->{$aPfix . 'url'} = $url; + + $testCmd->{$aPfix . 'upload_file'} = + $self->replaceParametersInArg($testCmd->{$aPfix . 'upload_file'}, $testCmd, $log); + + $testCmd->{$aPfix . 'user_name'} = + $self->replaceParametersInArg($testCmd->{$aPfix . 'user_name'}, $testCmd, $log); + + if (defined $testCmd->{$aPfix . 'post_options'}) { + my @options = @{$testCmd->{$aPfix . 'post_options'}}; + my @new_options = (); + foreach my $option (@options) { + $option = $self->replaceParametersInArg($option, $testCmd, $log); + push @new_options, ($option); + } + $testCmd->{$aPfix . 'post_options'} = \@new_options; + } + if (defined $testCmd->{$aPfix . 'json_field_substr_match'}) { + my $json_matches = $testCmd->{$aPfix . 'json_field_substr_match'}; + my @keys = keys %{$json_matches}; + + foreach my $key (@keys) { + my $new_value = $self->replaceParametersInArg($json_matches->{$key}, $testCmd, $log); + $json_matches->{$key} = $new_value; + } + } + + + } + +############################################################################### +sub replaceParametersInArg + { + my ($self, $arg, $testCmd, $log) = @_; + if(! defined $arg){ + return $arg; + } + my $outdir = $testCmd->{'outpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'}; + $arg =~ s/:UNAME:/$testCmd->{'current_user'}/g; + $arg =~ s/:UNAME_GROUP:/$testCmd->{'current_group_user'}/g; + $arg =~ s/:UNAME_OTHER:/$testCmd->{'current_other_user'}/g; + $arg =~ s/:UGROUP:/$testCmd->{'current_group'}/g; + $arg =~ s/:OUTDIR:/$outdir/g; + $arg =~ s/:INPDIR_HDFS:/$testCmd->{'inpdir_hdfs'}/g; + $arg =~ s/:INPDIR_LOCAL:/$testCmd->{'inpdir_local'}/g; + $arg =~ s/:TNUM:/$testCmd->{'num'}/g; + return $arg; + } + + + +############################################################################### + +sub getBaseCurlCmd(){ + my ($self) = @_; + my @curl_cmd = ("curl", '--silent','--show-error', '-H','Expect:'); + if (defined $ENV{'SOCKS_PROXY'}) { + push @curl_cmd, ('--socks5-hostname', $ENV{'SOCKS_PROXY'}); + } + return @curl_cmd; + +} + +############################################################################### +sub runCurlCmd(){ + my ($self, $testCmd, $log) = @_; + if (defined $testCmd->{'upload_file'}) { + return $self->upload_file($testCmd,$log); + } else { + #if there are setup steps, run them first + if (defined $testCmd->{'setup'}) { + my $i = 0; + foreach my $setupCmd (@{$testCmd->{'setup'}}){ + $i++; + print $log "\nRUNNING SETUP COMMAND: $i\n"; + my $pfix = "setup_${i}_"; + my $setupTestCmd = $self->createSetupCmd($testCmd, $setupCmd, $pfix, $log); + my $setupResult = $self->execCurlCmd($setupTestCmd, $pfix, $log); + + #if status code is set in setup, check if it matches results + if(defined $setupTestCmd->{"${pfix}status_code"}){ + $self->checkResStatusCode($setupResult, $setupTestCmd->{"${pfix}status_code"}, $log); + } + } + } + return $self->execCurlCmd($testCmd, "", $log); + } +} +############################################################################### +sub createSetupCmd(){ + my ($self, $testCmd, $setupCmd, $pfix, $log) = @_; + my $newTestCmd = dclone ($testCmd); + for my $key (keys %$setupCmd){ + $newTestCmd->{$pfix . $key} = $setupCmd->{$key}; + } + return $newTestCmd; +} + +############################################################################### +sub upload_file(){ + my ($self, $testCmd, $log) = @_; + $testCmd->{'method'} = 'PUT'; + my $result = $self->execCurlCmd($testCmd, "", $log); + my $checkRes = $self->checkResStatusCode($result, 100, $log); + if ($checkRes == 0) { + #fail + return 0; + } + my $header = $result->{'header_fields'}; + + #final url where the file should be stored + my $location = $header->{'Location'}; + $testCmd->{'url'} = $location; + + $result = $self->execCurlCmd($testCmd, "", $log); + return $result; +} + +############################################################################### +sub execCurlCmd(){ + my ($self, $testCmd, $argPrefix, $log) = @_; + my @curl_cmd = $self->getBaseCurlCmd(); + # Set up file locations + my $subName = (caller(0))[3]; + + my $filePrefix = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $argPrefix . $testCmd->{'num'}; + my $cmd_body = $filePrefix . ".cmd_body"; + + #results + my $res_header = $filePrefix . ".res_header"; + my $res_body = $filePrefix . ".res_body"; + + my $outdir = $filePrefix . ".out"; + my $stdoutfile = "$outdir/stdout"; + my $stderrfile = "$outdir/stderr"; + + mkpath( [ $outdir ] , 0, 0755) if ( ! -e outdir ); + if ( ! -e $outdir ) { + print $log "$0.$subName FATAL could not mkdir $outdir\n"; + die "$0.$subName FATAL could not mkdir $outdir\n"; + } + + $self->replaceParameters($testCmd, $argPrefix, $log ); + + my $method = $testCmd->{ $argPrefix . 'method'}; + + my $url = $testCmd->{ $argPrefix . 'url'}; + + my @options = (); + if (defined $testCmd->{$argPrefix . 'post_options'}) { + @options = @{$testCmd->{$argPrefix . 'post_options'}}; + } + + #handle authentication based on secure mode + my $user_name = $testCmd->{ $argPrefix . 'user_name' }; + if (defined $testCmd->{'is_secure_mode'} && $testCmd->{'is_secure_mode'} =~ /y.*/i) { + push @curl_cmd, ('--negotiate', '-u', ':'); + + #if keytab dir is defined, look for a keytab file for user and do a kinit + if(defined $testCmd->{'keytab_dir'} && defined $user_name){ + $user_name =~ /(.*?)(\/|$)/; + my $just_uname = $1; #uname without principal + my $keytab_dir = $testCmd->{'keytab_dir'}; + print $log "regex " . "${keytab_dir}/*${just_uname}\.*keytab"; + my @files = bsd_glob( "${keytab_dir}/*${just_uname}\.*keytab" ); + if(scalar @files == 0){ + die "Could not find keytab file for user $user_name in $keytab_dir"; + } elsif(scalar @files > 1){ + die "More than one keytab file found for user $user_name in $keytab_dir"; + } + my @cmd = ('kinit', '-k', '-t', $files[0], $user_name); + print $log "Command @cmd"; + IPC::Run::run(\@cmd, \undef, $log, $log) or + die "Could not kinit as $user_name using " . $files[0] . " $ERRNO"; + } + + } else { + #if mode is unsecure + if (defined $user_name) { + my $user_param = "user.name=${user_name}"; + if ($method eq 'POST' ) { + push @options, $user_param; + } else { + if ($url =~ /\?/) { + #has some parameters in url + $url = $url . '&' . $user_param; + } else { + $url = $url . '?' . $user_param; + } + } + } + + } + + if (defined $testCmd->{'format_header'}) { + push @curl_cmd, ('-H', $testCmd->{'format_header'}); + } + + + + + if (defined $testCmd->{$argPrefix . 'format_header'}) { + push @curl_cmd, ('-H', $testCmd->{$argPrefix . 'format_header'}); + } + + if (defined $testCmd->{$argPrefix . 'upload_file'}) { + push @curl_cmd, ('-T', $testCmd->{$argPrefix . 'upload_file'}); + } + + # if(!defined $testCmd->{'post_options'}){ + # $testCmd->{'post_options'} = \(); + # } + + if (defined $testCmd->{$argPrefix . 'check_call_back'}) { + my $d = HTTP::Daemon->new || die; + $testCmd->{'http_daemon'} = $d; + $testCmd->{'callback_url'} = $d->url . 'templeton/$jobId'; + push @curl_cmd, ('-d', 'callback=' . $testCmd->{'callback_url'}); + # push ${testCmd->{'post_options'}}, ('callback=' . $testCmd->{'callback_url'}); + # #my @options = @{$testCmd->{'post_options'}}; + # print $log "post options @options\n"; + } + + foreach my $option (@options) { + push @curl_cmd, ('-d', $option); + } + + push @curl_cmd, ("-X", $method, "-o", $res_body, "-D", $res_header); + push @curl_cmd, ($url); + + print $log "$0:$subName Going to run command : " . join (' ', @curl_cmd); + print $log "\n"; + + + my %result; + my $out; + my $err; + IPC::Run::run(\@curl_cmd, \undef, $out, $err) + or die "Failed running curl cmd " . join ' ', @curl_cmd; + + $result{'rc'} = $? >> 8; + $result{'stderr'} = $err; + $result{'stdout'} = $out; + $result{'body'} = `cat $res_body`; + + my @full_header = `cat $res_header`; + $result{'full_header'} = join '\n', @full_header; + + #find the final http status code + for my $line ( @full_header){ + if($line =~ /.*(HTTP\/1.1)\s+(\S+)/){ + $result{'status_code'} = $2; + } + } + + my %header_field; + foreach my $line (@full_header) { + chomp $line; + $line =~ /(.*?)\s*:\s*(.*)/; + if (defined $1 && defined $2 ) { + $header_field{$1} = $2; + } + } + $result{'header_fields'} = \%header_field; + + print $log "result : " . dump(%result); + #dump(%result); + + return \%result; + +} + + +############################################################################### +# Sub: generateBenchmark +# Generate benchmark results. This is a pure virtual function. +# +# Parameters: +# benchmarkcmd - reference to hash with meta tags and command to +# generate benchmark. Interpretation of the tags and the command are up to +# log - reference to a stream pointer for the logs +# the subclass. +# +# Returns: +# @returns reference to hash. Contents of hash are defined by the subclass. +# +sub generateBenchmark + { + my %result; + return \%result; + } + +############################################################################### +# Sub: compare +# Compare the results of the test run with the generated benchmark results. +# This is a pure virtual function. +# +# Parameters: +# benchmarkcmd - reference to hash with meta tags and command to +# testResult - reference to hash returned by runTest. +# benchmarkResult - reference to hash returned by generateBenchmark. +# log - reference to a stream pointer for the logs +# testHash - reference to hash with meta tags and commands +# +# Returns: +# @returns reference true if results are the same, false otherwise. "the +# same" is defined by the subclass. +# +sub compare + { + + my ($self, $testResult, $benchmarkResult, $log, $testCmd) = @_; + my $subName = (caller(0))[3]; + + my $result = 1; # until proven wrong... + if (defined $testCmd->{'status_code'}) { + my $res = $self->checkResStatusCode($testResult, $testCmd->{'status_code'}, $log); + if ($res == 0) { + $result = 0; + } + } + + my $json_hash; + my %json_info; + if (defined $testCmd->{'json_field_substr_match'} || $testCmd->{'json_field_match_object'}) { + my $json = new JSON; + $json_hash = $json->utf8->decode($testResult->{'body'}); + my $json_matches = $testCmd->{'json_field_substr_match'}; + my $json_matches_object = $testCmd->{'json_field_match_object'}; + + %json_info = %$json_hash; + if (defined $json_info{'info'}) { + %json_info = %{$json_info{'info'}}; + + } + print $log "\n\n json_info"; + print $log dump(%json_info); + print $log "\n\n"; + + if (defined $json_hash->{'id'}) { + print STDERR "jobid " . $json_hash->{'id'} . "\n"; + $json_info{'id'} = $json_hash->{'id'}; + } + + if(defined $json_matches->{'location_perms'} || defined $json_matches->{'location_group'}){ + $self->setLocationPermGroup(\%json_info, $testCmd, $log); + } + + foreach my $key (keys %$json_matches) { + my $json_field_val = $json_info{$key}; + if( (ref($json_field_val) && ! UNIVERSAL::isa($json_field_val,'SCALAR')) || + (!ref($json_field_val) && ! UNIVERSAL::isa(\$json_field_val,'SCALAR')) ){ + #flatten the object into a string + $json_field_val = dump($json_field_val); + } + my $regex_expected_value = $json_matches->{$key}; + print $log "Comparing $key: $json_field_val with regex /$regex_expected_value/\n"; + + if ($json_field_val !~ /$regex_expected_value/s) { + print $log "$0::$subName INFO check failed:" + . " json pattern check failed. For field " + . "$key, regex <" . $regex_expected_value + . "> did not match the result <" . $json_field_val + . ">\n"; + $result = 0; + } + } + + foreach my $key (keys %$json_matches_object) { + my $json_field_val = $json_info{$key}; + my $regex_expected_obj = $json->utf8->decode($json_matches_object->{$key}); + print $log "Comparing $key: " . dump($json_field_val) . ",expected value: " . dump($regex_expected_obj); + + if (!Compare($json_field_val, $regex_expected_obj)) { + print $log "$0::$subName INFO check failed:" + . " json compare failed. For field " + . "$key, regex <" . dump($regex_expected_obj) + . "> did not match the result <" . dump($json_field_val) + . ">\n"; + $result = 0; + } + } + + + } + + #kill it if there is a request to kill + if($testCmd->{'kill_job_timeout'}){ + sleep $testCmd->{'kill_job_timeout'}; + my $jobid = $json_hash->{'id'}; + if (!defined $jobid) { + print $log "$0::$subName INFO check failed: " + . "no jobid (id field)found in result"; + $result = 0; + } else { + $self->killJob($testCmd, $jobid, $log); + } + } + + + #try to get the call back url request until timeout + if ($result == 1 && defined $testCmd->{'check_call_back'}) { + my $d = $testCmd->{'http_daemon'}; + $d->timeout(300); #wait for 5 mins + my $url_requested; + $testCmd->{'callback_url'} =~ s/\$jobId/$json_hash->{'id'}/g; + print $log "Expanded callback url : <" . $testCmd->{'callback_url'} . ">\n"; + do{ + print $log "Waiting for call back url request\n"; + if (my $c = $d->accept) { + if (my $r = $c->get_request) { + my $durl = $d->url; + chop $durl; + $url_requested = $durl . $r->uri->path ; + print $log "Got request at url <" . $url_requested . ">\n"; + $c->send_status_line(200); + $c->close; + } + undef($c); + } else { + print $log "Timeout on wait on call back url" . "\n"; + $result = 0; + } + }while (defined $url_requested && $url_requested ne $testCmd->{'callback_url'}); + $d->close; + if (!defined $url_requested || $url_requested ne $testCmd->{'callback_url'}) { + print $log "failed to recieve request on call back url"; + $result = 0; + } + + } + + + if (defined $testCmd->{'check_job_created'} || defined $testCmd->{'check_job_complete'}) { + my $jobid = $json_hash->{'id'}; + if (!defined $jobid) { + print $log "$0::$subName INFO check failed: " + . "no jobid (id field)found in result"; + $result = 0; + } else { + my $jobResult = $self->getJobResult($testCmd, $jobid, $log); + my $json = new JSON; + my $res_hash = $json->utf8->decode($jobResult->{'body'}); + if (! defined $res_hash->{'status'}) { + print $log "$0::$subName INFO check failed: " + . "jobresult not defined "; + $result = 0; + } + if ($testCmd->{'check_job_complete'}) { + my $jobComplete; + my $NUM_RETRIES = 60; + my $SLEEP_BETWEEN_RETRIES = 5; + + #first wait for job completion + while ($NUM_RETRIES-- > 0) { + $jobComplete = $res_hash->{'status'}->{'jobComplete'}; + if (defined $jobComplete && lc($jobComplete) eq "true") { + last; + } + sleep $SLEEP_BETWEEN_RETRIES; + $jobResult = $self->getJobResult($testCmd, $jobid, $log); + $json = new JSON; + $res_hash = $json->utf8->decode($jobResult->{'body'}); + } + if ( (!defined $jobComplete) || lc($jobComplete) ne "true") { + print $log "$0::$subName INFO check failed: " + . " timeout on wait for job completion "; + $result = 0; + } else { + # job has completed, check the runState value + my $runState = $res_hash->{'status'}->{'runState'}; + my $runStateVal = $self->getRunStateNum($testCmd->{'check_job_complete'}); + if ( (!defined $runState) || $runState ne $runStateVal) { + print $log "check_job_complete failed. got runState $runState, expected $runStateVal"; + $result = 0; + } + } + } + } + } + return $result; + } + +############################################################################### +sub setLocationPermGroup{ + my ($self, $job_info, $testCmd, $log) = @_; + my $location = $job_info->{'location'}; + $location =~ /hdfs.*:\d+(\/.*)\/(.*)/; + my $dir = $1; + my $file = $2; + + my $testCmdBasics = $self->copyTestBasicConfig($testCmd); + $testCmdBasics->{'method'} = 'GET'; + $testCmdBasics->{'num'} = $testCmdBasics->{'num'} . "_checkFile"; + $testCmdBasics->{'url'} = ':WEBHDFS_URL:/webhdfs/v1' + . $dir . '?op=LISTSTATUS'; + + + my $result = $self->execCurlCmd($testCmdBasics, "", $log); + + my $json = new JSON; + my $json_hash = $json->utf8->decode($result->{'body'}); + my @filestatuses = @{$json_hash->{'FileStatuses'}->{'FileStatus'}}; + foreach my $filestatus (@filestatuses){ + if($filestatus->{'pathSuffix'} eq $file){ + $job_info->{'location_perms'} = numPermToStringPerm($filestatus->{'permission'}); + $job_info->{'location_group'} = $filestatus->{'group'}; + $job_info->{'location_owner'} = $filestatus->{'owner'}; + last; + } + + } + +} + +############################################################################### +#convert decimal string to binary string +sub dec2bin { + my $decimal = shift; + my $binary = unpack("B32", pack("N", $decimal)); + $binary =~ s/^0+(?=\d)//; # remove leading zeros + return $binary; +} + +############################################################################### +#convert single digit of the numeric permission format to string format +sub digitPermToStringPerm{ + my $numPerm = shift; + my $binaryPerm = dec2bin($numPerm); + my $stringPerm = ""; + if($binaryPerm =~ /1\d\d$/){ + $stringPerm .= "r"; + }else{ + $stringPerm .= "-"; + } + + if($binaryPerm =~ /\d1\d$/){ + $stringPerm .= "w"; + }else{ + $stringPerm .= "-"; + } + + if($binaryPerm =~ /\d\d1$/){ + $stringPerm .= "x"; + }else{ + $stringPerm .= "-"; + } + +} + +############################################################################### +# convert numeric permission format to string format +sub numPermToStringPerm{ + my $numPerm = shift; + $numPerm =~ /(\d)(\d)(\d)$/; + return digitPermToStringPerm($1) + . digitPermToStringPerm($2) + . digitPermToStringPerm($3); + +} + +############################################################################### +sub getRunStateNum{ + my ($self, $job_complete_state) = @_; + if (lc($job_complete_state) eq 'success') { + return 2; + } elsif (lc($job_complete_state) eq 'failure') { + return 3; + } elsif (lc($job_complete_state) eq 'killed') { + return 5; + } + +} + + +############################################################################### +sub getJobResult{ + my ($self, $testCmd, $jobid, $log) = @_; + my $testCmdBasics = $self->copyTestBasicConfig($testCmd); + $testCmdBasics->{'method'} = 'GET'; + $testCmdBasics->{'num'} = $testCmdBasics->{'num'} . "_jobStatusCheck"; + $testCmdBasics->{'url'} = ':TEMPLETON_URL:/templeton/v1/queue/' + . $jobid . '?' . "user.name=:UNAME:" ; + return $self->execCurlCmd($testCmdBasics, "", $log); +} +############################################################################### +sub killJob{ + my ($self, $testCmd, $jobid, $log) = @_; + my $testCmdBasics = $self->copyTestBasicConfig($testCmd); + $testCmdBasics->{'method'} = 'DELETE'; + $testCmdBasics->{'num'} = $testCmdBasics->{'num'} . "_killJob"; + $testCmdBasics->{'url'} = ':TEMPLETON_URL:/templeton/v1/queue/' + . $jobid . '?' . "user.name=:UNAME:" ; + return $self->execCurlCmd($testCmdBasics, "", $log); +} +############################################################################### +#Copy test config essential for running a sub command +sub copyTestBasicConfig{ + my ($self, $testCmd) = @_; + my %testCmdBasics; + foreach my $key (keys %$testCmd) { + if ($key ne 'method' + && $key ne 'url' + && $key ne 'upload_file' + && $key ne 'post_options' + ) { + $testCmdBasics{$key} = $testCmd->{$key}; + } + } + # $testCmdBasics{'localpath'} = $testCmd->{'localpath'}; + # $testCmdBasics{'group'} = $testCmd->{'group'}; + # $testCmdBasics{'num'} = $testCmd->{'num'}; + return \%testCmdBasics; +} +############################################################################### +sub checkResStatusCode{ + my ($self, $testResult, $e_status_code, $log) = @_; + my $subName = (caller(0))[3]; + + # print STDERR "expected " . $e_status_code . " was " . $testResult->{'status_code'}; + + if (!defined $testResult->{'status_code'} || + $testResult->{'status_code'} != $e_status_code) { + print $log "$0::$subName INFO Check failed: status_code " . + "$e_status_code expected, test returned " . + "<$testResult->{'status_code'}>\n"; + return 0; + } + return 1; + +} + +############################################################################### +# Sub: recordResults +# Record results of the test run. The required fields are filled in by the +# test harness. This call gives an individual driver a chance to fill in +# additional fields of cmd, cmd_id, expected_results, and actual_results. +# this function does not have to be implemened. +# This is a virtual function. +# +# Parameters: +# status - status of test passing, true or false +# testResult - reference to hash returned by runTest. +# benchmarkResult - reference to hash returned by generateBenchmark. +# dbinfo - reference to hash that will be used to fill in db. +# log - reference to hash that will be used to fill in db. +# +# Returns: +# None +# +sub recordResults + { + } + +############################################################################### +# Sub: cleanup +# Clean up after a test. This gives each individual driver a chance to do +# cleanup. A driver need not implement it. It is a virtual function. +# +# Parameters: +# status - status of test passing, true or false +# testHash - reference to hash that was passed to runTest() and +# generateBenchmark(). +# testResult - reference to hash returned by runTest. +# benchmarkResult - reference to hash returned by generateBenchmark. +# log - reference to a stream pointer for the logs +# +# Returns: +# None +# +sub cleanup + { + } + +############################################################################### +# Sub: run +# Run all the tests in the configuration file. +# +# Parameters: +# testsToRun - reference to array of test groups and ids to run +# testsToMatch - reference to array of test groups and ids to match. +# If a test group_num matches any of these regular expressions it will be run. +# cfg - reference to contents of cfg file +# log - reference to a stream pointer for the logs +# dbh - reference database connection +# testStatuses- reference to hash of test statuses +# confFile - config file name +# startat - test to start at. +# logname - name of the xml log for reporting results +# +# Returns: +# @returns nothing +# failed. +# +sub run + { + my ($self, $testsToRun, $testsToMatch, $cfg, $log, $dbh, $testStatuses, + $confFile, $startat, $logname ) = @_; + + my $subName = (caller(0))[3]; + my $msg=""; + my $duration=0; + my $totalDuration=0; + my $groupDuration=0; + my $sawstart = !(defined $startat); + # Rather than make each driver handle our multi-level cfg, we'll flatten + # the hashes into one for it. + my %globalHash; + + my $runAll = ((scalar(@$testsToRun) == 0) && (scalar(@$testsToMatch) == 0)); + + # Read the global keys + foreach (keys(%$cfg)) { + next if $_ eq 'groups'; + $globalHash{$_} = $cfg->{$_}; + } + + $globalHash{$_} = $cfg->{$_}; + # Do the global setup + $self->globalSetup(\%globalHash, $log); + + my $report=0; + # my $properties= new Properties(0, $globalHash{'propertiesFile'}); + + my %groupExecuted; + foreach my $group (@{$cfg->{'groups'}}) { + + print $log "INFO $subName at ".__LINE__.": Running TEST GROUP(".$group->{'name'}.")\n"; + + my %groupHash = %globalHash; + $groupHash{'group'} = $group->{'name'}; + + # Read the group keys + foreach (keys(%$group)) { + next if $_ eq 'tests'; + $groupHash{$_} = $group->{$_}; + } + + + # Run each test + foreach my $test (@{$group->{'tests'}}) { + # Check if we're supposed to run this one or not. + if (!$runAll) { + # check if we are supposed to run this test or not. + my $foundIt = 0; + foreach (@$testsToRun) { + if (/^$groupHash{'group'}(_[0-9]+)?$/) { + if (not defined $1) { + # In this case it's just the group name, so we'll + # run every test in the group + $foundIt = 1; + last; + } else { + # maybe, it at least matches the group + my $num = "_" . $test->{'num'}; + if ($num eq $1) { + $foundIt = 1; + last; + } + } + } + } + foreach (@$testsToMatch) { + my $protoName = $groupHash{'group'} . "_" . $test->{'num'}; + if ($protoName =~ /$_/) { + if (not defined $1) { + # In this case it's just the group name, so we'll + # run every test in the group + $foundIt = 1; + last; + } else { + # maybe, it at least matches the group + my $num = "_" . $test->{'num'}; + if ($num eq $1) { + $foundIt = 1; + last; + } + } + } + } + + next unless $foundIt; + } + + # This is a test, so run it. + my %testHash = %groupHash; + foreach (keys(%$test)) { + $testHash{$_} = $test->{$_}; + } + + my $testName = $testHash{'group'} . "_" . $testHash{'num'}; + + # if ( $groupExecuted{ $group->{'name'} }== 0 ){ + # $groupExecuted{ $group->{'name'} }=1; + # + # my $xmlDir= $globalHash{'localxmlpathbase'}."/run".$globalHash->{'UID'}; + # mkpath( [ $xmlDir ] , 1, 0777) if ( ! -e $xmlDir ); + # + # my $filename = $group->{'name'}.".xml"; + # $report = new TestReport ( $properties, "$xmlDir/$filename" ); + # $report->purge(); + # } + + # Check that ignore isn't set for this file, group, or test + if (defined $testHash{'ignore'}) { + print $log "Ignoring test $testName, ignore message: " . + $testHash{'ignore'} . "\n"; + next; + } + + # Have we not reached the starting point yet? + if (!$sawstart) { + if ($testName eq $startat) { + $sawstart = 1; + } else { + next; + } + } + + # Check that this test doesn't depend on an earlier test or tests + # that failed. Don't abort if that test wasn't run, just assume the + # user knew what they were doing and set it up right. + my $skipThisOne = 0; + foreach (keys(%testHash)) { + if (/^depends_on/ && defined($testStatuses->{$testHash{$_}}) && + $testStatuses->{$testHash{$_}} ne $passedStr) { + print $log "Skipping test $testName, it depended on " . + "$testHash{$_} which returned a status of " . + "$testStatuses->{$testHash{$_}}\n"; + $testStatuses->{$testName} = $dependStr; + $skipThisOne = 1; + last; + } + } + if ($skipThisOne) { + printResults($testStatuses, $log, "Results so far"); + next; + } + + print $log "\n******************************************************\n"; + print $log "\nTEST: $confFile::$testName\n"; + print $log "******************************************************\n"; + print $log "Beginning test $testName at " . time . "\n"; + my %dbinfo = ( + 'testrun_id' => $testHash{'trid'}, + 'test_type' => $testHash{'driver'}, + #'test_file' => $testHash{'file'}, + 'test_file' => $confFile, + 'test_group' => $testHash{'group'}, + 'test_num' => $testHash{'num'}, + ); + my $beginTime = time; + my $endTime = 0; + my ($testResult, $benchmarkResult); + eval { + $testResult = $self->runTest(\%testHash, $log); + $endTime = time; + $benchmarkResult = $self->generateBenchmark(\%testHash, $log); + my $result = + $self->compare($testResult, $benchmarkResult, $log, \%testHash); + $msg = "INFO: $subName() at ".__LINE__.":Test $testName"; + + if ($result eq $self->{'wrong_execution_mode'}) { + $msg .= " SKIPPED"; + $testStatuses->{$testName} = $skippedStr; + } elsif ($result) { + $msg .= " SUCCEEDED"; + $testStatuses->{$testName} = $passedStr; + + } else { + $msg .= " FAILED"; + $testStatuses->{$testName} = $failedStr; + + } + $msg= "$msg at " . time . "\n"; + #print $msg; + print $log $msg; + $duration = $endTime - $beginTime; + $dbinfo{'duration'} = $duration; + $self->recordResults($result, $testResult + , $benchmarkResult, \%dbinfo, $log); + + }; + + + if ($@) { + $msg= "ERROR $subName at : ".__LINE__." Failed to run test $testName <$@>\n"; + #print $msg; + print $log $msg; + $testStatuses->{$testName} = $abortedStr; + $dbinfo{'duration'} = $duration; + } + + + eval { + $dbinfo{'status'} = $testStatuses->{$testName}; + if ($dbh) { + $dbh->insertTestCase(\%dbinfo); + } + }; + if ($@) { + chomp $@; + warn "Failed to insert test case info, error <$@>\n"; + } + + $self->cleanup($testStatuses->{$testName}, \%testHash, $testResult, + $benchmarkResult, $log); + #$report->testcase( $group->{'name'}, $testName, $duration, $msg, $testStatuses->{$testName}, $testResult ) if ( $report ); + $report->testcase( $group->{'name'}, $testName, $duration, $msg, $testStatuses->{$testName} ) if ( $report ); + $groupDuration = $groupDuration + $duration; + $totalDuration = $totalDuration + $duration; + printResults( $testStatuses, $log, "Results so far" ); + } + + if ( $report ) { + $report->systemOut( $logname, $group->{'name'}); + printGroupResultsXml( $report, $group->{'name'}, $testStatuses, $groupDuration ); + } + $report = 0; + $groupDuration=0; + + + } + + # Do the global cleanup + $self->globalCleanup(\%globalHash, $log); + } + +# TODO These should be removed + +sub tmpIPCRun(){ + + my $self = shift; + my $subName = (caller(0))[3]; + my $runningSubName= shift; + my $refCmd = shift; + my @cmd = @$refCmd; + my $log = shift; + my $msg = shift; + + print $log "$0::$subName INFO Running ( @cmd )\n"; + + my $result= `@cmd`; + if ( $@ ) { + my $msg= "$0::$subName FATAL Failed to run from $runningSubName $msg < $@ >\n$result\n"; + print $log $msg; + die "$msg"; + } + + return $?; +} + +sub tmpIPCRunSplitStdoe { + + my $self = shift; + my $subName = (caller(0))[3]; + my $runningSubName= shift; + my $refCmd = shift; + my @cmd = @$refCmd; + my $dir = shift; + my $log = shift; + my $msg = shift; + my $die = shift; + + + my $failed = 0; + + my $outfilename = $dir."out.tmp"; + my $errfilename = $dir."err.tmp"; + print $log "$0::$subName INFO Running from $runningSubName ( @cmd 1>$outfilename 2>$errfilename )\n"; + #make sure files are writeable + open( TMP, ">$outfilename" ) || die "$0::$subName FATAL: Cannot open $outfilename for writing\n"; + close( TMP ); + open( TMP, ">$errfilename" ) || die "$0::$subName FATAL: Cannot open $errfilename for writing\n"; + close( TMP ); + + #RUN CMD + my $msg; + print $log `@cmd 1>$outfilename 2>$errfilename`; + + my $failed=0; + if ( $@ ) { + $msg= "$0::$subName FATAL < $@ >\n"; + $failed++; + } + + #READ FILES + my $stdout=""; + my $stderr="";; + open( TMP, "$outfilename" ) || die "$0::$subName FATAL: Cannot open $outfilename for reading\n"; + while ( ) { + $stdout .= $_; + } + close( TMP ); + + open( TMP, "$errfilename" ) || die "$0::$subName FATAL: Cannot open $errfilename for reading\n"; + while ( ) { + $stderr .= $_; + } + close( TMP ); + + #DIE IF Test Failed, otherwise return stdout and stderr + if ( $failed ) { + + $msg = "$0::$subName FATAL: Faied from $runningSubName \nSTDOUT:" . $stdout . "\nSTDERR:" . $stderr . "\n" if ( $failed ); + print $log "$msg"; + die $msg if ( $die != "1" ); #die by defaultast + return ( -1, $stdout, $stderr ); + + } + + return ( $?, $stdout, $stderr); +} + +sub tmpIPCRunJoinStdoe { + + my $self = shift; + my $subName = (caller(0))[3]; + my $runningSubName= shift; + my $refCmd = shift; + my @cmd = @$refCmd; + my $outfilename= shift; + my $log = shift; + my $msg = shift; + my $die = shift; + + #make sure files are writeable + open( TMP, ">$outfilename" ) || die "$0::$subName FATAL: Cannot open $outfilename for writing\n"; + close( TMP ); + + #RUN CMD + my $msg; + my $failed=0; + print $log "$0::$subName INFO Running ( @cmd 2>&1$outfilename 2>/dev/null )\n"; + print $log `@cmd 2>&1 > $outfilename 2>/dev/null`; + if ( $@ ) { + $failed++; + $msg= "$0::$subName FATAL < $@ >\n"; + } + + #READ FILES + my $stdoe=""; + open( TMP, "$outfilename" ) || die "$0::$subName FATAL: Cannot open $outfilename for reading\n"; + while ( ) { + $stdoe .= $_; + } + close( TMP ); + + if ( $failed ) { + print $log "$msg"; + die $msg if ( $die != "1" ); #die by default + return ( -1 ); + } + return ( $? ); +} + + +1; Index: src/java/org/apache/hcatalog/mapreduce/MultiOutputFormat.java =================================================================== --- src/java/org/apache/hcatalog/mapreduce/MultiOutputFormat.java (revision 1360995) +++ src/java/org/apache/hcatalog/mapreduce/MultiOutputFormat.java (working copy) @@ -160,7 +160,7 @@ * for multiple output formats. * * @param job the mapreduce job to be submitted - * @return + * @return JobConfigurer */ public static JobConfigurer createConfigurer(Job job) { return JobConfigurer.create(job); @@ -361,7 +361,7 @@ * * @param alias the name used for the OutputFormat during * addOutputFormat - * @return + * @return Job */ public Job getJob(String alias) { Job copy = outputConfigs.get(alias); Index: src/docs/src/documentation/content/xdocs/createtablelike.xml =================================================================== --- src/docs/src/documentation/content/xdocs/createtablelike.xml (revision 0) +++ src/docs/src/documentation/content/xdocs/createtablelike.xml (revision 0) @@ -0,0 +1,127 @@ + + + + + +
+ PUT ddl/database/:db/table/:existingtable/like/:newtable +
+ + +
+ Description +

Create a new HCatalog table like an existing one

+
+ +
+ URL +

http://www.myserver.com/templeton/v1/ddl/database/:db/table/:existingtable/like/:newtable

+
+ +
+ Parameters + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescriptionRequired?Default
:dbThe database nameRequiredNone
:existingtableThe existing table nameRequiredNone
:newtableThe new table name.RequiredNone
groupThe user group to use when creating a tableOptionalNone
permissionsThe permissions string to use when creating a table.OptionalNone
externalAllows you to specify a location so that Hive does not use the default + location for this table.Optionalfalse
ifNotExistsIf true, you will not receive an error if the table already exists.Optionalfalse
locationThe HDFS pathOptionalNone
+
+ +
+ Results + + + + + + + + + + + +
NameDescription
tableThe new table name
databaseThe database name
+
+ +
+ Example + +

Curl Command

+ +% curl -s -X PUT -HContent-type:application/json -d {} \ + 'http://localhost:50111/templeton/v1/ddl/database/default/table/test_table/like/test_table_2?user.name=ctdean' + + +

JSON Output

+ +{ + "table": "test_table_2", + "database": "default" +} + +
+ +
Index: src/docs/src/documentation/content/xdocs/responsetypes.xml =================================================================== --- src/docs/src/documentation/content/xdocs/responsetypes.xml (revision 0) +++ src/docs/src/documentation/content/xdocs/responsetypes.xml (revision 0) @@ -0,0 +1,86 @@ + + + + + +
+ GET :version +
+ + +
+ Description +

Returns a list of the response types supported by Templeton.

+
+ +
+ URL +

http://www.myserver.com/templeton/:version

+
+ +
+ Parameters + + + + + + + + +
NameDescriptionRequired?Default
:versionThe Templeton version number. (Currently this must be "v1")RequiredNone
+
+ +
+ Results + + + + + + + +
NameDescription
responseTypesA list of all supported response types
+
+ +
+ Example + +

Curl Command

+ +% curl -s 'http://localhost:50111/templeton/v1' + + +

JSON Output

+ +{ + "responseTypes": [ + "application/json" + ] +} + + +

JSON Output (error)

+ +{ + "error": "null for uri: http://localhost:50111/templeton/v2" +} + +
+ +
Index: src/docs/src/documentation/content/xdocs/queue.xml =================================================================== --- src/docs/src/documentation/content/xdocs/queue.xml (revision 0) +++ src/docs/src/documentation/content/xdocs/queue.xml (revision 0) @@ -0,0 +1,151 @@ + + + + + +
+ GET queue/:jobid +
+ + +
+ Description +

Check the status of a job and get related job information given its job ID. + Substitute ":jobid" with the job ID received when the job was created.

+
+ +
+ URL +

http://www.myserver.com/templeton/v1/queue/:jobid

+
+ +
+ Parameters + + + + + + + + +
NameDescriptionRequired?Default
:jobidThe job ID to check. This is the ID received when the + job was created.RequiredNone
+
+ +
+ Results + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
statusA JSON object containing the job status information. + See the Hadoop documentation + (Class + JobStatus) for more information.
profileA JSON object containing the job profile information. + See the Hadoop documentation + (Class + JobProfile) for more information. +
idThe job ID.
parentIdThe parent job ID.
percentCompleteThe job completion percentage, for example "75% complete".
exitValueThe job's exit value.
userUser name of the job creator.
callbackThe callback URL, if any.
completedA string representing completed status, for example "done".
+
+ +
+ Example + +

Curl Command

+ +% curl -s 'http://localhost:50111/templeton/v1/queue/job_201112212038_0003?user.name=ctdean' + + +

JSON Output

+ +{ + "status": { + "startTime": 1324529476131, + "username": "ctdean", + "jobID": { + "jtIdentifier": "201112212038", + "id": 4 + }, + "jobACLs": { + }, + "schedulingInfo": "NA", + "failureInfo": "NA", + "jobId": "job_201112212038_0004", + "jobPriority": "NORMAL", + "runState": 2, + "jobComplete": true + }, + "profile": { + "url": "http://localhost:50030/jobdetails.jsp?jobid=job_201112212038_0004", + "jobID": { + "jtIdentifier": "201112212038", + "id": 4 + }, + "user": "ctdean", + "queueName": "default", + "jobFile": "hdfs://localhost:9000/tmp/hadoop-ctdean/mapred/staging/ctdean/.staging/job_201112212038_0004/job.xml", + "jobName": "PigLatin:DefaultJobName", + "jobId": "job_201112212038_0004" + }, + "id": "job_201112212038_0004", + "parentId": "job_201112212038_0003", + "percentComplete": "100% complete", + "exitValue": 0, + "user": "ctdean", + "callback": null, + "completed": "done" +} + +
+ +
Index: src/docs/src/documentation/content/xdocs/listpartitions.xml =================================================================== --- src/docs/src/documentation/content/xdocs/listpartitions.xml (revision 0) +++ src/docs/src/documentation/content/xdocs/listpartitions.xml (revision 0) @@ -0,0 +1,107 @@ + + + + + +
+ GET ddl/database/:db/table/:table/partition +
+ + +
+ Description +

List all the partitions in an HCatalog table.

+
+ +
+ URL +

http://www.myserver.com/templeton/v1/ddl/database/:db/table/:table/partition

+
+ +
+ Parameters + + + + + + + + + + + + + + +
NameDescriptionRequired?Default
:dbThe database nameRequiredNone
:tableThe table nameRequiredNone
+
+ +
+ Results + + + + + + + + + + + + + + + +
NameDescription
partitionsA list of partition values and of partition names
databaseThe database name
tableThe table name
+
+ +
+ Example + +

Curl Command

+ +% curl -s 'http://localhost:50111/templeton/v1/ddl/database/default/table/my_table/partition?user.name=ctdean' + + +

JSON Output

+ +{ + "partitions": [ + { + "values": [ + { + "columnName": "dt", + "columnValue": "20120101" + }, + { + "columnName": "country", + "columnValue": "US" + } + ], + "name": "dt='20120101',country='US'" + } + ], + "database": "default", + "table": "my_table" +} + +
+ +
Index: src/docs/src/documentation/content/xdocs/hive.xml =================================================================== --- src/docs/src/documentation/content/xdocs/hive.xml (revision 0) +++ src/docs/src/documentation/content/xdocs/hive.xml (revision 0) @@ -0,0 +1,135 @@ + + + + + +
+ POST hive +
+ + +
+ Description +

Runs a Hive query or set of commands.

+
+ +
+ URL +

http://www.myserver.com/templeton/v1/hive

+
+ +
+ Parameters + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescriptionRequired?Default
executeString containing an entire, short hive program to run.One of either "execute" or "file" is requiredNone
fileHDFS file name of a hive program to run.One of either "exec" or "file" is requiredNone
defineSet a Hive configuration variable using the syntax + define=NAME=VALUE.OptionalNone
statusdirA directory where Templeton will write the status of the + Hive job. If provided, it is the caller's responsibility + to remove this directory when done.OptionalNone
callbackDefine a URL to be called upon job completion. You may embed a specific + job ID into this URL using $jobId. This tag + will be replaced in the callback URL with this job's job ID.OptionalNone
+
+ +
+ Results + + + + + + + + + + +
NameDescription
idA string containing the job ID similar to "job_201110132141_0001".
infoA JSON object containing the information returned when the job was queued. + See the Hadoop documentation + (Class + TaskController) for more information.
+
+ +
+ Example + +

Curl Command

+ +% curl -s -d user.name=ctdean \ + -d execute="select+*+from+pokes;" \ + -d statusdir="pokes.output" \ + 'http://localhost:50111/templeton/v1/hive' + + +

JSON Output

+ +{ + "id": "job_201111111311_0005", + "info": { + "stdout": "templeton-job-id:job_201111111311_0005 + ", + "stderr": "", + "exitcode": 0 + } +} + + +

Results

+ +% hadoop fs -ls pokes.output +Found 2 items +-rw-r--r-- 1 ctdean supergroup 610 2011-11-11 13:22 /user/ctdean/pokes.output/stderr +-rw-r--r-- 1 ctdean supergroup 15 2011-11-11 13:22 /user/ctdean/pokes.output/stdout + +% hadoop fs -cat pokes.output/stdout +1 a +2 bb +3 ccc + +
+ +
Index: src/docs/src/documentation/content/xdocs/deletepartition.xml =================================================================== --- src/docs/src/documentation/content/xdocs/deletepartition.xml (revision 0) +++ src/docs/src/documentation/content/xdocs/deletepartition.xml (revision 0) @@ -0,0 +1,122 @@ + + + + + +
+ DELETE ddl/database/:db/table/:table/partition/:partition +
+ + +
+ Description +

Delete (drop) a partition in an HCatalog table.

+
+ +
+ URL +

http://www.myserver.com/templeton/v1/ddl/database/:db/table/:table/partition/:partition

+
+ +
+ Parameters + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescriptionRequired?Default
:dbThe database nameRequiredNone
:tableThe table nameRequiredNone
:partitionThe partition name, col_name='value' list. Be careful to properly + encode the quote for http, for example, country=%27algeria%27.RequiredNone
ifExistsHive returns an error if the partition specified does not exist, + unless ifExists is set to true.Optionalfalse
groupThe user group to useOptionalNone
permissionsThe permissions string to use. The format is + "rwxrw-r-x".OptionalNone
+
+ +
+ Results + + + + + + + + + + + + + + + +
NameDescription
partitionThe partition name
tableThe table name
databaseThe database name
+
+ +
+ Example + +

Curl Command

+ +% curl -s -X DELETE \ + 'http://localhost:50111/templeton/v1/ddl/database/default/table/test_table/partition/country=%27algeria%27?user.name=ctdean' + + +

JSON Output

+ +{ + "partition": "country='algeria'", + "table": "test_table", + "database": "default" +} + +
+ +
Index: src/docs/src/documentation/content/xdocs/addproperty.xml =================================================================== --- src/docs/src/documentation/content/xdocs/addproperty.xml (revision 0) +++ src/docs/src/documentation/content/xdocs/addproperty.xml (revision 0) @@ -0,0 +1,119 @@ + + + + + +
+ PUT ddl/database/:db/table/:table/property/:property +
+ + +
+ Description +

Add a single property on an HCatalog table. + This will also reset an existing property.

+
+ +
+ URL +

http://www.myserver.com/templeton/v1/ddl/database/:db/table/:table/property/:property

+
+ +
+ Parameters + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescriptionRequired?Default
:dbThe database nameRequiredNone
:tableThe table nameRequiredNone
:propertyThe property nameRequiredNone
groupThe user group to useOptionalNone
permissionsThe permissions string to useOptionalNone
valueThe property valueRequiredNone
+
+ +
+ Results + + + + + + + + + + + + + + + +
NameDescription
databaseThe database name
tableThe table name
propertyThe property name
+
+ +
+ Example + +

Curl Command

+ +% curl -s -X PUT -HContent-type:application/json -d '{ "value": "apples" }' \ + 'http://localhost:50111/templeton/v1/ddl/database/default/table/test_table/property/fruit?user.name=ctdean' + + +

JSON Output

+ +{ + "property": "fruit", + "table": "test_table", + "database": "default" +} + +
+ +
Index: src/docs/src/documentation/content/xdocs/ddl.xml =================================================================== --- src/docs/src/documentation/content/xdocs/ddl.xml (revision 0) +++ src/docs/src/documentation/content/xdocs/ddl.xml (revision 0) @@ -0,0 +1,128 @@ + + + + + +
+ POST ddl +
+ + +
+ Description +

Performs an + HCatalog DDL command. The command is executed immediately upon request. + Responses are limited to 1MB. For requests which may return longer results + consider using the Hive resource as an alternative.

+
+ +
+ URL +

http://www.myserver.com/templeton/v1/ddl

+
+ +
+ Parameters + + + + + + + + + + + + + + + + + + + + +
NameDescriptionRequired?Default
execThe HCatalog ddl string to executeRequiredNone
groupThe user group to use when creating a tableOptionalNone
permissionsThe permissions string to use when creating a table. The format is + "rwxrw-r-x".OptionalNone
+
+ +
+ Results + + + + + + + + + + + + + + +
NameDescription
stdoutA string containing the result HCatalog sent to standard out (possibly empty).
stderrA string containing the result HCatalog sent to standard error + (possibly empty).
exitcodeThe exitcode HCatalog returned.
+
+ +
+ Example + +

Curl Command

+ +% curl -s -d user.name=ctdean \ + -d 'exec=show tables;' \ + 'http://localhost:50111/templeton/v1/ddl' + + +

JSON Output

+ +{ + "stdout": "important_table + my_other_table + my_table + my_table_2 + pokes + ", + "stderr": "WARNING: org.apache.hadoop.metrics.jvm.EventCounter is deprecated... + Hive history file=/tmp/ctdean/hive_job_log_ctdean_201111111258_2117356679.txt + OK + Time taken: 1.202 seconds + ", + "exitcode": 0 +} + + +

JSON Output (error)

+ +{ + "stdout": "", + "stderr": "WARNING: org.apache.hadoop.metrics.jvm.EventCounter is deprecated... + Hive history file=/tmp/ctdean/hive_job_log_ctdean_201204051246_689834593.txt + FAILED: Parse Error: line 1:5 Failed to recognize predicate 'tab'... + + ", + "exitcode": 11 +} + + +
+ +
Index: src/docs/src/documentation/content/xdocs/images/templetonlogov2.png =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/octet-stream Property changes on: src/docs/src/documentation/content/xdocs/images/templetonlogov2.png ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Index: src/docs/src/documentation/content/xdocs/images/TempletonArch.jpg =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/octet-stream Property changes on: src/docs/src/documentation/content/xdocs/images/TempletonArch.jpg ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Index: src/docs/src/documentation/content/xdocs/images/templetontitle.jpg =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/octet-stream Property changes on: src/docs/src/documentation/content/xdocs/images/templetontitle.jpg ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Index: src/docs/src/documentation/content/xdocs/images/templetonlogo.png =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/octet-stream Property changes on: src/docs/src/documentation/content/xdocs/images/templetonlogo.png ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Index: src/docs/src/documentation/content/xdocs/listtables.xml =================================================================== --- src/docs/src/documentation/content/xdocs/listtables.xml (revision 0) +++ src/docs/src/documentation/content/xdocs/listtables.xml (revision 0) @@ -0,0 +1,122 @@ + + + + + +
+ GET ddl/database/:db/table +
+ + +
+ Description +

List the tables in an HCatalog database.

+
+ +
+ URL +

http://www.myserver.com/templeton/v1/ddl/database/:db/table

+
+ +
+ Parameters + + + + + + + + + + + + + + +
NameDescriptionRequired?Default
:dbThe database nameRequiredNone
likeList only tables whose names match the specified patternOptional"*" (List all tables)
+
+ +
+ Results + + + + + + + + + + + +
NameDescription
tablesA list of table names
databaseThe database name
+
+ +
+ Example + +

Curl Command

+ +% curl -s 'http://localhost:50111/templeton/v1/ddl/database/default/table?user.name=ctdean&like=m*' + + +

JSON Output

+ +{ + "tables": [ + "my_table", + "my_table_2", + "my_table_3" + ], + "database": "default" +} + + +

JSON Output (error)

+ +{ + "errorDetail": " + org.apache.hadoop.hive.ql.metadata.HiveException: ERROR: The database defaultsd does not exist. + at org.apache.hadoop.hive.ql.exec.DDLTask.switchDatabase(DDLTask.java:3122) + at org.apache.hadoop.hive.ql.exec.DDLTask.execute(DDLTask.java:224) + at org.apache.hadoop.hive.ql.exec.Task.executeTask(Task.java:134) + at org.apache.hadoop.hive.ql.exec.TaskRunner.runSequential(TaskRunner.java:57) + at org.apache.hadoop.hive.ql.Driver.launchTask(Driver.java:1332) + at org.apache.hadoop.hive.ql.Driver.execute(Driver.java:1123) + at org.apache.hadoop.hive.ql.Driver.run(Driver.java:931) + at org.apache.hcatalog.cli.HCatDriver.run(HCatDriver.java:42) + at org.apache.hcatalog.cli.HCatCli.processCmd(HCatCli.java:247) + at org.apache.hcatalog.cli.HCatCli.processLine(HCatCli.java:203) + at org.apache.hcatalog.cli.HCatCli.main(HCatCli.java:162) + at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39) + at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) + at java.lang.reflect.Method.invoke(Method.java:597) + at org.apache.hadoop.util.RunJar.main(RunJar.java:156) + ", + "error": "FAILED: Error in metadata: ERROR: The database defaultsd does not exist.", + "errorCode": 500, + "database": "defaultsd" +} + + + +
+ +
Index: src/docs/src/documentation/content/xdocs/deletedb.xml =================================================================== --- src/docs/src/documentation/content/xdocs/deletedb.xml (revision 0) +++ src/docs/src/documentation/content/xdocs/deletedb.xml (revision 0) @@ -0,0 +1,120 @@ + + + + + +
+ DELETE ddl/database/:db +
+ + +
+ Description +

Delete a database.

+
+ +
+ URL +

http://www.myserver.com/templeton/v1/ddl/database/:db

+
+ +
+ Parameters + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescriptionRequired?Default
:dbThe database nameRequiredNone
ifExistsHive returns an error if the database specified does not exist, + unless ifExists is set to true.Optionalfalse
optionParameter set to either "restrict" or "cascade". Restrict will remove the + schema if all the tables are empty. Cascade removes everything including + data and definitions.OptionalNone
groupThe user group to useOptionalNone
permissionsThe permissions string to use. The format is + "rwxrw-r-x".OptionalNone
+
+ +
+ Results + + + + + + + +
NameDescription
databaseThe database name
+
+ +
+ Example + +

Curl Command

+ +% curl -s -X DELETE "http://localhost:50111/templeton/v1/ddl/database/newdb?user.name=ctdean" + + +

JSON Output

+ +{ + "database":"newdb" +} + + +

JSON Output (error)

+ +{ + "errorDetail": " + NoSuchObjectException(message:There is no database named my_db) + at org.apache.hadoop.hive.metastor... + ", + "error": "There is no database named newdb", + "errorCode": 404, + "database": "newdb" +} + + +
+ +
Index: src/docs/src/documentation/content/xdocs/descpartition.xml =================================================================== --- src/docs/src/documentation/content/xdocs/descpartition.xml (revision 0) +++ src/docs/src/documentation/content/xdocs/descpartition.xml (revision 0) @@ -0,0 +1,155 @@ + + + + + +
+ GET ddl/database/:db/table/:table/partition/:partition +
+ + +
+ Description +

Describe a single partition in an HCatalog table.

+
+ +
+ URL +

http://www.myserver.com/templeton/v1/ddl/database/:db/table/:table/partition/:partition

+
+ +
+ Parameters + + + + + + + + + + + + + + + + + + + + +
NameDescriptionRequired?Default
:dbThe database nameRequiredNone
:tableThe table nameRequiredNone
:partitionThe partition name, col_name='value' list. Be careful to properly + encode the quote for http, for example, country=%27algeria%27.RequiredNone
+
+ +
+ Results + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
databaseThe database name
tableThe table name
partitionThe partition name
partitionedTrue if the table is partitioned
locationLocation of table
outputFormatOutput format
columnslist of column names, types, and comments
ownerThe owner's user name
partitionColumnsList of the partition columns
inputFormatInput format
+
+ +
+ Example + +

Curl Command

+ +% curl -s \ + 'http://localhost:50111/templeton/v1/ddl/database/default/table/mytest/partition/country=%27US%27?user.name=ctdean' + + +

JSON Output

+ +{ + "partitioned": true, + "location": "hdfs://ip-10-77-6-151.ec2.internal:8020/apps/hive/warehouse/mytest/loc1", + "outputFormat": "org.apache.hadoop.hive.ql.io.RCFileOutputFormat", + "columns": [ + { + "name": "i", + "type": "int" + }, + { + "name": "j", + "type": "bigint" + }, + { + "name": "ip", + "comment": "IP Address of the User", + "type": "string" + } + ], + "owner": "rachel", + "partitionColumns": [ + { + "name": "country", + "type": "string" + } + ], + "inputFormat": "org.apache.hadoop.hive.ql.io.RCFileInputFormat", + "database": "default", + "table": "mytest", + "partition": "country='US'" +} + +
+ +
Index: src/docs/src/documentation/content/xdocs/createdb.xml =================================================================== --- src/docs/src/documentation/content/xdocs/createdb.xml (revision 0) +++ src/docs/src/documentation/content/xdocs/createdb.xml (revision 0) @@ -0,0 +1,112 @@ + + + + + +
+ PUT ddl/database/:db +
+ + +
+ Description +

Create a database.

+
+ +
+ URL +

http://www.myserver.com/templeton/v1/ddl/database/:db

+
+ +
+ Parameters + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescriptionRequired?Default
:dbThe database nameRequiredNone
groupThe user group to useOptionalNone
permissionsThe permissions string to useOptionalNone
locationThe database locationOptionalNone
commentA comment for the database, like a descriptionOptionalNone
propertiesThe database propertiesOptionalNone
+
+ +
+ Results + + + + + + + +
NameDescription
databaseThe database name
+
+ +
+ Example + +

Curl Command

+ +% curl -s -X PUT -HContent-type:application/json \ + -d '{ "comment":"Hello there", + "location":"hdfs://localhost:9000/user/hive/my_warehouse", + "properties":{"a":"b"}}' \ + 'http://localhost:50111/templeton/v1/ddl/database/newdb?user.name=rachel' + + +

JSON Output

+ +{ + "database":"newdb" +} + +
+ +
Index: src/docs/src/documentation/content/xdocs/resources.xml =================================================================== --- src/docs/src/documentation/content/xdocs/resources.xml (revision 0) +++ src/docs/src/documentation/content/xdocs/resources.xml (revision 0) @@ -0,0 +1,123 @@ + + + + + +
+ Templeton Resources +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ResourceDescription
:versionReturns a list of supported response types.
statusReturns the Templeton server status.
versionReturns the a list of supported versions and the current version.
ddlPerforms an HCatalog DDL command.
ddl/databaseList HCatalog databases.
ddl/database/:db (GET)Describe an HCatalog database.
ddl/database/:db (PUT)Create an HCatalog database.
ddl/database/:db (DELETE)Delete (drop) an HCatalog database.
ddl/database/:db/tableList the tables in an HCatalog database.
ddl/database/:db/table/:table (GET)Describe an HCatalog table.
ddl/database/:db/table/:table (PUT)Create a new HCatalog table.
ddl/database/:db/table/:table (POST)Rename an HCatalog table.
ddl/database/:db/table/:table (DELETE)Delete (drop) an HCatalog table.
ddl/database/:db/table/:existingtable/like/:newtable (PUT)Create a new HCatalog table like an existing one.
ddl/database/:db/table/:table/partionList all partitions in an HCatalog table.
ddl/database/:db/table/:table/partion/:partition (GET)Describe a single partition in an HCatalog table.
ddl/database/:db/table/:table/partion/:partition (PUT)Create a partition in an HCatalog table.
ddl/database/:db/table/:table/partion/:partition (DELETE)Delete (drop) a partition in an HCatalog table.
ddl/database/:db/table/:table/columnList the columns in an HCatalog table.
ddl/database/:db/table/:table/column/:column (GET)Describe a single column in an HCatalog table.
ddl/database/:db/table/:table/column/:column (PUT)Create a column in an HCatalog table.
ddl/database/:db/table/:table/property (GET)List table properties.
ddl/database/:db/table/:table/property/:property (GET)Return the value of a single table property.
ddl/database/:db/table/:table/property/:property (PUT)Set a table property.
mapreduce/streamingCreates and queues Hadoop streaming MapReduce jobs.
mapreduce/jarCreates and queues standard Hadoop MapReduce jobs.
pigCreates and queues Pig jobs.
hiveRuns Hive queries and commands.
queueReturns a list of all jobids registered for the user.
queue/:jobid (GET)Returns the status of a job given its ID.
queue/:jobid (DELETE)Kill a job given its ID.
+ +
Index: src/docs/src/documentation/content/xdocs/deletetable.xml =================================================================== --- src/docs/src/documentation/content/xdocs/deletetable.xml (revision 0) +++ src/docs/src/documentation/content/xdocs/deletetable.xml (revision 0) @@ -0,0 +1,109 @@ + + + + + +
+ DELETE ddl/database/:db/table/:table +
+ + +
+ Description +

Delete (drop) an HCatalog table.

+
+ +
+ URL +

http://www.myserver.com/templeton/v1/ddl/database/:db/table/:table

+
+ +
+ Parameters + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescriptionRequired?Default
:dbThe database nameRequiredNone
:tableThe table nameRequiredNone
ifExistsHive 0.70 and later returns an error if the table specified does not exist, + unless ifExists is set to true.Optionalfalse
groupThe user group to useOptionalNone
permissionsThe permissions string to use. The format is + "rwxrw-r-x".OptionalNone
+
+ +
+ Results + + + + + + + + + + + +
NameDescription
tableThe table name
databaseThe database name
+
+ +
+ Example + +

Curl Command

+ +% curl -s -X DELETE 'http://localhost:50111/templeton/v1/ddl/database/default/table/test_table?user.name=ctdean' + + +

JSON Output

+ +{ + "table": "test_table", + "database": "default" +} + +
+ +
Index: src/docs/src/documentation/content/xdocs/renametable.xml =================================================================== --- src/docs/src/documentation/content/xdocs/renametable.xml (revision 0) +++ src/docs/src/documentation/content/xdocs/renametable.xml (revision 0) @@ -0,0 +1,120 @@ + + + + + +
+ POST ddl/database/:db/table/:table +
+ + +
+ Description +

Rename an HCatalog table.

+
+ +
+ URL +

http://www.myserver.com/templeton/v1/ddl/database/:db/table/:table

+
+ +
+ Parameters + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescriptionRequired?Default
:dbThe database nameRequiredNone
:tableThe existing (old) table nameRequiredNone
renameThe new table nameRequiredNone
groupThe user group to useOptionalNone
permissionsThe permissions string to use. The format is + "rwxrw-r-x".OptionalNone
+
+ +
+ Results + + + + + + + + + + + +
NameDescription
tableThe new table name
databaseThe database name
+
+ +
+ Example + +

Curl Command

+ +% curl -s -d user.name=ctdean \ + -d rename=test_table_2 \ + 'http://localhost:50111/templeton/v1/ddl/database/default/table/test_table' + + +

JSON Output

+ +{ + "table": "test_table_2", + "database": "default" +} + + +

JSON Output (error)

+ +{ + "error": "Table test_table does not exist", + "errorCode": 404, + "database": "default", + "table": "test_table_2" +} + +
+ +
Index: src/docs/src/documentation/content/xdocs/mapreducestreaming.xml =================================================================== --- src/docs/src/documentation/content/xdocs/mapreducestreaming.xml (revision 0) +++ src/docs/src/documentation/content/xdocs/mapreducestreaming.xml (revision 0) @@ -0,0 +1,189 @@ + + + + + +
+ POST mapreduce/streaming +
+ + + +
+ Description +

Create and queue an + Hadoop + streaming MapReduce job.

+
+ +
+ URL +

http://www.myserver.com/templeton/v1/mapreduce/streaming

+
+ +
+ Parameters + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescriptionRequired?Default
inputLocation of the input data in Hadoop.RequiredNone
outputLocation in which to store the output data. If not specified, + Templeton will store the output in a location that can be discovered + using the queue resource.OptionalSee description
mapperLocation of the mapper program in Hadoop.RequiredNone
reducerLocation of the reducer program in Hadoop.RequiredNone
fileAdd an HDFS file to the distributed cache.OptionalNone
defineSet an Hadoop configuration variable using the syntax + define=NAME=VALUEOptionalNone
cmdenvSet an environment variable using the syntax + cmdenv=NAME=VALUEOptionalNone
argSet a program argument.OptionalNone
statusdirA directory where Templeton will write the status of the + Map Reduce job. If provided, it is the caller's responsibility + to remove this directory when done.OptionalNone
callbackDefine a URL to be called upon job completion. You may embed a specific + job ID into this URL using $jobId. This tag + will be replaced in the callback URL with this job's job ID.OptionalNone
+
+ +
+ Results + + + + + + + + + + +
NameDescription
idA string containing the job ID similar to "job_201110132141_0001".
infoA JSON object containing the information returned when the job was queued. + See the Hadoop documentation + (Class + TaskController) for more information.
+
+ +
+ Example + +

Code and Data Setup

+ +% cat mydata/file01 mydata/file02 +Hello World Bye World +Hello Hadoop Goodbye Hadoop + +% hadoop fs -put mydata/ . + +% hadoop fs -ls mydata +Found 2 items +-rw-r--r-- 1 ctdean supergroup 23 2011-11-11 13:29 /user/ctdean/mydata/file01 +-rw-r--r-- 1 ctdean supergroup 28 2011-11-11 13:29 /user/ctdean/mydata/file02 + + +

Curl Command

+ +% curl -s -d user.name=ctdean \ + -d input=mydata \ + -d output=mycounts \ + -d mapper=/bin/cat \ + -d reducer="/usr/bin/wc -w" \ + 'http://localhost:50111/templeton/v1/mapreduce/streaming' + + +

JSON Output

+ +{ + "id": "job_201111111311_0008", + "info": { + "stdout": "packageJobJar: [] [/Users/ctdean/var/hadoop/hadoop-0.20.205.0/share/hadoop/contrib/streaming/hadoop-streaming-0.20.205.0.jar... + templeton-job-id:job_201111111311_0008 + ", + "stderr": "11/11/11 13:26:43 WARN mapred.JobClient: Use GenericOptionsParser for parsing the arguments + 11/11/11 13:26:43 INFO mapred.FileInputFormat: Total input paths to process : 2 + ", + "exitcode": 0 + } +} + + +

Results

+ +% hadoop fs -ls mycounts +Found 3 items +-rw-r--r-- 1 ctdean supergroup 0 2011-11-11 13:27 /user/ctdean/mycounts/_SUCCESS +drwxr-xr-x - ctdean supergroup 0 2011-11-11 13:26 /user/ctdean/mycounts/_logs +-rw-r--r-- 1 ctdean supergroup 10 2011-11-11 13:27 /user/ctdean/mycounts/part-00000 + +% hadoop fs -cat mycounts/part-00000 + 8 + +
+ +
Index: src/docs/src/documentation/content/xdocs/queuelist.xml =================================================================== --- src/docs/src/documentation/content/xdocs/queuelist.xml (revision 0) +++ src/docs/src/documentation/content/xdocs/queuelist.xml (revision 0) @@ -0,0 +1,69 @@ + + + + + +
+ GET queue +
+ + +
+ Description +

Return a list of all job IDs registered to the user.

+
+ +
+ URL +

http://www.myserver.com/templeton/v1/queue

+
+ +
+ Parameters +

Only the standard parameters + are accepted.

+
+ +
+ Results + + + + + +
NameDescription
idsA list of all job IDs registered to the user.
+
+ +
+ Example + +

Curl Command

+ +% curl -s 'http://localhost:50111/templeton/v1/queue?user.name=ctdean' + + +

JSON Output

+ +{ + "job_201111111311_0008", + "job_201111111311_0012" +} + +
+ +
Index: src/docs/src/documentation/content/xdocs/ddlsummary.xml =================================================================== --- src/docs/src/documentation/content/xdocs/ddlsummary.xml (revision 0) +++ src/docs/src/documentation/content/xdocs/ddlsummary.xml (revision 0) @@ -0,0 +1,93 @@ + + + + + +
+ Templeton DDL Resources +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ResourceDescription
ddlPerforms an HCatalog DDL command.
ddl/databaseList HCatalog databases.
ddl/database/:db (GET)Describe an HCatalog database.
ddl/database/:db (PUT)Create an HCatalog database.
ddl/database/:db (DELETE)Delete (drop) an HCatalog database.
ddl/database/:db/tableList the tables in an HCatalog database.
ddl/database/:db/table/:table (GET)Describe an HCatalog table.
ddl/database/:db/table/:table (PUT)Create a new HCatalog table.
ddl/database/:db/table/:table (POST)Rename an HCatalog table.
ddl/database/:db/table/:table (DELETE)Delete (drop) an HCatalog table.
ddl/database/:db/table/:existingtable/like/:newtable (PUT)Create a new HCatalog table like an existing one.
ddl/database/:db/table/:table/partionList all partitions in an HCatalog table.
ddl/database/:db/table/:table/partion/:partition (GET)Describe a single partition in an HCatalog table.
ddl/database/:db/table/:table/partion/:partition (PUT)Create a partition in an HCatalog table.
ddl/database/:db/table/:table/partion/:partition (DELETE)Delete (drop) a partition in an HCatalog table.
ddl/database/:db/table/:table/columnList the columns in an HCatalog table.
ddl/database/:db/table/:table/column/:column (GET)Describe a single column in an HCatalog table.
ddl/database/:db/table/:table/column/:column (PUT)Create a column in an HCatalog table.
ddl/database/:db/table/:table/property (GET)List table properties.
ddl/database/:db/table/:table/property/:property (GET)Return the value of a single table property.
ddl/database/:db/table/:table/property/:property (PUT)Set a table property.
+ +
Index: src/docs/src/documentation/content/xdocs/createcolumn.xml =================================================================== --- src/docs/src/documentation/content/xdocs/createcolumn.xml (revision 0) +++ src/docs/src/documentation/content/xdocs/createcolumn.xml (revision 0) @@ -0,0 +1,126 @@ + + + + + +
+ PUT ddl/database/:db/table/:table/column/:column +
+ + +
+ Description +

Create a column in an HCatalog table.

+
+ +
+ URL +

http://www.myserver.com/templeton/v1/ddl/database/:db/table/:table/column/:column

+
+ +
+ Parameters + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescriptionRequired?Default
:dbThe database nameRequiredNone
:tableThe table nameRequiredNone
:columnThe column nameRequiredNone
groupThe user group to useOptionalNone
permissionsThe permissions string to useOptionalNone
typeThe type of column to add, like "string" or "int"RequiredNone
commentThe column comment, like a descriptionOptionalNone
+
+ +
+ Results + + + + + + + + + + + + + + + +
NameDescription
columnThe column name
tableThe table name
databaseThe database name
+
+ +
+ Example + +

Curl Command

+ +% curl -s -X PUT -HContent-type:application/json \ + -d '{"type": "string", "comment": "The brand name"}' \ + 'http://localhost:50111/templeton/v1/ddl/database/default/table/test_table/column/brand?user.name=ctdean' + + +

JSON Output

+ +{ + "column": "brand", + "table": "test_table", + "database": "default" +} + +
+ +
Index: src/docs/src/documentation/content/xdocs/status.xml =================================================================== --- src/docs/src/documentation/content/xdocs/status.xml (revision 0) +++ src/docs/src/documentation/content/xdocs/status.xml (revision 0) @@ -0,0 +1,75 @@ + + + + + +
+ GET status +
+ + +
+ Description +

Returns the current status of the Templeton server. + Useful for heartbeat monitoring.

+
+ +
+ URL +

http://www.myserver.com/templeton/v1/status

+
+ +
+ Parameters +

Only the standard parameters + are accepted.

+
+ +
+ Results + + + + + + + + + + +
NameDescription
status"ok" if the Templeton server was contacted.
versionString containing the version number similar to "v1".
+
+ +
+ Example + +

Curl Command

+ +% curl -s 'http://localhost:50111/templeton/v1/status' + + +

JSON Output

+ +{ + "status": "ok", + "version": "v1" +} + +
+ +
Index: src/docs/src/documentation/content/xdocs/descproperty.xml =================================================================== --- src/docs/src/documentation/content/xdocs/descproperty.xml (revision 0) +++ src/docs/src/documentation/content/xdocs/descproperty.xml (revision 0) @@ -0,0 +1,112 @@ + + + + + +
+ GET ddl/database/:db/table/:table/property/:property +
+ + +
+ Description +

Return the value of a single table property.

+
+ +
+ URL +

http://www.myserver.com/templeton/v1/ddl/database/:db/table/:table/property/:property

+
+ +
+ Parameters + + + + + + + + + + + + + + + + + + + + + +
NameDescriptionRequired?Default
:dbThe database nameRequiredNone
:tableThe table nameRequiredNone
:propertyThe property nameRequiredNone
+
+ +
+ Results + + + + + + + + + + + + + + + +
NameDescription
propertyThe requested property's name: value pair
databaseThe database name
tableThe table name
+
+ +
+ Example + +

Curl Command

+ +% curl -s 'http://localhost:50111/templeton/v1/ddl/database/default/table/test_table/property/fruit?user.name=ctdean' + + +

JSON Output

+ +{ + "property": { + "fruit": "apple" + }, + "table": "test_table", + "database": "default" +} + + +

JSON Output (error)

+ +{ + "error": "Table test_table does not exist", + "errorCode": 404, + "database": "default", + "table": "test_table" +} + +
+ +
Index: src/docs/src/documentation/content/xdocs/descdb.xml =================================================================== --- src/docs/src/documentation/content/xdocs/descdb.xml (revision 0) +++ src/docs/src/documentation/content/xdocs/descdb.xml (revision 0) @@ -0,0 +1,103 @@ + + + + + +
+ GET ddl/database/:db +
+ + +
+ Description +

Describe a database. (Note: this resource has a "format=extended" parameter however + the output structure does not change if it is used.)

+
+ +
+ URL +

http://www.myserver.com/templeton/v1/ddl/database/:db

+
+ +
+ Parameters + + + + + + + + + +
NameDescriptionRequired?Default
:dbThe database nameRequiredNone
+
+ +
+ Results + + + + + + + + + + + + + + + + + + + +
NameDescription
locationThe database location
paramsThe database parameters
commentThe database comment
databaseThe database name
+
+ +
+ Example + +

Curl Command

+ +% curl -s 'http://localhost:50111/templeton/v1/ddl/database/newdb?user.name=ctdean' + + +

JSON Output

+ +{ + "location":"hdfs://localhost:9000/warehouse/newdb.db", + "params":"{a=b}", + "comment":"Hello there", + "database":"newdb" +} + + +

JSON Output (error)

+ +{ + "error": "No such database: newdb", + "errorCode": 404 +} + + +
+ +
Index: src/docs/src/documentation/content/xdocs/desctable.xml =================================================================== --- src/docs/src/documentation/content/xdocs/desctable.xml (revision 0) +++ src/docs/src/documentation/content/xdocs/desctable.xml (revision 0) @@ -0,0 +1,191 @@ + + + + + +
+ GET ddl/database/:db/table/:table +
+ + +
+ Description +

Describe an HCatalog table. Normally returns a simple list of columns + (using "desc table"), but the extended format will show more information (using + "show table extended like").

+
+ +
+ URL +

http://www.myserver.com/templeton/v1/ddl/database/:db/table/:table

+

http://www.myserver.com/templeton/v1/ddl/database/:db/table/:table?format=extended

+
+ +
+ Parameters + + + + + + + + + + + + + + + + + + + + +
NameDescriptionRequired?Default
:dbThe database nameRequiredNone
:tableThe table nameRequiredNone
formatSet "format=extended" to see additional information (using "show table + extended like")OptionalNot extended
+
+ +
+ Results + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
columnsA list of column names and types
databaseThe database name
tableThe table name
partitioned (extended only)True if the table is partitioned
location (extended only)Location of table
outputFormat (extended only)Output format
owner (extended only)The owner's user name
partitionColumns (extended only)List of the partition columns
inputFormat (extended only)Input format
+
+ +
+ Example + +

Curl Command (simple)

+ +% curl -s 'http://localhost:50111/templeton/v1/ddl/database/default/table/my_table?user.name=ctdean' + + +

JSON Output (simple)

+ +{ + "columns": [ + { + "name": "id", + "type": "bigint" + }, + { + "name": "user", + "comment": "The user name", + "type": "string" + }, + { + "name": "my_p", + "type": "string" + }, + { + "name": "my_q", + "type": "string" + } + ], + "database": "default", + "table": "my_table" +} + + +

Curl Command (extended)

+ +% curl -s 'http://localhost:50111/templeton/v1/ddl/database/default/table/test_table?user.name=ctdean&format=extended' + + +

JSON Output (extended)

+ +{ + "partitioned": true, + "location": "hdfs://ip-10-77-6-151.ec2.internal:8020/apps/hive/warehouse/test_table", + "outputFormat": "org.apache.hadoop.hive.ql.io.RCFileOutputFormat", + "columns": [ + { + "name": "id", + "type": "bigint" + }, + { + "name": "price", + "comment": "The unit price", + "type": "float" + } + ], + "owner": "ctdean", + "partitionColumns": [ + { + "name": "country", + "type": "string" + } + ], + "inputFormat": "org.apache.hadoop.hive.ql.io.RCFileInputFormat", + "database": "default", + "table": "test_table" +} + + +

JSON Output (error)

+ +{ + "error": "Table xtest_table does not exist", + "errorCode": 404, + "database": "default", + "table": "xtest_table" +} + + +
+ +
Index: src/docs/src/documentation/content/xdocs/listproperties.xml =================================================================== --- src/docs/src/documentation/content/xdocs/listproperties.xml (revision 0) +++ src/docs/src/documentation/content/xdocs/listproperties.xml (revision 0) @@ -0,0 +1,103 @@ + + + + + +
+ GET ddl/database/:db/table/:table/property +
+ + +
+ Description +

List all the properties of an HCatalog table.

+
+ +
+ URL +

http://www.myserver.com/templeton/v1/ddl/database/:db/table/:table/property

+
+ +
+ Parameters + + + + + + + + + + + + + + +
NameDescriptionRequired?Default
:dbThe database nameRequiredNone
:tableThe table nameRequiredNone
+
+ +
+ Results + + + + + + + + + + + + + + + +
NameDescription
propertiesA list of the tables properties in name: value pairs
databaseThe database name
tableThe table name
+
+ +
+ Example + +

Curl Command

+ +% curl -s 'http://localhost:50111/templeton/v1/ddl/database/default/table/test_table/property?user.name=ctdean' + + +

JSON Output

+ +{ + "properties": { + "fruit": "apple", + "last_modified_by": "ctdean", + "hcat.osd": "org.apache.hcatalog.rcfile.RCFileOutputDriver", + "color": "blue", + "last_modified_time": "1331620706", + "hcat.isd": "org.apache.hcatalog.rcfile.RCFileInputDriver", + "transient_lastDdlTime": "1331620706", + "comment": "Best table made today", + "country": "Albania" + }, + "table": "test_table", + "database": "default" +} + +
+ +
Index: src/docs/src/documentation/content/xdocs/rest_server_install.xml =================================================================== --- src/docs/src/documentation/content/xdocs/rest_server_install.xml (revision 0) +++ src/docs/src/documentation/content/xdocs/rest_server_install.xml (revision 0) @@ -0,0 +1,258 @@ + + + + + +
+ Installation +
+ + +
+ Procedure +
    +
  1. Ensure that the + required related installations + are in place, and place required files into the + Hadoop distributed cache.
  2. +
  3. Download and unpack the HCatalog distribution.
  4. +
  5. Set the TEMPLETON_HOME environment variable to the base of the + HCatalog REST server installation. This will usually be HCATALOG_HOME/webhcat. + This is used to find the Templeton + configuration.
  6. +
  7. Review the configuration + and update or create templeton-site.xml as required. Ensure that + site specific component installation locations are accurate, especially + the Hadoop configuration path. Configuration variables that use a filesystem + path try to have reasonable defaults, but it's always safe to specify a full + and complete path.
  8. +
  9. Verify that HCatalog is installed and that the hcat + executable is in the PATH.
  10. +
  11. Build HCatalog using the command ant jar from the + top level HCatalog directory.
  12. +
  13. Start the REST server with the command + bin/templeton_server.sh start.
  14. +
  15. Check that your local install works. Assuming that the server is running + on port 8080, the following command would give output similar to that shown. + +% curl -i http://localhost:50111/templeton/v1/status +HTTP/1.1 200 OK +Content-Type: application/json +Transfer-Encoding: chunked +Server: Jetty(7.6.0.v20120127) + +{"status":"ok","version":"v1"} +% +
  16. +
+
+ +
+ Server Commands +
    +
  • Start the server: bin/templeton_server.sh start
  • +
  • Stop the server: bin/templeton_server.sh stop
  • +
  • End-to-end build, run, test: ant e2e
  • +
+
+ +
+ Requirements +
    +
  • Ant, version 1.8 or higher
  • +
  • Hadoop, version 0.20.205.0
  • +
  • ZooKeeper is required if you are using + the ZooKeeper storage class. (Be sure to review + and update the ZooKeeper related Templeton + configuration.)
  • +
  • + HCatalog. + Version 0.4.1 or higher. The hcat executable must be both in the + PATH and properly configured in the Templeton configuration. +
  • +
  • + Permissions must be given to the user running the server. + (see below) +
  • +
  • + If running a secure cluster, Kerberos keys and principals must be + created. (see below) +
  • +
  • Hadoop Distributed Cache. + To use the Hive, + Pig, or + hadoop/streaming + resources, see instructions below for placing the required files in the + Hadoop Distributed Cache.
  • +
+
+ +
+ Hadoop Distributed Cache + +

The server requires some files be accessible on the + + Hadoop distributed cache. For example, to avoid the installation of Pig and Hive + everywhere on the cluster, the server gathers a version of Pig or Hive from the + Hadoop distributed cache whenever those resources are invoked. After placing the + following components into HDFS please update the site configuration as required for + each.

+ +
    +
  • Hive: + Download + the HCatalog tar.gz file and place it in HDFS. (If you need a version + that is not yet released, you may need to build it yourself following the HCatalog + instructions.) + +hadoop fs -put /tmp/hcatalog-0.3.0.tar.gz /user/templeton/hcatalog-0.3.0.tar.gz + +
  • + +
  • Pig: + Download + the Pig tar.gz file and place it into HDFS. For example: + +hadoop fs -put /tmp/pig-0.9.2.tar.gz /user/templeton/pig-0.9.2.tar.gz + +
  • + +
  • Hadoop Streaming: + Place hadoop-streaming.jar into HDFS. For example, use the + following command, substituting your path to the jar for the one below. + +hadoop fs -put $HADOOP_PREFIX/hadoop-0.20.205.0/contrib/streaming/hadoop-streaming-0.20.205.0.jar \ + /user/templeton/hadoop-streaming.jar + +
  • + +
  • Override Jars: + Place override jars required (if any) into HDFS. Note: As of this writing, all + released versions of Hadoop require a patch to properly run Templeton. This patch is + distributed with Templeton (located at + templeton/src/hadoop_temp_fix/ugi.jar) + and should be placed into HDFS, as reflected in the current default configuration. + +hadoop fs -put ugi.jar /user/templeton/ugi.jar + +
  • +
+ +

The location of these files in the cache, and the location + of the installations inside the archives, can be specified using the following + Templeton configuration variables. (See the + Configuration documentation for more information + on changing Templeton configuration parameters.)

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDefaultDescription
templeton.pig.archivehdfs:///user/templeton/pig-0.9.2.tar.gzThe path to the Pig archive.
templeton.pig.pathpig-0.9.2.tar.gz/pig-0.9.2/bin/pigThe path to the Pig executable.
templeton.hive.archivehdfs:///user/templeton/hcatalog-0.3.0.tar.gzThe path to the Hive archive.
templeton.hive.pathhcatalog-0.3.0.tar.gz/hcatalog-0.3.0/bin/hiveThe path to the Hive executable.
templeton.streaming.jarhdfs:///user/templeton/hadoop-streaming.jarThe path to the Hadoop streaming jar file.
templeton.override.jarshdfs:///user/templeton/ugi.jarJars to add to the HADOOP_CLASSPATH for all Map Reduce jobs. + These jars must exist on HDFS.
+
+ +
+ Permissions +

+ Permission must given for the user running the templeton + executable to run jobs for other users. That is, the templeton + server will impersonate users on the Hadoop cluster. +

+ +

+ Create (or assign) a Unix user who will run the Templeton server. + Call this USER. See the Secure Cluster section below for choosing + a user on a Kerberos cluster. +

+ +

+ Modify the Hadoop core-site.xml file and set these properties: +

+ + + + + + + + + + + +
VariableValue
hadoop.proxyuser.USER.groups + A comma separated list of the Unix groups whose users will be + impersonated. +
hadoop.proxyuser.USER.hosts + A comma separated list of the hosts that will run the hcat and + JobTracker servers. +
+
+ +
+ Secure Cluster +

+ To run Templeton on a secure cluster follow the Permissions + instructions above but create a Kerberos principal for the + Templeton server with the name USER/host@realm +

+ +

+ Also, set the templeton configuration variables + templeton.kerberos.principal and + templeton.kerberos.keytab +

+
+ + +
Index: src/docs/src/documentation/content/xdocs/example.xml =================================================================== --- src/docs/src/documentation/content/xdocs/example.xml (revision 0) +++ src/docs/src/documentation/content/xdocs/example.xml (revision 0) @@ -0,0 +1,86 @@ + + + + + +
+ Example +
+ +

The following example, extracted from the HCatalog documentation, shows how people + might use HCatalog along with various other Hadoop tools to move data from the grid + into a database and ultimately analyze it.

+ +

Without Templeton there are three main steps to completing + the task.

+ +

First, Joe in data acquisition uses distcp to get data + onto the grid.

+ + +hadoop distcp file:///file.dat hdfs://data/rawevents/20100819/data + +hcat "alter table rawevents add partition 20100819 hdfs://data/rawevents/20100819/data" + + +

Second, Sally in data processing uses Pig to cleanse and prepare the + data. Oozie will be notified by HCatalog that data is available and can then + start the Pig job

+ + +A = load 'rawevents' using HCatLoader; +B = filter A by date = '20100819' and by bot_finder(zeta) = 0; +… +store Z into 'processedevents' using HCatStorer("date=20100819"); + + +

Third, Robert in client management uses Hive to analyze his + clients' results.

+ + +insert overwrite table 20100819events +select advertiser_id, count(clicks) +from processedevents +where date = ‘20100819’ +group by adverstiser_id; + + +

With Templeton all these steps can be easily performed programatcally + upon receipt of the initial data. Sally and Robert can still maintain their own scripts + and simply push them into HDFS to be accessed when required by Templeton.

+ + +??Still need to add web hdfs push! + +>POST /v1/templeton/ddl.json?exec="alter table rawevents add partition 20100819 hdfs://data/rawevents/20100819/data" +>{"result":"ok"} +> +>POST /v1/templeton/pig.json?src="hdfs://scripts/cleanse.pig" +>{"result": "ok", "jobid": "123"} +> +>... +>GET /v1/templeton/queue/123.json +>{"result": "ok", "status" "completed"} +> +>POST /v1/templeton/hive.json?src="hdfs://scripts/analyze.hive" +>{"result": "ok", "jobid": "456"} +> + + + +
Index: src/docs/src/documentation/content/xdocs/desccolumn.xml =================================================================== --- src/docs/src/documentation/content/xdocs/desccolumn.xml (revision 0) +++ src/docs/src/documentation/content/xdocs/desccolumn.xml (revision 0) @@ -0,0 +1,103 @@ + + + + + +
+ GET ddl/database/:db/table/:table/column/:column +
+ + +
+ Description +

Describe a single column in an HCatalog table.

+
+ +
+ URL +

http://www.myserver.com/templeton/v1/ddl/database/:db/table/:table/column/:column

+
+ +
+ Parameters + + + + + + + + + + + + + + + + + + + + +
NameDescriptionRequired?Default
:dbThe database nameRequiredNone
:tableThe table nameRequiredNone
:columnThe column nameRequiredNone
+
+ +
+ Results + + + + + + + + + + + + + + + +
NameDescription
databaseThe database name
tableThe table name
columnA JSON object containing the column name, type, and comment (if any)
+
+ +
+ Example + +

Curl Command

+ +% curl -s 'http://localhost:50111/templeton/v1/ddl/database/default/table/test_table/column/price?user.name=ctdean' + + +

JSON Output

+ +{ + "database": "default", + "table": "test_table", + "column": { + "name": "price", + "comment": "The unit price", + "type": "float" + } +} + +
+ +
Index: src/docs/src/documentation/content/xdocs/createpartition.xml =================================================================== --- src/docs/src/documentation/content/xdocs/createpartition.xml (revision 0) +++ src/docs/src/documentation/content/xdocs/createpartition.xml (revision 0) @@ -0,0 +1,126 @@ + + + + + +
+ PUT ddl/database/:db/table/:table/partition/:partition +
+ + +
+ Description +

Create a partition in an HCatalog table.

+
+ +
+ URL +

http://www.myserver.com/templeton/v1/ddl/database/:db/table/:table/partition/:partition

+
+ +
+ Parameters + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescriptionRequired?Default
:dbThe database nameRequiredNone
:tableThe table nameRequiredNone
:partitionThe partition name, col_name='value' list. Be careful to properly + encode the quote for http, for example, country=%27algeria%27.RequiredNone
groupThe user group to useOptionalNone
permissionsThe permissions string to useOptionalNone
locationThe location for partition creationRequiredNone
ifNotExistsIf true, return an error if the partition already exists.OptionalFalse
+
+ +
+ Results + + + + + + + + + + + + + + + +
NameDescription
partitionThe partition name
tableThe table name
databaseThe database name
+
+ +
+ Example + +

Curl Command

+ +% curl -s -X PUT -HContent-type:application/json -d '{"location": "loc_a"}' \ + 'http://localhost:50111/templeton/v1/ddl/database/default/table/test_table/partition/country=%27algeria%27?user.name=ctdean' + + +

JSON Output

+ +{ + "partition": "country='algeria'", + "table": "test_table", + "database": "default" +} + +
+ +
Index: src/docs/src/documentation/content/xdocs/mapreducejar.xml =================================================================== --- src/docs/src/documentation/content/xdocs/mapreducejar.xml (revision 0) +++ src/docs/src/documentation/content/xdocs/mapreducejar.xml (revision 0) @@ -0,0 +1,156 @@ + + + + + +
+ POST mapreduce/jar +
+ + + +
+ Description +

Creates and queues a standard + + Hadoop MapReduce job.

+
+ +
+ URL +

http://www.myserver.com/templeton/v1/mapreduce/jar

+
+ +
+ Parameters + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescriptionRequired?Default
jarName of the jar file for Map Reduce to use.RequiredNone
className of the class for Map Reduce to use.RequiredNone
libjarsComma separated jar files to include in the classpath.OptionalNone
filesComma separated files to be copied to the map reduce clusterOptionalNone
argSet a program argument.OptionalNone
defineSet an Hadoop configuration variable using the syntax + define=NAME=VALUEOptionalNone
statusdirA directory where Templeton will write the status of the + Map Reduce job. If provided, it is the caller's responsibility + to remove this directory when done.OptionalNone
callbackDefine a URL to be called upon job completion. You may embed a specific + job ID into this URL using $jobId. This tag + will be replaced in the callback URL with this job's job ID.OptionalNone
+
+ +
+ Results + + + + + + + + + + +
NameDescription
idA string containing the job ID similar to "job_201110132141_0001".
infoA JSON object containing the information returned when the job was queued. + See the Hadoop documentation + (Class + TaskController) for more information.
+
+ +
+ Example + +

Code and Data Setup

+ +% hadoop fs -put wordcount.jar . +% hadoop fs -put transform.jar . + +% hadoop fs -ls . +Found 2 items +-rw-r--r-- 1 ctdean supergroup 23 2011-11-11 13:29 /user/ctdean/wordcount.jar +-rw-r--r-- 1 ctdean supergroup 28 2011-11-11 13:29 /user/ctdean/transform.jar + + +

Curl Command

+ +% curl -s -d user.name=ctdean \ + -d jar=wordcount.jar \ + -d class=org.myorg.WordCount \ + -d libjars=transform.jar \ + -d arg=wordcount/input \ + -d arg=wordcount/output \ + 'http://localhost:50111/templeton/v1/mapreduce/jar' + + +

JSON Output

+ +{ + "id": "job_201111121211_0001", + "info": { + "stdout": "templeton-job-id:job_201111121211_0001 + ", + "stderr": "", + "exitcode": 0 + } +} + +
+ +
Index: src/docs/src/documentation/content/xdocs/site.xml =================================================================== --- src/docs/src/documentation/content/xdocs/site.xml (revision 1360995) +++ src/docs/src/documentation/content/xdocs/site.xml (working copy) @@ -47,6 +47,48 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Index: src/docs/src/documentation/content/xdocs/listdbs.xml =================================================================== --- src/docs/src/documentation/content/xdocs/listdbs.xml (revision 0) +++ src/docs/src/documentation/content/xdocs/listdbs.xml (revision 0) @@ -0,0 +1,80 @@ + + + + + +
+ GET ddl/database +
+ + +
+ Description +

List the databases in HCatalog.

+
+ +
+ URL +

http://www.myserver.com/templeton/v1/ddl/database

+
+ +
+ Parameters + + + + + + + + +
NameDescriptionRequired?Default
likeList only databases whose names match the specified patternOptional"*" (List all)
+
+ +
+ Results + + + + + + + +
NameDescription
databasesA list of database names
+
+ +
+ Example + +

Curl Command

+ +% curl -s 'http://localhost:50111/templeton/v1/ddl/database?user.name=ctdean&like=n*' + + +

JSON Output

+ +{ + "databases": [ + "newdb", + "newdb2" + ] +} + +
+ +
Index: src/docs/src/documentation/content/xdocs/configuration.xml =================================================================== --- src/docs/src/documentation/content/xdocs/configuration.xml (revision 0) +++ src/docs/src/documentation/content/xdocs/configuration.xml (revision 0) @@ -0,0 +1,284 @@ + + + + + +
+ Configuration +
+ +

The configuration for Templeton merges the normal Hadoop configuration with + the Templeton specific variables. Because Templeton is designed to connect services + that are not normally connected, the configuration is more complex than might be + desirable.

+ +

The Templeton specific configuration is split into two layers:

+ +
    +
  1. templeton-default.xml - All the configuration variables + that Templeton needs. This file sets the defaults that ship with Templeton and + should only be changed by Templeton developers. Do not copy this file and/or + change it to maintain local installation settings. Because templeton-default.xml + is present in the Templeton war file, editing a local copy of it will not change the + configuration.
  2. + +
  3. templeton-site.xml - The (possibly empty) configuration + file in which the system administrator can set variables for their Hadoop cluster. + Create this file and maintain entries in it for configuration variables + that require you to override default values based on your local installation.
  4. +
+ +

The configuration files are loaded in this order with later files overriding + earlier ones.

+ +

Note: the Templeton server will require restart + after any change to the configuration.

+ +

To find the configuration files, Templeton first attempts to load a file from the + CLASSPATH and then looks in the directory specified in the + TEMPLETON_HOME environment variable.

+ +

Configuration files may access the special environment variable + env for all environment variables. For example, the pig executable + could be specified using:

+ + +${env.PIG_HOME}/bin/pig + + +

Configuration variables that use a filesystem path try to have reasonable defaults. + However, it's always safe to specify the full and complete path if there is any + uncertainty.

+ +

Note: The location of the log files created by Templeton and some other properties + of the logging system are set in the templeton-log4j.properties file.

+ +
+ Variables + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDefaultDescription
templeton.port50111The HTTP port for the main server.
templeton.hadoop.config.dir$(env.HADOOP_CONFIG_DIR)The path to the Hadoop configuration.
templeton.jar${env.TEMPLETON_HOME}/templeton/templeton-0.1.0-dev.jarThe path to the Templeton jar file.
templeton.libjars${env.TEMPLETON_HOME}/lib/zookeeper-3.3.4.jarJars to add to the classpath.
templeton.override.jarshdfs:///user/templeton/ugi.jarJars to add to the HADOOP_CLASSPATH for all Map Reduce jobs. + These jars must exist on HDFS.
templeton.override.enabledtrueEnable the override path in templeton.override.jars
templeton.streaming.jarhdfs:///user/templeton/hadoop-streaming.jarThe hdfs path to the Hadoop streaming jar file.
templeton.hadoop${env.HADOOP_PREFIX}/bin/hadoopThe path to the Hadoop executable.
templeton.pig.archivehdfs:///user/templeton/pig-0.9.2.tar.gzThe path to the Pig archive.
templeton.pig.pathpig-0.9.2.tar.gz/pig-0.9.2/bin/pigThe path to the Pig executable.
templeton.hcat${env.HCAT_PREFIX}/bin/hcatThe path to the Hcatalog executable.
templeton.hive.archivehdfs:///user/templeton/hcatalog-0.3.0.tar.gzThe path to the Hive archive.
templeton.hive.pathhcatalog-0.3.0.tar.gz/hcatalog-0.3.0/bin/hiveThe path to the Hive executable.
templeton.hive.properties +hive.metastore.local=false, +hive.metastore.uris=thrift://localhost:9933, +hive.metastore.sasl.enabled=falseProperties to set when running hive.
templeton.exec.encodingUTF-8The encoding of the stdout and stderr data.
templeton.exec.timeout10000How long in milliseconds a program is allowed to run on the + Templeton box. +
templeton.exec.max-procs16The maximum number of processes allowed to run at once.
templeton.exec.max-output-bytes1048576The maximum number of bytes from stdout or stderr stored in ram.
templeton.exec.envsHADOOP_PREFIX,HADOOP_HOME,JAVA_HOMEThe environment variables passed through to exec.
templeton.zookeeper.hosts127.0.0.1:2181ZooKeeper servers, as comma separated host:port pairs
templeton.zookeeper.session-timeout30000ZooKeeper session timeout in milliseconds
templeton.callback.retry.interval10000How long to wait between callback retry attempts in milliseconds
templeton.callback.retry.attempts5How many times to retry the callback
templeton.storage.classorg.apache.hcatalog.templeton.tool.ZooKeeperStorageThe class to use as storage
templeton.storage.root/templeton-hadoopThe path to the directory to use for storage
templeton.hdfs.cleanup.interval43200000The maximum delay between a thread's cleanup checks
templeton.hdfs.cleanup.maxage604800000The maximum age of a templeton job
templeton.zookeeper.cleanup.interval43200000The maximum delay between a thread's cleanup checks
templeton.zookeeper.cleanup.maxage604800000The maximum age of a templeton job
templeton.kerberos.secretA random valueThe secret used to sign the HTTP cookie value. The default + value is a random value. Unless multiple Templeton instances + need to share the secret the random value is adequate.
templeton.kerberos.principalNoneThe Kerberos principal to used by the server. As stated by the + Kerberos SPNEGO specification, it should be + USER/${HOSTNAME}@{REALM}. It does not have a + default value.
templeton.kerberos.keytabNoneThe keytab file containing the credentials for the Kerberos + principal.
+
+ + +
Index: src/docs/src/documentation/content/xdocs/queuedelete.xml =================================================================== --- src/docs/src/documentation/content/xdocs/queuedelete.xml (revision 0) +++ src/docs/src/documentation/content/xdocs/queuedelete.xml (revision 0) @@ -0,0 +1,156 @@ + + + + + +
+ DELETE queue/:jobid +
+ + +
+ Description +

Kill a job given its job ID. + Substitute ":jobid" with the job ID received when the job was created.

+
+ +
+ URL +

http://www.myserver.com/templeton/v1/queue/:jobid

+
+ +
+ Parameters + + + + + + + + +
NameDescriptionRequired?Default
:jobidThe job ID to delete. This is the ID received when the job + job was created.RequiredNone
+
+ +
+ Results + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
statusA JSON object containing the job status information. + See the Hadoop documentation + (Class + JobStatus) for more information.
profileA JSON object containing the job profile information. + See the Hadoop documentation + (Class + JobProfile) for more information. +
idThe job ID.
parentIdThe parent job ID.
percentCompleteThe job completion percentage, for example "75% complete".
exitValueThe job's exit value.
userUser name of the job creator.
callbackThe callback URL, if any.
completedA string representing completed status, for example "done".
+
+ + +
+ Example + +

Curl Command

+ +% curl -s -X DELETE 'http://localhost:50111/templeton/v1/queue/job_201111111311_0009?user.name=ctdean' + + +

JSON Output

+ +{ + "status": { + "startTime": 1321047216471, + "username": "ctdean", + "jobID": { + "jtIdentifier": "201111111311", + "id": 9 + }, + "jobACLs": { + }, + "schedulingInfo": "NA", + "failureInfo": "NA", + "jobId": "job_201111111311_0009", + "jobPriority": "NORMAL", + "runState": 1, + "jobComplete": false + }, + "profile": { + "url": "http://localhost:50030/jobdetails.jsp?jobid=job_201111111311_0009", + "user": "ctdean", + "jobID": { + "jtIdentifier": "201111111311", + "id": 9 + }, + "queueName": "default", + "jobFile": "hdfs://localhost:9000/tmp/hadoop-ctdean/mapred/staging/ctdean/.staging/job_201111111311_0009/job.xml", + "jobName": "streamjob3322518350676530377.jar", + "jobId": "job_201111111311_0009" + } + "id": "job_201111111311_0009", + "parentId": "job_201111111311_0008", + "percentComplete": "10% complete", + "exitValue": 0, + "user": "ctdean", + "callback": null, + "completed": "false" +} + +

Note: The job is not immediately deleted, therefore the + information returned may not reflect deletion, as in our example. + Use GET queue/:jobid + to monitor the job and confirm that it is eventually deleted.

+
+ +
Index: src/docs/src/documentation/content/xdocs/versions.xml =================================================================== --- src/docs/src/documentation/content/xdocs/versions.xml (revision 0) +++ src/docs/src/documentation/content/xdocs/versions.xml (revision 0) @@ -0,0 +1,77 @@ + + + + + +
+ GET version +
+ + +
+ Description +

Returns a list of supported versions and the current version.

+
+ +
+ URL +

http://www.myserver.com/templeton/v1/version

+
+ +
+ Parameters +

Only the standard parameters + are accepted.

+
+ +
+ Results + + + + + + + + + + + +
NameDescription
supportedVersionsA list of all supported versions.
versionThe current version.
+
+ +
+ Example + +

Curl Command

+ +% curl -s 'http://localhost:50111/templeton/v1/version' + + +

JSON Output

+ +{ + "supportedVersions": [ + "v1" + ], + "version": "v1" +} + +
+ +
Index: src/docs/src/documentation/content/xdocs/createtable.xml =================================================================== --- src/docs/src/documentation/content/xdocs/createtable.xml (revision 0) +++ src/docs/src/documentation/content/xdocs/createtable.xml (revision 0) @@ -0,0 +1,222 @@ + + + + + +
+ PUT ddl/database/:db/table/:table +
+ + +
+ Description +

Create a new HCatalog table. For more information, please refer to the + Hive documentation.

+
+ +
+ URL +

http://www.myserver.com/templeton/v1/ddl/database/:db/table/:table

+
+ +
+ Parameters + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescriptionRequired?Default
:dbThe database nameRequiredNone
:tableThe new table nameRequiredNone
groupThe user group to use when creating a tableOptionalNone
permissionsThe permissions string to use when creating a table.OptionalNone
externalAllows you to specify a location so that Hive does not use the default + location for this table.Optionalfalse
ifNotExistsIf true, you will not receive an error if the table already exists.Optionalfalse
commentComment for the tableOptionalNone
columnsA list of column descriptions, including name, type, and an optional comment.OptionalNone
partitionedByA list of column descriptions used to partition the table. Like the columns + parameter this is a list of name, type, and comment fields.OptionalNone
clusteredByAn object describing how to cluster the table including the parameters + columnNames, sortedBy, numberOfBuckets. The sortedBy parameter includes + the parameters columnName and order. For further information + please refer to the examples below or to the + + Hive documentation.OptionalNone
formatStorage format description including paraeters for rowFormat, storedAs + and storedBy. For further information please refer to the examples below or to the + + Hive documentation.OptionalNone
locationThe HDFS pathOptionalNone
tablePropertiesA list of table property names and values (key/value pairs)OptionalNone
+
+ +
+ Results + + + + + + + + + + + +
NameDescription
tableThe new table name
databaseThe database name
+
+ +
+ Example + +

Curl Command

+ +% curl -s -X PUT -HContent-type:application/json -d '{ + "comment": "Best table made today", + "columns": [ + { "name": "id", "type": "bigint" }, + { "name": "price", "type": "float", "comment": "The unit price" } ], + "partitionedBy": [ + { "name": "country", "type": "string" } ], + "format": { "storedAs": "rcfile" } }' \ + 'http://localhost:50111/templeton/v1/ddl/database/default/table/test_table?user.name=ctdean' + + +

Curl Command (using clusteredBy)

+ +% curl -s -X PUT -HContent-type:application/json -d '{ + "comment": "Best table made today", + "columns": [ + { "name": "id", "type": "bigint"}, + { "name": "price", "type": "float", "comment": "The unit price" } ], + "partitionedBy": [ + { "name": "country", "type": "string" } ], + "clusteredBy": { + "columnNames": ["id"], + "sortedBy": [ + { "columnName": "id", "order": "ASC" } ], + "numberOfBuckets": 10 }, + "format": { + "storedAs": "rcfile", + "rowFormat": { + "fieldsTerminatedBy": "\u0001", + "serde": { + "name": "org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe", + "properties": { + "key": "value" } } } } + } ' \ + 'http://localhost:50111/templeton/v1/ddl/database/default/table/test_table_c?user.name=ctdean' + + +

JSON Output

+ +{ + "table": "test_table", + "database": "default" +} + + +

JSON Output (error)

+ +{ + "statement": "use default; create table test_table_c(id bigint, price float comment ...", + "error": "unable to create table: test_table_c", + "exec": { + "stdout": "", + "stderr": "WARNING: org.apache.hadoop.metrics.jvm.EventCounter is deprecated... + Hive history file=/tmp/ctdean/hive_job_log_ctdean_201204051335_2016086186.txt + SLF4J: Class path contains multiple SLF4J bindings. + SLF4J: Found binding in ... + SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation. + OK + Time taken: 0.448 seconds + FAILED: Error in semantic analysis: Operation not supported. HCatalog doesn't allow Clustered By in create table. + ", + "exitcode": 10 + } +} + + +
+ +
Index: src/docs/src/documentation/content/xdocs/listcolumns.xml =================================================================== --- src/docs/src/documentation/content/xdocs/listcolumns.xml (revision 0) +++ src/docs/src/documentation/content/xdocs/listcolumns.xml (revision 0) @@ -0,0 +1,110 @@ + + + + + +
+ GET ddl/database/:db/table/:table/column +
+ + +
+ Description +

List the columns in an HCatalog table.

+
+ +
+ URL +

http://www.myserver.com/templeton/v1/ddl/database/:db/table/:table/column

+
+ +
+ Parameters + + + + + + + + + + + + + + +
NameDescriptionRequired?Default
:dbThe database nameRequiredNone
:tableThe table nameRequiredNone
+
+ +
+ Results + + + + + + + + + + + + + + +
NameDescription
columnsA list of column names and types
databaseThe database name
tableThe table name
+
+ +
+ Example + +

Curl Command

+ +% curl -s 'http://localhost:50111/templeton/v1/ddl/database/default/table/my_table/column?user.name=ctdean' + + +

JSON Output

+ +{ + "columns": [ + { + "name": "id", + "type": "bigint" + }, + { + "name": "user", + "comment":"The user name", + "type": "string" + }, + { + "name": "my_p", + "type": "string" + }, + { + "name": "my_q", + "type": "string" + } + ], + "database": "default", + "table": "my_table" +} + +
+ +
Index: src/docs/src/documentation/content/xdocs/rest.xml =================================================================== --- src/docs/src/documentation/content/xdocs/rest.xml (revision 0) +++ src/docs/src/documentation/content/xdocs/rest.xml (revision 0) @@ -0,0 +1,140 @@ + + + + + +
+ HCatalog REST API +
+ + +
+ Introduction +

This document describes HCatalog REST API. + As shown in the figure below, developers make HTTP requests to access + Hadoop MapReduce, + Pig, + Hive, and + + HCatalog DDL from within applications. + Data and code used by this API is maintained in + HDFS. HCatalog DDL commands + are executed directly when requested. + MapReduce, Pig, and Hive jobs are placed in queue by + and can be monitored for progress or stopped as required. + Developers specify a location + in HDFS into which Pig, Hive, and MapReduce results should be placed.

+
+
+ +
+ URL format +

HCatalog's REST resources are accessed using the following URL format:

+

http://yourserver/templeton/v1/resource

+

where "yourserver" is replaced with your server name, and + "resource" is replaced by the HCatalog + resource name.

+

For example, to check if the server is running you could + access the following URL:

+

http://www.myserver.com/templeton/v1/status

+
+ +
+ Security +

The current version supports two types of security:

+
    +
  • Default security (without additional authentication)
  • +
  • Authentication via + Kerberos
  • +
+
+ Standard Parameters +

Every REST resource can accept the following parameters to + aid in authentication:

+
    +
  • user.name: The user name as a string. + Only valid when using default security.
  • +
  • SPNEGO credentials: When running with Kerberos authentication.
  • +
+
+ +
+ Security Error Response +

If the user.name parameter is not supplied when required, + the following error will be returned:

+ +{ + "error": "No user found. Missing user.name parameter." +} + +
+ +
+ +
+ WebHDFS and Code Push +

Data and code that are used by HCatalog's REST resources must first be placed in + Hadoop. When placing files into HDFS is required you can use + whatever method is most convienient for you. We suggest WebHDFS since it provides + a REST interface for moving files into and out of HDFS.

+
+ +
+ Error Codes and Responses +

The server returns the following HTTP status codes.

+
    +
  • 200 OK: Success!
  • +
  • 400 Bad Request: The request was invalid.
  • +
  • 401 Unauthorized: Credentials were missing or incorrect.
  • +
  • 404 Not Found: The URI requested is invalid or the + resource requested does not exist.
  • +
  • 500 Internal Server Error: We received an unexpected result.
  • +
  • 503 Busy, please retry: The server is busy.
  • +
+

Other data returned directly by the server is returned in JSON format. + JSON responses are limited to 1MB in size. Responses over this limit must be + stored into HDFS using provided options instead of being directly returned. + If an HCatalog DDL command might return results greater than 1MB, it's + suggested that a corresponding Hive request be executed instead.

+
+ +
+ Log Files +

The server creates three log files when in operation:

+
    +
  • templeton.log is the log4j log. This the main log the application + writes to.
  • +
  • templeton-console.log is what Java writes to stdout when the server is + started. It is a small amount of data, similar to "hcat.out".
  • +
  • tempelton-console-error.log is what Java writes to stderr, similar to + "hcat.err".
  • +
+

In the tempelton-log4j.properties file you can set the location of these logs using the + variable templeton.log.dir. This log4j.properties file is set in the server startup script.

+
+ +
+ Project Name +

The original work to add REST APIs to HCatalog was called Templeton. For backward compatibility + the name still appears in URLs, log file names, etc. The Templeton name is taken from a character + in the award-winning children's novel Charlotte's Web, by E. B. White. The novel's protagonist is a pig named + Wilber. Templeton is a rat who helps Wilber by running errands and making deliveries as + requested by Charlotte while spinning her web.

+
+ +
Index: src/docs/src/documentation/content/xdocs/pig.xml =================================================================== --- src/docs/src/documentation/content/xdocs/pig.xml (revision 0) +++ src/docs/src/documentation/content/xdocs/pig.xml (revision 0) @@ -0,0 +1,144 @@ + + + + + +
+ POST pig +
+ + +
+ Description +

Create and queue a Pig job.

+
+ +
+ URL +

http://www.myserver.com/templeton/v1/pig

+
+ +
+ Parameters + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescriptionRequired?Default
executeString containing an entire, short pig program to run.One of either "execcute" or "file" is requiredNone
fileHDFS file name of a pig program to run.One of either "exec" or "file" is requiredNone
argSet a program argument.OptionalNone
filesComma separated files to be copied to the map reduce clusterOptionalNone
statusdirA directory where Templeton will write the status of the + Pig job. If provided, it is the caller's responsibility + to remove this directory when done.OptionalNone
callbackDefine a URL to be called upon job completion. You may embed a specific + job ID into this URL using $jobId. This tag + will be replaced in the callback URL with this job's job ID.OptionalNone
+
+ +
+ Results + + + + + + + + + + +
NameDescription
idA string containing the job ID similar to "job_201110132141_0001".
infoA JSON object containing the information returned when the job was queued. + See the Hadoop documentation + (Class + TaskController) for more information.
+
+ +
+ Example + +

Code and Data Setup

+ +% cat id.pig +A = load 'passwd' using PigStorage(':'); +B = foreach A generate $0 as id; +dump B; + +% cat fake-passwd +ctdean:Chris Dean:secret +pauls:Paul Stolorz:good +carmas:Carlos Armas:evil +dra:Deirdre McClure:marvelous + +% hadoop fs -put id.pig . +% hadoop fs -put fake-passwd passwd + + +

Curl Command

+ +% curl -s -d user.name=ctdean \ + -d file=id.pig \ + -d arg=-v \ + 'http://localhost:50111/templeton/v1/pig' + + +

JSON Output

+ +{ + "id": "job_201111101627_0018", + "info": { + "stdout": "templeton-job-id:job_201111101627_0018 + ", + "stderr": "", + "exitcode": 0 + } +} + +
+ +
Index: build.xml =================================================================== --- build.xml (revision 1360995) +++ build.xml (working copy) @@ -250,6 +250,17 @@ + + + + + + + @@ -402,8 +414,14 @@ Tests failed! + + + + + + + description="Generate Javadoc and Forrest documentation"> + @@ -533,9 +557,16 @@ doctitle="HCatalog ${hcatalog.version} API" failonerror="true"> + + + + + + @@ -578,6 +609,9 @@ + + + @@ -618,6 +652,9 @@ + + + @@ -657,7 +694,8 @@ - + + Index: storage-handlers/hbase/ivy.xml =================================================================== --- storage-handlers/hbase/ivy.xml (revision 1360995) +++ storage-handlers/hbase/ivy.xml (working copy) @@ -48,9 +48,9 @@ + rev="${hadoop.jars.version}" conf="common->master" /> + rev="${hadoop.jars.version}" conf="common->master" /> + rev="${hadoop.jars.version}" conf="common->master" /> + rev="${hadoop.jars.version}" conf="default" /> + rev="${hadoop.jars.version}" conf="common->master" /> - @@ -78,7 +78,7 @@ + rev="${hive.version}" conf="default"/> - - Index: ivy/libraries.properties =================================================================== --- ivy/libraries.properties (revision 1360995) +++ ivy/libraries.properties (working copy) @@ -13,14 +13,19 @@ #This properties file lists the versions of the various artifacts used by hadoop and components. #It drives ivy and the generation of a maven POM +activation.version=1.1 activemq.version=5.5.0 antlr.version=3.0.1 +asm-all.version=3.2 commons-cli.version=1.2 +commons-codec.version=1.4 commons-configuration.version=1.6 commons-compress.version=1.4.1 commons-dbcp.version=1.4 +commons-exec.version=1.1 commons-httpclient.version=3.0.1 commons-lang.version=2.4 +commons-logging.version=1.1.1 commons-pool.version=1.5.4 datanucleus-connectionpool.version=2.0.3 datanucleus-core.version=2.0.3 @@ -29,23 +34,31 @@ derby.version=10.4.2.0 fb303.version=0.7.0 guava.version=11.0 -hadoop-core.version=1.0.2 -hadoop-test.version=1.0.2 -hadoop-tools.version=1.0.2 +hadoop.jars.version=1.0.3 hbase.version=0.92.0 high-scale-lib.version=1.1.1 hive.version=0.10.0-SNAPSHOT ivy.version=2.1.0 jackson.version=1.7.3 javax-mgmt.version=1.1-rev-1 +jaxb-api.version=2.2.2 +jaxb-impl.version=2.2.3-1 jdeb.version=0.8 jdo.version=2.3-ec +jersey.version=1.9.1 +jettison.version=1.1 jetty.version=6.1.26 +jetty.webhcat.version=7.6.0.v20120127 jms.version=1.1 junit.version=4.10 log4j.version=1.2.16 +oro.version=2.0.8 pig.version=0.8.0 rats-lib.version=0.5.1 +servlet-api.version=2.5 slf4j.version=1.6.1 +stax-api.version=1.0-2 +wadl-resourcedoc-doclet.version=1.4 +xerces.version=2.9.1 zookeeper.version=3.4.3 Index: build-common-new.xml =================================================================== --- build-common-new.xml (revision 1360995) +++ build-common-new.xml (working copy) @@ -18,7 +18,7 @@ --> - + @@ -86,7 +86,7 @@ - + @@ -139,5 +139,5 @@ classPathRef="findbugs.class.path"/> - + Index: hcatalog-pig-adapter/ivy.xml =================================================================== --- hcatalog-pig-adapter/ivy.xml (revision 1360995) +++ hcatalog-pig-adapter/ivy.xml (working copy) @@ -30,7 +30,7 @@ - + Index: hcatalog-pig-adapter/build.xml =================================================================== --- hcatalog-pig-adapter/build.xml (revision 1360995) +++ hcatalog-pig-adapter/build.xml (working copy) @@ -18,6 +18,7 @@ --> + Index: webhcat/svr/ivy.xml =================================================================== --- webhcat/svr/ivy.xml (revision 0) +++ webhcat/svr/ivy.xml (revision 0) @@ -0,0 +1,124 @@ + + + + + + + Apache HCatalog + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Index: webhcat/svr/src/test/java/org/apache/hcatalog/templeton/TestServer.java =================================================================== --- webhcat/svr/src/test/java/org/apache/hcatalog/templeton/TestServer.java (revision 0) +++ webhcat/svr/src/test/java/org/apache/hcatalog/templeton/TestServer.java (revision 0) @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton; + +import junit.framework.TestCase; + +import org.apache.hcatalog.templeton.Main; +import org.apache.hcatalog.templeton.mock.MockServer; +import java.util.List; + +/* + * Test that the server code exists, and responds to basic requests. + */ +public class TestServer extends TestCase { + + MockServer server; + + public void setUp() { + new Main(null); // Initialize the config + server = new MockServer(); + } + + public void testServer() { + assertNotNull(server); + } + + public void testStatus() { + assertEquals(server.status().get("status"), "ok"); + } + + public void testVersions() { + assertEquals(server.version().get("version"), "v1"); + } + + public void testFormats() { + assertEquals(1, server.requestFormats().size()); + assertEquals( ((List)server.requestFormats().get("responseTypes")).get(0), "application/json"); + } +} Index: webhcat/svr/src/test/java/org/apache/hcatalog/templeton/mock/MockExecService.java =================================================================== --- webhcat/svr/src/test/java/org/apache/hcatalog/templeton/mock/MockExecService.java (revision 0) +++ webhcat/svr/src/test/java/org/apache/hcatalog/templeton/mock/MockExecService.java (revision 0) @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton.mock; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import org.apache.commons.exec.ExecuteException; +import org.apache.hcatalog.templeton.ExecBean; +import org.apache.hcatalog.templeton.ExecService; +import org.apache.hcatalog.templeton.NotAuthorizedException; + +public class MockExecService implements ExecService { + + public ExecBean run(String program, List args, + Map env) { + ExecBean bean = new ExecBean(); + bean.stdout = program; + bean.stderr = args.toString(); + return bean; + } + + @Override + public ExecBean runUnlimited(String program, + List args, Map env) + throws NotAuthorizedException, ExecuteException, IOException { + ExecBean bean = new ExecBean(); + bean.stdout = program; + bean.stderr = args.toString(); + return null; + } +} Index: webhcat/svr/src/test/java/org/apache/hcatalog/templeton/mock/MockServer.java =================================================================== --- webhcat/svr/src/test/java/org/apache/hcatalog/templeton/mock/MockServer.java (revision 0) +++ webhcat/svr/src/test/java/org/apache/hcatalog/templeton/mock/MockServer.java (revision 0) @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton.mock; + +import org.apache.hcatalog.templeton.Server; + +/* + * Test that the server code exists. + */ +public class MockServer extends Server { + public String user; + public MockServer() { + execService = new MockExecService(); + resetUser(); + } + + public void resetUser() { + user = System.getenv("USER"); + } + + public String getUser() { + return user; + } +} Index: webhcat/svr/src/test/java/org/apache/hcatalog/templeton/mock/MockUriInfo.java =================================================================== --- webhcat/svr/src/test/java/org/apache/hcatalog/templeton/mock/MockUriInfo.java (revision 0) +++ webhcat/svr/src/test/java/org/apache/hcatalog/templeton/mock/MockUriInfo.java (revision 0) @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hcatalog.templeton.mock; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.List; + +import javax.ws.rs.core.MultivaluedMap; +import javax.ws.rs.core.PathSegment; +import javax.ws.rs.core.UriBuilder; +import javax.ws.rs.core.UriInfo; + +public class MockUriInfo implements UriInfo { + + @Override + public URI getAbsolutePath() { + // TODO Auto-generated method stub + return null; + } + + @Override + public UriBuilder getAbsolutePathBuilder() { + // TODO Auto-generated method stub + return null; + } + + @Override + public URI getBaseUri() { + try { + return new URI("http://fakeuri/templeton"); + } catch (URISyntaxException e) { + e.printStackTrace(); + } + return null; + } + + @Override + public UriBuilder getBaseUriBuilder() { + // TODO Auto-generated method stub + return null; + } + + @Override + public List getMatchedResources() { + // TODO Auto-generated method stub + return null; + } + + @Override + public List getMatchedURIs() { + // TODO Auto-generated method stub + return null; + } + + @Override + public List getMatchedURIs(boolean arg0) { + // TODO Auto-generated method stub + return null; + } + + @Override + public String getPath() { + // TODO Auto-generated method stub + return null; + } + + @Override + public String getPath(boolean arg0) { + // TODO Auto-generated method stub + return null; + } + + @Override + public MultivaluedMap getPathParameters() { + // TODO Auto-generated method stub + return null; + } + + @Override + public MultivaluedMap getPathParameters(boolean arg0) { + // TODO Auto-generated method stub + return null; + } + + @Override + public List getPathSegments() { + // TODO Auto-generated method stub + return null; + } + + @Override + public List getPathSegments(boolean arg0) { + // TODO Auto-generated method stub + return null; + } + + @Override + public MultivaluedMap getQueryParameters() { + // TODO Auto-generated method stub + return null; + } + + @Override + public MultivaluedMap getQueryParameters(boolean arg0) { + // TODO Auto-generated method stub + return null; + } + + @Override + public URI getRequestUri() { + // TODO Auto-generated method stub + return null; + } + + @Override + public UriBuilder getRequestUriBuilder() { + // TODO Auto-generated method stub + return null; + } + +} Index: webhcat/svr/src/test/java/org/apache/hcatalog/templeton/tool/TestTrivialExecService.java =================================================================== --- webhcat/svr/src/test/java/org/apache/hcatalog/templeton/tool/TestTrivialExecService.java (revision 0) +++ webhcat/svr/src/test/java/org/apache/hcatalog/templeton/tool/TestTrivialExecService.java (revision 0) @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton.tool; + +import static org.junit.Assert.*; + +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.HashMap; +import org.junit.Test; + +public class TestTrivialExecService { + @Test + public void test() { + ArrayList list = new ArrayList(); + list.add("echo"); + list.add("success"); + BufferedReader out = null; + BufferedReader err = null; + try { + Process process = TrivialExecService.getInstance() + .run(list, + new ArrayList(), + new HashMap()); + out = new BufferedReader(new InputStreamReader( + process.getInputStream())); + err = new BufferedReader(new InputStreamReader( + process.getErrorStream())); + assertEquals("success", out.readLine()); + out.close(); + String line; + while ((line = err.readLine()) != null) { + fail(line); + } + process.waitFor(); + } catch (Exception e) { + e.printStackTrace(); + fail("Process caused exception."); + } finally { + try { + out.close(); + } catch (Exception ex) { + // Whatever. + } + try { + err.close(); + } catch (Exception ex) { + // Whatever + } + } + } +} Index: webhcat/svr/src/test/java/org/apache/hcatalog/templeton/tool/TestTempletonUtils.java =================================================================== --- webhcat/svr/src/test/java/org/apache/hcatalog/templeton/tool/TestTempletonUtils.java (revision 0) +++ webhcat/svr/src/test/java/org/apache/hcatalog/templeton/tool/TestTempletonUtils.java (revision 0) @@ -0,0 +1,209 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton.tool; + +import static org.junit.Assert.*; + +import java.io.FileNotFoundException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.util.StringUtils; +import org.apache.hcatalog.templeton.tool.TempletonUtils; +import org.junit.Test; + +public class TestTempletonUtils { + public static final String[] CONTROLLER_LINES = { + "2011-12-15 18:12:21,758 [main] INFO org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceLauncher - More information at: http://localhost:50030/jobdetails.jsp?jobid=job_201112140012_0047", + "2011-12-15 18:12:46,907 [main] INFO org.apache.pig.tools.pigstats.SimplePigStats - Script Statistics: " + }; + + @Test + public void testIssetString() { + assertFalse(TempletonUtils.isset((String)null)); + assertFalse(TempletonUtils.isset("")); + assertTrue(TempletonUtils.isset("hello")); + } + + @Test + public void testIssetTArray() { + assertFalse(TempletonUtils.isset((Long[]) null)); + assertFalse(TempletonUtils.isset(new String[0])); + String[] parts = new String("hello.world").split("\\."); + assertTrue(TempletonUtils.isset(parts)); + } + + @Test + public void testPrintTaggedJobID() { + //JobID job = new JobID(); + // TODO -- capture System.out? + } + + + @Test + public void testExtractPercentComplete() { + assertNull(TempletonUtils.extractPercentComplete("fred")); + for (String line : CONTROLLER_LINES) + assertNull(TempletonUtils.extractPercentComplete(line)); + + String fifty = "2011-12-15 18:12:36,333 [main] INFO org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceLauncher - 50% complete"; + assertEquals("50% complete", TempletonUtils.extractPercentComplete(fifty)); + } + + @Test + public void testEncodeArray() { + assertEquals(null, TempletonUtils.encodeArray((String []) null)); + String[] tmp = new String[0]; + assertTrue(TempletonUtils.encodeArray(new String[0]).length() == 0); + tmp = new String[3]; + tmp[0] = "fred"; + tmp[1] = null; + tmp[2] = "peter,lisa,, barney"; + assertEquals("fred,,peter" + + StringUtils.ESCAPE_CHAR + ",lisa" + StringUtils.ESCAPE_CHAR + "," + + StringUtils.ESCAPE_CHAR + ", barney", + TempletonUtils.encodeArray(tmp)); + } + + @Test + public void testDecodeArray() { + assertTrue(TempletonUtils.encodeArray((String[]) null) == null); + String[] tmp = new String[3]; + tmp[0] = "fred"; + tmp[1] = null; + tmp[2] = "peter,lisa,, barney"; + String[] tmp2 = TempletonUtils.decodeArray(TempletonUtils.encodeArray(tmp)); + try { + for (int i=0; i< tmp.length; i++) { + assertEquals((String) tmp[i], (String)tmp2[i]); + } + } catch (Exception e) { + fail("Arrays were not equal" + e.getMessage()); + } + } + + @Test + public void testHadoopFsPath() { + try { + TempletonUtils.hadoopFsPath(null, null, null); + TempletonUtils.hadoopFsPath("/tmp", null, null); + TempletonUtils.hadoopFsPath("/tmp", new Configuration(), null); + } catch (FileNotFoundException e) { + fail("Couldn't find /tmp"); + } catch (Exception e) { + // This is our problem -- it means the configuration was wrong. + e.printStackTrace(); + } + try { + TempletonUtils.hadoopFsPath("/scoobydoo/teddybear", + new Configuration(), null); + fail("Should not have found /scoobydoo/teddybear"); + } catch (FileNotFoundException e) { + // Should go here. + } catch (Exception e) { + // This is our problem -- it means the configuration was wrong. + e.printStackTrace(); + } + } + + @Test + public void testHadoopFsFilename() { + try { + assertEquals(null, TempletonUtils.hadoopFsFilename(null, null, null)); + assertEquals(null, TempletonUtils.hadoopFsFilename("/tmp", null, null)); + assertEquals("file:/tmp", + TempletonUtils.hadoopFsFilename("/tmp", + new Configuration(), + null)); + } catch (FileNotFoundException e) { + fail("Couldn't find name for /tmp"); + } catch (Exception e) { + // Something else is wrong + e.printStackTrace(); + } + try { + TempletonUtils.hadoopFsFilename("/scoobydoo/teddybear", + new Configuration(), null); + fail("Should not have found /scoobydoo/teddybear"); + } catch (FileNotFoundException e) { + // Should go here. + } catch (Exception e) { + // Something else is wrong. + e.printStackTrace(); + } + } + + @Test + public void testHadoopFsListAsArray() { + try { + assertTrue(TempletonUtils.hadoopFsListAsArray(null, null, null) == null); + assertTrue(TempletonUtils.hadoopFsListAsArray("/tmp, /usr", + null, null) == null); + String[] tmp2 + = TempletonUtils.hadoopFsListAsArray("/tmp,/usr", + new Configuration(), null); + assertEquals("file:/tmp", tmp2[0]); + assertEquals("file:/usr", tmp2[1]); + } catch (FileNotFoundException e) { + fail("Couldn't find name for /tmp"); + } catch (Exception e) { + // Something else is wrong + e.printStackTrace(); + } + try { + TempletonUtils.hadoopFsListAsArray("/scoobydoo/teddybear,joe", + new Configuration(), + null); + fail("Should not have found /scoobydoo/teddybear"); + } catch (FileNotFoundException e) { + // Should go here. + } catch (Exception e) { + // Something else is wrong. + e.printStackTrace(); + } + } + + @Test + public void testHadoopFsListAsString() { + try { + assertTrue(TempletonUtils.hadoopFsListAsString(null, null, null) == null); + assertTrue(TempletonUtils.hadoopFsListAsString("/tmp,/usr", + null, null) == null); + assertEquals("file:/tmp,file:/usr", TempletonUtils.hadoopFsListAsString + ("/tmp,/usr", new Configuration(), null)); + } catch (FileNotFoundException e) { + fail("Couldn't find name for /tmp"); + } catch (Exception e) { + // Something else is wrong + e.printStackTrace(); + } + try { + TempletonUtils.hadoopFsListAsString("/scoobydoo/teddybear,joe", + new Configuration(), + null); + fail("Should not have found /scoobydoo/teddybear"); + } catch (FileNotFoundException e) { + // Should go here. + } catch (Exception e) { + // Something else is wrong. + e.printStackTrace(); + } + } + +} Index: webhcat/svr/src/test/java/org/apache/hcatalog/templeton/TestDesc.java =================================================================== --- webhcat/svr/src/test/java/org/apache/hcatalog/templeton/TestDesc.java (revision 0) +++ webhcat/svr/src/test/java/org/apache/hcatalog/templeton/TestDesc.java (revision 0) @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton; + +import java.io.ByteArrayOutputStream; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import junit.framework.TestCase; +import org.apache.hcatalog.templeton.ColumnDesc; +import org.apache.hcatalog.templeton.TableDesc; +import org.codehaus.jackson.map.ObjectMapper; + +/** + * TestDesc - Test the desc objects that are correctly converted to + * and from json. This also sets every field of the TableDesc object. + */ +public class TestDesc extends TestCase { + public void testTableDesc() + throws Exception + { + TableDesc td = buildTableDesc(); + assertNotNull(td); + + String json = toJson(td); + assertTrue(json.length() > 100); + + TableDesc tdCopy = (TableDesc) fromJson(json, TableDesc.class); + assertEquals(td, tdCopy); + } + + private TableDesc buildTableDesc() { + TableDesc x = new TableDesc(); + x.group = "staff"; + x.permissions = "755"; + x.external = true; + x.ifNotExists = true; + x.table = "a_table"; + x.comment = "a comment"; + x.columns = buildColumns(); + x.partitionedBy = buildPartitionedBy(); + x.clusteredBy = buildClusterBy(); + x.format = buildStorageFormat(); + x.location = "hdfs://localhost:9000/user/me/a_table"; + x.tableProperties = buildGenericProperties(); + return x; + } + + public List buildColumns() { + ArrayList x = new ArrayList(); + x.add(new ColumnDesc("id", "bigint", null)); + x.add(new ColumnDesc("price", "float", "The unit price")); + x.add(new ColumnDesc("name", "string", "The item name")); + return x; + } + + public List buildPartitionedBy() { + ArrayList x = new ArrayList(); + x.add(new ColumnDesc("country", "string", "The country of origin")); + return x; + } + + public TableDesc.ClusteredByDesc buildClusterBy() { + TableDesc.ClusteredByDesc x = new TableDesc.ClusteredByDesc(); + x.columnNames = new ArrayList(); + x.columnNames.add("id"); + x.sortedBy = buildSortedBy(); + x.numberOfBuckets = 16; + return x; + } + + public List buildSortedBy() { + ArrayList x + = new ArrayList(); + x.add(new TableDesc.ClusterSortOrderDesc("id", TableDesc.SortDirectionDesc.ASC)); + return x; + } + + public TableDesc.StorageFormatDesc buildStorageFormat() { + TableDesc.StorageFormatDesc x = new TableDesc.StorageFormatDesc(); + x.rowFormat = buildRowFormat(); + x.storedAs = "rcfile"; + x.storedBy = buildStoredBy(); + return x; + } + + public TableDesc.RowFormatDesc buildRowFormat() { + TableDesc.RowFormatDesc x = new TableDesc.RowFormatDesc(); + x.fieldsTerminatedBy = "\u0001"; + x.collectionItemsTerminatedBy = "\u0002"; + x.mapKeysTerminatedBy = "\u0003"; + x.linesTerminatedBy = "\u0004"; + x.serde = buildSerde(); + return x; + } + + public TableDesc.SerdeDesc buildSerde() { + TableDesc.SerdeDesc x = new TableDesc.SerdeDesc(); + x.name = "org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe"; + x.properties = new HashMap(); + x.properties.put("field.delim", ","); + return x; + } + + public TableDesc.StoredByDesc buildStoredBy() { + TableDesc.StoredByDesc x = new TableDesc.StoredByDesc(); + x.className = "org.apache.hadoop.hive.hbase.HBaseStorageHandler"; + x.properties = new HashMap(); + x.properties.put("hbase.columns.mapping", "cf:string"); + x.properties.put("hbase.table.name", "hbase_table_0"); + return x; + } + + public Map buildGenericProperties() { + HashMap x = new HashMap(); + x.put("carmas", "evil"); + x.put("rachel", "better"); + x.put("ctdean", "angelic"); + x.put("paul", "dangerously unbalanced"); + x.put("dra", "organic"); + return x; + } + + private String toJson(Object obj) + throws Exception + { + ObjectMapper mapper = new ObjectMapper(); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + mapper.writeValue(out, obj); + return out.toString(); + } + + private Object fromJson(String json, Class klass) + throws Exception + { + ObjectMapper mapper = new ObjectMapper(); + return mapper.readValue(json, klass); + } +} Index: webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CompleteDelegator.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CompleteDelegator.java (revision 0) +++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CompleteDelegator.java (revision 0) @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton; + +import java.io.IOException; +import java.net.URL; +import java.net.MalformedURLException; +import java.util.Date; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.mapred.JobID; +import org.apache.hadoop.mapred.JobProfile; +import org.apache.hadoop.mapred.JobStatus; +import org.apache.hadoop.mapred.JobTracker; +import org.apache.hadoop.mapred.TempletonJobTracker; +import org.apache.hcatalog.templeton.tool.JobState; +import org.apache.hcatalog.templeton.tool.TempletonUtils; + +/** + * Complete a job. This will run the callback if + * + * - the job is done + * - there is a callback + * - the callback has not yet been called + * + * There is a small chance for a race condition if two callers run + * this at the same time. That should never happen. + * + * We use a Hadoop config var to notify this class on the completion + * of a job. Hadoop will call use multiple times in the event of + * failure. Even if the failure is that the client callback failed. + * + * See LauncherDelegator for the HADOOP_END_RETRY* vars that are set. + */ +public class CompleteDelegator extends TempletonDelegator { + private static final Log LOG = LogFactory.getLog(CompleteDelegator.class); + + public CompleteDelegator(AppConfig appConf) { + super(appConf); + } + + public CompleteBean run(String id) + throws CallbackFailedException, IOException + { + if (id == null) + acceptWithError("No jobid given"); + + JobState state = null; + try { + state = new JobState(id, Main.getAppConfigInstance()); + if (state.getCompleteStatus() == null) + failed("Job not yet complete", null); + + Long notified = state.getNotifiedTime(); + if (notified != null) + return acceptWithError("Callback already run on " + + new Date(notified.longValue())); + + String callback = state.getCallback(); + if (callback == null) + return new CompleteBean("No callback registered"); + + try { + doCallback(state.getId(), callback); + } catch (Exception e) { + failed("Callback failed " + callback + " for " + id, e); + } + + state.setNotifiedTime(System.currentTimeMillis()); + return new CompleteBean("Callback sent"); + } finally { + if (state != null) + state.close(); + } + } + + /** + * Call the callback url with the jobid to let them know it's + * finished. If the url has the string $jobId in it, it will be + * replaced with the completed jobid. + */ + public static void doCallback(String jobid, String url) + throws MalformedURLException, IOException + { + if (url.contains("$jobId")) + url = url.replace("$jobId", jobid); + TempletonUtils.fetchUrl(new URL(url)); + } + + private void failed(String msg, Exception e) + throws CallbackFailedException + { + if (e != null) + LOG.error(msg, e); + else + LOG.error(msg); + throw new CallbackFailedException(msg); + } + + private CompleteBean acceptWithError(String msg) { + LOG.error(msg); + return new CompleteBean(msg); + } +} Index: webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ListDelegator.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ListDelegator.java (revision 0) +++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ListDelegator.java (revision 0) @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton; + +import java.io.IOException; +import java.util.List; +import java.util.ArrayList; +import org.apache.hadoop.mapred.JobID; +import org.apache.hadoop.mapred.JobProfile; +import org.apache.hadoop.mapred.JobStatus; +import org.apache.hadoop.mapred.JobTracker; +import org.apache.hadoop.mapred.TempletonJobTracker; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hcatalog.templeton.tool.JobState; +import org.apache.hcatalog.templeton.tool.TempletonUtils; +import org.apache.zookeeper.ZooKeeper; + +/** + * List jobs owned by a user. + */ +public class ListDelegator extends TempletonDelegator { + public ListDelegator(AppConfig appConf) { + super(appConf); + } + + public List run(String user) + throws NotAuthorizedException, BadParam, IOException + { + UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user); + TempletonJobTracker tracker = null; + try { + tracker = new TempletonJobTracker(ugi, + JobTracker.getAddress(appConf), + appConf); + + ArrayList ids = new ArrayList(); + + JobStatus[] jobs = tracker.getAllJobs(); + + if (jobs != null) { + for (JobStatus job : jobs) { + JobState state = null; + try { + String id = job.getJobID().toString(); + state = new JobState(id, Main.getAppConfigInstance()); + if (user.equals(state.getUser())) + ids.add(id); + } finally { + if (state != null) { + state.close(); + } + } + } + } + + return ids; + } catch (IllegalStateException e) { + throw new BadParam(e.getMessage()); + } finally { + if (tracker != null) + tracker.close(); + } + } +} Index: webhcat/svr/src/main/java/org/apache/hcatalog/templeton/JarDelegator.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/JarDelegator.java (revision 0) +++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/JarDelegator.java (revision 0) @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.List; +import org.apache.commons.exec.ExecuteException; +import org.apache.hcatalog.templeton.tool.TempletonUtils; + +/** + * Submit a job to the MapReduce queue. + * + * This is the backend of the mapreduce/jar web service. + */ +public class JarDelegator extends LauncherDelegator { + public JarDelegator(AppConfig appConf) { + super(appConf); + } + + public EnqueueBean run(String user, String jar, String mainClass, + String libjars, String files, + List jarArgs, List defines, + String statusdir, String callback, String completedUrl) + throws NotAuthorizedException, BadParam, BusyException, QueueException, + ExecuteException, IOException, InterruptedException + { + runAs = user; + List args = makeArgs(jar, mainClass, + libjars, files, jarArgs, defines, + statusdir, completedUrl); + + return enqueueController(user, callback, args); + } + + private List makeArgs(String jar, String mainClass, + String libjars, String files, + List jarArgs, List defines, + String statusdir, String completedUrl) + throws BadParam, IOException, InterruptedException + { + ArrayList args = new ArrayList(); + try { + ArrayList allFiles = new ArrayList(); + allFiles.add(TempletonUtils.hadoopFsFilename(jar, appConf, runAs)); + + args.addAll(makeLauncherArgs(appConf, statusdir, + completedUrl, allFiles)); + args.add("--"); + args.add(appConf.clusterHadoop()); + args.add("jar"); + args.add(TempletonUtils.hadoopFsPath(jar, appConf, runAs).getName()); + if (TempletonUtils.isset(mainClass)) + args.add(mainClass); + if (TempletonUtils.isset(libjars)) { + args.add("-libjars"); + args.add(TempletonUtils.hadoopFsListAsString(libjars, appConf, + runAs)); + } + if (TempletonUtils.isset(files)) { + args.add("-files"); + args.add(TempletonUtils.hadoopFsListAsString(files, appConf, + runAs)); + } + + for (String d : defines) + args.add("-D" + d); + + args.addAll(jarArgs); + } catch (FileNotFoundException e) { + throw new BadParam(e.getMessage()); + } catch (URISyntaxException e) { + throw new BadParam(e.getMessage()); + } + + return args; + } +} Index: webhcat/svr/src/main/java/org/apache/hcatalog/templeton/MaxByteArrayOutputStream.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/MaxByteArrayOutputStream.java (revision 0) +++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/MaxByteArrayOutputStream.java (revision 0) @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton; + +import java.io.ByteArrayOutputStream; + +/** + * An output stream that will only accept the first N bytes of data. + */ +public class MaxByteArrayOutputStream extends ByteArrayOutputStream { + /** + * The max number of bytes stored. + */ + private int maxBytes; + + /** + * The number of bytes currently stored. + */ + private int nBytes; + + /** + * Create. + */ + public MaxByteArrayOutputStream(int maxBytes) { + this.maxBytes = maxBytes; + nBytes = 0; + } + + /** + * Writes the specified byte to this byte array output stream. + * Any bytes after the first maxBytes will be ignored. + * + * @param b the byte to be written. + */ + public synchronized void write(int b) { + if (nBytes < maxBytes) { + ++nBytes; + super.write(b); + } + } + + /** + * Writes len bytes from the specified byte array + * starting at offset off to this byte array output stream. + * Any bytes after the first maxBytes will be ignored. + * + * @param b the data. + * @param off the start offset in the data. + * @param len the number of bytes to write. + */ + public synchronized void write(byte b[], int off, int len) { + int storable = Math.min(maxBytes - nBytes, len); + if (storable > 0) { + nBytes += storable; + super.write(b, off, storable); + } + } + + +} Index: webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ExecService.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ExecService.java (revision 0) +++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ExecService.java (revision 0) @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import org.apache.commons.exec.ExecuteException; + +public interface ExecService { + public ExecBean run(String program, List args, + Map env) + throws NotAuthorizedException, BusyException, ExecuteException, IOException; + + public ExecBean runUnlimited(String program, List args, + Map env) + throws NotAuthorizedException, ExecuteException, IOException; +} Index: webhcat/svr/src/main/java/org/apache/hcatalog/templeton/PigDelegator.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/PigDelegator.java (revision 0) +++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/PigDelegator.java (revision 0) @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import org.apache.commons.exec.ExecuteException; +import org.apache.hcatalog.templeton.tool.TempletonUtils; + +/** + * Submit a Pig job. + * + * This is the backend of the pig web service. + */ +public class PigDelegator extends LauncherDelegator { + public PigDelegator(AppConfig appConf) { + super(appConf); + } + + public EnqueueBean run(String user, + String execute, String srcFile, + List pigArgs, String otherFiles, + String statusdir, String callback, String completedUrl) + throws NotAuthorizedException, BadParam, BusyException, QueueException, + ExecuteException, IOException, InterruptedException + { + runAs = user; + List args = makeArgs(execute, + srcFile, pigArgs, + otherFiles, statusdir, completedUrl); + + return enqueueController(user, callback, args); + } + + private List makeArgs(String execute, String srcFile, + List pigArgs, String otherFiles, + String statusdir, String completedUrl) + throws BadParam, IOException, InterruptedException + { + ArrayList args = new ArrayList(); + try { + ArrayList allFiles = new ArrayList(); + if (TempletonUtils.isset(srcFile)) + allFiles.add(TempletonUtils.hadoopFsFilename + (srcFile, appConf, runAs)); + if (TempletonUtils.isset(otherFiles)) { + String[] ofs = TempletonUtils.hadoopFsListAsArray + (otherFiles, appConf, runAs); + allFiles.addAll(Arrays.asList(ofs)); + } + + args.addAll(makeLauncherArgs(appConf, statusdir, completedUrl, allFiles)); + args.add("-archives"); + args.add(appConf.pigArchive()); + + args.add("--"); + args.add(appConf.pigPath()); + args.addAll(pigArgs); + if (TempletonUtils.isset(execute)) { + args.add("-execute"); + args.add(execute); + } else if (TempletonUtils.isset(srcFile)) { + args.add("-file"); + args.add(TempletonUtils.hadoopFsPath(srcFile, appConf, runAs) + .getName()); + } + } catch (FileNotFoundException e) { + throw new BadParam(e.getMessage()); + } catch (URISyntaxException e) { + throw new BadParam(e.getMessage()); + } + + return args; + } +} Index: webhcat/svr/src/main/java/org/apache/hcatalog/templeton/SimpleWebException.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/SimpleWebException.java (revision 0) +++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/SimpleWebException.java (revision 0) @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton; + +import java.io.IOException; +import java.util.Map; +import java.util.HashMap; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response.ResponseBuilder; +import javax.ws.rs.core.Response; +import org.codehaus.jackson.map.ObjectMapper; + +/** + * Simple exception that will return a json error payload if thrown + * from a JAX web server. We skip using WebApplicationException and + * instead map our own so that Jersey doesn't log our exceptions as + * error in the output log. See SimpleExceptionMapper. + */ +public class SimpleWebException extends Throwable { + public int httpCode; + public Map params; + + public SimpleWebException(int httpCode, String msg) { + super(msg); + this.httpCode = httpCode; + } + + public SimpleWebException(int httpCode, String msg, Map params) { + super(msg); + this.httpCode = httpCode; + this.params = params; + } + + public Response getResponse() { + return buildMessage(httpCode, params, getMessage()); + } + + public static Response buildMessage(int httpCode, Map params, + String msg) + { + HashMap err = new HashMap(); + err.put("error", msg); + if (params != null) + err.putAll(params); + + String json = "\"error\""; + try { + json = new ObjectMapper().writeValueAsString(err); + } catch (IOException e) { + } + + return Response.status(httpCode) + .entity(json) + .type(MediaType.APPLICATION_JSON) + .build(); + } +} Index: webhcat/svr/src/main/java/org/apache/hcatalog/templeton/BadParam.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/BadParam.java (revision 0) +++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/BadParam.java (revision 0) @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton; + +/** + * Missing required or badly configured paramater. + */ +public class BadParam extends SimpleWebException { + public BadParam(String msg) { + super(400, msg); + } +} Index: webhcat/svr/src/main/java/org/apache/hcatalog/templeton/PartitionDesc.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/PartitionDesc.java (revision 0) +++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/PartitionDesc.java (revision 0) @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton; + +import javax.xml.bind.annotation.XmlRootElement; + +/** + * A description of the partition to create. + */ +@XmlRootElement +public class PartitionDesc extends GroupPermissionsDesc { + public String partition; + public String location; + public boolean ifNotExists = false; + + public PartitionDesc() {} + + public String toString() { + return String.format("PartitionDesc(partition=%s, location=%s, ifNotExists=%s)", + partition, location, ifNotExists); + } +} Index: webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ExecBean.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ExecBean.java (revision 0) +++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ExecBean.java (revision 0) @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton; + +/** + * ExecBean - The results of an exec call. + */ +public class ExecBean { + public String stdout; + public String stderr; + public int exitcode; + + public ExecBean() {} + + /** + * Create a new ExecBean. + * + * @param stdout standard output of the the program. + * @param stderr error output of the the program. + * @param exitcode exit code of the program. + */ + public ExecBean(String stdout, String stderr, int exitcode) { + this.stdout = stdout; + this.stderr = stderr; + this.exitcode = exitcode; + } + + public String toString() { + return String.format("ExecBean(stdout=%s, stderr=%s, exitcode=%s)", + stdout, stderr, exitcode); + } +} Index: webhcat/svr/src/main/java/org/apache/hcatalog/templeton/UgiFactory.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/UgiFactory.java (revision 0) +++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/UgiFactory.java (revision 0) @@ -0,0 +1,32 @@ +package org.apache.hcatalog.templeton; + +import java.io.IOException; +import java.util.concurrent.ConcurrentHashMap; + +import org.apache.hadoop.security.UserGroupInformation; + +public class UgiFactory { + private static ConcurrentHashMap userUgiMap = + new ConcurrentHashMap(); + + static UserGroupInformation getUgi(String user) throws IOException{ + UserGroupInformation ugi = userUgiMap.get(user); + if(ugi == null){ + //create new ugi and add to map + final UserGroupInformation newUgi = + UserGroupInformation.createProxyUser(user, + UserGroupInformation.getLoginUser()); + + //if another thread adds an entry before the check in this one + // the one created here will not be added. + userUgiMap.putIfAbsent(user, newUgi); + + //use the UGI object that got added + return userUgiMap.get(user); + + } + return ugi; + } + + +} Index: webhcat/svr/src/main/java/org/apache/hcatalog/templeton/SimpleExceptionMapper.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/SimpleExceptionMapper.java (revision 0) +++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/SimpleExceptionMapper.java (revision 0) @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton; + +import javax.ws.rs.core.Response; +import javax.ws.rs.ext.ExceptionMapper; +import javax.ws.rs.ext.Provider; + +/** + * Map our exceptions to the Jersey response. This lets us have nice + * results in the error body. + */ +@Provider +public class SimpleExceptionMapper + implements ExceptionMapper +{ + public Response toResponse(SimpleWebException e) { + return e.getResponse(); + } +} Index: webhcat/svr/src/main/java/org/apache/hcatalog/templeton/WadlConfig.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/WadlConfig.java (revision 0) +++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/WadlConfig.java (revision 0) @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton; + +import java.util.List; + +import com.sun.jersey.api.wadl.config.WadlGeneratorConfig; +import com.sun.jersey.api.wadl.config.WadlGeneratorDescription; +import com.sun.jersey.server.wadl.generators.resourcedoc.WadlGeneratorResourceDocSupport; + +/** + * Simple class that incorporates javadoc information into the + * wadl produced by jersey. + * + */ +public class WadlConfig extends WadlGeneratorConfig { + + @Override + public List configure() { + return generator( WadlGeneratorResourceDocSupport.class ) + .prop( "resourceDocStream", "resourcedoc.xml" ) + .descriptions(); + } + +} Index: webhcat/svr/src/main/java/org/apache/hcatalog/templeton/NotAuthorizedException.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/NotAuthorizedException.java (revision 0) +++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/NotAuthorizedException.java (revision 0) @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton; + +/** + * Simple "user not found" type exception. + */ +public class NotAuthorizedException extends SimpleWebException { + public NotAuthorizedException(String msg) { + super(401, msg); + } +} Index: webhcat/svr/src/main/java/org/apache/hcatalog/templeton/HiveDelegator.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/HiveDelegator.java (revision 0) +++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/HiveDelegator.java (revision 0) @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.List; +import org.apache.commons.exec.ExecuteException; +import org.apache.hcatalog.templeton.tool.TempletonUtils; + +/** + * Submit a Hive job. + * + * This is the backend of the hive web service. + */ +public class HiveDelegator extends LauncherDelegator { + + public HiveDelegator(AppConfig appConf) { + super(appConf); + } + + public EnqueueBean run(String user, + String execute, String srcFile, List defines, + String statusdir, String callback, String completedUrl) + throws NotAuthorizedException, BadParam, BusyException, QueueException, + ExecuteException, IOException, InterruptedException + { + runAs = user; + List args = makeArgs(execute, srcFile, defines, statusdir, + completedUrl); + + return enqueueController(user, callback, args); + } + + private List makeArgs(String execute, String srcFile, + List defines, String statusdir, String completedUrl) + throws BadParam, IOException, InterruptedException + { + ArrayList args = new ArrayList(); + try { + args.addAll(makeBasicArgs(execute, srcFile, statusdir, completedUrl)); + args.add("--"); + args.add(appConf.hivePath()); + args.add("--service"); + args.add("cli"); + for (String prop : appConf.getStrings(AppConfig.HIVE_PROPS_NAME)) { + args.add("--hiveconf"); + args.add(prop); + } + for (String prop : defines) { + args.add("--hiveconf"); + args.add(prop); + } + if (TempletonUtils.isset(execute)) { + args.add("-e"); + args.add(execute); + } else if (TempletonUtils.isset(srcFile)) { + args.add("-f"); + args.add(TempletonUtils.hadoopFsPath(srcFile, appConf, runAs) + .getName()); + } + } catch (FileNotFoundException e) { + throw new BadParam(e.getMessage()); + } catch (URISyntaxException e) { + throw new BadParam(e.getMessage()); + } + + return args; + } + + private List makeBasicArgs(String execute, String srcFile, + String statusdir, String completedUrl) + throws URISyntaxException, FileNotFoundException, IOException, + InterruptedException + { + ArrayList args = new ArrayList(); + + ArrayList allFiles = new ArrayList(); + if (TempletonUtils.isset(srcFile)) + allFiles.add(TempletonUtils.hadoopFsFilename(srcFile, appConf, + runAs)); + + args.addAll(makeLauncherArgs(appConf, statusdir, completedUrl, allFiles)); + + args.add("-archives"); + args.add(appConf.hiveArchive()); + + return args; + } +} Index: webhcat/svr/src/main/java/org/apache/hcatalog/templeton/TableDesc.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/TableDesc.java (revision 0) +++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/TableDesc.java (revision 0) @@ -0,0 +1,237 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton; + +import java.util.List; +import java.util.Map; +import javax.xml.bind.annotation.XmlRootElement; + +/** + * A description of the table to create. + */ +@XmlRootElement +public class TableDesc extends GroupPermissionsDesc { + public boolean external = false; + public boolean ifNotExists = false; + public String table; + public String comment; + public List columns; + public List partitionedBy; + public ClusteredByDesc clusteredBy; + public StorageFormatDesc format; + public String location; + public Map tableProperties; + + /** + * Create a new TableDesc + */ + public TableDesc() {} + + public String toString() { + return String.format("TableDesc(table=%s, columns=%s)", table, columns); + } + + public boolean equals(Object o) { + if (this == o) + return true; + if (! (o instanceof TableDesc)) + return false; + TableDesc that = (TableDesc) o; + return xequals(this.external, that.external) + && xequals(this.ifNotExists, that.ifNotExists) + && xequals(this.table, that.table) + && xequals(this.comment, that.comment) + && xequals(this.columns, that.columns) + && xequals(this.partitionedBy, that.partitionedBy) + && xequals(this.clusteredBy, that.clusteredBy) + && xequals(this.format, that.format) + && xequals(this.location, that.location) + && xequals(this.tableProperties, that.tableProperties) + && super.equals(that) + ; + } + + /** + * How to cluster the table. + */ + @XmlRootElement + public static class ClusteredByDesc { + public List columnNames; + public List sortedBy; + public int numberOfBuckets; + + public ClusteredByDesc() {} + + public String toString() { + String fmt + = "ClusteredByDesc(columnNames=%s, sortedBy=%s, numberOfBuckets=%s)"; + return String.format(fmt, columnNames, sortedBy, numberOfBuckets); + } + + public boolean equals(Object o) { + if (this == o) + return true; + if (! (o instanceof ClusteredByDesc)) + return false; + ClusteredByDesc that = (ClusteredByDesc) o; + return xequals(this.columnNames, that.columnNames) + && xequals(this.sortedBy, that.sortedBy) + && xequals(this.numberOfBuckets, that.numberOfBuckets) + ; + } + } + + /** + * The clustered sort order. + */ + @XmlRootElement + public static class ClusterSortOrderDesc { + public String columnName; + public SortDirectionDesc order; + + public ClusterSortOrderDesc() {} + + public ClusterSortOrderDesc(String columnName, SortDirectionDesc order) { + this.columnName = columnName; + this.order = order; + } + + public String toString() { + return String + .format("ClusterSortOrderDesc(columnName=%s, order=%s)", + columnName, order); + } + + public boolean equals(Object o) { + if (this == o) + return true; + if (! (o instanceof ClusterSortOrderDesc)) + return false; + ClusterSortOrderDesc that = (ClusterSortOrderDesc) o; + return xequals(this.columnName, that.columnName) + && xequals(this.order, that.order) + ; + } + } + + /** + * Ther ASC or DESC sort order. + */ + @XmlRootElement + public static enum SortDirectionDesc { + ASC, DESC + } + + /** + * The storage format. + */ + @XmlRootElement + public static class StorageFormatDesc { + public RowFormatDesc rowFormat; + public String storedAs; + public StoredByDesc storedBy; + + public StorageFormatDesc() {} + + public boolean equals(Object o) { + if (this == o) + return true; + if (! (o instanceof StorageFormatDesc)) + return false; + StorageFormatDesc that = (StorageFormatDesc) o; + return xequals(this.rowFormat, that.rowFormat) + && xequals(this.storedAs, that.storedAs) + && xequals(this.storedBy, that.storedBy) + ; + } + } + + /** + * The Row Format. + */ + @XmlRootElement + public static class RowFormatDesc { + public String fieldsTerminatedBy; + public String collectionItemsTerminatedBy; + public String mapKeysTerminatedBy; + public String linesTerminatedBy; + public SerdeDesc serde; + + public RowFormatDesc() {} + + public boolean equals(Object o) { + if (this == o) + return true; + if (! (o instanceof RowFormatDesc)) + return false; + RowFormatDesc that = (RowFormatDesc) o; + return xequals(this.fieldsTerminatedBy, that.fieldsTerminatedBy) + && xequals(this.collectionItemsTerminatedBy, + that.collectionItemsTerminatedBy) + && xequals(this.mapKeysTerminatedBy, that.mapKeysTerminatedBy) + && xequals(this.linesTerminatedBy, that.linesTerminatedBy) + && xequals(this.serde, that.serde) + ; + } + } + + /** + * The SERDE Row Format. + */ + @XmlRootElement + public static class SerdeDesc { + public String name; + public Map properties; + + public SerdeDesc() {} + + public boolean equals(Object o) { + if (this == o) + return true; + if (! (o instanceof SerdeDesc)) + return false; + SerdeDesc that = (SerdeDesc) o; + return xequals(this.name, that.name) + && xequals(this.properties, that.properties) + ; + } + } + + /** + * How to store the table. + */ + @XmlRootElement + public static class StoredByDesc { + public String className; + public Map properties; + + public StoredByDesc() {} + + public boolean equals(Object o) { + if (this == o) + return true; + if (! (o instanceof StoredByDesc)) + return false; + StoredByDesc that = (StoredByDesc) o; + return xequals(this.className, that.className) + && xequals(this.properties, that.properties) + ; + } + } + +} Index: webhcat/svr/src/main/java/org/apache/hcatalog/templeton/GroupPermissionsDesc.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/GroupPermissionsDesc.java (revision 0) +++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/GroupPermissionsDesc.java (revision 0) @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton; + +import javax.xml.bind.annotation.XmlRootElement; + +/** + * The base create permissions for ddl objects. + */ +public abstract class GroupPermissionsDesc { + public String group; + public String permissions; + + public GroupPermissionsDesc() {} + + protected static boolean xequals(Object a, Object b) { + if (a == null) { + if (b == null) + return true; + else + return false; + } + + return a.equals(b); + } + + protected static boolean xequals(boolean a, boolean b) { return a == b; } + protected static boolean xequals(int a, int b) { return a == b; } + protected static boolean xequals(char a, char b) { return a == b; } + + public boolean equals(Object o) { + if (this == o) + return true; + if (! (o instanceof GroupPermissionsDesc)) + return false; + GroupPermissionsDesc that = (GroupPermissionsDesc) o; + return xequals(this.group, that.group) + && xequals(this.permissions, that.permissions) + ; + } +} Index: webhcat/svr/src/main/java/org/apache/hcatalog/templeton/LauncherDelegator.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/LauncherDelegator.java (revision 0) +++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/LauncherDelegator.java (revision 0) @@ -0,0 +1,198 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton; + +import java.io.IOException; +import java.security.PrivilegedExceptionAction; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import org.apache.commons.exec.ExecuteException; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.util.ToolRunner; +import org.apache.hcatalog.templeton.tool.JobState; +import org.apache.hcatalog.templeton.tool.TempletonControllerJob; +import org.apache.hcatalog.templeton.tool.TempletonStorage; +import org.apache.hcatalog.templeton.tool.TempletonUtils; +import org.apache.hcatalog.templeton.tool.ZooKeeperStorage; + +/** + * The helper class for all the Templeton delegator classes that + * launch child jobs. + */ +public class LauncherDelegator extends TempletonDelegator { + private static final Log LOG = LogFactory.getLog(Server.class); + public static final String JAR_CLASS = TempletonControllerJob.class.getName(); + protected String runAs = null; + + public LauncherDelegator(AppConfig appConf) { + super(appConf); + } + + public void registerJob(String id, String user, String callback) + throws IOException + { + JobState state = null; + try { + state = new JobState(id, Main.getAppConfigInstance()); + state.setUser(user); + state.setCallback(callback); + } finally { + if (state != null) + state.close(); + } + } + + /** + * Enqueue the TempletonControllerJob directly calling doAs. + */ + public EnqueueBean enqueueController(String user, String callback, + List args) + throws NotAuthorizedException, BusyException, ExecuteException, + IOException, QueueException + { + try { + UserGroupInformation ugi = UgiFactory.getUgi(user); + + final long startTime = System.nanoTime(); + + String id = queueAsUser(ugi, args); + + long elapsed = ((System.nanoTime() - startTime) / ((int) 1e6)); + LOG.debug("queued job " + id + " in " + elapsed + " ms"); + + if (id == null) + throw new QueueException("Unable to get job id"); + + registerJob(id, user, callback); + + return new EnqueueBean(id); + } catch (InterruptedException e) { + throw new QueueException("Unable to launch job " + e); + } + } + + private String queueAsUser(UserGroupInformation ugi, final List args) + throws IOException, InterruptedException + { + String id = ugi.doAs(new PrivilegedExceptionAction() { + public String run() throws Exception { + String[] array = new String[args.size()]; + TempletonControllerJob ctrl = new TempletonControllerJob(); + ToolRunner.run(ctrl, args.toArray(array)); + return ctrl.getSubmittedId(); + } + }); + + return id; + } + + public List makeLauncherArgs(AppConfig appConf, String statusdir, + String completedUrl, + List copyFiles) + { + ArrayList args = new ArrayList(); + + args.add("-libjars"); + args.add(appConf.libJars()); + addCacheFiles(args, appConf); + + // Hadoop vars + addDef(args, "user.name", runAs); + addDef(args, AppConfig.HADOOP_SPECULATIVE_NAME, "false"); + + // Internal vars + addDef(args, TempletonControllerJob.STATUSDIR_NAME, statusdir); + addDef(args, TempletonControllerJob.COPY_NAME, + TempletonUtils.encodeArray(copyFiles)); + addDef(args, TempletonControllerJob.OVERRIDE_CLASSPATH, + makeOverrideClasspath(appConf)); + + // Job vars + addStorageVars(args); + addCompletionVars(args, completedUrl); + + return args; + } + + // Storage vars + private void addStorageVars(List args) { + addDef(args, TempletonStorage.STORAGE_CLASS, + appConf.get(TempletonStorage.STORAGE_CLASS)); + addDef(args, TempletonStorage.STORAGE_ROOT, + appConf.get(TempletonStorage.STORAGE_ROOT)); + addDef(args, ZooKeeperStorage.ZK_HOSTS, + appConf.get(ZooKeeperStorage.ZK_HOSTS)); + addDef(args, ZooKeeperStorage.ZK_SESSION_TIMEOUT, + appConf.get(ZooKeeperStorage.ZK_SESSION_TIMEOUT)); + } + + // Completion notifier vars + private void addCompletionVars(List args, String completedUrl) { + addDef(args, AppConfig.HADOOP_END_RETRY_NAME, + appConf.get(AppConfig.CALLBACK_RETRY_NAME)); + addDef(args, AppConfig.HADOOP_END_INTERVAL_NAME, + appConf.get(AppConfig.CALLBACK_INTERVAL_NAME)); + addDef(args, AppConfig.HADOOP_END_URL_NAME, completedUrl); + } + + /** + * Add files to the Distributed Cache for the controller job. + */ + public static void addCacheFiles(List args, AppConfig appConf) { + String overrides = appConf.overrideJarsString(); + if (overrides != null) { + args.add("-files"); + args.add(overrides); + } + } + + /** + * Create the override classpath, which will be added to + * HADOOP_CLASSPATH at runtime by the controller job. + */ + public static String makeOverrideClasspath(AppConfig appConf) { + String[] overrides = appConf.overrideJars(); + if (overrides == null) + return null; + + ArrayList cp = new ArrayList(); + for (String fname : overrides) { + Path p = new Path(fname); + cp.add(p.getName()); + } + return StringUtils.join(":", cp); + } + + + /** + * Add a Hadoop command line definition to args if the value is + * not null. + */ + public static void addDef(List args, String name, String val) { + if (val != null) { + args.add("-D"); + args.add(name + "=" + val); + } + } + +} Index: webhcat/svr/src/main/java/org/apache/hcatalog/templeton/DeleteDelegator.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/DeleteDelegator.java (revision 0) +++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/DeleteDelegator.java (revision 0) @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton; + +import java.io.IOException; +import org.apache.hadoop.mapred.JobID; +import org.apache.hadoop.mapred.JobProfile; +import org.apache.hadoop.mapred.JobStatus; +import org.apache.hadoop.mapred.JobTracker; +import org.apache.hadoop.mapred.TempletonJobTracker; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hcatalog.templeton.tool.JobState; + +/** + * Delete a job + */ +public class DeleteDelegator extends TempletonDelegator { + public DeleteDelegator(AppConfig appConf) { + super(appConf); + } + + public QueueStatusBean run(String user, String id) + throws NotAuthorizedException, BadParam, IOException + { + UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user); + TempletonJobTracker tracker = null; + JobState state = null; + try { + tracker = new TempletonJobTracker(ugi, + JobTracker.getAddress(appConf), + appConf); + JobID jobid = StatusDelegator.StringToJobID(id); + if (jobid == null) + throw new BadParam("Invalid jobid: " + id); + tracker.killJob(jobid); + state = new JobState(id, Main.getAppConfigInstance()); + String childid = state.getChildId(); + if (childid != null) + tracker.killJob(StatusDelegator.StringToJobID(childid)); + return StatusDelegator.makeStatus(tracker, jobid, state); + } catch (IllegalStateException e) { + throw new BadParam(e.getMessage()); + } finally { + if (tracker != null) + tracker.close(); + if (state != null) + state.close(); + } + } +} Index: webhcat/svr/src/main/java/org/apache/hcatalog/templeton/TablePropertyDesc.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/TablePropertyDesc.java (revision 0) +++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/TablePropertyDesc.java (revision 0) @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton; + +import javax.xml.bind.annotation.XmlRootElement; + +/** + * A description of a table property. + */ +@XmlRootElement +public class TablePropertyDesc extends GroupPermissionsDesc { + public String name; + public String value; + + public TablePropertyDesc() {} + + public String toString() { + return String.format("TablePropertyDesc(name=%s, value=%s)", + name, value); + } +} Index: webhcat/svr/src/main/java/org/apache/hcatalog/templeton/TableLikeDesc.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/TableLikeDesc.java (revision 0) +++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/TableLikeDesc.java (revision 0) @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton; + +import javax.xml.bind.annotation.XmlRootElement; + +/** + * A description of the table to create that's like another table. + */ +@XmlRootElement +public class TableLikeDesc extends GroupPermissionsDesc { + public boolean external = false; + public boolean ifNotExists = false; + public String location; + public String existingTable; + public String newTable; + + public TableLikeDesc() {} + + public String toString() { + return String.format("TableLikeDesc(existingTable=%s, newTable=%s, location=%s", + existingTable, newTable, location); + } +} Index: webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/TrivialExecService.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/TrivialExecService.java (revision 0) +++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/TrivialExecService.java (revision 0) @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton.tool; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +/** + * Execute a local program. This is a singleton service that will + * execute a programs on the local box. + */ +public class TrivialExecService { + private static volatile TrivialExecService theSingleton; + + /** + * Retrieve the singleton. + */ + public static synchronized TrivialExecService getInstance() { + if (theSingleton == null) + theSingleton = new TrivialExecService(); + return theSingleton; + } + + public Process run(List cmd, List removeEnv, + Map environmentVariables) + throws IOException + { + System.err.println("templeton: starting " + cmd); + System.err.print("With environment variables: " ); + for(Map.Entry keyVal : environmentVariables.entrySet()){ + System.err.println(keyVal.getKey() + "=" + keyVal.getValue()); + } + ProcessBuilder pb = new ProcessBuilder(cmd); + for (String key : removeEnv) + pb.environment().remove(key); + pb.environment().putAll(environmentVariables); + return pb.start(); + } + +} Index: webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/NullSplit.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/NullSplit.java (revision 0) +++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/NullSplit.java (revision 0) @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton.tool; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.mapreduce.InputSplit; + +/** + * An empty splitter. + */ +public class NullSplit extends InputSplit implements Writable { + public long getLength() { return 0; } + + public String[] getLocations() throws IOException { + return new String[]{}; + } + + @Override + public void write(DataOutput out) throws IOException {} + + @Override + public void readFields(DataInput in) throws IOException {} +} + Index: webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/TempletonControllerJob.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/TempletonControllerJob.java (revision 0) +++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/TempletonControllerJob.java (revision 0) @@ -0,0 +1,339 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton.tool; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.PrintWriter; +import java.security.PrivilegedExceptionAction; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapred.JobClient; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapreduce.Counter; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.JobID; +import org.apache.hadoop.mapreduce.Mapper; +import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; +import org.apache.hcatalog.templeton.SecureProxySupport; +import org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier; + +/** + * A Map Reduce job that will start another job. + * + * We have a single Mapper job that starts a child MR job. The parent + * monitors the child child job and ends when the child job exits. In + * addition, we + * + * - write out the parent job id so the caller can record it. + * - run a keep alive thread so the job doesn't end. + * - Optionally, store the stdout, stderr, and exit value of the child + * in hdfs files. + */ +public class TempletonControllerJob extends Configured implements Tool { + static enum ControllerCounters { SIMPLE_COUNTER }; + + public static final String COPY_NAME = "templeton.copy"; + public static final String STATUSDIR_NAME = "templeton.statusdir"; + public static final String JAR_ARGS_NAME = "templeton.args"; + public static final String OVERRIDE_CLASSPATH = "templeton.override-classpath"; + + public static final String STDOUT_FNAME = "stdout"; + public static final String STDERR_FNAME = "stderr"; + public static final String EXIT_FNAME = "exit"; + + public static final int WATCHER_TIMEOUT_SECS = 10; + public static final int KEEP_ALIVE_MSEC = 60 * 1000; + + private static TrivialExecService execService = TrivialExecService.getInstance(); + + private static final Log LOG = LogFactory.getLog(TempletonControllerJob.class); + + + public static class LaunchMapper + extends Mapper + { + protected Process startJob(Context context, String user, + String overrideClasspath) + throws IOException, InterruptedException + { + Configuration conf = context.getConfiguration(); + copyLocal(COPY_NAME, conf); + String[] jarArgs + = TempletonUtils.decodeArray(conf.get(JAR_ARGS_NAME)); + + ArrayList removeEnv = new ArrayList(); + removeEnv.add("HADOOP_ROOT_LOGGER"); + Map env = TempletonUtils.hadoopUserEnv(user, + overrideClasspath); + List jarArgsList = new LinkedList(Arrays.asList(jarArgs)); + String tokenFile = System.getenv("HADOOP_TOKEN_FILE_LOCATION"); + if(tokenFile != null){ + jarArgsList.add(1, "-Dmapreduce.job.credentials.binary=" + tokenFile ); + } + return execService.run(jarArgsList, removeEnv, env); + } + + private void copyLocal(String var, Configuration conf) + throws IOException + { + String[] filenames = TempletonUtils.decodeArray(conf.get(var)); + if (filenames != null) { + for (String filename : filenames) { + Path src = new Path(filename); + Path dst = new Path(src.getName()); + FileSystem fs = src.getFileSystem(conf); + System.err.println("templeton: copy " + src + " => " + dst); + fs.copyToLocalFile(src, dst); + } + } + } + + @Override + public void run(Context context) + throws IOException, InterruptedException + { + + Configuration conf = context.getConfiguration(); + + Process proc = startJob(context, + conf.get("user.name"), + conf.get(OVERRIDE_CLASSPATH)); + + String statusdir = conf.get(STATUSDIR_NAME); + Counter cnt = context.getCounter(ControllerCounters.SIMPLE_COUNTER); + + ExecutorService pool = Executors.newCachedThreadPool(); + executeWatcher(pool, conf, context.getJobID(), + proc.getInputStream(), statusdir, STDOUT_FNAME); + executeWatcher(pool, conf, context.getJobID(), + proc.getErrorStream(), statusdir, STDERR_FNAME); + KeepAlive keepAlive = startCounterKeepAlive(pool, cnt); + + proc.waitFor(); + keepAlive.sendReport = false; + pool.shutdown(); + if (! pool.awaitTermination(WATCHER_TIMEOUT_SECS, TimeUnit.SECONDS)) + pool.shutdownNow(); + + writeExitValue(conf, proc.exitValue(), statusdir); + JobState state = new JobState(context.getJobID().toString(), conf); + state.setExitValue(proc.exitValue()); + state.setCompleteStatus("done"); + state.close(); + + if (proc.exitValue() != 0) + System.err.println("templeton: job failed with exit code " + + proc.exitValue()); + else + System.err.println("templeton: job completed with exit code 0"); + } + + private void executeWatcher(ExecutorService pool, Configuration conf, + JobID jobid, InputStream in, String statusdir, + String name) + throws IOException + { + Watcher w = new Watcher(conf, jobid, in, statusdir, name); + pool.execute(w); + } + + private KeepAlive startCounterKeepAlive(ExecutorService pool, Counter cnt) + throws IOException + { + KeepAlive k = new KeepAlive(cnt); + pool.execute(k); + return k; + } + + private void writeExitValue(Configuration conf, int exitValue, String statusdir) + throws IOException + { + if (TempletonUtils.isset(statusdir)) { + Path p = new Path(statusdir, EXIT_FNAME); + FileSystem fs = p.getFileSystem(conf); + OutputStream out = fs.create(p); + System.err.println("templeton: Writing exit value " + + exitValue + " to " + p); + PrintWriter writer = new PrintWriter(out); + writer.println(exitValue); + writer.close(); + } + } + } + + public static class Watcher implements Runnable { + private InputStream in; + private OutputStream out; + private JobID jobid; + private Configuration conf; + + public Watcher(Configuration conf, JobID jobid, InputStream in, + String statusdir, String name) + throws IOException + { + this.conf = conf; + this.jobid = jobid; + this.in = in; + + if (name.equals(STDERR_FNAME)) + out = System.err; + else + out = System.out; + + if (TempletonUtils.isset(statusdir)) { + Path p = new Path(statusdir, name); + FileSystem fs = p.getFileSystem(conf); + out = fs.create(p); + System.err.println("templeton: Writing status to " + p); + } + } + + @Override + public void run() { + try { + InputStreamReader isr = new InputStreamReader(in); + BufferedReader reader = new BufferedReader(isr); + PrintWriter writer = new PrintWriter(out); + + String line; + while ((line = reader.readLine()) != null) { + writer.println(line); + JobState state = null; + try { + String percent = TempletonUtils.extractPercentComplete(line); + String childid = TempletonUtils.extractChildJobId(line); + + if (percent != null || childid != null) { + state = new JobState(jobid.toString(), conf); + state.setPercentComplete(percent); + state.setChildId(childid); + } + } catch (IOException e) { + System.err.println("templeton: state error: " + e); + } finally { + if (state != null) { + try { + state.close(); + } catch (IOException e) { + } + } + } + } + writer.flush(); + } catch (IOException e) { + System.err.println("templeton: execute error: " + e); + } + } + } + + public static class KeepAlive implements Runnable { + private Counter cnt; + public boolean sendReport; + + public KeepAlive(Counter cnt) + { + this.cnt = cnt; + this.sendReport = true; + } + + @Override + public void run() { + try { + while (sendReport) { + cnt.increment(1); + Thread.sleep(KEEP_ALIVE_MSEC); + } + } catch (InterruptedException e) { + // Ok to be interrupted + } + } + } + + private JobID submittedJobId; + public String getSubmittedId() { + if (submittedJobId == null) + return null; + else + return submittedJobId.toString(); + } + + /** + * Enqueue the job and print out the job id for later collection. + */ + @Override + public int run(String[] args) + throws IOException, InterruptedException, ClassNotFoundException + { + Configuration conf = getConf(); + conf.set(JAR_ARGS_NAME, TempletonUtils.encodeArray(args)); + conf.set("user.name", UserGroupInformation.getCurrentUser().getShortUserName()); + Job job = new Job(conf); + job.setJarByClass(TempletonControllerJob.class); + job.setJobName("TempletonControllerJob"); + job.setMapperClass(LaunchMapper.class); + job.setMapOutputKeyClass(Text.class); + job.setMapOutputValueClass(Text.class); + job.setInputFormatClass(SingleInputFormat.class); + NullOutputFormat of + = new NullOutputFormat(); + job.setOutputFormatClass(of.getClass()); + job.setNumReduceTasks(0); + + JobClient jc = new JobClient(new JobConf(job.getConfiguration())); + + Token mrdt = jc.getDelegationToken(new Text("mr token")); + job.getCredentials().addToken(new Text("mr token"), mrdt); + job.submit(); + + submittedJobId = job.getJobID(); + + return 0; + } + + + public static void main(String[] args) throws Exception { + int ret = ToolRunner.run(new TempletonControllerJob(), args); + if (ret != 0) + System.err.println("TempletonControllerJob failed!"); + System.exit(ret); + } +} Index: webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/JobStateTracker.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/JobStateTracker.java (revision 0) +++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/JobStateTracker.java (revision 0) @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton.tool; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.ZooKeeper; +import org.apache.zookeeper.ZooDefs.Ids; +import org.apache.zookeeper.data.Stat; + +public class JobStateTracker { + // The path to the tracking root + private String job_trackingroot = null; + + // The zookeeper connection to use + private ZooKeeper zk; + + // The id of the tracking node -- must be a SEQUENTIAL node + private String trackingnode; + + // The id of the job this tracking node represents + private String jobid; + + // The logger + private static final Log LOG = LogFactory.getLog(JobStateTracker.class); + + /** + * Constructor for a new node -- takes the jobid of an existing job + * + */ + public JobStateTracker(String node, ZooKeeper zk, boolean nodeIsTracker, + String job_trackingpath) { + this.zk = zk; + if (nodeIsTracker) { + trackingnode = node; + } else { + jobid = node; + } + job_trackingroot = job_trackingpath; + } + + /** + * Create the parent znode for this job state. + */ + public void create() + throws IOException + { + String[] paths = ZooKeeperStorage.getPaths(job_trackingroot); + for (String znode : paths) { + try { + zk.create(znode, new byte[0], + Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); + } catch (KeeperException.NodeExistsException e) { + } catch (Exception e) { + throw new IOException("Unable to create parent nodes"); + } + } + try { + trackingnode = zk.create(makeTrackingZnode(), jobid.getBytes(), + Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT_SEQUENTIAL); + } catch (Exception e) { + throw new IOException("Unable to create " + makeTrackingZnode()); + } + } + + public void delete() + throws IOException + { + try { + zk.delete(makeTrackingJobZnode(trackingnode), -1); + } catch (Exception e) { + // Might have been deleted already + LOG.info("Couldn't delete " + makeTrackingJobZnode(trackingnode)); + } + } + + /** + * Get the jobid for this tracking node + * @throws IOException + */ + public String getJobID() throws IOException { + try { + return new String(zk.getData(makeTrackingJobZnode(trackingnode), + false, new Stat())); + } catch (KeeperException e) { + // It was deleted during the transaction + throw new IOException("Node already deleted " + trackingnode); + } catch (InterruptedException e) { + throw new IOException("Couldn't read node " + trackingnode); + } + } + + /** + * Make a ZK path to a new tracking node + */ + public String makeTrackingZnode() { + return job_trackingroot + "/"; + } + + /** + * Make a ZK path to an existing tracking node + */ + public String makeTrackingJobZnode(String nodename) { + return job_trackingroot + "/" + nodename; + } + + /* + * Get the list of tracking jobs. These can be used to determine which jobs have + * expired. + */ + public static List getTrackingJobs(Configuration conf, ZooKeeper zk) + throws IOException { + ArrayList jobs = new ArrayList(); + try { + for (String myid : zk.getChildren( + conf.get(TempletonStorage.STORAGE_ROOT) + + ZooKeeperStorage.TRACKINGDIR, false)) { + jobs.add(myid); + } + } catch (Exception e) { + throw new IOException("Can't get tracking children", e); + } + return jobs; + } +} Index: webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/TempletonUtils.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/TempletonUtils.java (revision 0) +++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/TempletonUtils.java (revision 0) @@ -0,0 +1,269 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton.tool; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.io.Reader; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.net.URLConnection; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.mapreduce.JobID; +import org.apache.hadoop.util.StringUtils; + +/** + * General utility methods. + */ +public class TempletonUtils { + /** + * Is the object non-empty? + */ + public static boolean isset(String s) { + return (s != null) && (s.length() > 0); + } + + /** + * Is the object non-empty? + */ + public static boolean isset(char ch) { + return (ch != 0); + } + + /** + * Is the object non-empty? + */ + public static boolean isset(T[] a) { + return (a != null) && (a.length > 0); + } + + + /** + * Is the object non-empty? + */ + public static boolean isset(Collection col) { + return (col != null) && (! col.isEmpty()); + } + + /** + * Is the object non-empty? + */ + public static boolean isset(Map col) { + return (col != null) && (! col.isEmpty()); + } + + + public static final Pattern JAR_COMPLETE + = Pattern.compile(" map \\d+%\\s+reduce \\d+%$"); + public static final Pattern PIG_COMPLETE = Pattern.compile(" \\d+% complete$"); + + /** + * Extract the percent complete line from Pig or Jar jobs. + */ + public static String extractPercentComplete(String line) { + Matcher jar = JAR_COMPLETE.matcher(line); + if (jar.find()) + return jar.group().trim(); + + Matcher pig = PIG_COMPLETE.matcher(line); + if (pig.find()) + return pig.group().trim(); + + return null; + } + + public static final Pattern JAR_ID = Pattern.compile(" Running job: (\\S+)$"); + public static final Pattern PIG_ID = Pattern.compile(" HadoopJobId: (\\S+)$"); + public static final Pattern[] ID_PATTERNS = {JAR_ID, PIG_ID}; + + /** + * Extract the job id from jar jobs. + */ + public static String extractChildJobId(String line) { + for (Pattern p : ID_PATTERNS) { + Matcher m = p.matcher(line); + if (m.find()) + return m.group(1); + } + + return null; + } + + /** + * Take an array of strings and encode it into one string. + */ + public static String encodeArray(String[] plain) { + if (plain == null) + return null; + + String[] escaped = new String[plain.length]; + + for (int i = 0; i < plain.length; ++i) { + if (plain[i] == null) { + plain[i] = ""; + } + escaped[i] = StringUtils.escapeString(plain[i]); + } + + return StringUtils.arrayToString(escaped); + } + + /** + * Encode a List into a string. + */ + public static String encodeArray(List list) { + if (list == null) + return null; + String[] array = new String[list.size()]; + return encodeArray(list.toArray(array)); + } + + /** + * Take an encode strings and decode it into an array of strings. + */ + public static String[] decodeArray(String s) { + if (s == null) + return null; + + String[] escaped = StringUtils.split(s); + String[] plain = new String[escaped.length]; + + for (int i = 0; i < escaped.length; ++i) + plain[i] = StringUtils.unEscapeString(escaped[i]); + + return plain; + } + + public static String[] hadoopFsListAsArray(String files, Configuration conf, + String user) + throws URISyntaxException, FileNotFoundException, IOException, + InterruptedException + { + if (files == null || conf == null) { + return null; + } + String[] dirty = files.split(","); + String[] clean = new String[dirty.length]; + + for (int i = 0; i < dirty.length; ++i) + clean[i] = hadoopFsFilename(dirty[i], conf, user); + + return clean; + } + + public static String hadoopFsListAsString(String files, Configuration conf, + String user) + throws URISyntaxException, FileNotFoundException, IOException, + InterruptedException + { + if (files == null || conf == null) { + return null; + } + return StringUtils.arrayToString(hadoopFsListAsArray(files, conf, user)); + } + + public static String hadoopFsFilename(String fname, Configuration conf, String user) + throws URISyntaxException, FileNotFoundException, IOException, + InterruptedException + { + Path p = hadoopFsPath(fname, conf, user); + if (p == null) + return null; + else + return p.toString(); + } + + /** + * @return true iff we are sure the file is not there. + */ + public static boolean hadoopFsIsMissing(FileSystem fs, Path p) { + try { + return ! fs.exists(p); + } catch(Throwable t) { + // Got an error, might be there anyway due to a + // permissions problem. + return false; + } + } + + public static Path hadoopFsPath(String fname, Configuration conf, String user) + throws URISyntaxException, FileNotFoundException, IOException, + InterruptedException + { + if (fname == null || conf == null) { + return null; + } + FileSystem defaultFs = FileSystem.get(new URI(fname), conf, user); + URI u = new URI(fname); + Path p = new Path(u).makeQualified(defaultFs); + + FileSystem fs = p.getFileSystem(conf); + if (hadoopFsIsMissing(fs, p)) + throw new FileNotFoundException("File " + fname + " does not exist."); + + return p; + } + + /** + * GET the given url. Returns the number of bytes received. + */ + public static int fetchUrl(URL url) + throws IOException + { + URLConnection cnx = url.openConnection(); + InputStream in = cnx.getInputStream(); + + byte[] buf = new byte[8192]; + int total = 0; + int len = 0; + while ((len = in.read(buf)) >= 0) + total += len; + + return total; + } + + /** + * Set the environment variables to specify the hadoop user. + */ + public static Map hadoopUserEnv(String user, + String overrideClasspath) + { + HashMap env = new HashMap(); + env.put("HADOOP_USER_NAME", user); + + if (overrideClasspath != null) { + env.put("HADOOP_USER_CLASSPATH_FIRST", "true"); + String cur = System.getenv("HADOOP_CLASSPATH"); + if (TempletonUtils.isset(cur)) + overrideClasspath = overrideClasspath + ":" + cur; + env.put("HADOOP_CLASSPATH", overrideClasspath); + } + + return env; + } +} Index: webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/ZooKeeperStorage.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/ZooKeeperStorage.java (revision 0) +++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/ZooKeeperStorage.java (revision 0) @@ -0,0 +1,376 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton.tool; + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.WatchedEvent; +import org.apache.zookeeper.Watcher; +import org.apache.zookeeper.ZooDefs.Ids; +import org.apache.zookeeper.ZooKeeper; + +/** + * A storage implementation based on storing everything in ZooKeeper. + * This keeps everything in a central location that is guaranteed + * to be available and accessible. + * + * Data is stored with each key/value pair being a node in ZooKeeper. + */ +public class ZooKeeperStorage implements TempletonStorage { + + public static final String TRACKINGDIR = "/created"; + + // Locations for each of the storage types + public String storage_root = null; + public String job_path = null; + public String job_trackingpath = null; + public String overhead_path = null; + + public static final String ZK_HOSTS = "templeton.zookeeper.hosts"; + public static final String ZK_SESSION_TIMEOUT + = "templeton.zookeeper.session-timeout"; + + public static final String ENCODING = "UTF-8"; + + private static final Log LOG = LogFactory.getLog(ZooKeeperStorage.class); + + private ZooKeeper zk; + + /** + * Open a ZooKeeper connection for the JobState. + */ + public static ZooKeeper zkOpen(String zkHosts, int zkSessionTimeout) + throws IOException + { + return new ZooKeeper(zkHosts, + zkSessionTimeout, + new Watcher() { + @Override + synchronized public void process(WatchedEvent event) { + } + }); + } + + /** + * Open a ZooKeeper connection for the JobState. + */ + public static ZooKeeper zkOpen(Configuration conf) + throws IOException + { + return zkOpen(conf.get(ZK_HOSTS), + conf.getInt(ZK_SESSION_TIMEOUT, 30000)); + } + + public ZooKeeperStorage() { + // No-op -- this is needed to be able to instantiate the + // class from the name. + } + + /** + * Close this ZK connection. + */ + public void close() + throws IOException + { + if (zk != null) { + try { + zk.close(); + zk = null; + } catch (InterruptedException e) { + throw new IOException("Closing ZooKeeper connection", e); + } + } + } + + public void startCleanup(Configuration config) { + try { + ZooKeeperCleanup.startInstance(config); + } catch (Exception e) { + LOG.warn("Cleanup instance didn't start."); + } + } + + /** + * Create a node in ZooKeeper + */ + public void create(Type type, String id) + throws IOException + { + try { + String[] paths = getPaths(makeZnode(type, id)); + boolean wasCreated = false; + for (String znode : paths) { + try { + zk.create(znode, new byte[0], + Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); + wasCreated = true; + } catch (KeeperException.NodeExistsException e) { + } + } + if (wasCreated) { + try { + // Really not sure if this should go here. Will have + // to see how the storage mechanism evolves. + if (type.equals(Type.JOB)) { + JobStateTracker jt = new JobStateTracker(id, zk, false, + job_trackingpath); + jt.create(); + } + } catch (Exception e) { + LOG.warn("Error tracking: " + e.getMessage()); + // If we couldn't create the tracker node, don't + // create the main node. + zk.delete(makeZnode(type, id), -1); + } + } + if (zk.exists(makeZnode(type, id), false) == null) + throw new IOException("Unable to create " + makeZnode(type, id)); + if (wasCreated) { + try { + saveField(type, id, "created", + Long.toString(System.currentTimeMillis())); + } catch (NotFoundException nfe) { + // Wow, something's really wrong. + throw new IOException("Couldn't write to node " + id, nfe); + } + } + } catch (KeeperException e) { + throw new IOException("Creating " + id, e); + } catch (InterruptedException e) { + throw new IOException("Creating " + id, e); + } + } + + /** + * Get the path based on the job type. + * + * @param type + */ + public String getPath(Type type) { + String typepath = overhead_path; + switch (type) { + case JOB: + typepath = job_path; + break; + case JOBTRACKING: + typepath = job_trackingpath; + break; + } + return typepath; + } + + public static String[] getPaths(String fullpath) { + ArrayList paths = new ArrayList(); + if (fullpath.length() < 2) { + paths.add(fullpath); + } else { + int location = 0; + while ((location = fullpath.indexOf("/", location + 1)) > 0) { + paths.add(fullpath.substring(0, location)); + } + paths.add(fullpath); + } + String[] strings = new String[paths.size()]; + return paths.toArray(strings); + } + + /** + * A helper method that sets a field value. + * @param type + * @param id + * @param name + * @param val + * @throws KeeperException + * @throws UnsupportedEncodingException + * @throws InterruptedException + */ + private void setFieldData(Type type, String id, String name, String val) + throws KeeperException, UnsupportedEncodingException, InterruptedException + { + try { + zk.create(makeFieldZnode(type, id, name), + val.getBytes(ENCODING), + Ids.OPEN_ACL_UNSAFE, + CreateMode.PERSISTENT); + } catch(KeeperException.NodeExistsException e) { + zk.setData(makeFieldZnode(type, id, name), + val.getBytes(ENCODING), + -1); + } + } + + /** + * Make a ZK path to the named field. + */ + public String makeFieldZnode(Type type, String id, String name) { + return makeZnode(type, id) + "/" + name; + } + + /** + * Make a ZK path to job + */ + public String makeZnode(Type type, String id) { + return getPath(type) + "/" + id; + } + + @Override + public void saveField(Type type, String id, String key, String val) + throws NotFoundException { + try { + if (val != null) { + create(type, id); + setFieldData(type, id, key, val); + } + } catch(Exception e) { + throw new NotFoundException("Writing " + key + ": " + val + ", " + + e.getMessage()); + } + } + + @Override + public String getField(Type type, String id, String key) { + try { + byte[] b = zk.getData(makeFieldZnode(type, id, key), false, null); + return new String(b, ENCODING); + } catch(Exception e) { + return null; + } + } + + @Override + public Map getFields(Type type, String id) { + HashMap map = new HashMap(); + try { + for (String node: zk.getChildren(makeZnode(type, id), false)) { + byte[] b = zk.getData(makeFieldZnode(type, id, node), + false, null); + map.put(node, new String(b, ENCODING)); + } + } catch(Exception e) { + return map; + } + return map; + } + + @Override + public boolean delete(Type type, String id) throws NotFoundException { + try { + for (String child : zk.getChildren(makeZnode(type, id), false)) { + try { + zk.delete(makeFieldZnode(type, id, child), -1); + } catch (Exception e) { + // Other nodes may be trying to delete this at the same time, + // so just log errors and skip them. + throw new NotFoundException("Couldn't delete " + + makeFieldZnode(type, id, child)); + } + } + try { + zk.delete(makeZnode(type, id), -1); + } catch (Exception e) { + // Same thing -- might be deleted by other nodes, so just go on. + throw new NotFoundException("Couldn't delete " + + makeZnode(type, id)); + } + } catch (Exception e) { + // Error getting children of node -- probably node has been deleted + throw new NotFoundException("Couldn't get children of " + + makeZnode(type, id)); + } + return true; + } + + @Override + public List getAll() { + ArrayList allNodes = new ArrayList(); + for (Type type: Type.values()) { + allNodes.addAll(getAllForType(type)); + } + return allNodes; + } + + @Override + public List getAllForType(Type type) { + try { + return zk.getChildren(getPath(type), false); + } catch (Exception e) { + return new ArrayList(); + } + } + + @Override + public List getAllForKey(String key, String value) { + ArrayList allNodes = new ArrayList(); + try { + for (Type type: Type.values()) { + allNodes.addAll(getAllForTypeAndKey(type, key, value)); + } + } catch (Exception e) { + LOG.info("Couldn't find children."); + } + return allNodes; + } + + @Override + public List getAllForTypeAndKey(Type type, String key, String value) { + ArrayList allNodes = new ArrayList(); + try { + for (String id : zk.getChildren(getPath(type), false)) { + for (String field : zk.getChildren(id, false)) { + if (field.endsWith("/" + key)) { + byte[] b = zk.getData(field, false, null); + if (new String(b, ENCODING).equals(value)) { + allNodes.add(id); + } + } + } + } + } catch (Exception e) { + // Log and go to the next type -- this one might not exist + LOG.info("Couldn't find children of " + getPath(type)); + } + return allNodes; + } + + @Override + public void openStorage(Configuration config) throws IOException { + storage_root = config.get(STORAGE_ROOT); + job_path = storage_root + "/jobs"; + job_trackingpath = storage_root + TRACKINGDIR; + overhead_path = storage_root + "/overhead"; + + if (zk == null) { + zk = zkOpen(config); + } + } + + @Override + public void closeStorage() throws IOException { + close(); + } +} Index: webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/HDFSStorage.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/HDFSStorage.java (revision 0) +++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/HDFSStorage.java (revision 0) @@ -0,0 +1,256 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton.tool; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; + +/** + * HDFS implementation of templeton storage. + * + * This implementation assumes that all keys in key/value pairs are + * chosen such that they don't have any newlines in them. + * + */ +public class HDFSStorage implements TempletonStorage { + FileSystem fs = null; + + public String storage_root = null; + + public static final String JOB_PATH = "/jobs"; + public static final String JOB_TRACKINGPATH = "/created"; + public static final String OVERHEAD_PATH = "/overhead"; + + private static final Log LOG = LogFactory.getLog(HDFSStorage.class); + + public void startCleanup(Configuration config) { + try { + HDFSCleanup.startInstance(config); + } catch (Exception e) { + LOG.warn("Cleanup instance didn't start."); + } + } + + @Override + public void saveField(Type type, String id, String key, String val) + throws NotFoundException { + if (val == null) { + return; + } + PrintWriter out = null; + try { + Path keyfile = new Path(getPath(type) + "/" + id + "/" + key); + // This will replace the old value if there is one + // Overwrite the existing file + out = new PrintWriter(new OutputStreamWriter(fs.create(keyfile))); + out.write(val); + } catch (IOException e) { + LOG.info("Couldn't write to " + getPath(type) + "/" + id + ": " + + e.getMessage()); + } finally { + try { + out.flush(); + out.close(); + } catch (Exception e) { + // fail + } + } + } + + @Override + public String getField(Type type, String id, String key) { + BufferedReader in = null; + try { + in = new BufferedReader(new InputStreamReader + (fs.open(new Path(getPath(type) + "/" + + id + "/" + key)))); + String line = null; + String val = ""; + while ((line = in.readLine()) != null) { + if (! val.equals("")) { + val += "\n"; + } + val += line; + } + return val; + } catch (IOException e) { + LOG.trace("Couldn't find " + getPath(type) + "/" + id + "/" + key + + ": " + e.getMessage()); + } finally { + try { + in.close(); + } catch (Exception e) { + // fail + } + } + return null; + } + + @Override + public Map getFields(Type type, String id) { + HashMap map = new HashMap(); + BufferedReader in = null; + try { + for (FileStatus status : fs.listStatus + (new Path(getPath(type) + "/" + id))) { + in = new BufferedReader(new InputStreamReader + (fs.open(status.getPath()))); + String line = null; + String val = ""; + while ((line = in.readLine()) != null) { + if (!val.equals("")) { + val += "\n"; + } + val += line; + } + map.put(status.getPath().getName(), val); + } + } catch (IOException e) { + LOG.trace("Couldn't find " + getPath(type) + "/" + id); + } finally { + try { + in.close(); + } catch (Exception e) { + // fail + } + } + return map; + } + + @Override + public boolean delete(Type type, String id) throws NotFoundException { + try { + fs.delete(new Path(getPath(type) + "/" + id), true); + } catch (IOException e) { + throw new NotFoundException("Node " + id + " was not found: " + + e.getMessage()); + } + return false; + } + + @Override + public List getAll() { + ArrayList allNodes = new ArrayList(); + for (Type type: Type.values()) { + allNodes.addAll(getAllForType(type)); + } + return allNodes; + } + + @Override + public List getAllForType(Type type) { + ArrayList allNodes = new ArrayList(); + try { + for (FileStatus status : fs.listStatus(new Path(getPath(type)))) { + allNodes.add(status.getPath().getName()); + } + return null; + } catch (Exception e) { + LOG.trace("Couldn't find children for type " + type.toString()); + } + return allNodes; + } + + @Override + public List getAllForKey(String key, String value) { + ArrayList allNodes = new ArrayList(); + try { + for (Type type : Type.values()) { + allNodes.addAll(getAllForTypeAndKey(type, key, value)); + } + } catch (Exception e) { + LOG.trace("Couldn't find children for key " + key + ": " + + e.getMessage()); + } + return allNodes; + } + + @Override + public List getAllForTypeAndKey(Type type, String key, String value) { + ArrayList allNodes = new ArrayList(); + HashMap map = new HashMap(); + try { + for (FileStatus status : + fs.listStatus(new Path(getPath(type)))) { + map = (HashMap) + getFields(type, status.getPath().getName()); + if (map.get(key).equals(value)) { + allNodes.add(status.getPath().getName()); + } + } + } catch (Exception e) { + LOG.trace("Couldn't find children for key " + key + ": " + + e.getMessage()); + } + return allNodes; + } + + @Override + public void openStorage(Configuration config) throws IOException { + storage_root = config.get(TempletonStorage.STORAGE_ROOT); + if (fs == null) { + fs = FileSystem.get(config); + } + } + + @Override + public void closeStorage() throws IOException { + // Nothing to do here + } + + /** + * Get the path to storage based on the type. + * @param type + */ + public String getPath(Type type) { + return getPath(type, storage_root); + } + + /** + * Static method to get the path based on the type. + * + * @param type + * @param root + */ + public static String getPath(Type type, String root) { + String typepath = root + OVERHEAD_PATH; + switch (type) { + case JOB: + typepath = root + JOB_PATH; + break; + case JOBTRACKING: + typepath = root + JOB_TRACKINGPATH; + break; + } + return typepath; + } +} Index: webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/SingleInputFormat.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/SingleInputFormat.java (revision 0) +++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/SingleInputFormat.java (revision 0) @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton.tool; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.mapreduce.InputFormat; +import org.apache.hadoop.mapreduce.InputSplit; +import org.apache.hadoop.mapreduce.JobContext; +import org.apache.hadoop.mapreduce.RecordReader; +import org.apache.hadoop.mapreduce.TaskAttemptContext; + +/** + * An empty InputFormat. + */ +public class SingleInputFormat + extends InputFormat +{ + public List getSplits(JobContext job) + throws IOException + { + List res = new ArrayList(); + res.add(new NullSplit()); + return res; + } + + public RecordReader + createRecordReader(InputSplit split, + TaskAttemptContext context) + throws IOException + { + return new NullRecordReader(); + } +} + Index: webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/JobState.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/JobState.java (revision 0) +++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/JobState.java (revision 0) @@ -0,0 +1,357 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton.tool; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; + +/** + * The persistent state of a job. The state is stored in one of the + * supported storage systems. + */ +public class JobState { + + private static final Log LOG = LogFactory.getLog(JobState.class); + + private String id; + + // Storage is instantiated in the constructor + private TempletonStorage storage = null; + + private static TempletonStorage.Type type = TempletonStorage.Type.JOB; + + private Configuration config = null; + + public JobState(String id, Configuration conf) + throws IOException + { + this.id = id; + config = conf; + storage = getStorage(conf); + } + + public void delete() + throws IOException + { + try { + storage.delete(type, id); + } catch (Exception e) { + // Error getting children of node -- probably node has been deleted + LOG.info("Couldn't delete " + id); + } + } + + /** + * Get an instance of the selected storage class. Defaults to + * HDFS storage if none is specified. + */ + public static TempletonStorage getStorageInstance(Configuration conf) { + TempletonStorage storage = null; + try { + storage = (TempletonStorage) + Class.forName(conf.get(TempletonStorage.STORAGE_CLASS)) + .newInstance(); + } catch (Exception e) { + LOG.warn("No storage method found: " + e.getMessage()); + try { + storage = new HDFSStorage(); + } catch (Exception ex) { + LOG.error("Couldn't create storage."); + } + } + return storage; + } + + /** + * Get an open instance of the selected storage class. Defaults + * to HDFS storage if none is specified. + */ + public static TempletonStorage getStorage(Configuration conf) throws IOException { + TempletonStorage storage = getStorageInstance(conf); + storage.openStorage(conf); + return storage; + } + + /** + * For storage methods that require a connection, this is a hint + * that it's time to close the connection. + */ + public void close() throws IOException { + storage.closeStorage(); + } + + // + // Properties + // + + /** + * This job id. + */ + public String getId() { + return id; + } + + /** + * The percent complete of a job + */ + public String getPercentComplete() + throws IOException + { + return getField("percentComplete"); + } + public void setPercentComplete(String percent) + throws IOException + { + setField("percentComplete", percent); + } + + /** + * The child id of TempletonControllerJob + */ + public String getChildId() + throws IOException + { + return getField("childid"); + } + public void setChildId(String childid) + throws IOException + { + setField("childid", childid); + } + + /** + * Add a jobid to the list of children of this job. + * + * @param jobid + * @throws IOException + */ + public void addChild(String jobid) throws IOException { + String jobids = ""; + try { + jobids = getField("children"); + } catch (Exception e) { + // There are none or they're not readable. + } + if (!jobids.equals("")) { + jobids += ","; + } + jobids += jobid; + setField("children", jobids); + } + + /** + * Get a list of jobstates for jobs that are children of this job. + * @throws IOException + */ + public List getChildren() throws IOException { + ArrayList children = new ArrayList(); + for (String jobid : getField("children").split(",")) { + children.add(new JobState(jobid, config)); + } + return children; + } + + /** + * Save a comma-separated list of jobids that are children + * of this job. + * @param jobids + * @throws IOException + */ + public void setChildren(String jobids) throws IOException { + setField("children", jobids); + } + + /** + * Set the list of child jobs of this job + * @param children + */ + public void setChildren(List children) throws IOException { + String val = ""; + for (JobState jobstate : children) { + if (!val.equals("")) { + val += ","; + } + val += jobstate.getId(); + } + setField("children", val); + } + + /** + * The system exit value of the job. + */ + public Long getExitValue() + throws IOException + { + return getLongField("exitValue"); + } + public void setExitValue(long exitValue) + throws IOException + { + setLongField("exitValue", exitValue); + } + + /** + * When this job was created. + */ + public Long getCreated() + throws IOException + { + return getLongField("created"); + } + public void setCreated(long created) + throws IOException + { + setLongField("created", created); + } + + /** + * The user who started this job. + */ + public String getUser() + throws IOException + { + return getField("user"); + } + public void setUser(String user) + throws IOException + { + setField("user", user); + } + + /** + * The url callback + */ + public String getCallback() + throws IOException + { + return getField("callback"); + } + public void setCallback(String callback) + throws IOException + { + setField("callback", callback); + } + + /** + * The status of a job once it is completed. + */ + public String getCompleteStatus() + throws IOException + { + return getField("completed"); + } + public void setCompleteStatus(String complete) + throws IOException + { + setField("completed", complete); + } + + /** + * The time when the callback was sent. + */ + public Long getNotifiedTime() + throws IOException + { + return getLongField("notified"); + } + public void setNotifiedTime(long notified) + throws IOException + { + setLongField("notified", notified); + } + + // + // Helpers + // + + /** + * Fetch an integer field from the store. + */ + public Long getLongField(String name) + throws IOException + { + String s = storage.getField(type, id, name); + if (s == null) + return null; + else { + try { + return new Long(s); + } catch (NumberFormatException e) { + LOG.error("templeton: bug " + name + " " + s + " : "+ e); + return null; + } + } + } + + /** + * Store a String field from the store. + */ + public void setField(String name, String val) + throws IOException + { + try { + storage.saveField(type, id, name, val); + } catch (NotFoundException ne) { + throw new IOException(ne.getMessage()); + } + } + + public String getField(String name) + throws IOException + { + return storage.getField(type, id, name); + } + + /** + * Store a long field. + * + * @param name + * @param val + * @throws IOException + */ + public void setLongField(String name, long val) + throws IOException + { + try { + storage.saveField(type, id, name, String.valueOf(val)); + } catch (NotFoundException ne) { + throw new IOException("Job " + id + " was not found: " + + ne.getMessage()); + } + } + + /** + * Get an id for each currently existing job, which can be used to create + * a JobState object. + * + * @param conf + * @throws IOException + */ + public static List getJobs(Configuration conf) throws IOException { + try { + return getStorage(conf).getAllForType(type); + } catch (Exception e) { + throw new IOException("Can't get jobs", e); + } + } +} Index: webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/NotFoundException.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/NotFoundException.java (revision 0) +++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/NotFoundException.java (revision 0) @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton.tool; + +/** + * Simple not found exception. + */ +public class NotFoundException extends Exception { + private static final long serialVersionUID = 1L; + + public NotFoundException(String msg) { + super(msg); + } +} Index: webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/ZooKeeperCleanup.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/ZooKeeperCleanup.java (revision 0) +++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/ZooKeeperCleanup.java (revision 0) @@ -0,0 +1,198 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton.tool; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Date; + +import org.apache.hadoop.conf.Configuration; +import org.apache.zookeeper.ZooKeeper; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +/** + * This does periodic cleanup + */ +public class ZooKeeperCleanup extends Thread { + protected Configuration appConf; + + // The interval to wake up and check the queue + public static final String ZK_CLEANUP_INTERVAL = + "templeton.zookeeper.cleanup.interval"; // 12 hours + + // The max age of a task allowed + public static final String ZK_CLEANUP_MAX_AGE = + "templeton.zookeeper.cleanup.maxage"; // ~ 1 week + + protected static long interval = 1000L * 60L * 60L * 12L; + protected static long maxage = 1000L * 60L * 60L * 24L * 7L; + + // The logger + private static final Log LOG = LogFactory.getLog(ZooKeeperCleanup.class); + + // Handle to cancel loop + private boolean stop = false; + + // The instance + private static ZooKeeperCleanup thisclass = null; + + // Whether the cycle is running + private static boolean isRunning = false; + + /** + * Create a cleanup object. We use the appConfig to configure JobState. + * @param appConf + */ + private ZooKeeperCleanup(Configuration appConf) { + this.appConf = appConf; + interval = appConf.getLong(ZK_CLEANUP_INTERVAL, interval); + maxage = appConf.getLong(ZK_CLEANUP_MAX_AGE, maxage); + } + + public static ZooKeeperCleanup getInstance(Configuration appConf) { + if (thisclass != null) { + return thisclass; + } + thisclass = new ZooKeeperCleanup(appConf); + return thisclass; + } + + public static void startInstance(Configuration appConf) throws IOException { + if (! isRunning) { + getInstance(appConf).start(); + } + } + + /** + * Run the cleanup loop. + * + * @throws IOException + */ + public void run() { + ZooKeeper zk = null; + List nodes = null; + isRunning = true; + while (!stop) { + try { + // Put each check in a separate try/catch, so if that particular + // cycle fails, it'll try again on the next cycle. + try { + zk = ZooKeeperStorage.zkOpen(appConf); + + nodes = getChildList(zk); + + for (String node : nodes) { + boolean deleted = checkAndDelete(node, zk); + if (!deleted) { + break; + } + } + + zk.close(); + } catch (Exception e) { + LOG.error("Cleanup cycle failed: " + e.getMessage()); + } finally { + if (zk != null) { + try { + zk.close(); + } catch (InterruptedException e) { + // We're trying to exit anyway, just ignore. + } + } + } + + long sleepMillis = (long) (Math.random() * interval); + LOG.info("Next execution: " + new Date(new Date().getTime() + + sleepMillis)); + Thread.sleep(sleepMillis); + + } catch (Exception e) { + // If sleep fails, we should exit now before things get worse. + isRunning = false; + LOG.error("Cleanup failed: " + e.getMessage(), e); + } + } + isRunning = false; + } + + /** + * Get the list of jobs from JobState + * + * @throws IOException + */ + public List getChildList(ZooKeeper zk) { + try { + List jobs = JobStateTracker.getTrackingJobs(appConf, zk); + Collections.sort(jobs); + return jobs; + } catch (IOException e) { + LOG.info("No jobs to check."); + } + return new ArrayList(); + } + + /** + * Check to see if a job is more than maxage old, and delete it if so. + */ + public boolean checkAndDelete(String node, ZooKeeper zk) { + JobState state = null; + try { + JobStateTracker tracker = new JobStateTracker(node, zk, true, + appConf.get(TempletonStorage.STORAGE_ROOT + + ZooKeeperStorage.TRACKINGDIR)); + long now = new Date().getTime(); + state = new JobState(tracker.getJobID(), appConf); + + // Set the default to 0 -- if the created date is null, there was + // an error in creation, and we want to delete it anyway. + long then = 0; + if (state.getCreated() != null) { + then = state.getCreated(); + } + if (now - then > maxage) { + LOG.info("Deleting " + tracker.getJobID()); + state.delete(); + tracker.delete(); + return true; + } + return false; + } catch (Exception e) { + LOG.info("checkAndDelete failed for " + node); + // We don't throw a new exception for this -- just keep going with the + // next one. + return true; + } finally { + if (state != null) { + try { + state.close(); + } catch (IOException e) { + LOG.info("Couldn't close job state."); + } + } + } + } + + // Handle to stop this process from the outside if needed. + public void exit() { + stop = true; + } +} Index: webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/HDFSCleanup.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/HDFSCleanup.java (revision 0) +++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/HDFSCleanup.java (revision 0) @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton.tool; + +import java.io.IOException; +import java.util.Date; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hcatalog.templeton.tool.TempletonStorage.Type; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +/** + * This does periodic cleanup + */ +public class HDFSCleanup extends Thread { + protected Configuration appConf; + + // The interval to wake up and check the queue + public static final String HDFS_CLEANUP_INTERVAL = + "templeton.hdfs.cleanup.interval"; // 12 hours + + // The max age of a task allowed + public static final String HDFS_CLEANUP_MAX_AGE = + "templeton.hdfs.cleanup.maxage"; // ~ 1 week + + protected static long interval = 1000L * 60L * 60L * 12L; + protected static long maxage = 1000L * 60L * 60L * 24L * 7L; + + // The logger + private static final Log LOG = LogFactory.getLog(HDFSCleanup.class); + + // Handle to cancel loop + private boolean stop = false; + + // The instance + private static HDFSCleanup thisclass = null; + + // Whether the cycle is running + private static boolean isRunning = false; + + // The storage root + private String storage_root; + + /** + * Create a cleanup object. + */ + private HDFSCleanup(Configuration appConf) { + this.appConf = appConf; + interval = appConf.getLong(HDFS_CLEANUP_INTERVAL, interval); + maxage = appConf.getLong(HDFS_CLEANUP_MAX_AGE, maxage); + storage_root = appConf.get(TempletonStorage.STORAGE_ROOT); + } + + public static HDFSCleanup getInstance(Configuration appConf) { + if (thisclass != null) { + return thisclass; + } + thisclass = new HDFSCleanup(appConf); + return thisclass; + } + + public static void startInstance(Configuration appConf) throws IOException { + if (!isRunning) { + getInstance(appConf).start(); + } + } + + /** + * Run the cleanup loop. + * + */ + public void run() { + FileSystem fs = null; + while (!stop) { + try { + // Put each check in a separate try/catch, so if that particular + // cycle fails, it'll try again on the next cycle. + try { + if (fs == null) { + fs = FileSystem.get(appConf); + } + checkFiles(fs); + } catch (Exception e) { + LOG.error("Cleanup cycle failed: " + e.getMessage()); + } + + long sleepMillis = (long) (Math.random() * interval); + LOG.info("Next execution: " + new Date(new Date().getTime() + + sleepMillis)); + Thread.sleep(sleepMillis); + + } catch (Exception e) { + // If sleep fails, we should exit now before things get worse. + isRunning = false; + LOG.error("Cleanup failed: " + e.getMessage(), e); + } + } + isRunning = false; + } + + /** + * Loop through all the files, deleting any that are older than + * maxage. + * + * @param fs + * @throws IOException + */ + private void checkFiles(FileSystem fs) throws IOException { + long now = new Date().getTime(); + for (Type type : Type.values()) { + try { + for (FileStatus status : fs.listStatus(new Path( + HDFSStorage.getPath(type, storage_root)))) { + if (now - status.getModificationTime() > maxage) { + LOG.info("Deleting " + status.getPath().toString()); + fs.delete(status.getPath(), true); + } + } + } catch (Exception e) { + // Nothing to find for this type. + } + } + } + + // Handle to stop this process from the outside if needed. + public void exit() { + stop = true; + } + +} Index: webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/NullRecordReader.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/NullRecordReader.java (revision 0) +++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/NullRecordReader.java (revision 0) @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton.tool; + +import java.io.IOException; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.mapreduce.InputSplit; +import org.apache.hadoop.mapreduce.RecordReader; +import org.apache.hadoop.mapreduce.TaskAttemptContext; + +/** + * An empty record reader. + */ +public class NullRecordReader + extends RecordReader +{ + @Override + public void initialize(InputSplit genericSplit, TaskAttemptContext context) + throws IOException + {} + + @Override + public void close() throws IOException {} + + @Override + public NullWritable getCurrentKey() { + return NullWritable.get(); + } + + @Override + public NullWritable getCurrentValue() { + return NullWritable.get(); + } + + @Override + public float getProgress() { return 1.0f; } + + @Override + public boolean nextKeyValue() throws IOException { + return false; + } +} + Index: webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/TempletonStorage.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/TempletonStorage.java (revision 0) +++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/TempletonStorage.java (revision 0) @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton.tool; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.conf.Configuration; + +/** + * An interface to handle different Templeton storage methods, including + * ZooKeeper and HDFS. Any storage scheme must be able to handle being + * run in an HDFS environment, where specific file systems and virtual + * machines may not be available. + * + * Storage is done individually in a hierarchy: type (the data type, + * as listed below), then the id (a given jobid, jobtrackingid, etc.), + * then the key/value pairs. So an entry might look like: + * + * JOB + * jobid00035 + * user -> rachel + * datecreated -> 2/5/12 + * etc. + * + * Each field must be available to be fetched/changed individually. + */ +public interface TempletonStorage { + // These are the possible types referenced by 'type' below. + public enum Type { + UNKNOWN, JOB, JOBTRACKING, TEMPLETONOVERHEAD + } + + public static final String STORAGE_CLASS = "templeton.storage.class"; + public static final String STORAGE_ROOT = "templeton.storage.root"; + + /** + * Start the cleanup process for this storage type. + * @param config + */ + public void startCleanup(Configuration config); + + /** + * Save a single key/value pair for a specific job id. + * @param type The data type (as listed above) + * @param id The String id of this data grouping (jobid, etc.) + * @param key The name of the field to save + * @param val The value of the field to save + */ + public void saveField(Type type, String id, String key, String val) + throws NotFoundException; + + /** + * Get the value of one field for a given data type. If the type + * is UNKNOWN, search for the id in all types. + * @param type The data type (as listed above) + * @param id The String id of this data grouping (jobid, etc.) + * @param key The name of the field to retrieve + * @return The value of the field requested, or null if not + * found. + */ + public String getField(Type type, String id, String key); + + /** + * Get all the name/value pairs stored for this id. + * Be careful using getFields() -- optimistic locking will mean that + * your odds of a conflict are decreased if you read/write one field + * at a time. getFields() is intended for read-only usage. + * + * If the type is UNKNOWN, search for the id in all types. + * + * @param type The data type (as listed above) + * @param id The String id of this data grouping (jobid, etc.) + * @return A Map of key/value pairs found for this type/id. + */ + public Map getFields(Type type, String id); + + /** + * Delete a data grouping (all data for a jobid, all tracking data + * for a job, etc.). If the type is UNKNOWN, search for the id + * in all types. + * + * @param type The data type (as listed above) + * @param id The String id of this data grouping (jobid, etc.) + * @return True if successful, false if not, throws NotFoundException + * if the id wasn't found. + */ + public boolean delete(Type type, String id) throws NotFoundException; + + /** + * Get the id of each data grouping in the storage system. + * + * @return An ArrayList of ids. + */ + public List getAll(); + + /** + * Get the id of each data grouping of a given type in the storage + * system. + * @param type The data type (as listed above) + * @return An ArrayList of ids. + */ + public List getAllForType(Type type); + + /** + * Get the id of each data grouping that has the specific key/value + * pair. + * @param key The name of the field to search for + * @param value The value of the field to search for + * @return An ArrayList of ids. + */ + public List getAllForKey(String key, String value); + + /** + * Get the id of each data grouping of a given type that has the + * specific key/value pair. + * @param type The data type (as listed above) + * @param key The name of the field to search for + * @param value The value of the field to search for + * @return An ArrayList of ids. + */ + public List getAllForTypeAndKey(Type type, String key, + String value); + + /** + * For storage methods that require a connection, this is a hint + * that it's time to open a connection. + */ + public void openStorage(Configuration config) throws IOException; + + /** + * For storage methods that require a connection, this is a hint + * that it's time to close the connection. + */ + public void closeStorage() throws IOException; +} Index: webhcat/svr/src/main/java/org/apache/hcatalog/templeton/TempletonDelegator.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/TempletonDelegator.java (revision 0) +++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/TempletonDelegator.java (revision 0) @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton; + +/** + * The helper class for all the Templeton delegator classes. A + * delegator will call the underlying Templeton service such as hcat + * or hive. + */ +public class TempletonDelegator { + protected AppConfig appConf; + + public TempletonDelegator(AppConfig appConf) { + this.appConf = appConf; + } +} Index: webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CatchallExceptionMapper.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CatchallExceptionMapper.java (revision 0) +++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CatchallExceptionMapper.java (revision 0) @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton; + +import javax.ws.rs.core.Response; +import javax.ws.rs.ext.ExceptionMapper; +import javax.ws.rs.ext.Provider; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +/** + * Map all exceptions to the Jersey response. This lets us have nice + * results in the error body. + */ +@Provider +public class CatchallExceptionMapper + implements ExceptionMapper +{ + private static final Log LOG = LogFactory.getLog(CatchallExceptionMapper.class); + + public Response toResponse(Exception e) { + LOG.error(e.getMessage(), e); + return SimpleWebException.buildMessage(500, null, e.getMessage()); + } +} Index: webhcat/svr/src/main/java/org/apache/hcatalog/templeton/BusyException.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/BusyException.java (revision 0) +++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/BusyException.java (revision 0) @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton; + +/** + * Simple "we are busy, try again" exception. + */ +public class BusyException extends SimpleWebException { + public BusyException() { + super(503, "Busy, please retry"); + } +} Index: webhcat/svr/src/main/java/org/apache/hcatalog/templeton/DatabaseDesc.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/DatabaseDesc.java (revision 0) +++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/DatabaseDesc.java (revision 0) @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton; + +import java.util.Map; + +import javax.xml.bind.annotation.XmlRootElement; + +/** + * A description of the database to create. + */ +@XmlRootElement +public class DatabaseDesc extends GroupPermissionsDesc { + public boolean ifNotExists; + public String database; + public String comment; + public String location; + public Map properties; + + public DatabaseDesc() {} + + public String toString() { + return String.format("DatabaseDesc(database=%s, comment=%s, location=%s, " + + "properties=%s)", database, comment, location, properties); + } +} Index: webhcat/svr/src/main/java/org/apache/hcatalog/templeton/QueueException.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/QueueException.java (revision 0) +++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/QueueException.java (revision 0) @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton; + +import java.util.HashMap; + +/** + * Unable to queue the job + */ +public class QueueException extends SimpleWebException { + public QueueException(String msg) { + super(500, msg); + } + +} Index: webhcat/svr/src/main/java/org/apache/hcatalog/templeton/AppConfig.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/AppConfig.java (revision 0) +++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/AppConfig.java (revision 0) @@ -0,0 +1,202 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton; + +import java.io.File; +import java.net.URL; +import java.util.Map; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.util.VersionInfo; +import org.apache.hcatalog.templeton.tool.JobState; +import org.apache.hcatalog.templeton.tool.ZooKeeperCleanup; +import org.apache.hcatalog.templeton.tool.ZooKeeperStorage; + +/** + * The configuration for Templeton. This merges the normal Hadoop + * configuration with the Templeton specific variables. + * + * The Templeton configuration variables are described in + * templeton-default.xml + * + * The Templeton specific configuration is split into two layers + * + * 1. templeton-default.xml - All the configuration variables that + * Templeton needs. These are the defaults that ship with the app + * and should only be changed be the app developers. + * + * 2. templeton-site.xml - The (possibly empty) configuration that the + * system administrator can set variables for their Hadoop cluster. + * + * The configuration files are loaded in this order with later files + * overriding earlier ones. + * + * To find the configuration files, we first attempt to load a file + * from the CLASSPATH and then look in the directory specified in the + * TEMPLETON_HOME environment variable. + * + * In addition the configuration files may access the special env + * variable env for all environment variables. For example, the + * hadoop executable could be specified using: + *
+ *      ${env.HADOOP_PREFIX}/bin/hadoop
+ *
+ */ +public class AppConfig extends Configuration { + public static final String[] HADOOP_CONF_FILENAMES = { + "core-default.xml", "core-site.xml", "mapred-default.xml", "mapred-site.xml" + }; + + public static final String[] HADOOP_PREFIX_VARS = { + "HADOOP_PREFIX", "HADOOP_HOME" + }; + + public static final String TEMPLETON_HOME_VAR = "TEMPLETON_HOME"; + + public static final String[] TEMPLETON_CONF_FILENAMES = { + "templeton-default.xml", + "templeton-site.xml" + }; + + public static final String PORT = "templeton.port"; + public static final String EXEC_ENCODING_NAME = "templeton.exec.encoding"; + public static final String EXEC_ENVS_NAME = "templeton.exec.envs"; + public static final String EXEC_MAX_BYTES_NAME = "templeton.exec.max-output-bytes"; + public static final String EXEC_MAX_PROCS_NAME = "templeton.exec.max-procs"; + public static final String EXEC_TIMEOUT_NAME = "templeton.exec.timeout"; + public static final String HADOOP_NAME = "templeton.hadoop"; + public static final String HADOOP_CONF_DIR = "templeton.hadoop.conf.dir"; + public static final String HCAT_NAME = "templeton.hcat"; + public static final String HIVE_ARCHIVE_NAME = "templeton.hive.archive"; + public static final String HIVE_PATH_NAME = "templeton.hive.path"; + public static final String HIVE_PROPS_NAME = "templeton.hive.properties"; + public static final String LIB_JARS_NAME = "templeton.libjars"; + public static final String PIG_ARCHIVE_NAME = "templeton.pig.archive"; + public static final String PIG_PATH_NAME = "templeton.pig.path"; + public static final String STREAMING_JAR_NAME = "templeton.streaming.jar"; + public static final String TEMPLETON_JAR_NAME = "templeton.jar"; + public static final String OVERRIDE_JARS_NAME = "templeton.override.jars"; + public static final String OVERRIDE_JARS_ENABLED = "templeton.override.enabled"; + public static final String KERBEROS_SECRET = "templeton.kerberos.secret"; + public static final String KERBEROS_PRINCIPAL = "templeton.kerberos.principal"; + public static final String KERBEROS_KEYTAB = "templeton.kerberos.keytab"; + + public static final String CALLBACK_INTERVAL_NAME + = "templeton.callback.retry.interval"; + public static final String CALLBACK_RETRY_NAME + = "templeton.callback.retry.attempts"; + public static final String HADOOP_END_INTERVAL_NAME = "job.end.retry.interval"; + public static final String HADOOP_END_RETRY_NAME = "job.end.retry.attempts"; + public static final String HADOOP_END_URL_NAME = "job.end.notification.url"; + public static final String HADOOP_SPECULATIVE_NAME + = "mapred.map.tasks.speculative.execution"; + + private static final Log LOG = LogFactory.getLog(AppConfig.class); + + public AppConfig() { + init(); + LOG.info("Using Hadoop version " + VersionInfo.getVersion()); + } + + private void init() { + for (Map.Entry e : System.getenv().entrySet()) + set("env." + e.getKey(), e.getValue()); + + String templetonDir = getTempletonDir(); + for (String fname : TEMPLETON_CONF_FILENAMES) + if (! loadOneClasspathConfig(fname)) + loadOneFileConfig(templetonDir, fname); + + String hadoopConfDir = getHadoopConfDir(); + for (String fname : HADOOP_CONF_FILENAMES) + loadOneFileConfig(hadoopConfDir, fname); + } + + public void startCleanup() { + JobState.getStorageInstance(this).startCleanup(this); + } + + public String getHadoopConfDir() { + return get(HADOOP_CONF_DIR); + } + + public static String getTempletonDir() { + return System.getenv(TEMPLETON_HOME_VAR); + } + + private boolean loadOneFileConfig(String dir, String fname) { + if (dir != null) { + File f = new File(dir, fname); + if (f.exists()) { + addResource(new Path(f.getAbsolutePath())); + LOG.debug("loaded config file " + f.getAbsolutePath()); + return true; + } + } + return false; + } + + private boolean loadOneClasspathConfig(String fname) { + URL x = getResource(fname); + if (x != null) { + addResource(x); + LOG.debug("loaded config from classpath " + x); + return true; + } + + return false; + } + + public String templetonJar() { return get(TEMPLETON_JAR_NAME); } + public String libJars() { return get(LIB_JARS_NAME); } + public String clusterHadoop() { return get(HADOOP_NAME); } + public String clusterHcat() { return get(HCAT_NAME); } + public String pigPath() { return get(PIG_PATH_NAME); } + public String pigArchive() { return get(PIG_ARCHIVE_NAME); } + public String hivePath() { return get(HIVE_PATH_NAME); } + public String hiveArchive() { return get(HIVE_ARCHIVE_NAME); } + public String streamingJar() { return get(STREAMING_JAR_NAME); } + public String kerberosSecret() { return get(KERBEROS_SECRET); } + public String kerberosPrincipal(){ return get(KERBEROS_PRINCIPAL); } + public String kerberosKeytab() { return get(KERBEROS_KEYTAB); } + + public String[] overrideJars() { + if (getBoolean(OVERRIDE_JARS_ENABLED, true)) + return getStrings(OVERRIDE_JARS_NAME); + else + return null; + } + public String overrideJarsString() { + if (getBoolean(OVERRIDE_JARS_ENABLED, true)) + return get(OVERRIDE_JARS_NAME); + else + return null; + } + + public long zkCleanupInterval() { + return getLong(ZooKeeperCleanup.ZK_CLEANUP_INTERVAL, + (1000L * 60L * 60L * 12L)); } + public long zkMaxAge() { + return getLong(ZooKeeperCleanup.ZK_CLEANUP_MAX_AGE, + (1000L * 60L * 60L * 24L * 7L)); } + public String zkHosts() { return get(ZooKeeperStorage.ZK_HOSTS); } + public int zkSessionTimeout() { return getInt(ZooKeeperStorage.ZK_SESSION_TIMEOUT, + 30000); } +} Index: webhcat/svr/src/main/java/org/apache/hcatalog/templeton/HcatDelegator.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/HcatDelegator.java (revision 0) +++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/HcatDelegator.java (revision 0) @@ -0,0 +1,856 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton; + +import java.io.IOException; +import java.net.URI; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import javax.ws.rs.core.Response; +import org.apache.commons.exec.ExecuteException; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hcatalog.templeton.tool.TempletonUtils; + + +/** + * Run hcat on the local server using the ExecService. This is + * the backend of the ddl web service. + */ +public class HcatDelegator extends LauncherDelegator { + private static final Log LOG = LogFactory.getLog(HcatDelegator.class); + private ExecService execService; + + public HcatDelegator(AppConfig appConf, ExecService execService) { + super(appConf); + this.execService = execService; + } + + /** + * Run the local hcat executable. + */ + public ExecBean run(String user, String exec, boolean format, + String group, String permissions) + throws NotAuthorizedException, BusyException, ExecuteException, IOException + { + SecureProxySupport proxy = new SecureProxySupport(); + try { + List args = makeArgs(exec, format, group, permissions); + proxy.open(user, appConf); + + // Setup the hadoop vars to specify the user. + String cp = makeOverrideClasspath(appConf); + Map env = TempletonUtils.hadoopUserEnv(user, cp); + proxy.addEnv(env); + proxy.addArgs(args); + return execService.run(appConf.clusterHcat(), args, env); + } catch (InterruptedException e) { + throw new IOException(e); + } finally { + if (proxy != null) + proxy.close(); + } + } + + private List makeArgs(String exec, boolean format, + String group, String permissions) { + ArrayList args = new ArrayList(); + args.add("-e"); + args.add(exec); + if (TempletonUtils.isset(group)) { + args.add("-g"); + args.add(group); + } + if (TempletonUtils.isset(permissions)) { + args.add("-p"); + args.add(permissions); + } + if (format) { + args.add("-D"); + args.add("hive.ddl.output.format=json"); + // Use both args to ease development. Delete this one on + // May 1. + args.add("-D"); + args.add("hive.format=json"); + } + + return args; + } + + /** + * Return a json description of the database. + */ + public Response descDatabase(String user, String db, boolean extended) + throws HcatException, NotAuthorizedException, BusyException, + ExecuteException, IOException + { + String exec = "desc database " + db + "; "; + if (extended) + exec = "desc database extended " + db + "; "; + + try { + String res = jsonRun(user, exec); + return JsonBuilder.create(res).build(); + } catch (HcatException e) { + if (e.execBean.stderr.indexOf("Error in semantic analysis") > -1) { + return JsonBuilder.create(). + put("error", "Database " + db + " does not exist") + .put("errorCode", "404") + .put("database", db).build(); + } + throw new HcatException("unable to describe database: " + db, + e.execBean, exec); + } + } + + /** + * Return a json "show databases like". This will return a list of + * databases. + */ + public Response listDatabases(String user, String dbPattern) + throws HcatException, NotAuthorizedException, BusyException, + ExecuteException, IOException + { + String exec = String.format("show databases like '%s';", dbPattern); + try { + String res = jsonRun(user, exec); + return JsonBuilder.create(res) + .build(); + } catch (HcatException e) { + throw new HcatException("unable to show databases for: " + dbPattern, + e.execBean, exec); + } + } + + /** + * Create a database with the given name + */ + public Response createDatabase(String user, DatabaseDesc desc) + throws HcatException, NotAuthorizedException, BusyException, + ExecuteException, IOException + { + String exec = "create database"; + if (desc.ifNotExists) + exec += " if not exists"; + exec += " " + desc.database; + if (TempletonUtils.isset(desc.comment)) + exec += String.format(" comment '%s'", desc.comment); + if (TempletonUtils.isset(desc.location)) + exec += String.format(" location '%s'", desc.location); + if (TempletonUtils.isset(desc.properties)) + exec += String.format(" with dbproperties (%s)", + makePropertiesStatement(desc.properties)); + exec += ";"; + + String res = jsonRun(user, exec, desc.group, desc.permissions); + return JsonBuilder.create(res) + .put("database", desc.database) + .build(); + } + + /** + * Drop the given database + */ + public Response dropDatabase(String user, String db, + boolean ifExists, String option, + String group, String permissions) + throws HcatException, NotAuthorizedException, BusyException, + ExecuteException, IOException + { + String exec = "drop database"; + if (ifExists) + exec += " if exists"; + exec += " " + db; + if (TempletonUtils.isset(option)) + exec += " " + option; + exec += ";"; + + String res = jsonRun(user, exec, group, permissions); + return JsonBuilder.create(res) + .put("database", db) + .build(); + } + + /** + * Create a table. + */ + public Response createTable(String user, String db, TableDesc desc) + throws HcatException, NotAuthorizedException, BusyException, + ExecuteException, IOException + { + String exec = makeCreateTable(db, desc); + + try { + String res = jsonRun(user, exec, desc.group, desc.permissions, true); + + return JsonBuilder.create(res) + .put("database", db) + .put("table", desc.table) + .build(); + } catch (final HcatException e) { + throw new HcatException("unable to create table: " + desc.table, + e.execBean, exec); + } + } + + /** + * Create a table like another. + */ + public Response createTableLike(String user, String db, TableLikeDesc desc) + throws HcatException, NotAuthorizedException, BusyException, + ExecuteException, IOException + { + String exec = String.format("use %s; create", db); + + if (desc.external) + exec += " external"; + exec += String.format(" table %s like %s", desc.newTable, desc.existingTable); + if (TempletonUtils.isset(desc.location)) + exec += String.format(" location '%s'", desc.location); + exec += ";"; + + try { + String res = jsonRun(user, exec, desc.group, desc.permissions, true); + + return JsonBuilder.create(res) + .put("database", db) + .put("table", desc.newTable) + .build(); + } catch (final HcatException e) { + throw new HcatException("unable to create table: " + desc.newTable, + e.execBean, exec); + } + } + + /** + * Return a json description of the table. + */ + public Response descTable(String user, String db, String table, boolean extended) + throws HcatException, NotAuthorizedException, BusyException, + ExecuteException, IOException + { + String exec = "use " + db + "; "; + if (extended) + exec += "desc extended " + table + "; "; + else + exec += "desc " + table + "; "; + try { + String res = jsonRun(user, exec); + return JsonBuilder.create(res) + .put("database", db) + .put("table", table) + .build(); + } catch (HcatException e) { + throw new HcatException("unable to describe table: " + table, + e.execBean, exec); + } + } + + /** + * Return a json "show table like". This will return a list of + * tables. + */ + public Response listTables(String user, String db, String tablePattern) + throws HcatException, NotAuthorizedException, BusyException, + ExecuteException, IOException + { + String exec = String.format("use %s; show tables like '%s';", + db, tablePattern); + try { + String res = jsonRun(user, exec); + return JsonBuilder.create(res) + .put("database", db) + .build(); + } catch (HcatException e) { + throw new HcatException("unable to show tables for: " + tablePattern, + e.execBean, exec); + } + } + + /** + * Return a json "show table extended like". This will return + * only the first single table. + */ + public Response descExtendedTable(String user, String db, String table) + throws HcatException, NotAuthorizedException, BusyException, + ExecuteException, IOException + { + String exec = String.format("use %s; show table extended like %s;", + db, table); + try { + String res = jsonRun(user, exec); + JsonBuilder jb = JsonBuilder.create(singleTable(res, table)) + .remove("tableName") + .put("database", db) + .put("table", table); + + // If we can get them from HDFS, add group and permission + String loc = (String) jb.getMap().get("location"); + if (loc != null && loc.startsWith("hdfs://")) { + try { + FileSystem fs = FileSystem.get(appConf); + FileStatus status = fs.getFileStatus(new Path(new URI(loc))); + jb.put("group", status.getGroup()); + jb.put("permission", status.getPermission().toString()); + } catch (Exception e) { + LOG.warn(e.getMessage() + " Couldn't get permissions for " + loc); + } + } + return jb.build(); + } catch (HcatException e) { + throw new HcatException("unable to show table: " + table, e.execBean, exec); + } + } + + // Format a list of Columns for a create statement + private String makeCols(List cols) { + ArrayList res = new ArrayList(); + for (ColumnDesc col : cols) + res.add(makeOneCol(col)); + return StringUtils.join(res, ", "); + } + + // Format a Column for a create statement + private String makeOneCol(ColumnDesc col) { + String res = String.format("%s %s", col.name, col.type); + if (TempletonUtils.isset(col.comment)) + res += String.format(" comment '%s'", col.comment); + return res; + } + + // Make a create table statement + private String makeCreateTable(String db, TableDesc desc) { + String exec = String.format("use %s; create", db); + + if (desc.external) + exec += " external"; + exec += " table"; + if (desc.ifNotExists) + exec += " if not exists"; + exec += " " + desc.table; + + if (TempletonUtils.isset(desc.columns)) + exec += String.format("(%s)", makeCols(desc.columns)); + if (TempletonUtils.isset(desc.comment)) + exec += String.format(" comment '%s'", desc.comment); + if (TempletonUtils.isset(desc.partitionedBy)) + exec += String.format(" partitioned by (%s)", makeCols(desc.partitionedBy)); + if (desc.clusteredBy != null) + exec += String.format(" clustered by %s", makeClusteredBy(desc.clusteredBy)); + if (desc.format != null) + exec += " " + makeStorageFormat(desc.format); + if (TempletonUtils.isset(desc.location)) + exec += String.format(" location '%s'", desc.location); + if (TempletonUtils.isset(desc.tableProperties)) + exec += String.format(" tblproperties (%s)", + makePropertiesStatement(desc.tableProperties)); + exec += ";"; + + return exec; + } + + // Format a clustered by statement + private String makeClusteredBy(TableDesc.ClusteredByDesc desc) { + String res = String.format("(%s)", StringUtils.join(desc.columnNames, ", ")); + if (TempletonUtils.isset(desc.sortedBy)) + res += String.format(" sorted by (%s)", makeClusterSortList(desc.sortedBy)); + res += String.format(" into %s buckets", desc.numberOfBuckets); + + return res; + } + + // Format a sorted by statement + private String makeClusterSortList(List descs) { + ArrayList res = new ArrayList(); + for (TableDesc.ClusterSortOrderDesc desc : descs) + res.add(makeOneClusterSort(desc)); + return StringUtils.join(res, ", "); + } + + // Format a single cluster sort statement + private String makeOneClusterSort(TableDesc.ClusterSortOrderDesc desc) { + return String.format("%s %s", desc.columnName, desc.order.toString()); + } + + // Format the storage format statements + private String makeStorageFormat(TableDesc.StorageFormatDesc desc) { + String res = ""; + + if (desc.rowFormat != null) + res += makeRowFormat(desc.rowFormat); + if (TempletonUtils.isset(desc.storedAs)) + res += String.format(" stored as %s", desc.storedAs); + if (desc.storedBy != null) + res += " " + makeStoredBy(desc.storedBy); + + return res; + } + + // Format the row format statement + private String makeRowFormat(TableDesc.RowFormatDesc desc) { + String res = + makeTermBy(desc.fieldsTerminatedBy, "fields") + + makeTermBy(desc.collectionItemsTerminatedBy, "collection items") + + makeTermBy(desc.mapKeysTerminatedBy, "map keys") + + makeTermBy(desc.linesTerminatedBy, "lines"); + + if (TempletonUtils.isset(res)) + return "row format delimited" + res; + else if (desc.serde != null) + return makeSerdeFormat(desc.serde); + else + return ""; + } + + // A row format terminated by clause + private String makeTermBy(String sep, String fieldName) { + + if (TempletonUtils.isset(sep)) + return String.format(" %s terminated by '%s'", fieldName, sep); + else + return ""; + } + + // Format the serde statement + private String makeSerdeFormat(TableDesc.SerdeDesc desc) { + String res = "row format serde " + desc.name; + if (TempletonUtils.isset(desc.properties)) + res += String.format(" with serdeproperties (%s)", + makePropertiesStatement(desc.properties)); + return res; + } + + // Format the properties statement + private String makePropertiesStatement(Map properties) { + ArrayList res = new ArrayList(); + for (Map.Entry e : properties.entrySet()) + res.add(String.format("'%s'='%s'", e.getKey(), e.getValue())); + return StringUtils.join(res, ", "); + } + + // Format the stored by statement + private String makeStoredBy(TableDesc.StoredByDesc desc) { + String res = String.format("stored by '%s'", desc.className); + if (TempletonUtils.isset(desc.properties)) + res += String.format(" with serdeproperties (%s)", + makePropertiesStatement(desc.properties)); + return res; + } + + // Pull out the first table from the "show extended" json. + private String singleTable(String json, String table) + throws IOException + { + Map obj = JsonBuilder.jsonToMap(json); + if (JsonBuilder.isError(obj)) + return json; + + List tables = (List) obj.get("tables"); + if (TempletonUtils.isset(tables)) + return JsonBuilder.mapToJson(tables.get(0)); + else { + return JsonBuilder + .createError(String.format("Table %s does not exist", table), + JsonBuilder.MISSING). + buildJson(); + } + } + + /** + * Drop a table. + */ + public Response dropTable(String user, String db, + String table, boolean ifExists, + String group, String permissions) + throws HcatException, NotAuthorizedException, BusyException, + ExecuteException, IOException + { + String exec = String.format("use %s; drop table", db); + if (ifExists) + exec += " if exists"; + exec += String.format(" %s;", table); + + try { + String res = jsonRun(user, exec, group, permissions, true); + return JsonBuilder.create(res) + .put("database", db) + .put("table", table) + .build(); + } catch (HcatException e) { + throw new HcatException("unable to drop table: " + table, e.execBean, exec); + } + } + + /** + * Rename a table. + */ + public Response renameTable(String user, String db, + String oldTable, String newTable, + String group, String permissions) + throws HcatException, NotAuthorizedException, BusyException, + ExecuteException, IOException + { + String exec = String.format("use %s; alter table %s rename to %s;", + db, oldTable, newTable); + try { + String res = jsonRun(user, exec, group, permissions, true); + return JsonBuilder.create(res) + .put("database", db) + .put("table", newTable) + .build(); + } catch (HcatException e) { + throw new HcatException("unable to rename table: " + oldTable, + e.execBean, exec); + } + } + + /** + * Describe one table property. + */ + public Response descTableProperty(String user, String db, + String table, String property) + throws HcatException, NotAuthorizedException, BusyException, + ExecuteException, IOException + { + Response res = descTable(user, db, table, true); + if (res.getStatus() != JsonBuilder.OK) + return res; + Map props = tableProperties(res.getEntity()); + Map found = null; + if (props != null) { + String value = (String) props.get(property); + if (value != null) { + found = new HashMap(); + found.put(property, value); + } + } + + return JsonBuilder.create() + .put("database", db) + .put("table", table) + .put("property", found) + .build(); + } + + /** + * List the table properties. + */ + public Response listTableProperties(String user, String db, String table) + throws HcatException, NotAuthorizedException, BusyException, + ExecuteException, IOException + { + Response res = descTable(user, db, table, true); + if (res.getStatus() != JsonBuilder.OK) + return res; + Map props = tableProperties(res.getEntity()); + return JsonBuilder.create() + .put("database", db) + .put("table", table) + .put("properties", props) + .build(); + } + + /** + * Add one table property. + */ + public Response addOneTableProperty(String user, String db, String table, + TablePropertyDesc desc) + throws HcatException, NotAuthorizedException, BusyException, + ExecuteException, IOException + { + String exec + = String.format("use %s; alter table %s set tblproperties ('%s'='%s');", + db, table, desc.name, desc.value); + try { + String res = jsonRun(user, exec, desc.group, desc.permissions, true); + return JsonBuilder.create(res) + .put("database", db) + .put("table", table) + .put("property", desc.name) + .build(); + } catch (HcatException e) { + throw new HcatException("unable to add table property: " + table, + e.execBean, exec); + } + } + + private Map tableProperties(Object extendedTable) { + if (! (extendedTable instanceof Map)) + return null; + Map m = (Map) extendedTable; + Map tableInfo = (Map) m.get("tableInfo"); + if (tableInfo == null) + return null; + + return (Map) tableInfo.get("parameters"); + } + + /** + * Return a json description of the partitions. + */ + public Response listPartitions(String user, String db, String table) + throws HcatException, NotAuthorizedException, BusyException, + ExecuteException, IOException + { + String exec = "use " + db + "; "; + exec += "show partitions " + table + "; "; + try { + String res = jsonRun(user, exec); + return JsonBuilder.create(res) + .put("database", db) + .put("table", table) + .build(); + } catch (HcatException e) { + throw new HcatException("unable to show partitions for table: " + table, + e.execBean, exec); + } + } + + /** + * Return a json description of one partition. + */ + public Response descOnePartition(String user, String db, String table, + String partition) + throws HcatException, NotAuthorizedException, BusyException, + ExecuteException, IOException + { + String exec = "use " + db + "; "; + exec += "show table extended like " + table + + " partition (" + partition + "); "; + try { + String res = jsonRun(user, exec); + return JsonBuilder.create(singleTable(res, table)) + .remove("tableName") + .put("database", db) + .put("table", table) + .put("partition", partition) + .build(); + } catch (HcatException e) { + throw new HcatException("unable to show partition: " + + table + " " + partition, + e.execBean, + exec); + } + } + + /** + * Add one partition. + */ + public Response addOnePartition(String user, String db, String table, + PartitionDesc desc) + throws HcatException, NotAuthorizedException, BusyException, + ExecuteException, IOException + { + String exec = String.format("use %s; alter table %s add", db, table); + if (desc.ifNotExists) + exec += " if not exists"; + exec += String.format(" partition (%s)", desc.partition); + if (TempletonUtils.isset(desc.location)) + exec += String.format(" location '%s'", desc.location); + exec += ";"; + try { + String res = jsonRun(user, exec, desc.group, desc.permissions, true); + if (res.indexOf("AlreadyExistsException") > -1) { + return JsonBuilder.create(). + put("error", "Partition already exists") + .put("errorCode", "409") + .put("database", db) + .put("table", table) + .put("partition", desc.partition).build(); + } + return JsonBuilder.create(res) + .put("database", db) + .put("table", table) + .put("partition", desc.partition) + .build(); + } catch (HcatException e) { + throw new HcatException("unable to add partition: " + desc, + e.execBean, exec); + } + } + + /** + * Drop a partition. + */ + public Response dropPartition(String user, String db, + String table, String partition, boolean ifExists, + String group, String permissions) + throws HcatException, NotAuthorizedException, BusyException, + ExecuteException, IOException + { + String exec = String.format("use %s; alter table %s drop", db, table); + if (ifExists) + exec += " if exists"; + exec += String.format(" partition (%s);", partition); + + try { + String res = jsonRun(user, exec, group, permissions, true); + return JsonBuilder.create(res) + .put("database", db) + .put("table", table) + .put("partition", partition) + .build(); + } catch (HcatException e) { + throw new HcatException("unable to drop partition: " + partition, + e.execBean, exec); + } + } + + /** + * Return a json description of the columns. Same as + * describeTable. + */ + public Response listColumns(String user, String db, String table) + throws HcatException, NotAuthorizedException, BusyException, + ExecuteException, IOException + { + try { + return descTable(user, db, table, false); + } catch (HcatException e) { + throw new HcatException("unable to show columns for table: " + table, + e.execBean, e.statement); + } + } + + /** + * Return a json description of one column. + */ + public Response descOneColumn(String user, String db, String table, String column) + throws SimpleWebException, NotAuthorizedException, BusyException, + ExecuteException, IOException + { + Response res = listColumns(user, db, table); + if (res.getStatus() != JsonBuilder.OK) + return res; + + Object o = res.getEntity(); + final Map fields = (o != null && (o instanceof Map)) ? (Map) o : null; + if (fields == null) + throw new SimpleWebException(500, "Internal error, unable to find column " + + column); + + + List cols = (List) fields.get("columns"); + Map found = null; + if (cols != null) { + for (Map col : cols) { + if (column.equals(col.get("name"))) { + found = col; + break; + } + } + } + if (found == null) + throw new SimpleWebException(500, "unable to find column " + column, + new HashMap() {{ + put("description", fields); + }}); + fields.remove("columns"); + fields.put("column", found); + return Response.fromResponse(res).entity(fields).build(); + } + + /** + * Add one column. + */ + public Response addOneColumn(String user, String db, String table, + ColumnDesc desc) + throws HcatException, NotAuthorizedException, BusyException, + ExecuteException, IOException + { + String exec = String.format("use %s; alter table %s add columns (%s %s", + db, table, desc.name, desc.type); + if (TempletonUtils.isset(desc.comment)) + exec += String.format(" comment '%s'", desc.comment); + exec += ");"; + try { + String res = jsonRun(user, exec, desc.group, desc.permissions, true); + return JsonBuilder.create(res) + .put("database", db) + .put("table", table) + .put("column", desc.name) + .build(); + } catch (HcatException e) { + throw new HcatException("unable to add column: " + desc, + e.execBean, exec); + } + } + + // Check that the hcat result is valid and or has a valid json + // error + private boolean isValid(ExecBean eb, boolean requireEmptyOutput) { + if (eb == null) + return false; + + try { + Map m = JsonBuilder.jsonToMap(eb.stdout); + if (m.containsKey("error")) // This is a valid error message. + return true; + } catch (IOException e) { + return false; + } + + if (eb.exitcode != 0) + return false; + + if (requireEmptyOutput) + if (TempletonUtils.isset(eb.stdout)) + return false; + + return true; + } + + // Run an hcat expression and return just the json outout. + private String jsonRun(String user, String exec, + String group, String permissions, + boolean requireEmptyOutput) + throws HcatException, NotAuthorizedException, BusyException, + ExecuteException, IOException + { + ExecBean res = run(user, exec, true, group, permissions); + + if (! isValid(res, requireEmptyOutput)) + throw new HcatException("Failure calling hcat: " + exec, res, exec); + + return res.stdout; + } + + // Run an hcat expression and return just the json outout. No + // permissions set. + private String jsonRun(String user, String exec) + throws HcatException, NotAuthorizedException, BusyException, + ExecuteException, IOException + { + return jsonRun(user, exec, null, null); + } + + // Run an hcat expression and return just the json outout. + private String jsonRun(String user, String exec, + String group, String permissions) + throws HcatException, NotAuthorizedException, BusyException, + ExecuteException, IOException + { + return jsonRun(user, exec, group, permissions, false); + } +} Index: webhcat/svr/src/main/java/org/apache/hcatalog/templeton/Server.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/Server.java (revision 0) +++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/Server.java (revision 0) @@ -0,0 +1,827 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import javax.ws.rs.DELETE; +import javax.ws.rs.FormParam; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.PUT; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.SecurityContext; +import javax.ws.rs.core.UriInfo; +import org.apache.commons.exec.ExecuteException; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authentication.client.PseudoAuthenticator; +import org.apache.hcatalog.templeton.tool.TempletonUtils; + +/** + * The Templeton Web API server. + */ +@Path("/v1") +public class Server { + public static final String VERSION = "v1"; + + /** + * The status message. Always "ok" + */ + public static final Map STATUS_OK = createStatusMsg(); + + /** + * The list of supported api versions. + */ + public static final Map SUPPORTED_VERSIONS = createVersions(); + + /** + * The list of supported return formats. Always json. + */ + public static final Map SUPPORTED_FORMATS = createFormats(); + + // Build the status message for the /status call. + private static Map createStatusMsg() { + HashMap res = new HashMap(); + res.put("status", "ok"); + res.put("version", VERSION); + + return Collections.unmodifiableMap(res); + } + + // Build the versions list. + private static Map createVersions() { + ArrayList versions = new ArrayList(); + versions.add(VERSION); + + HashMap res = new HashMap(); + res.put("supportedVersions", versions); + res.put("version", VERSION); + + return Collections.unmodifiableMap(res); + } + + // Build the supported formats list + private static Map createFormats() { + ArrayList formats = new ArrayList(); + formats.add(MediaType.APPLICATION_JSON); + HashMap res = new HashMap(); + res.put("responseTypes", formats); + + return Collections.unmodifiableMap(res); + } + + protected static ExecService execService = ExecServiceImpl.getInstance(); + private static AppConfig appConf = Main.getAppConfigInstance(); + + // The SecurityContext set by AuthFilter + private @Context SecurityContext theSecurityContext; + + // The uri requested + private @Context UriInfo theUriInfo; + + private static final Log LOG = LogFactory.getLog(Server.class); + + /** + * Check the status of this server. Always OK. + */ + @GET + @Path("status") + @Produces({MediaType.APPLICATION_JSON}) + public Map status() { + return STATUS_OK; + } + + /** + * Check the supported request formats of this server. + */ + @GET + @Produces({MediaType.APPLICATION_JSON}) + public Map requestFormats() { + return SUPPORTED_FORMATS; + } + + /** + * Check the version(s) supported by this server. + */ + @GET + @Path("version") + @Produces({MediaType.APPLICATION_JSON}) + public Map version() { + return SUPPORTED_VERSIONS; + } + + /** + * Execute an hcat ddl expression on the local box. It is run + * as the authenticated user and rate limited. + */ + @POST + @Path("ddl") + @Produces({MediaType.APPLICATION_JSON}) + public ExecBean ddl(@FormParam("exec") String exec, + @FormParam("group") String group, + @FormParam("permissions") String permissions) + throws NotAuthorizedException, BusyException, BadParam, + ExecuteException, IOException + { + verifyUser(); + verifyParam(exec, "exec"); + + HcatDelegator d = new HcatDelegator(appConf, execService); + return d.run(getUser(), exec, false, group, permissions); + } + + /** + * List all the tables in an hcat database. + */ + @GET + @Path("ddl/database/{db}/table") + @Produces(MediaType.APPLICATION_JSON) + public Response listTables(@PathParam("db") String db, + @QueryParam("like") String tablePattern) + throws HcatException, NotAuthorizedException, BusyException, + BadParam, ExecuteException, IOException + { + verifyUser(); + verifyDdlParam(db, ":db"); + + HcatDelegator d = new HcatDelegator(appConf, execService); + if (! TempletonUtils.isset(tablePattern)) + tablePattern = "*"; + return d.listTables(getUser(), db, tablePattern); + } + + /** + * Create a new table. + */ + @PUT + @Path("ddl/database/{db}/table/{table}") + @Produces(MediaType.APPLICATION_JSON) + public Response createTable(@PathParam("db") String db, + @PathParam("table") String table, + TableDesc desc) + throws SimpleWebException, NotAuthorizedException, BusyException, + BadParam, ExecuteException, IOException + { + verifyUser(); + verifyDdlParam(db, ":db"); + verifyDdlParam(table, ":table"); + desc.table = table; + + HcatDelegator d = new HcatDelegator(appConf, execService); + return d.createTable(getUser(), db, desc); + } + + /** + * Create a new table like another table. + */ + @PUT + @Path("ddl/database/{db}/table/{existingTable}/like/{newTable}") + @Produces(MediaType.APPLICATION_JSON) + public Response createTableLike(@PathParam("db") String db, + @PathParam("existingTable") String existingTable, + @PathParam("newTable") String newTable, + TableLikeDesc desc) + throws SimpleWebException, NotAuthorizedException, BusyException, + BadParam, ExecuteException, IOException + { + verifyUser(); + verifyDdlParam(db, ":db"); + verifyDdlParam(existingTable, ":existingTable"); + verifyDdlParam(newTable, ":newTable"); + desc.existingTable = existingTable; + desc.newTable = newTable; + + HcatDelegator d = new HcatDelegator(appConf, execService); + return d.createTableLike(getUser(), db, desc); + } + + /** + * Describe an hcat table. This is normally a simple list of + * columns (using "desc table"), but the extended format will show + * more information (using "show table extended like"). + */ + @GET + @Path("ddl/database/{db}/table/{table}") + @Produces(MediaType.APPLICATION_JSON) + public Response descTable(@PathParam("db") String db, + @PathParam("table") String table, + @QueryParam("format") String format) + throws HcatException, NotAuthorizedException, BusyException, + BadParam, ExecuteException, IOException + { + verifyUser(); + verifyDdlParam(db, ":db"); + verifyDdlParam(table, ":table"); + + HcatDelegator d = new HcatDelegator(appConf, execService); + if ("extended".equals(format)) + return d.descExtendedTable(getUser(), db, table); + else + return d.descTable(getUser(), db, table, false); + } + + /** + * Drop an hcat table. + */ + @DELETE + @Path("ddl/database/{db}/table/{table}") + @Produces(MediaType.APPLICATION_JSON) + public Response dropTable(@PathParam("db") String db, + @PathParam("table") String table, + @QueryParam("ifExists") boolean ifExists, + @QueryParam("group") String group, + @QueryParam("permissions") String permissions) + throws HcatException, NotAuthorizedException, BusyException, + BadParam, ExecuteException, IOException + { + verifyUser(); + verifyDdlParam(db, ":db"); + verifyDdlParam(table, ":table"); + + HcatDelegator d = new HcatDelegator(appConf, execService); + return d.dropTable(getUser(), db, table, ifExists, group, permissions); + } + + /** + * Rename an hcat table. + */ + @POST + @Path("ddl/database/{db}/table/{table}") + @Produces(MediaType.APPLICATION_JSON) + public Response renameTable(@PathParam("db") String db, + @PathParam("table") String oldTable, + @FormParam("rename") String newTable, + @FormParam("group") String group, + @FormParam("permissions") String permissions) + throws HcatException, NotAuthorizedException, BusyException, + BadParam, ExecuteException, IOException + { + verifyUser(); + verifyDdlParam(db, ":db"); + verifyDdlParam(oldTable, ":table"); + verifyDdlParam(newTable, "rename"); + + HcatDelegator d = new HcatDelegator(appConf, execService); + return d.renameTable(getUser(), db, oldTable, newTable, group, permissions); + } + + /** + * Describe a single property on an hcat table. + */ + @GET + @Path("ddl/database/{db}/table/{table}/property/{property}") + @Produces(MediaType.APPLICATION_JSON) + public Response descOneTableProperty(@PathParam("db") String db, + @PathParam("table") String table, + @PathParam("property") String property) + throws HcatException, NotAuthorizedException, BusyException, + BadParam, ExecuteException, IOException + { + verifyUser(); + verifyDdlParam(db, ":db"); + verifyDdlParam(table, ":table"); + verifyDdlParam(property, ":property"); + + HcatDelegator d = new HcatDelegator(appConf, execService); + return d.descTableProperty(getUser(), db, table, property); + } + + /** + * List all the properties on an hcat table. + */ + @GET + @Path("ddl/database/{db}/table/{table}/property") + @Produces(MediaType.APPLICATION_JSON) + public Response listTableProperties(@PathParam("db") String db, + @PathParam("table") String table) + throws HcatException, NotAuthorizedException, BusyException, + BadParam, ExecuteException, IOException + { + verifyUser(); + verifyDdlParam(db, ":db"); + verifyDdlParam(table, ":table"); + + HcatDelegator d = new HcatDelegator(appConf, execService); + return d.listTableProperties(getUser(), db, table); + } + + /** + * Add a single property on an hcat table. + */ + @PUT + @Path("ddl/database/{db}/table/{table}/property/{property}") + @Produces(MediaType.APPLICATION_JSON) + public Response addOneTableProperty(@PathParam("db") String db, + @PathParam("table") String table, + @PathParam("property") String property, + TablePropertyDesc desc) + throws HcatException, NotAuthorizedException, BusyException, + BadParam, ExecuteException, IOException + { + verifyUser(); + verifyDdlParam(db, ":db"); + verifyDdlParam(table, ":table"); + verifyDdlParam(property, ":property"); + desc.name = property; + + HcatDelegator d = new HcatDelegator(appConf, execService); + return d.addOneTableProperty(getUser(), db, table, desc); + } + + /** + * List all the partitions in an hcat table. + */ + @GET + @Path("ddl/database/{db}/table/{table}/partition") + @Produces(MediaType.APPLICATION_JSON) + public Response listPartitions(@PathParam("db") String db, + @PathParam("table") String table) + throws HcatException, NotAuthorizedException, BusyException, + BadParam, ExecuteException, IOException + { + verifyUser(); + verifyDdlParam(db, ":db"); + verifyDdlParam(table, ":table"); + + HcatDelegator d = new HcatDelegator(appConf, execService); + return d.listPartitions(getUser(), db, table); + } + + /** + * Describe a single partition in an hcat table. + */ + @GET + @Path("ddl/database/{db}/table/{table}/partition/{partition}") + @Produces(MediaType.APPLICATION_JSON) + public Response descPartition(@PathParam("db") String db, + @PathParam("table") String table, + @PathParam("partition") String partition) + throws HcatException, NotAuthorizedException, BusyException, + BadParam, ExecuteException, IOException + { + verifyUser(); + verifyDdlParam(db, ":db"); + verifyDdlParam(table, ":table"); + verifyParam(partition, ":partition"); + + HcatDelegator d = new HcatDelegator(appConf, execService); + return d.descOnePartition(getUser(), db, table, partition); + } + + /** + * Create a partition in an hcat table. + */ + @PUT + @Path("ddl/database/{db}/table/{table}/partition/{partition}") + @Produces(MediaType.APPLICATION_JSON) + public Response addOnePartition(@PathParam("db") String db, + @PathParam("table") String table, + @PathParam("partition") String partition, + PartitionDesc desc) + throws HcatException, NotAuthorizedException, BusyException, + BadParam, ExecuteException, IOException + { + verifyUser(); + verifyDdlParam(db, ":db"); + verifyDdlParam(table, ":table"); + verifyParam(partition, ":partition"); + desc.partition = partition; + HcatDelegator d = new HcatDelegator(appConf, execService); + return d.addOnePartition(getUser(), db, table, desc); + } + + /** + * Drop a partition in an hcat table. + */ + @DELETE + @Path("ddl/database/{db}/table/{table}/partition/{partition}") + @Produces(MediaType.APPLICATION_JSON) + public Response dropPartition(@PathParam("db") String db, + @PathParam("table") String table, + @PathParam("partition") String partition, + @QueryParam("ifExists") boolean ifExists, + @QueryParam("group") String group, + @QueryParam("permissions") String permissions) + throws HcatException, NotAuthorizedException, BusyException, + BadParam, ExecuteException, IOException + { + verifyUser(); + verifyDdlParam(db, ":db"); + verifyDdlParam(table, ":table"); + verifyParam(partition, ":partition"); + HcatDelegator d = new HcatDelegator(appConf, execService); + return d.dropPartition(getUser(), db, table, partition, ifExists, + group, permissions); + } + + /** + * List all databases, or those that match a pattern. + */ + @GET + @Path("ddl/database/") + @Produces(MediaType.APPLICATION_JSON) + public Response listDatabases(@QueryParam("like") String dbPattern) + throws HcatException, NotAuthorizedException, BusyException, + BadParam, ExecuteException, IOException + { + verifyUser(); + + HcatDelegator d = new HcatDelegator(appConf, execService); + if (! TempletonUtils.isset(dbPattern)) + dbPattern = "*"; + return d.listDatabases(getUser(), dbPattern); + } + + /** + * Describe a database + */ + @GET + @Path("ddl/database/{db}") + @Produces(MediaType.APPLICATION_JSON) + public Response descDatabase(@PathParam("db") String db, + @QueryParam("format") String format) + throws HcatException, NotAuthorizedException, BusyException, + BadParam, ExecuteException, IOException + { + verifyUser(); + verifyDdlParam(db, ":db"); + HcatDelegator d = new HcatDelegator(appConf, execService); + return d.descDatabase(getUser(), db, "extended".equals(format)); + } + + /** + * Create a database + */ + @PUT + @Path("ddl/database/{db}") + @Produces(MediaType.APPLICATION_JSON) + public Response createDatabase(@PathParam("db") String db, + DatabaseDesc desc) + throws HcatException, NotAuthorizedException, BusyException, + BadParam, ExecuteException, IOException + { + verifyUser(); + verifyDdlParam(db, ":db"); + desc.database = db; + HcatDelegator d = new HcatDelegator(appConf, execService); + return d.createDatabase(getUser(), desc); + } + + /** + * Drop a database + */ + @DELETE + @Path("ddl/database/{db}") + @Produces(MediaType.APPLICATION_JSON) + public Response dropDatabase(@PathParam("db") String db, + @QueryParam("ifExists") boolean ifExists, + @QueryParam("option") String option, + @QueryParam("group") String group, + @QueryParam("permissions") String permissions) + throws HcatException, NotAuthorizedException, BusyException, + BadParam, ExecuteException, IOException + { + verifyUser(); + verifyDdlParam(db, ":db"); + if (TempletonUtils.isset(option)) + verifyDdlParam(option, "option"); + HcatDelegator d = new HcatDelegator(appConf, execService); + return d.dropDatabase(getUser(), db, ifExists, option, + group, permissions); + } + + /** + * List the columns in an hcat table. Currently the same as + * describe table. + */ + @GET + @Path("ddl/database/{db}/table/{table}/column") + @Produces(MediaType.APPLICATION_JSON) + public Response listColumns(@PathParam("db") String db, + @PathParam("table") String table) + throws HcatException, NotAuthorizedException, BusyException, + BadParam, ExecuteException, IOException + { + verifyUser(); + verifyDdlParam(db, ":db"); + verifyDdlParam(table, ":table"); + + HcatDelegator d = new HcatDelegator(appConf, execService); + return d.listColumns(getUser(), db, table); + } + + /** + * Describe a single column in an hcat table. + */ + @GET + @Path("ddl/database/{db}/table/{table}/column/{column}") + @Produces(MediaType.APPLICATION_JSON) + public Response descColumn(@PathParam("db") String db, + @PathParam("table") String table, + @PathParam("column") String column) + throws SimpleWebException, NotAuthorizedException, BusyException, + BadParam, ExecuteException, IOException + { + verifyUser(); + verifyDdlParam(db, ":db"); + verifyDdlParam(table, ":table"); + verifyParam(column, ":column"); + + HcatDelegator d = new HcatDelegator(appConf, execService); + return d.descOneColumn(getUser(), db, table, column); + } + + /** + * Create a column in an hcat table. + */ + @PUT + @Path("ddl/database/{db}/table/{table}/column/{column}") + @Produces(MediaType.APPLICATION_JSON) + public Response addOneColumn(@PathParam("db") String db, + @PathParam("table") String table, + @PathParam("column") String column, + ColumnDesc desc) + throws HcatException, NotAuthorizedException, BusyException, + BadParam, ExecuteException, IOException + { + verifyUser(); + verifyDdlParam(db, ":db"); + verifyDdlParam(table, ":table"); + verifyParam(column, ":column"); + verifyParam(desc.type, "type"); + desc.name = column; + + HcatDelegator d = new HcatDelegator(appConf, execService); + return d.addOneColumn(getUser(), db, table, desc); + } + + /** + * Run a MapReduce Streaming job. + */ + @POST + @Path("mapreduce/streaming") + @Produces({MediaType.APPLICATION_JSON}) + public EnqueueBean mapReduceStreaming(@FormParam("input") List inputs, + @FormParam("output") String output, + @FormParam("mapper") String mapper, + @FormParam("reducer") String reducer, + @FormParam("file") List files, + @FormParam("define") List defines, + @FormParam("cmdenv") List cmdenvs, + @FormParam("arg") List args, + @FormParam("statusdir") String statusdir, + @FormParam("callback") String callback) + throws NotAuthorizedException, BusyException, BadParam, QueueException, + ExecuteException, IOException, InterruptedException + { + verifyUser(); + verifyParam(inputs, "input"); + verifyParam(mapper, "mapper"); + verifyParam(reducer, "reducer"); + + StreamingDelegator d = new StreamingDelegator(appConf); + return d.run(getUser(), inputs, output, mapper, reducer, + files, defines, cmdenvs, args, + statusdir, callback, getCompletedUrl()); + } + + /** + * Run a MapReduce Jar job. + */ + @POST + @Path("mapreduce/jar") + @Produces({MediaType.APPLICATION_JSON}) + public EnqueueBean mapReduceJar(@FormParam("jar") String jar, + @FormParam("class") String mainClass, + @FormParam("libjars") String libjars, + @FormParam("files") String files, + @FormParam("arg") List args, + @FormParam("define") List defines, + @FormParam("statusdir") String statusdir, + @FormParam("callback") String callback) + throws NotAuthorizedException, BusyException, BadParam, QueueException, + ExecuteException, IOException, InterruptedException + { + verifyUser(); + verifyParam(jar, "jar"); + verifyParam(mainClass, "class"); + + JarDelegator d = new JarDelegator(appConf); + return d.run(getUser(), + jar, mainClass, + libjars, files, args, defines, + statusdir, callback, getCompletedUrl()); + } + + /** + * Run a Pig job. + */ + @POST + @Path("pig") + @Produces({MediaType.APPLICATION_JSON}) + public EnqueueBean pig(@FormParam("execute") String execute, + @FormParam("file") String srcFile, + @FormParam("arg") List pigArgs, + @FormParam("files") String otherFiles, + @FormParam("statusdir") String statusdir, + @FormParam("callback") String callback) + throws NotAuthorizedException, BusyException, BadParam, QueueException, + ExecuteException, IOException, InterruptedException + { + verifyUser(); + if (execute == null && srcFile == null) + throw new BadParam("Either execute or file parameter required"); + + PigDelegator d = new PigDelegator(appConf); + return d.run(getUser(), + execute, srcFile, + pigArgs, otherFiles, + statusdir, callback, getCompletedUrl()); + } + + /** + * Run a Hive job. + */ + @POST + @Path("hive") + @Produces({MediaType.APPLICATION_JSON}) + public EnqueueBean hive(@FormParam("execute") String execute, + @FormParam("file") String srcFile, + @FormParam("define") List defines, + @FormParam("statusdir") String statusdir, + @FormParam("callback") String callback) + throws NotAuthorizedException, BusyException, BadParam, QueueException, + ExecuteException, IOException, InterruptedException + { + verifyUser(); + if (execute == null && srcFile == null) + throw new BadParam("Either execute or file parameter required"); + + HiveDelegator d = new HiveDelegator(appConf); + return d.run(getUser(), execute, srcFile, defines, + statusdir, callback, getCompletedUrl()); + } + + /** + * Return the status of the jobid. + */ + @GET + @Path("queue/{jobid}") + @Produces({MediaType.APPLICATION_JSON}) + public QueueStatusBean showQueueId(@PathParam("jobid") String jobid) + throws NotAuthorizedException, BadParam, IOException + { + verifyUser(); + verifyParam(jobid, ":jobid"); + + StatusDelegator d = new StatusDelegator(appConf); + return d.run(getUser(), jobid); + } + + /** + * Kill a job in the queue. + */ + @DELETE + @Path("queue/{jobid}") + @Produces({MediaType.APPLICATION_JSON}) + public QueueStatusBean deleteQueueId(@PathParam("jobid") String jobid) + throws NotAuthorizedException, BadParam, IOException + { + verifyUser(); + verifyParam(jobid, ":jobid"); + + DeleteDelegator d = new DeleteDelegator(appConf); + return d.run(getUser(), jobid); + } + + /** + * Return all the known job ids for this user. + */ + @GET + @Path("queue") + @Produces({MediaType.APPLICATION_JSON}) + public List showQueueList() + throws NotAuthorizedException, BadParam, IOException + { + verifyUser(); + + ListDelegator d = new ListDelegator(appConf); + return d.run(getUser()); + } + + /** + * Notify on a completed job. + */ + @GET + @Path("internal/complete/{jobid}") + @Produces({MediaType.APPLICATION_JSON}) + public CompleteBean completeJob(@PathParam("jobid") String jobid) + throws CallbackFailedException, IOException + { + CompleteDelegator d = new CompleteDelegator(appConf); + return d.run(jobid); + } + + /** + * Verify that we have a valid user. Throw an exception if invalid. + */ + public void verifyUser() + throws NotAuthorizedException + { + if (getUser() == null) { + String msg = "No user found."; + if (! UserGroupInformation.isSecurityEnabled()) + msg += " Missing " + PseudoAuthenticator.USER_NAME + " parameter."; + throw new NotAuthorizedException(msg); + } + } + + /** + * Verify that the parameter exists. Throw an exception if invalid. + */ + public void verifyParam(String param, String name) + throws BadParam + { + if (param == null) + throw new BadParam("Missing " + name + " parameter"); + } + + /** + * Verify that the parameter exists. Throw an exception if invalid. + */ + public void verifyParam(List param, String name) + throws BadParam + { + if (param == null || param.isEmpty()) + throw new BadParam("Missing " + name + " parameter"); + } + + public static final Pattern DDL_ID = Pattern.compile("[a-zA-Z]\\w*"); + + /** + * Verify that the parameter exists and is a simple DDL identifier + * name. Throw an exception if invalid. + * + * Bug: This needs to allow for quoted ddl identifiers. + */ + public void verifyDdlParam(String param, String name) + throws BadParam + { + verifyParam(param, name); + Matcher m = DDL_ID.matcher(param); + if (! m.matches()) + throw new BadParam("Invalid DDL identifier " + name ); + } + + /** + * Get the user name from the security context. + */ + public String getUser() { + if (theSecurityContext == null) + return null; + if (theSecurityContext.getUserPrincipal() == null) + return null; + return theSecurityContext.getUserPrincipal().getName(); + } + + /** + * The callback url on this server when a task is completed. + */ + public String getCompletedUrl() { + if (theUriInfo == null) + return null; + if (theUriInfo.getBaseUri() == null) + return null; + return theUriInfo.getBaseUri() + VERSION + + "/internal/complete/$jobId"; + } +} Index: webhcat/svr/src/main/java/org/apache/hcatalog/templeton/StatusDelegator.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/StatusDelegator.java (revision 0) +++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/StatusDelegator.java (revision 0) @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton; + +import java.io.IOException; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.mapred.JobID; +import org.apache.hadoop.mapred.JobProfile; +import org.apache.hadoop.mapred.JobStatus; +import org.apache.hadoop.mapred.JobTracker; +import org.apache.hadoop.mapred.TempletonJobTracker; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hcatalog.templeton.tool.JobState; + +/** + * Fetch the status of a given job id in the queue. + */ +public class StatusDelegator extends TempletonDelegator { + private static final Log LOG = LogFactory.getLog(StatusDelegator.class); + + public StatusDelegator(AppConfig appConf) { + super(appConf); + } + + public QueueStatusBean run(String user, String id) + throws NotAuthorizedException, BadParam, IOException + { + UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user); + TempletonJobTracker tracker = null; + JobState state = null; + try { + tracker = new TempletonJobTracker(ugi, + JobTracker.getAddress(appConf), + appConf); + JobID jobid = StatusDelegator.StringToJobID(id); + if (jobid == null) + throw new BadParam("Invalid jobid: " + id); + state = new JobState(id, Main.getAppConfigInstance()); + return StatusDelegator.makeStatus(tracker, jobid, state); + } catch (IllegalStateException e) { + throw new BadParam(e.getMessage()); + } finally { + if (tracker != null) + tracker.close(); + if (state != null) + state.close(); + } + } + + public static QueueStatusBean makeStatus(TempletonJobTracker tracker, + JobID jobid, + String childid, + JobState state) + throws BadParam, IOException + { + JobID bestid = jobid; + if (childid != null) + bestid = StatusDelegator.StringToJobID(childid); + + JobStatus status = tracker.getJobStatus(bestid); + JobProfile profile = tracker.getJobProfile(bestid); + + if (status == null || profile == null) { + if (bestid != jobid) { // Corrupt childid, retry. + LOG.error("Corrupt child id " + childid + " for " + jobid); + bestid = jobid; + status = tracker.getJobStatus(bestid); + profile = tracker.getJobProfile(bestid); + } + } + + if (status == null || profile == null) // No such job. + throw new BadParam("Could not find job " + bestid); + + return new QueueStatusBean(state, status, profile); + } + + public static QueueStatusBean makeStatus(TempletonJobTracker tracker, + JobID jobid, + JobState state) + throws BadParam, IOException + { + return makeStatus(tracker, jobid, state.getChildId(), state); + } + + /** + * A version of JobID.forName with our app specific error handling. + */ + public static JobID StringToJobID(String id) + throws BadParam + { + try { + return JobID.forName(id); + } catch (IllegalArgumentException e) { + throw new BadParam(e.getMessage()); + } + } +} Index: webhcat/svr/src/main/java/org/apache/hcatalog/templeton/QueueStatusBean.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/QueueStatusBean.java (revision 0) +++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/QueueStatusBean.java (revision 0) @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton; + +import java.io.IOException; +import org.apache.hadoop.mapred.JobStatus; +import org.apache.hadoop.mapred.JobProfile; +import org.apache.hcatalog.templeton.tool.JobState; + +/** + * QueueStatusBean - The results of an exec call. + */ +public class QueueStatusBean { + public JobStatus status; + public JobProfile profile; + + public String id; + public String parentId; + public String percentComplete; + public Long exitValue; + public String user; + public String callback; + public String completed; + + public QueueStatusBean() {} + + /** + * Create a new QueueStatusBean + * + * @param state store job state + * @param status job status + * @param profile job profile + */ + public QueueStatusBean(JobState state, JobStatus status, JobProfile profile) + throws IOException + { + this.status = status; + this.profile = profile; + + id = profile.getJobID().toString(); + parentId = state.getId(); + if (id.equals(parentId)) + parentId = null; + percentComplete = state.getPercentComplete(); + exitValue = state.getExitValue(); + user = state.getUser(); + callback = state.getCallback(); + completed = state.getCompleteStatus(); + } +} Index: webhcat/svr/src/main/java/org/apache/hcatalog/templeton/JsonBuilder.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/JsonBuilder.java (revision 0) +++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/JsonBuilder.java (revision 0) @@ -0,0 +1,186 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import org.apache.hcatalog.templeton.tool.TempletonUtils; +import org.codehaus.jackson.map.ObjectMapper; + +/** + * Helper class to build new json objects with new top level + * properties. Only add non-null entries. + */ +public class JsonBuilder { + static final int OK = 200; + static final int MISSING = 404; + static final int SERVER_ERROR = 500; + + // The map we're building. + private Map map; + + // Parse the json map. + private JsonBuilder(String json) + throws IOException + { + map = jsonToMap(json); + } + + /** + * Create a new map object from the existing json. + */ + public static JsonBuilder create(String json) + throws IOException + { + return new JsonBuilder(json); + } + + /** + * Create a new map object. + */ + public static JsonBuilder create() + throws IOException + { + return new JsonBuilder(null); + } + + /** + * Create a new map error object. + */ + public static JsonBuilder createError(String msg, int code) + throws IOException + { + return new JsonBuilder(null) + .put("error", msg) + .put("errorCode", code); + } + + /** + * Add a non-null value to the map. + */ + public JsonBuilder put(String name, Object val) { + if (val != null) + map.put(name, val); + return this; + } + + /** + * Remove a value from the map. + */ + public JsonBuilder remove(String name) { + map.remove(name); + return this; + } + + /** + * Get the underlying map. + */ + public Map getMap() { + return map; + } + + /** + * Turn the map back to response object. + */ + public Response build() { + return buildResponse(); + } + + /** + * Turn the map back to json. + */ + public String buildJson() + throws IOException + { + return mapToJson(map); + } + + /** + * Turn the map back to response object. + */ + public Response buildResponse() { + int status = OK; // Server ok. + if (map.containsKey("error")) + status = SERVER_ERROR; // Generic http server error. + Object o = map.get("errorCode"); + if (o != null) { + try { + status = Integer.parseInt(o.toString()); + } catch (Exception e) { + if (o instanceof Number) + status = ((Number) o).intValue(); + } + } + return buildResponse(status); + } + + /** + * Turn the map back to response object. + */ + public Response buildResponse(int status) { + return Response.status(status) + .entity(map) + .type(MediaType.APPLICATION_JSON) + .build(); + } + + /** + * Is the object non-empty? + */ + public boolean isset() { + return TempletonUtils.isset(map); + } + + /** + * Check if this is an error doc. + */ + public static boolean isError(Map obj) + { + return (obj != null) && obj.containsKey("error"); + } + + /** + * Convert a json string to a Map. + */ + public static Map jsonToMap(String json) + throws IOException + { + if (! TempletonUtils.isset(json)) + return new HashMap(); + else { + ObjectMapper mapper = new ObjectMapper(); + return mapper.readValue(json, Map.class); + } + } + + /** + * Convert a map to a json string. + */ + public static String mapToJson(Object obj) + throws IOException + { + ObjectMapper mapper = new ObjectMapper(); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + mapper.writeValue(out, obj); + return out.toString(); + } +} Index: webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CallbackFailedException.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CallbackFailedException.java (revision 0) +++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CallbackFailedException.java (revision 0) @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton; + +/** + * The callback failed when it tried to reach the callback URL. + */ +public class CallbackFailedException extends SimpleWebException { + public CallbackFailedException(String msg) { + super(400, msg); + } +} Index: webhcat/svr/src/main/java/org/apache/hcatalog/templeton/StreamingDelegator.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/StreamingDelegator.java (revision 0) +++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/StreamingDelegator.java (revision 0) @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import org.apache.commons.exec.ExecuteException; + +/** + * Submit a streaming job to the MapReduce queue. Really just a front + end to the JarDelegator. + * + * This is the backend of the mapreduce/streaming web service. + */ +public class StreamingDelegator extends LauncherDelegator { + public StreamingDelegator(AppConfig appConf) { + super(appConf); + } + + public EnqueueBean run(String user, + List inputs, String output, + String mapper, String reducer, + List files, List defines, + List cmdenvs, + List jarArgs, + String statusdir, + String callback, + String completedUrl) + throws NotAuthorizedException, BadParam, BusyException, QueueException, + ExecuteException, IOException, InterruptedException + { + List args = makeArgs(inputs, output, mapper, reducer, + files, defines, cmdenvs, jarArgs); + + JarDelegator d = new JarDelegator(appConf); + return d.run(user, + appConf.streamingJar(), null, + null, null, args, defines, + statusdir, callback, completedUrl); + } + + private List makeArgs(List inputs, + String output, + String mapper, + String reducer, + List files, + List defines, + List cmdenvs, + List jarArgs) + { + ArrayList args = new ArrayList(); + for (String input : inputs) { + args.add("-input"); + args.add(input); + } + args.add("-output"); + args.add(output); + args.add("-mapper"); + args.add(mapper); + args.add("-reducer"); + args.add(reducer); + + for (String f : files) + args.add("-file" + f); + for (String d : defines) + args.add("-D" + d); + for (String e : cmdenvs) + args.add("-cmdenv" + e); + args.addAll(jarArgs); + + return args; + } +} Index: webhcat/svr/src/main/java/org/apache/hcatalog/templeton/Main.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/Main.java (revision 0) +++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/Main.java (revision 0) @@ -0,0 +1,217 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton; + +import com.sun.jersey.api.core.PackagesResourceConfig; +import com.sun.jersey.spi.container.servlet.ServletContainer; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hdfs.web.AuthFilter; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.util.GenericOptionsParser; +import org.eclipse.jetty.rewrite.handler.RedirectPatternRule; +import org.eclipse.jetty.rewrite.handler.RewriteHandler; +import org.eclipse.jetty.server.Handler; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.handler.HandlerList; +import org.eclipse.jetty.servlet.FilterHolder; +import org.eclipse.jetty.servlet.FilterMapping; +import org.eclipse.jetty.servlet.ServletContextHandler; +import org.eclipse.jetty.servlet.ServletHolder; +import org.slf4j.bridge.SLF4JBridgeHandler; + +/** + * The main executable that starts up and runs the Server. + */ +public class Main { + public static final String SERVLET_PATH = "templeton"; + private static final Log LOG = LogFactory.getLog(Main.class); + + public static final int DEFAULT_PORT = 8080; + + private static volatile AppConfig conf; + + /** + * Retrieve the config singleton. + */ + public static synchronized AppConfig getAppConfigInstance() { + if (conf == null) + LOG.error("Bug: configuration not yet loaded"); + return conf; + } + + public Main(String[] args) { + init(args); + } + + public void init(String[] args) { + initLogger(); + conf = loadConfig(args); + conf.startCleanup(); + LOG.debug("Loaded conf " + conf); + } + + // Jersey uses java.util.logging - bridge to slf4 + private void initLogger() { + java.util.logging.Logger rootLogger + = java.util.logging.LogManager.getLogManager().getLogger(""); + for (java.util.logging.Handler h : rootLogger.getHandlers()) + rootLogger.removeHandler(h); + + SLF4JBridgeHandler.install(); + } + + public AppConfig loadConfig(String[] args) { + AppConfig cf = new AppConfig(); + try { + GenericOptionsParser parser = new GenericOptionsParser(cf, args); + if (parser.getRemainingArgs().length > 0) + usage(); + } catch (IOException e) { + LOG.error("Unable to parse options: " + e); + usage(); + } + + return cf; + } + + public void usage() { + System.err.println("usage: templeton [-Dtempleton.port=N] [-D...]"); + System.exit(1); + } + + public void run() { + int port = conf.getInt(AppConfig.PORT, DEFAULT_PORT); + try { + checkEnv(); + runServer(port); + System.out.println("templeton: listening on port " + port); + LOG.info("Templeton listening on port " + port); + } catch (Exception e) { + System.err.println("templeton: Server failed to start: " + e.getMessage()); + LOG.fatal("Server failed to start: " + e); + System.exit(1); + } + } + + private void checkEnv() { + checkCurrentDirPermissions(); + + } + + private void checkCurrentDirPermissions() { + //org.apache.commons.exec.DefaultExecutor requires + // that current directory exists + File pwd = new File("."); + if(!pwd.exists()){ + String msg = "Server failed to start: templeton: Current working directory '.' does not exist!"; + System.err.println(msg); + LOG.fatal( msg); + System.exit(1); + } + } + + public Server runServer(int port) + throws Exception + { + + //Authenticate using keytab + if(UserGroupInformation.isSecurityEnabled()){ + UserGroupInformation.loginUserFromKeytab(conf.kerberosPrincipal(), + conf.kerberosKeytab()); + } + + // Create the Jetty server + Server server = new Server(port); + ServletContextHandler root = new ServletContextHandler(server, "/"); + + // Add the Auth filter + root.addFilter(makeAuthFilter(), "/*", FilterMapping.REQUEST); + + // Connect Jersey + ServletHolder h = new ServletHolder(new ServletContainer(makeJerseyConfig())); + root.addServlet(h, "/" + SERVLET_PATH + "/*"); + + // Add any redirects + addRedirects(server); + + // Start the server + server.start(); + return server; + } + + // Configure the AuthFilter with the Kerberos params iff security + // is enabled. + public FilterHolder makeAuthFilter() { + FilterHolder authFilter = new FilterHolder(AuthFilter.class); + if (UserGroupInformation.isSecurityEnabled()) { + authFilter.setInitParameter("dfs.web.authentication.signature.secret", + conf.kerberosSecret()); + authFilter.setInitParameter("dfs.web.authentication.kerberos.principal", + conf.kerberosPrincipal()); + authFilter.setInitParameter("dfs.web.authentication.kerberos.keytab", + conf.kerberosKeytab()); + } + return authFilter; + } + + public PackagesResourceConfig makeJerseyConfig() { + PackagesResourceConfig rc + = new PackagesResourceConfig("org.apache.hcatalog.templeton"); + HashMap props = new HashMap(); + props.put("com.sun.jersey.api.json.POJOMappingFeature", "true"); + props.put("com.sun.jersey.config.property.WadlGeneratorConfig", + "org.apache.hcatalog.templeton.WadlConfig"); + rc.setPropertiesAndFeatures(props); + + return rc; + } + + public void addRedirects(Server server) { + RewriteHandler rewrite = new RewriteHandler(); + + RedirectPatternRule redirect = new RedirectPatternRule(); + redirect.setPattern("/templeton/v1/application.wadl"); + redirect.setLocation("/templeton/application.wadl"); + rewrite.addRule(redirect); + + HandlerList handlerlist = new HandlerList(); + ArrayList handlers = new ArrayList(); + + // Any redirect handlers need to be added first + handlers.add(rewrite); + + // Now add all the default handlers + for (Handler handler : server.getHandlers()) { + handlers.add(handler); + } + Handler[] newlist = new Handler[handlers.size()]; + handlerlist.setHandlers(handlers.toArray(newlist)); + server.setHandler(handlerlist); + } + + public static void main(String[] args) { + Main templeton = new Main(args); + templeton.run(); + } +} Index: webhcat/svr/src/main/java/org/apache/hcatalog/templeton/SecureProxySupport.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/SecureProxySupport.java (revision 0) +++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/SecureProxySupport.java (revision 0) @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton; + +import java.io.File; +import java.io.IOException; +import java.security.PrivilegedExceptionAction; +import java.util.List; +import java.util.Map; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.io.Text; +import org.apache.thrift.TException; +import org.apache.hadoop.security.Credentials; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.Token; + +/** + * Helper class to run jobs using Kerberos security. Always safe to + * use these methods, it's a noop if security is not enabled. + */ +public class SecureProxySupport { + private Path tokenPath; + private final String HCAT_SERVICE = "hcat"; + private boolean isEnabled; + private String user; + + public SecureProxySupport() { + isEnabled = UserGroupInformation.isSecurityEnabled(); + } + + private static final Log LOG = LogFactory.getLog(SecureProxySupport.class); + + /** + * The file where we store the auth token + */ + public Path getTokenPath() { return( tokenPath ); } + + /** + * The token to pass to hcat. + */ + public String getHcatServiceStr() { return( HCAT_SERVICE ); } + + /** + * Create the delegation token. + */ + public Path open(String user, Configuration conf) + throws IOException, InterruptedException + { + close(); + if (isEnabled) { + this.user = user; + File t = File.createTempFile("templeton", null); + tokenPath = new Path(t.toURI()); + Token fsToken = getFSDelegationToken(user, conf); + String hcatTokenStr; + try { + hcatTokenStr = buildHcatDelegationToken(user); + } catch (Exception e) { + throw new IOException(e); + } + Token msToken = new Token(); + msToken.decodeFromUrlString(hcatTokenStr); + msToken.setService(new Text(HCAT_SERVICE)); + writeProxyDelegationTokens(fsToken, msToken, conf, user, tokenPath); + + } + return tokenPath; + } + + /** + * Cleanup + */ + public void close() { + if (tokenPath != null) { + new File(tokenPath.toUri()).delete(); + tokenPath = null; + } + } + + /** + * Add Hadoop env variables. + */ + public void addEnv(Map env) { + if (isEnabled) { + env.put(UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION, + getTokenPath().toUri().getPath()); + } + } + + /** + * Add hcat args. + */ + public void addArgs(List args) { + if (isEnabled) { + args.add("-D"); + args.add("hive.metastore.token.signature=" + getHcatServiceStr()); + args.add("-D"); + args.add("proxy.user.name=" + user); + } + } + + class TokenWrapper { + Token token; + } + + private Token getFSDelegationToken(String user, + final Configuration conf) + throws IOException, InterruptedException + { + LOG.info("user: " + user + " loginUser: " + UserGroupInformation.getLoginUser().getUserName()); + final UserGroupInformation ugi = UgiFactory.getUgi(user); + + final TokenWrapper twrapper = new TokenWrapper(); + ugi.doAs(new PrivilegedExceptionAction() { + public Object run() throws IOException { + FileSystem fs = FileSystem.get(conf); + twrapper.token = fs.getDelegationToken(ugi.getShortUserName()); + return null; + } + }); + return twrapper.token; + + } + + private void writeProxyDelegationTokens(final Token fsToken, + final Token msToken, + final Configuration conf, + String user, + final Path tokenPath) + throws IOException, InterruptedException{ + + + LOG.info("user: " + user + " loginUser: " + UserGroupInformation.getLoginUser().getUserName()); + final UserGroupInformation ugi = UgiFactory.getUgi(user); + + + ugi.doAs(new PrivilegedExceptionAction() { + public Object run() throws IOException { + Credentials cred = new Credentials(); + cred.addToken(fsToken.getService(), fsToken); + cred.addToken(msToken.getService(), msToken); + cred.writeTokenStorageFile(tokenPath, conf); + return null; + } + }); + + } + + private String buildHcatDelegationToken(String user) + throws IOException, InterruptedException, MetaException, TException + { + HiveConf c = new HiveConf(); + final HiveMetaStoreClient client = new HiveMetaStoreClient(c); + LOG.info("user: " + user + " loginUser: " + UserGroupInformation.getLoginUser().getUserName()); + final TokenWrapper twrapper = new TokenWrapper(); + final UserGroupInformation ugi = UgiFactory.getUgi(user); + String s = ugi.doAs(new PrivilegedExceptionAction() { + public String run() + throws IOException, MetaException, TException + { + String u = ugi.getUserName(); + return client.getDelegationToken(u); + } + }); + return s; + } +} Index: webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ExecServiceImpl.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ExecServiceImpl.java (revision 0) +++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ExecServiceImpl.java (revision 0) @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Semaphore; +import org.apache.commons.exec.CommandLine; +import org.apache.commons.exec.DefaultExecutor; +import org.apache.commons.exec.ExecuteException; +import org.apache.commons.exec.ExecuteWatchdog; +import org.apache.commons.exec.PumpStreamHandler; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +/** + * Execute a local program. This is a singleton service that will + * execute programs as non-privileged users on the local box. See + * ExecService.run and ExecService.runUnlimited for details. + */ +public class ExecServiceImpl implements ExecService { + private static final Log LOG = LogFactory.getLog(ExecServiceImpl.class); + private static AppConfig appConf = Main.getAppConfigInstance(); + + private static volatile ExecServiceImpl theSingleton; + + /** + * Retrieve the singleton. + */ + public static synchronized ExecServiceImpl getInstance() { + if (theSingleton == null) { + theSingleton = new ExecServiceImpl(); + } + return theSingleton; + } + + private Semaphore avail; + + private ExecServiceImpl() { + avail = new Semaphore(appConf.getInt(AppConfig.EXEC_MAX_PROCS_NAME, 16)); + } + + /** + * Run the program synchronously as the given user. We rate limit + * the number of processes that can simultaneously created for + * this instance. + * + * @param program The program to run + * @param args Arguments to pass to the program + * @param env Any extra environment variables to set + * @return The result of the run. + */ + public ExecBean run(String program, List args, + Map env) + throws NotAuthorizedException, BusyException, ExecuteException, IOException + { + boolean aquired = false; + try { + aquired = avail.tryAcquire(); + if (aquired) { + return runUnlimited(program, args, env); + } else { + throw new BusyException(); + } + } finally { + if (aquired) { + avail.release(); + } + } + } + + /** + * Run the program synchronously as the given user. Warning: + * CommandLine will trim the argument strings. + * + * @param program The program to run. + * @param args Arguments to pass to the program + * @param env Any extra environment variables to set + * @return The result of the run. + */ + public ExecBean runUnlimited(String program, List args, + Map env) + throws NotAuthorizedException, ExecuteException, IOException + { + try { + return auxRun(program, args, env); + } catch (IOException e) { + File cwd = new java.io.File("."); + if (cwd.canRead() && cwd.canWrite()) + throw e; + else + throw new IOException("Invalid permissions on Templeton directory: " + + cwd.getCanonicalPath()); + } + } + + private ExecBean auxRun(String program, List args, Map env) + throws NotAuthorizedException, ExecuteException, IOException + { + DefaultExecutor executor = new DefaultExecutor(); + executor.setExitValues(null); + + // Setup stdout and stderr + int nbytes = appConf.getInt(AppConfig.EXEC_MAX_BYTES_NAME, -1); + ByteArrayOutputStream outStream = new MaxByteArrayOutputStream(nbytes); + ByteArrayOutputStream errStream = new MaxByteArrayOutputStream(nbytes); + executor.setStreamHandler(new PumpStreamHandler(outStream, errStream)); + + // Only run for N milliseconds + int timeout = appConf.getInt(AppConfig.EXEC_TIMEOUT_NAME, 0); + ExecuteWatchdog watchdog = new ExecuteWatchdog(timeout); + executor.setWatchdog(watchdog); + + CommandLine cmd = makeCommandLine(program, args); + + LOG.info("Running: " + cmd); + ExecBean res = new ExecBean(); + res.exitcode = executor.execute(cmd, execEnv(env)); + String enc = appConf.get(AppConfig.EXEC_ENCODING_NAME); + res.stdout = outStream.toString(enc); + res.stderr = errStream.toString(enc); + + return res; + } + + private CommandLine makeCommandLine(String program, + List args) + throws NotAuthorizedException, IOException + { + String path = validateProgram(program); + CommandLine cmd = new CommandLine(path); + if (args != null) + for (String arg : args) + cmd.addArgument(arg, false); + + return cmd; + } + + /** + * Build the environment used for all exec calls. + * + * @return The environment variables. + */ + public Map execEnv(Map env) { + HashMap res = new HashMap(); + + for (String key : appConf.getStrings(AppConfig.EXEC_ENVS_NAME)) { + String val = System.getenv(key); + if (val != null) { + res.put(key, val); + } + } + if (env != null) + res.putAll(env); + for(Map.Entry envs : res.entrySet()){ + LOG.info("Env " + envs.getKey() + "=" + envs.getValue()); + } + return res; + } + + /** + * Given a program name, lookup the fully qualified path. Throws + * an exception if the program is missing or not authorized. + * + * @param path The path of the program. + * @return The path of the validated program. + */ + public String validateProgram(String path) + throws NotAuthorizedException, IOException + { + File f = new File(path); + if (f.canExecute()) { + return f.getCanonicalPath(); + } else { + throw new NotAuthorizedException("Unable to access program: " + path); + } + } +} Index: webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ColumnDesc.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ColumnDesc.java (revision 0) +++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ColumnDesc.java (revision 0) @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton; + +import javax.xml.bind.annotation.XmlRootElement; + +/** + * A description of the column to create. + */ +@XmlRootElement +public class ColumnDesc extends GroupPermissionsDesc { + public String name; + public String type; + public String comment; + + public ColumnDesc() {} + + /** + * Create a new ColumnDesc + */ + public ColumnDesc(String name, String type, String comment) { + this.name = name; + this.type = type; + this.comment = comment; + } + + public String toString() { + return String.format("ColumnDesc(name=%s, type=%s, comment=%s)", + name, type, comment); + } + + public boolean equals(Object o) { + if (this == o) + return true; + if (! (o instanceof ColumnDesc)) + return false; + ColumnDesc that = (ColumnDesc) o; + return xequals(this.name, that.name) + && xequals(this.type, that.type) + && xequals(this.comment, that.comment) + && super.equals(that) + ; + } +} Index: webhcat/svr/src/main/java/org/apache/hcatalog/templeton/EnqueueBean.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/EnqueueBean.java (revision 0) +++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/EnqueueBean.java (revision 0) @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton; + +/** + * EnqueueBean - The results of a call that enqueues a Hadoop job. + */ +public class EnqueueBean { + public String id; + + public EnqueueBean() {} + + /** + * Create a new EnqueueBean. + * + * @param id job id + */ + public EnqueueBean(String id) { + this.id = id; + } +} Index: webhcat/svr/src/main/java/org/apache/hcatalog/templeton/HcatException.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/HcatException.java (revision 0) +++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/HcatException.java (revision 0) @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton; + +import java.util.HashMap; + +/** + * Unable to run hcat on the job. + */ +public class HcatException extends SimpleWebException { + public ExecBean execBean; + public String statement; + + public HcatException(String msg, final ExecBean bean, final String statement) { + super(500, msg, new HashMap() {{ + put("exec", bean); + put("statement", statement); + }}); + execBean = bean; + this.statement = statement; + } +} Index: webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CompleteBean.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CompleteBean.java (revision 0) +++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CompleteBean.java (revision 0) @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton; + +/** + * CompleteBean - The results of an CompleteDelegator run. + */ +public class CompleteBean { + public String status; + + public CompleteBean() {} + + /** + * Create a new CompleteBean + * + * @param status run status + */ + public CompleteBean(String status) { + this.status = status; + } +} Index: webhcat/svr/src/main/java/org/apache/hadoop/mapred/TempletonJobTracker.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hadoop/mapred/TempletonJobTracker.java (revision 0) +++ webhcat/svr/src/main/java/org/apache/hadoop/mapred/TempletonJobTracker.java (revision 0) @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.mapred; + +import java.io.IOException; +import java.net.InetSocketAddress; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.ipc.RPC; +import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.security.UserGroupInformation; + +/* + * Communicate with the JobTracker as a specific user. + */ +public class TempletonJobTracker { + private JobSubmissionProtocol cnx; + + /** + * Create a connection to the Job Tracker. + */ + public TempletonJobTracker(UserGroupInformation ugi, + InetSocketAddress addr, + Configuration conf) + throws IOException + { + cnx = (JobSubmissionProtocol) + RPC.getProxy(JobSubmissionProtocol.class, + JobSubmissionProtocol.versionID, + addr, + ugi, + conf, + NetUtils.getSocketFactory(conf, + JobSubmissionProtocol.class)); + } + + /** + * Grab a handle to a job that is already known to the JobTracker. + * + * @return Profile of the job, or null if not found. + */ + public JobProfile getJobProfile(JobID jobid) + throws IOException + { + return cnx.getJobProfile(jobid); + } + + /** + * Grab a handle to a job that is already known to the JobTracker. + * + * @return Status of the job, or null if not found. + */ + public JobStatus getJobStatus(JobID jobid) + throws IOException + { + return cnx.getJobStatus(jobid); + } + + + /** + * Kill a job. + */ + public void killJob(JobID jobid) + throws IOException + { + cnx.killJob(jobid); + } + + /** + * Get all the jobs submitted. + */ + public JobStatus[] getAllJobs() + throws IOException + { + return cnx.getAllJobs(); + } + + /** + * Close the connection to the Job Tracker. + */ + public void close() { + RPC.stopProxy(cnx); + } +} Index: webhcat/svr/src/main/bin/templeton_config.sh =================================================================== --- webhcat/svr/src/main/bin/templeton_config.sh (revision 0) +++ webhcat/svr/src/main/bin/templeton_config.sh (revision 0) @@ -0,0 +1,93 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#==================================== +#Default config param values +#==================================== + +# The file containing the running pid +PID_FILE=./templeton.pid + +#default log directory +TEMPLETON_LOG_DIR=${TEMPLETON_LOG_DIR:-.} + +# The console error log +ERROR_LOG=${TEMPLETON_LOG_DIR}/templeton-console-error.log + +# The console log +CONSOLE_LOG=${TEMPLETON_LOG_DIR}/templeton-console.log + +# The name of the templeton jar file +TEMPLETON_JAR=templeton-0.1.0-dev.jar + +# How long to wait before testing that the process started correctly +SLEEP_TIME_AFTER_START=10 + +#================================================ +#See if the default configs have been overwritten +#================================================ + +#These parameters can be overriden by templeton-env.sh +# the root of the TEMPLETON installation +export TEMPLETON_PREFIX=`dirname "$this"`/.. + +#check to see if the conf dir is given as an optional argument +if [ $# -gt 1 ] +then + if [ "--config" = "$1" ] + then + shift + confdir=$1 + shift + TEMPLETON_CONF_DIR=$confdir + fi +fi + +# Allow alternate conf dir location. +if [ -e "${TEMPLETON_PREFIX}/conf/templeton-env.sh" ]; then + DEFAULT_CONF_DIR=${TEMPLETON_PREFIX}/"conf" +else + DEFAULT_CONF_DIR="/etc/templeton" +fi +TEMPLETON_CONF_DIR="${TEMPLETON_CONF_DIR:-$DEFAULT_CONF_DIR}" + +#users can add various env vars to templeton-env.sh in the conf +#rather than having to export them before running the command +if [ -f "${TEMPLETON_CONF_DIR}/templeton-env.sh" ]; then + . "${TEMPLETON_CONF_DIR}/templeton-env.sh" +fi + +#==================================== +#determine where hadoop is +#==================================== + +#check HADOOP_HOME and then check HADOOP_PREFIX +if [ -f ${HADOOP_HOME}/bin/hadoop ]; then + HADOOP_PREFIX=$HADOOP_HOME +#if this is an rpm install check for /usr/bin/hadoop +elif [ -f ${TEMPLETON_PREFIX}/bin/hadoop ]; then + HADOOP_PREFIX=$TEMPLETON_PREFIX +#otherwise see if HADOOP_PREFIX is defined +elif [ ! -f ${HADOOP_PREFIX}/bin/hadoop ]; then + echo "Hadoop not found." + exit 1 +fi + + + + + Index: webhcat/svr/src/main/bin/templeton_server.sh =================================================================== --- webhcat/svr/src/main/bin/templeton_server.sh (revision 0) +++ webhcat/svr/src/main/bin/templeton_server.sh (revision 0) @@ -0,0 +1,237 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Support functions +# + +# Follow symlinks on Linux and Darwin +function real_script_name() { + local base=$1 + local real + if readlink -f $base >/dev/null 2>&1; then + # Darwin/Mac OS X + real=`readlink -f $base` + fi + if [[ "$?" != "0" || -z "$real" ]]; then + # Linux + local bin=$(cd -P -- "$(dirname -- "$base")">/dev/null && pwd -P) + local script="$(basename -- "$base")" + real="$bin/$script" + fi + echo "$real" +} + +function usage() { + echo "usage: $0 [start|stop|foreground]" + echo " start Start the Templeton Server" + echo " stop Stop the Templeton Server" + echo " foreground Run the Templeton Server in the foreground" + exit 1 +} + +# Print an error message and exit +function die() { + echo "templeton: $@" 1>&2 + exit 1 +} + +# Print an message +function log() { + echo "templeton: $@" +} + +# Find the templeton jar +function find_jar_path() { + for dir in "." "build/templeton" "share/templeton/"; do + local jar="$base_dir/$dir/$TEMPLETON_JAR" + if [[ -f $jar ]]; then + echo $jar + break + fi + done +} + +# Find the templeton classpath +function find_classpath() { + local classpath="" + for dir in "share/templeton/lib" "build/ivy/lib/templeton" "conf" ; do + local path="$base_dir/$dir" + + if [[ -d $path ]]; then + for jar_or_conf in $path/*; do + if [[ -z "$classpath" ]]; then + classpath="$jar_or_conf" + else + classpath="$classpath:$jar_or_conf" + fi + done + fi + done + + if [[ -n "$TEMPLETON_CONF_DIR" ]]; then + if [[ -z "$classpath" ]]; then + classpath="$TEMPLETON_CONF_DIR" + else + classpath="$classpath:$TEMPLETON_CONF_DIR" + fi + fi + + # Append hcat classpath + local hcat_classpath + hcat_classpath=`hcat -classpath` + if [[ "$?" != "0" ]]; then + die "Unable to get the hcatalog classpath" + fi + echo "$classpath:$hcat_classpath" +} + +# Check if the pid is running +function check_pid() { + local pid=$1 + if ps -p $pid > /dev/null; then + return 0 + else + return 1 + fi +} + +# Start the templeton server in the foreground +function foreground_templeton() { + $start_cmd +} + +# Start the templeton server in the background. Record the PID for +# later use. +function start_templeton() { + if [[ -f $PID_FILE ]]; then + # Check if there is a server running + local pid=`cat $PID_FILE` + if check_pid $pid; then + die "already running on process $pid" + fi + fi + + log "starting ..." + log "$start_cmd" + nohup $start_cmd >>$CONSOLE_LOG 2>>$ERROR_LOG & + local pid=$! + + if [[ -z "${pid}" ]] ; then # we failed right off + die "failed to start. Check logs in " `dirname $ERROR_LOG` + fi + + sleep $SLEEP_TIME_AFTER_START + + if check_pid $pid; then + echo $pid > $PID_FILE + log "starting ... started." + else + die "failed to start. Check logs in " `dirname $ERROR_LOG` + fi +} + +# Stop a running server +function stop_templeton() { + local pid + if [[ -f $PID_FILE ]]; then + # Check if there is a server running + local check=`cat $PID_FILE` + if check_pid $check; then + pid=$check + fi + fi + + if [[ -z "$pid" ]]; then + log "no running server found" + else + log "stopping ..." + kill $pid + sleep $SLEEP_TIME_AFTER_START + if check_pid $pid; then + die "failed to stop" + else + log "stopping ... stopped" + fi + fi +} + +# +# Build command line and run +# + +this=`real_script_name "${BASH_SOURCE-$0}"` +this_bin=`dirname $this` +base_dir="$this_bin/.." + +if [[ -f "$base_dir/libexec/templeton_config.sh" ]]; then + . "$base_dir/libexec/templeton_config.sh" +else + . "$this_bin/templeton_config.sh" +fi + +JAR=`find_jar_path` +if [[ -z "$JAR" ]]; then + die "No templeton jar found" +fi + +CLASSPATH=`find_classpath` +if [[ -z "$CLASSPATH" ]]; then + die "No classpath or jars found" +fi +CLASSPATH="$JAR:$CLASSPATH" + +if [[ -z "$HADOOP_CLASSPATH" ]]; then + export HADOOP_CLASSPATH="$CLASSPATH" +else + export HADOOP_CLASSPATH="$CLASSPATH:$HADOOP_CLASSPATH" +fi + +if [[ -z "$TEMPLETON_LOG4J" ]]; then + if [[ -f "$base_dir/conf/templeton-log4j.properties" ]]; then + TEMPLETON_LOG4J="$base_dir/conf/templeton-log4j.properties"; + elif [[ -f "$base_dir/conf/templeton-log4j.properties" ]]; then + TEMPLETON_LOG4J="$base_dir/conf/templeton-log4j.properties"; + else + TEMPLETON_LOG4J="templeton-log4j.properties"; + fi +fi + +export HADOOP_USER_CLASSPATH_FIRST=true +export HADOOP_OPTS="-Dtempleton.log.dir=$TEMPLETON_LOG_DIR -Dlog4j.configuration=$TEMPLETON_LOG4J" + +start_cmd="$HADOOP_PREFIX/bin/hadoop jar $JAR org.apache.hcatalog.templeton.Main " + + +cmd=$1 +case $cmd in + start) + start_templeton + ;; + stop) + stop_templeton + ;; + foreground) + foreground_templeton + ;; + *) + usage + ;; +esac + +log "done" Property changes on: webhcat/svr/src/main/bin/templeton_server.sh ___________________________________________________________________ Added: svn:executable + * Index: webhcat/svr/build.xml =================================================================== --- webhcat/svr/build.xml (revision 0) +++ webhcat/svr/build.xml (revision 0) @@ -0,0 +1,31 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +