hive> dfs -lsr /apps/hive/warehouse/nyse_stocks_pcsb > ; drwx------ - sandbox hdfs 0 2013-01-22 22:16 /apps/hive/warehouse/nyse_stocks_pcsb/year=2000 -rw------- 1 sandbox hdfs 3975498 2013-01-22 22:16 /apps/hive/warehouse/nyse_stocks_pcsb/year=2000/000000_0 -rw------- 1 sandbox hdfs 3876608 2013-01-22 22:16 /apps/hive/warehouse/nyse_stocks_pcsb/year=2000/000001_0 -rw------- 1 sandbox hdfs 4158822 2013-01-22 22:16 /apps/hive/warehouse/nyse_stocks_pcsb/year=2000/000002_0 -rw------- 1 sandbox hdfs 4134833 2013-01-22 22:16 /apps/hive/warehouse/nyse_stocks_pcsb/year=2000/000003_0 drwx------ - sandbox hdfs 0 2013-01-22 22:16 /apps/hive/warehouse/nyse_stocks_pcsb/year=2001 -rw------- 1 sandbox hdfs 4211599 2013-01-22 22:16 /apps/hive/warehouse/nyse_stocks_pcsb/year=2001/000000_0 -rw------- 1 sandbox hdfs 4169889 2013-01-22 22:16 /apps/hive/warehouse/nyse_stocks_pcsb/year=2001/000001_0 -rw------- 1 sandbox hdfs 4380578 2013-01-22 22:16 /apps/hive/warehouse/nyse_stocks_pcsb/year=2001/000002_0 -rw------- 1 sandbox hdfs 4350816 2013-01-22 22:16 /apps/hive/warehouse/nyse_stocks_pcsb/year=2001/000003_0 hive> create table test AS > select /*+ MAPJOIN(l) */ > l.stock_price_open lo, > r.stock_price_open ro > from nyse_stocks_pcsb l JOIN nyse_stocks_pcsb_dup r ON (l.year = r.year and l.stock_symbol = r.stock_symbol and l.dte=r.dte) > > ; Total MapReduce jobs = 1 setting HADOOP_USER_NAME sandbox WARNING: org.apache.hadoop.metrics.jvm.EventCounter is deprecated. Please use org.apache.hadoop.log.metrics.EventCounter in all the log4j.properties files. Execution log at: /tmp/sandbox/sandbox_20130122231717_9fe307ed-ccc5-4a66-9d21-77700a7ad047.log 2013-01-22 11:17:44 Starting to launch local task to process map join; maximum memory = 1065484288 2013-01-22 11:17:49 Processing rows: 831 Hashtable size: 831 Memory usage: 13315208 rate: 0.012 2013-01-22 11:17:49 Dump the hashtable into file: file:/tmp/sandbox/hive_2013-01-22_23-17-17_444_6282099405469266683/-local-10002/HashTable-Stage-1/MapJoin-l-30-(year%3D2000)000000_0.hashtable 2013-01-22 11:17:49 Upload 1 File to: file:/tmp/sandbox/hive_2013-01-22_23-17-17_444_6282099405469266683/-local-10002/HashTable-Stage-1/MapJoin-l-30-(year%3D2000)000000_0.hashtable File size: 1851832 2013-01-22 11:17:52 Processing rows: 1648 Hashtable size: 817 Memory usage: 12882328 rate: 0.012 2013-01-22 11:17:52 Dump the hashtable into file: file:/tmp/sandbox/hive_2013-01-22_23-17-17_444_6282099405469266683/-local-10002/HashTable-Stage-1/MapJoin-l-30-(year%3D2000)000001_0.hashtable 2013-01-22 11:17:52 Upload 1 File to: file:/tmp/sandbox/hive_2013-01-22_23-17-17_444_6282099405469266683/-local-10002/HashTable-Stage-1/MapJoin-l-30-(year%3D2000)000001_0.hashtable File size: 1814452 2013-01-22 11:17:55 Processing rows: 2512 Hashtable size: 864 Memory usage: 13964248 rate: 0.013 2013-01-22 11:17:55 Dump the hashtable into file: file:/tmp/sandbox/hive_2013-01-22_23-17-17_444_6282099405469266683/-local-10002/HashTable-Stage-1/MapJoin-l-30-(year%3D2000)000002_0.hashtable 2013-01-22 11:17:55 Upload 1 File to: file:/tmp/sandbox/hive_2013-01-22_23-17-17_444_6282099405469266683/-local-10002/HashTable-Stage-1/MapJoin-l-30-(year%3D2000)000002_0.hashtable File size: 1923109 2013-01-22 11:17:57 Processing rows: 3372 Hashtable size: 860 Memory usage: 13823832 rate: 0.013 2013-01-22 11:17:57 Dump the hashtable into file: file:/tmp/sandbox/hive_2013-01-22_23-17-17_444_6282099405469266683/-local-10002/HashTable-Stage-1/MapJoin-l-30-(year%3D2000)000003_0.hashtable 2013-01-22 11:17:57 Upload 1 File to: file:/tmp/sandbox/hive_2013-01-22_23-17-17_444_6282099405469266683/-local-10002/HashTable-Stage-1/MapJoin-l-30-(year%3D2000)000003_0.hashtable File size: 1923062 2013-01-22 11:18:00 Processing rows: 4203 Hashtable size: 831 Memory usage: 12963864 rate: 0.012 2013-01-22 11:18:00 Dump the hashtable into file: file:/tmp/sandbox/hive_2013-01-22_23-17-17_444_6282099405469266683/-local-10002/HashTable-Stage-1/MapJoin-l-30-(year%3D2001)000000_0.hashtable 2013-01-22 11:18:00 Upload 1 File to: file:/tmp/sandbox/hive_2013-01-22_23-17-17_444_6282099405469266683/-local-10002/HashTable-Stage-1/MapJoin-l-30-(year%3D2001)000000_0.hashtable File size: 1851832 2013-01-22 11:18:02 Processing rows: 5020 Hashtable size: 817 Memory usage: 12751304 rate: 0.012 2013-01-22 11:18:02 Dump the hashtable into file: file:/tmp/sandbox/hive_2013-01-22_23-17-17_444_6282099405469266683/-local-10002/HashTable-Stage-1/MapJoin-l-30-(year%3D2001)000001_0.hashtable 2013-01-22 11:18:02 Upload 1 File to: file:/tmp/sandbox/hive_2013-01-22_23-17-17_444_6282099405469266683/-local-10002/HashTable-Stage-1/MapJoin-l-30-(year%3D2001)000001_0.hashtable File size: 1814452 2013-01-22 11:18:05 Processing rows: 5884 Hashtable size: 864 Memory usage: 13350712 rate: 0.013 2013-01-22 11:18:05 Dump the hashtable into file: file:/tmp/sandbox/hive_2013-01-22_23-17-17_444_6282099405469266683/-local-10002/HashTable-Stage-1/MapJoin-l-30-(year%3D2001)000002_0.hashtable 2013-01-22 11:18:05 Upload 1 File to: file:/tmp/sandbox/hive_2013-01-22_23-17-17_444_6282099405469266683/-local-10002/HashTable-Stage-1/MapJoin-l-30-(year%3D2001)000002_0.hashtable File size: 1923109 2013-01-22 11:18:07 Processing rows: 6744 Hashtable size: 860 Memory usage: 13348464 rate: 0.013 2013-01-22 11:18:07 Dump the hashtable into file: file:/tmp/sandbox/hive_2013-01-22_23-17-17_444_6282099405469266683/-local-10002/HashTable-Stage-1/MapJoin-l-30-(year%3D2001)000003_0.hashtable 2013-01-22 11:18:07 Upload 1 File to: file:/tmp/sandbox/hive_2013-01-22_23-17-17_444_6282099405469266683/-local-10002/HashTable-Stage-1/MapJoin-l-30-(year%3D2001)000003_0.hashtable File size: 1923062 2013-01-22 11:18:07 End of local task; Time Taken: 23.233 sec. Execution completed successfully Mapred Local Task Succeeded . Convert the Join into MapJoin Mapred Local Task Succeeded . Convert the Join into MapJoin Launching Job 1 out of 1 Number of reduce tasks is set to 0 since there's no reduce operator Starting Job = job_201301222050_0007, Tracking URL = http://sandbox:50030/jobdetails.jsp?jobid=job_201301222050_0007 Kill Command = /usr/lib/hadoop/libexec/../bin/hadoop job -kill job_201301222050_0007 Hadoop job information for Stage-1: number of mappers: 8; number of reducers: 0 2013-01-22 23:18:24,151 Stage-1 map = 0%, reduce = 0% 2013-01-22 23:18:39,426 Stage-1 map = 50%, reduce = 0% 2013-01-22 23:19:40,055 Stage-1 map = 50%, reduce = 0%, Cumulative CPU 97.38 sec 2013-01-22 23:19:41,308 Stage-1 map = 50%, reduce = 0%, Cumulative CPU 97.38 sec 2013-01-22 23:19:42,459 Stage-1 map = 50%, reduce = 0%, Cumulative CPU 97.38 sec 2013-01-22 23:19:43,911 Stage-1 map = 50%, reduce = 0%, Cumulative CPU 97.38 sec 2013-01-22 23:19:45,024 Stage-1 map = 50%, reduce = 0%, Cumulative CPU 97.38 sec 2013-01-22 23:19:46,076 Stage-1 map = 50%, reduce = 0%, Cumulative CPU 97.38 sec 2013-01-22 23:19:47,420 Stage-1 map = 50%, reduce = 0%, Cumulative CPU 97.38 sec 2013-01-22 23:19:48,438 Stage-1 map = 50%, reduce = 0%, Cumulative CPU 97.38 sec 2013-01-22 23:19:49,572 Stage-1 map = 50%, reduce = 0%, Cumulative CPU 97.38 sec 2013-01-22 23:19:50,633 Stage-1 map = 50%, reduce = 0%, Cumulative CPU 97.38 sec 2013-01-22 23:19:51,695 Stage-1 map = 50%, reduce = 0%, Cumulative CPU 97.38 sec 2013-01-22 23:19:52,704 Stage-1 map = 50%, reduce = 0%, Cumulative CPU 97.38 sec 2013-01-22 23:19:54,202 Stage-1 map = 50%, reduce = 0%, Cumulative CPU 97.38 sec 2013-01-22 23:19:55,225 Stage-1 map = 50%, reduce = 0%, Cumulative CPU 97.38 sec 2013-01-22 23:19:56,243 Stage-1 map = 50%, reduce = 0%, Cumulative CPU 97.38 sec 2013-01-22 23:19:57,326 Stage-1 map = 50%, reduce = 0%, Cumulative CPU 97.38 sec 2013-01-22 23:19:58,538 Stage-1 map = 50%, reduce = 0%, Cumulative CPU 97.38 sec 2013-01-22 23:19:59,623 Stage-1 map = 50%, reduce = 0%, Cumulative CPU 97.38 sec 2013-01-22 23:20:01,257 Stage-1 map = 50%, reduce = 0%, Cumulative CPU 97.38 sec 2013-01-22 23:20:02,281 Stage-1 map = 50%, reduce = 0%, Cumulative CPU 97.38 sec 2013-01-22 23:20:03,296 Stage-1 map = 50%, reduce = 0%, Cumulative CPU 97.38 sec 2013-01-22 23:20:04,315 Stage-1 map = 50%, reduce = 0%, Cumulative CPU 97.38 sec 2013-01-22 23:20:05,376 Stage-1 map = 50%, reduce = 0%, Cumulative CPU 97.38 sec 2013-01-22 23:20:06,549 Stage-1 map = 50%, reduce = 0%, Cumulative CPU 97.38 sec 2013-01-22 23:20:07,785 Stage-1 map = 50%, reduce = 0%, Cumulative CPU 97.38 sec 2013-01-22 23:20:10,185 Stage-1 map = 50%, reduce = 0%, Cumulative CPU 97.38 sec 2013-01-22 23:20:18,285 Stage-1 map = 50%, reduce = 0%, Cumulative CPU 97.38 sec 2013-01-22 23:20:19,777 Stage-1 map = 50%, reduce = 0%, Cumulative CPU 97.38 sec ... 2013-01-23 00:11:02,696 Stage-1 map = 100%, reduce = 0%, Cumulative CPU 5635.47 sec 2013-01-23 00:11:03,700 Stage-1 map = 100%, reduce = 0%, Cumulative CPU 5635.47 sec 2013-01-23 00:11:04,707 Stage-1 map = 100%, reduce = 0%, Cumulative CPU 5635.47 sec 2013-01-23 00:11:05,715 Stage-1 map = 100%, reduce = 0%, Cumulative CPU 5708.88 sec 2013-01-23 00:11:06,735 Stage-1 map = 100%, reduce = 0%, Cumulative CPU 5708.88 sec 2013-01-23 00:11:07,740 Stage-1 map = 100%, reduce = 0%, Cumulative CPU 5708.88 sec 2013-01-23 00:11:08,743 Stage-1 map = 100%, reduce = 0%, Cumulative CPU 5708.88 sec 2013-01-23 00:11:09,748 Stage-1 map = 100%, reduce = 0%, Cumulative CPU 5708.88 sec 2013-01-23 00:11:10,752 Stage-1 map = 100%, reduce = 0%, Cumulative CPU 5708.88 sec 2013-01-23 00:11:11,757 Stage-1 map = 100%, reduce = 0%, Cumulative CPU 5708.88 sec 2013-01-23 00:11:12,762 Stage-1 map = 100%, reduce = 0%, Cumulative CPU 5715.31 sec 2013-01-23 00:11:13,768 Stage-1 map = 100%, reduce = 0%, Cumulative CPU 5715.31 sec 2013-01-23 00:11:14,772 Stage-1 map = 100%, reduce = 0%, Cumulative CPU 5715.31 sec 2013-01-23 00:11:15,776 Stage-1 map = 100%, reduce = 0%, Cumulative CPU 5715.31 sec 2013-01-23 00:11:16,780 Stage-1 map = 100%, reduce = 0%, Cumulative CPU 5726.76 sec 2013-01-23 00:11:17,786 Stage-1 map = 100%, reduce = 0%, Cumulative CPU 5726.76 sec 2013-01-23 00:11:18,791 Stage-1 map = 100%, reduce = 0%, Cumulative CPU 5726.76 sec 2013-01-23 00:11:19,796 Stage-1 map = 100%, reduce = 0%, Cumulative CPU 5726.76 sec 2013-01-23 00:11:20,809 Stage-1 map = 100%, reduce = 0%, Cumulative CPU 5726.76 sec 2013-01-23 00:11:21,823 Stage-1 map = 100%, reduce = 0%, Cumulative CPU 5726.76 sec 2013-01-23 00:11:22,832 Stage-1 map = 100%, reduce = 100%, Cumulative CPU 5726.76 sec