From fc3117ffcd053fbc9d9c618e50900a1dbc0e005e Mon Sep 17 00:00:00 2001 From: Ashutosh Chauhan Date: Fri, 21 Apr 2017 18:51:35 -0700 Subject: [PATCH] HIVE-16536 : Various improvements in TestPerfCliDriver --- .../metastore_export/csv/TABLE_PARAMS.txt | 143 ------- .../metastore_export/csv/TABLE_PARAMS.txt.bz2 | Bin 0 -> 481 bytes .../metastore_export/csv/TAB_COL_STATS.txt | 425 --------------------- .../metastore_export/csv/TAB_COL_STATS.txt.bz2 | Bin 0 -> 6216 bytes .../java/org/apache/hadoop/hive/ql/QTestUtil.java | 99 +++-- .../test/results/clientpositive/perf/query14.q.out | 4 +- .../test/results/clientpositive/perf/query23.q.out | 2 +- .../test/results/clientpositive/perf/query88.q.out | 224 +++++------ 8 files changed, 188 insertions(+), 709 deletions(-) delete mode 100644 data/files/tpcds-perf/metastore_export/csv/TABLE_PARAMS.txt create mode 100644 data/files/tpcds-perf/metastore_export/csv/TABLE_PARAMS.txt.bz2 delete mode 100644 data/files/tpcds-perf/metastore_export/csv/TAB_COL_STATS.txt create mode 100644 data/files/tpcds-perf/metastore_export/csv/TAB_COL_STATS.txt.bz2 diff --git data/files/tpcds-perf/metastore_export/csv/TABLE_PARAMS.txt data/files/tpcds-perf/metastore_export/csv/TABLE_PARAMS.txt deleted file mode 100644 index 78020f100f..0000000000 --- data/files/tpcds-perf/metastore_export/csv/TABLE_PARAMS.txt +++ /dev/null @@ -1,143 +0,0 @@ -_store_@COLUMN_STATS_ACCURATE@{"COLUMN_STATS":{"s_store_sk":"true","s_store_id":"true","s_rec_start_date":"true","s_rec_end_date":"true","s_closed_date_sk":"true","s_store_name":"true","s_number_employees":"true","s_floor_space":"true","s_hours":"true","s_manager":"true","s_market_id":"true","s_geography_class":"true","s_market_desc":"true","s_market_manager":"true","s_division_id":"true","s_division_name":"true","s_company_id":"true","s_company_name":"true","s_street_number":"true","s_street_name":"true","s_street_type":"true","s_suite_number":"true","s_city":"true","s_county":"true","s_state":"true","s_zip":"true","s_country":"true","s_gmt_offset":"true","s_tax_precentage":"true"},"BASIC_STATS":"true"} -_store_@numFiles@1 -_store_@numRows@1704 -_store_@rawDataSize@3256276 -_store_@totalSize@101707 -_store_@transient_lastDdlTime@1434562098 -_call_center_@COLUMN_STATS_ACCURATE@{"COLUMN_STATS":{"cc_call_center_sk":"true","cc_call_center_id":"true","cc_rec_start_date":"true","cc_rec_end_date":"true","cc_closed_date_sk":"true","cc_open_date_sk":"true","cc_name":"true","cc_class":"true","cc_employees":"true","cc_sq_ft":"true","cc_hours":"true","cc_manager":"true","cc_mkt_id":"true","cc_mkt_class":"true","cc_mkt_desc":"true","cc_market_manager":"true","cc_division":"true","cc_division_name":"true","cc_company":"true","cc_company_name":"true","cc_street_number":"true","cc_street_name":"true","cc_street_type":"true","cc_suite_number":"true","cc_city":"true","cc_county":"true","cc_state":"true","cc_zip":"true","cc_country":"true","cc_gmt_offset":"true","cc_tax_percentage":"true"},"BASIC_STATS":"true"} -_call_center_@numFiles@1 -_call_center_@numRows@60 -_call_center_@rawDataSize@122700 -_call_center_@totalSize@10347 -_call_center_@transient_lastDdlTime@1434561922 -_catalog_page_@COLUMN_STATS_ACCURATE@{"COLUMN_STATS":{"cp_catalog_page_sk":"true","cp_catalog_page_id":"true","cp_start_date_sk":"true","cp_end_date_sk":"true","cp_department":"true","cp_catalog_number":"true","cp_catalog_page_number":"true","cp_description":"true","cp_type":"true"},"BASIC_STATS":"true"} -_catalog_page_@numFiles@1 -_catalog_page_@numRows@46000 -_catalog_page_@rawDataSize@21198808 -_catalog_page_@totalSize@1576662 -_catalog_page_@transient_lastDdlTime@1434561925 -_customer_@COLUMN_STATS_ACCURATE@{"COLUMN_STATS":{"c_customer_sk":"true","c_customer_id":"true","c_current_cdemo_sk":"true","c_current_hdemo_sk":"true","c_current_addr_sk":"true","c_first_shipto_date_sk":"true","c_first_sales_date_sk":"true","c_salutation":"true","c_first_name":"true","c_last_name":"true","c_preferred_cust_flag":"true","c_birth_day":"true","c_birth_month":"true","c_birth_year":"true","c_birth_country":"true","c_login":"true","c_email_address":"true","c_last_review_date":"true"},"BASIC_STATS":"true"} -_customer_@numFiles@538 -_customer_@numRows@80000000 -_customer_@rawDataSize@68801615852 -_customer_@totalSize@3143935054 -_customer_@transient_lastDdlTime@1434561966 -_customer_address_@COLUMN_STATS_ACCURATE@{"COLUMN_STATS":{"ca_address_sk":"true","ca_address_id":"true","ca_street_number":"true","ca_street_name":"true","ca_street_type":"true","ca_suite_number":"true","ca_city":"true","ca_county":"true","ca_state":"true","ca_zip":"true","ca_country":"true","ca_gmt_offset":"true","ca_location_type":"true"},"BASIC_STATS":"true"} -_customer_address_@numFiles@274 -_customer_address_@numRows@40000000 -_customer_address_@rawDataSize@40595195284 -_customer_address_@totalSize@530195843 -_customer_address_@transient_lastDdlTime@1434561994 -_customer_demographics_@COLUMN_STATS_ACCURATE@{"COLUMN_STATS":{"cd_demo_sk":"true","cd_gender":"true","cd_marital_status":"true","cd_education_status":"true","cd_purchase_estimate":"true","cd_credit_rating":"true","cd_dep_count":"true","cd_dep_employed_count":"true","cd_dep_college_count":"true"},"BASIC_STATS":"true"} -_customer_demographics_@numFiles@8 -_customer_demographics_@numRows@1861800 -_customer_demographics_@rawDataSize@717186159 -_customer_demographics_@totalSize@323062 -_customer_demographics_@transient_lastDdlTime@1434562071 -_date_dim_@COLUMN_STATS_ACCURATE@{"COLUMN_STATS":{"d_date_sk":"true","d_date_id":"true","d_date":"true","d_month_seq":"true","d_week_seq":"true","d_quarter_seq":"true","d_year":"true","d_dow":"true","d_moy":"true","d_dom":"true","d_qoy":"true","d_fy_year":"true","d_fy_quarter_seq":"true","d_fy_week_seq":"true","d_day_name":"true","d_quarter_name":"true","d_holiday":"true","d_weekend":"true","d_following_holiday":"true","d_first_dom":"true","d_last_dom":"true","d_same_day_ly":"true","d_same_day_lq":"true","d_current_day":"true","d_current_week":"true","d_current_month":"true","d_current_quarter":"true","d_current_year":"true"},"BASIC_STATS":"true"} -_date_dim_@numFiles@1 -_date_dim_@numRows@73049 -_date_dim_@rawDataSize@81741831 -_date_dim_@totalSize@362925 -_date_dim_@transient_lastDdlTime@1434562075 -_household_demographics_@COLUMN_STATS_ACCURATE@{"COLUMN_STATS":{"hd_demo_sk":"true","hd_income_band_sk":"true","hd_buy_potential":"true","hd_dep_count":"true","hd_vehicle_count":"true"},"BASIC_STATS":"true"} -_household_demographics_@numFiles@1 -_household_demographics_@numRows@7200 -_household_demographics_@rawDataSize@770400 -_household_demographics_@totalSize@901 -_household_demographics_@transient_lastDdlTime@1434562078 -_income_band_@COLUMN_STATS_ACCURATE@{"COLUMN_STATS":{"ib_income_band_sk":"true","ib_lower_bound":"true","ib_upper_bound":"true"},"BASIC_STATS":"true"} -_income_band_@numFiles@1 -_income_band_@numRows@20 -_income_band_@rawDataSize@240 -_income_band_@totalSize@399 -_income_band_@transient_lastDdlTime@1434562081 -_item_@COLUMN_STATS_ACCURATE@{"COLUMN_STATS":{"i_item_sk":"true","i_item_id":"true","i_rec_start_date":"true","i_rec_end_date":"true","i_item_desc":"true","i_current_price":"true","i_wholesale_cost":"true","i_brand_id":"true","i_brand":"true","i_class_id":"true","i_class":"true","i_category_id":"true","i_category":"true","i_manufact_id":"true","i_manufact":"true","i_size":"true","i_formulation":"true","i_color":"true","i_units":"true","i_container":"true","i_manager_id":"true","i_product_name":"true"},"BASIC_STATS":"true"} -_item_@numFiles@9 -_item_@numRows@462000 -_item_@rawDataSize@663560457 -_item_@totalSize@29760748 -_item_@transient_lastDdlTime@1434562091 -_promotion_@COLUMN_STATS_ACCURATE@{"COLUMN_STATS":{"p_promo_sk":"true","p_promo_id":"true","p_start_date_sk":"true","p_end_date_sk":"true","p_item_sk":"true","p_cost":"true","p_response_target":"true","p_promo_name":"true","p_channel_dmail":"true","p_channel_email":"true","p_channel_catalog":"true","p_channel_tv":"true","p_channel_radio":"true","p_channel_press":"true","p_channel_event":"true","p_channel_demo":"true","p_channel_details":"true","p_purpose":"true","p_discount_active":"true"},"BASIC_STATS":"true"} -_promotion_@numFiles@1 -_promotion_@numRows@2300 -_promotion_@rawDataSize@2713420 -_promotion_@totalSize@63964 -_promotion_@transient_lastDdlTime@1434562093 -_reason_@COLUMN_STATS_ACCURATE@{"COLUMN_STATS":{"r_reason_sk":"true","r_reason_id":"true","r_reason_desc":"true"},"BASIC_STATS":"true"} -_reason_@numFiles@1 -_reason_@numRows@72 -_reason_@rawDataSize@14400 -_reason_@totalSize@1024 -_reason_@transient_lastDdlTime@1434562095 -_ship_mode_@COLUMN_STATS_ACCURATE@{"COLUMN_STATS":{"sm_ship_mode_sk":"true","sm_ship_mode_id":"true","sm_type":"true","sm_code":"true","sm_carrier":"true","sm_contract":"true"},"BASIC_STATS":"true"} -_ship_mode_@numFiles@0 -_ship_mode_@numRows@0 -_ship_mode_@rawDataSize@0 -_ship_mode_@totalSize@0 -_ship_mode_@transient_lastDdlTime@1434562097 -_time_dim_@COLUMN_STATS_ACCURATE@{"COLUMN_STATS":{"t_time_sk":"true","t_time_id":"true","t_time":"true","t_hour":"true","t_minute":"true","t_second":"true","t_am_pm":"true","t_shift":"true","t_sub_shift":"true","t_meal_time":"true"},"BASIC_STATS":"true"} -_time_dim_@numFiles@1 -_time_dim_@numRows@86400 -_time_dim_@rawDataSize@40694400 -_time_dim_@totalSize@133902 -_time_dim_@transient_lastDdlTime@1434562099 -_warehouse_@COLUMN_STATS_ACCURATE@{"COLUMN_STATS":{"w_warehouse_sk":"true","w_warehouse_id":"true","w_warehouse_name":"true","w_warehouse_sq_ft":"true","w_street_number":"true","w_street_name":"true","w_street_type":"true","w_suite_number":"true","w_city":"true","w_county":"true","w_state":"true","w_zip":"true","w_country":"true","w_gmt_offset":"true"},"BASIC_STATS":"true"} -_warehouse_@numFiles@1 -_warehouse_@numRows@27 -_warehouse_@rawDataSize@27802 -_warehouse_@totalSize@2971 -_warehouse_@transient_lastDdlTime@1434562102 -_web_page_@COLUMN_STATS_ACCURATE@{"COLUMN_STATS":{"wp_web_page_sk":"true","wp_web_page_id":"true","wp_rec_start_date":"true","wp_rec_end_date":"true","wp_creation_date_sk":"true","wp_access_date_sk":"true","wp_autogen_flag":"true","wp_customer_sk":"true","wp_url":"true","wp_type":"true","wp_char_count":"true","wp_link_count":"true","wp_image_count":"true","wp_max_ad_count":"true"},"BASIC_STATS":"true"} -_web_page_@numFiles@1 -_web_page_@numRows@4602 -_web_page_@rawDataSize@2696178 -_web_page_@totalSize@50572 -_web_page_@transient_lastDdlTime@1434562104 -_web_site_@COLUMN_STATS_ACCURATE@{"COLUMN_STATS":{"web_site_sk":"true","web_site_id":"true","web_rec_start_date":"true","web_rec_end_date":"true","web_name":"true","web_open_date_sk":"true","web_close_date_sk":"true","web_class":"true","web_manager":"true","web_mkt_id":"true","web_mkt_class":"true","web_mkt_desc":"true","web_market_manager":"true","web_company_id":"true","web_company_name":"true","web_street_number":"true","web_street_name":"true","web_street_type":"true","web_suite_number":"true","web_city":"true","web_county":"true","web_state":"true","web_zip":"true","web_country":"true","web_gmt_offset":"true","web_tax_percentage":"true"},"BASIC_STATS":"true"} -_web_site_@numFiles@1 -_web_site_@numRows@84 -_web_site_@rawDataSize@155408 -_web_site_@totalSize@11271 -_web_site_@transient_lastDdlTime@1434562107 -_catalog_returns_@COLUMN_STATS_ACCURATE@{"COLUMN_STATS":{"cr_returned_date_sk":"true","cr_returned_time_sk":"true","cr_item_sk":"true","cr_refunded_customer_sk":"true","cr_refunded_cdemo_sk":"true","cr_refunded_hdemo_sk":"true","cr_refunded_addr_sk":"true","cr_returning_customer_sk":"true","cr_returning_cdemo_sk":"true","cr_returning_hdemo_sk":"true","cr_returning_addr_sk":"true","cr_call_center_sk":"true","cr_catalog_page_sk":"true","cr_ship_mode_sk":"true","cr_warehouse_sk":"true","cr_reason_sk":"true","cr_order_number":"true","cr_return_quantity":"true","cr_return_amount":"true","cr_return_tax":"true","cr_return_amt_inc_tax":"true","cr_fee":"true","cr_return_ship_cost":"true","cr_refunded_cash":"true","cr_reversed_charge":"true","cr_store_credit":"true","cr_net_loss":"true"},"BASIC_STATS":"true"} -_catalog_returns_@numFiles@400 -_catalog_returns_@numRows@28798881 -_catalog_returns_@rawDataSize@3057234680 -_catalog_returns_@totalSize@1651022096 -_catalog_sales_@COLUMN_STATS_ACCURATE@{"COLUMN_STATS":{"cs_sold_date_sk":"true","cs_sold_time_sk":"true","cs_ship_date_sk":"true","cs_bill_customer_sk":"true","cs_bill_cdemo_sk":"true","cs_bill_hdemo_sk":"true","cs_bill_addr_sk":"true","cs_ship_customer_sk":"true","cs_ship_cdemo_sk":"true","cs_ship_hdemo_sk":"true","cs_ship_addr_sk":"true","cs_call_center_sk":"true","cs_catalog_page_sk":"true","cs_ship_mode_sk":"true","cs_warehouse_sk":"true","cs_item_sk":"true","cs_promo_sk":"true","cs_order_number":"true","cs_quantity":"true","cs_wholesale_cost":"true","cs_list_price":"true","cs_sales_price":"true","cs_ext_discount_amt":"true","cs_ext_sales_price":"true","cs_ext_wholesale_cost":"true","cs_ext_list_price":"true","cs_ext_tax":"true","cs_coupon_amt":"true","cs_ext_ship_cost":"true","cs_net_paid":"true","cs_net_paid_inc_tax":"true","cs_net_paid_inc_ship":"true","cs_net_paid_inc_ship_tax":"true","cs_net_profit":"true"},"BASIC_STATS":"true"} -_catalog_sales_@numFiles@552 -_catalog_sales_@numRows@287989836 -_catalog_sales_@rawDataSize@38999608952 -_catalog_sales_@totalSize@16430853294 -_catalog_sales_@transient_lastDdlTime@1434700893 -_inventory_@COLUMN_STATS_ACCURATE@{"COLUMN_STATS":{"inv_date_sk":"true","inv_item_sk":"true","inv_warehouse_sk":"true","inv_quantity_on_hand":"true"},"BASIC_STATS":"true"} -_inventory_@numFiles@51 -_inventory_@numRows@37584000 -_inventory_@rawDataSize@593821104 -_inventory_@totalSize@58323594 -_inventory_@transient_lastDdlTime@1434681166 -_store_sales_@COLUMN_STATS_ACCURATE@{"COLUMN_STATS":{"ss_sold_date_sk":"true","ss_sold_time_sk":"true","ss_item_sk":"true","ss_customer_sk":"true","ss_cdemo_sk":"true","ss_hdemo_sk":"true","ss_addr_sk":"true","ss_store_sk":"true","ss_promo_sk":"true","ss_ticket_number":"true","ss_quantity":"true","ss_wholesale_cost":"true","ss_list_price":"true","ss_sales_price":"true","ss_ext_discount_amt":"true","ss_ext_sales_price":"true","ss_ext_wholesale_cost":"true","ss_ext_list_price":"true","ss_ext_tax":"true","ss_coupon_amt":"true","ss_net_paid":"true","ss_net_paid_inc_tax":"true","ss_net_profit":"true"},"BASIC_STATS":"true"} -_store_sales_@numFiles@600 -_store_sales_@numRows@575995635 -_store_sales_@rawDataSize@50814502088 -_store_sales_@totalSize@22288384284 -_store_sales_@transient_lastDdlTime@1434700760 -_web_sales_@COLUMN_STATS_ACCURATE@{"COLUMN_STATS":{"ws_sold_date_sk":"true","ws_sold_time_sk":"true","ws_ship_date_sk":"true","ws_item_sk":"true","ws_bill_customer_sk":"true","ws_bill_cdemo_sk":"true","ws_bill_hdemo_sk":"true","ws_bill_addr_sk":"true","ws_ship_customer_sk":"true","ws_ship_cdemo_sk":"true","ws_ship_hdemo_sk":"true","ws_ship_addr_sk":"true","ws_web_page_sk":"true","ws_web_site_sk":"true","ws_ship_mode_sk":"true","ws_warehouse_sk":"true","ws_promo_sk":"true","ws_order_number":"true","ws_quantity":"true","ws_wholesale_cost":"true","ws_list_price":"true","ws_sales_price":"true","ws_ext_discount_amt":"true","ws_ext_sales_price":"true","ws_ext_wholesale_cost":"true","ws_ext_list_price":"true","ws_ext_tax":"true","ws_coupon_amt":"true","ws_ext_ship_cost":"true","ws_net_paid":"true","ws_net_paid_inc_tax":"true","ws_net_paid_inc_ship":"true","ws_net_paid_inc_ship_tax":"true","ws_net_profit":"true"},"BASIC_STATS":"true"} -_web_sales_@numFiles@529 -_web_sales_@numRows@144002668 -_web_sales_@rawDataSize@19580198212 -_web_sales_@totalSize@8304889016 -_web_sales_@transient_lastDdlTime@1434700829 -_web_returns_@COLUMN_STATS_ACCURATE@{"COLUMN_STATS":{"wr_returned_date_sk":"true","wr_returned_time_sk":"true","wr_item_sk":"true","wr_refunded_customer_sk":"true","wr_refunded_cdemo_sk":"true","wr_refunded_hdemo_sk":"true","wr_refunded_addr_sk":"true","wr_returning_customer_sk":"true","wr_returning_cdemo_sk":"true","wr_returning_hdemo_sk":"true","wr_returning_addr_sk":"true","wr_web_page_sk":"true","wr_reason_sk":"true","wr_order_number":"true","wr_return_quantity":"true","wr_return_amt":"true","wr_return_tax":"true","wr_return_amt_inc_tax":"true","wr_fee":"true","wr_return_ship_cost":"true","wr_refunded_cash":"true","wr_reversed_charge":"true","wr_account_credit":"true","wr_net_loss":"true"},"BASIC_STATS":"true"} -_web_returns_@numFiles@135 -_web_returns_@numRows@14398467 -_web_returns_@rawDataSize@1325194184 -_web_returns_@totalSize@827734274 -_web_returns_@transient_lastDdlTime@1434700842 -_store_returns_@COLUMN_STATS_ACCURATE@{"COLUMN_STATS":{"sr_returned_date_sk":"true","sr_return_time_sk":"true","sr_item_sk":"true","sr_customer_sk":"true","sr_cdemo_sk":"true","sr_hdemo_sk":"true","sr_addr_sk":"true","sr_store_sk":"true","sr_reason_sk":"true","sr_ticket_number":"true","sr_return_quantity":"true","sr_return_amt":"true","sr_return_tax":"true","sr_return_amt_inc_tax":"true","sr_fee":"true","sr_return_ship_cost":"true","sr_refunded_cash":"true","sr_reversed_charge":"true","sr_store_credit":"true","sr_net_loss":"true"},"BASIC_STATS":"true"} -_store_returns_@numFiles@573 -_store_returns_@numRows@57591150 -_store_returns_@rawDataSize@4462194832 -_store_returns_@totalSize@2683203627 -_store_returns_@transient_lastDdlTime@1434700790 diff --git data/files/tpcds-perf/metastore_export/csv/TABLE_PARAMS.txt.bz2 data/files/tpcds-perf/metastore_export/csv/TABLE_PARAMS.txt.bz2 new file mode 100644 index 0000000000000000000000000000000000000000..7985f6289809b7829e253a02a5324296cb33849a GIT binary patch literal 481 zcmV<70UrKBT4*^jL0KkKSwg~?vH${f&wvOJf8ay_7y!QK->^Uea2jwZV+o0%!5D@{ z69$<9H7N}NfQC$iPgO<$00002r6xjTo|!b700E*HTV*@N9G-=ylHCx9G7;%CLmX*= z!W^`iAs)*p3pF(3u`(EEzg? zIB4*PrGpWQ5gK*)Nu=~*Q5@!Ogk^R!f2EKYoOQ=_Ha#4`2gw zUS^*wCG3q{9^zn}SBYVp+)=aas$A7`cjEz*u$0ej-0e%1w?(L>$;?@DtPP!&#*~gp zfbLBS9p-J#pT>t3N-Diq@SIsQN!U#K%2LWGo$8k*SGgsR(Ja}LP|>?o;KNlaFkp5P z%?{&cgQ>_(Qp?-1PHEADs0Qxq?w8poNS8^*2EaN9bROypu}hZ0}w=zfQ>R!$SPt00B8{+1SX=9Fe3=VK$#o5uHt~R;xLen2#r8V5D$L; z5C@+`5QFrS_I(f8MfRd^FLk+?$-J6QrB*kt2`FKBV>fZztGe6Nvo~_gPG(NGv9Ddy zo6M_o7m*1@?VN2=ie{p(6K<;GUSlU_>s-1n-fqPvF5BD6c=g$CYNl#V-|-fm>m!?4p+w`)UcqhfKbsBYfth|QeV z-rh#WIo870I5ePkZg4nvuT|9W#KlcIxW@J+=GeuVxo)1x+Gf*nyH75x?KZEx=QubA zIlNec+-;a;{4tLelNm6~Z&z~y?N!9$VVM-RZiaXq04y4ra~Q5lAo9o-p;k1kLA!MU zR)bG&2JSS{pU(VBg7-Un^bGDoexnpj2g&QHdnaG%V*@qEnbgsLX1Sj^l` zyB7zBHxg?328IE@9lggtFYTOn>DJ{r;W)k(rRpHN$4K(&=@;kD@8$d;cHjK#sTHqG z&Tbe_4xr(^VR?`3PvV#yvL#rXp^e7{n>~PxGJ)@Fka(1VJ;K4B5>bsGF3NSeN}xq( zRFI0Wky61677UsiI)Pu0XYKsxG8oU8RzFYD&NC{A>F&BCsRUqZ5yBWJOG(eG=aw9>PlQY#l9TACPCZ^~q zU9mR}LH+2eA0SVT{J?DT{&DiX@;i}yq+rDYpyVMJw2?WzLf~Q>u!d`suDY{uvKx~C z)K1+NGCjvcXMN`XC7X;Zy@E&C5qt+x+v>OGMaK;gi|eu3%ZRwqZAku9-}Z@2&F+~v zqJnCFBG&4?fhWlP=`~W@q*U$clvvb(49vmHE&p(x%gBh++pfOJ zx=>tKMDbC}hi7@wiI~b6q+wVu-~0C}@e~cWzNZ-VMs{sQM?ToNn%^VBUUd_;u-}_e zKhpc>t6hf^JVyU(k~~M{(ZNyi`#?%0hp%4;v%Fn+pm?q|zkC@LI5FN|KKQ;hsSM_S zr9Iy&?lOw0laB$oVd=H4+rpyPMBtJf-F>c51E7wAez`(Sgq#E;I@yRlh!lILR?k^d zK7!I;VGPjJ=Wl$rq?D+nI|FQt*lUl2PYE2{{eo-jKwNS9<UynS;;U4c z=DJTMLYvLVf^w)xLh!}%!q`nrY(gSnTR>b;od}v1?x><1wN+7PfI;Ut1y|7U;AU%>xmDZ^M~0KIbH`rY1s-JA(g5pl4$)k%1Z@WS6=GO#V1Q{L1;FwI~b z-5_PIBzc8+i)Mp8nGk}qc`1#yv)xs4x*hwApqQ8?G%1ZCso4fU#PEL$~7y?-`2g^ z_4VYpy%a3>+dNUxKsY42B91i*op^lEWI*tMKUDW>1`77UI{Marp|_eLWPW4a7Ncq> z(9OnXI{Q`5N5*0~Uf3*y+v1SZoHn*HK|>C-x|z5di2Bzr;!xb@z=d;O8Ec<(*S4X9 z+fkBBxKoV$xoRzEN^iSg6qvly++-T^>wWhWd@?~a2Za*ALIwbu7{nn67y@Wufe4Wx zMg)RMBP0k#B0z}=1_)pPCIB=f!2>`fBnTQp93+HD5QL0^KoKA%L^ECOQ4a6*$tyhA zMzgQCg$AoRAbfT5g4k$QX`=X2td0HBW|o^KXC@82qtMJl4A_7KuAE~1z3Fnk;05ZT zhKWAOv@x7`K%h{hPRlzKcBb8!F>klKg;;Mvk!fL@Fxgu`Bkb0t{mOLVmh|2c&}F5=Z}3|Oz;AP{&}Bz25Ba)V+2P- zQnIMGdViEn2#Clw?XPZa)MoreXIqE5-d@d9q^>L`*eU?H456YYZp6{SBy2n_FyTQ+wpI2Rmn|I|+OVg0+d1n5Zh8_FpgqNbAOs8-& z1V+0Ml#K5k7oG0sF|2XcJZB^kuIX-oBz@FCvEO{!;4s{Q&XBN_=4WcrB?A!eXoQ$z zf$LY&V|NqXV-j&9NHz1bC|>&ls@%eqlJ{NJ87nU{#IHT5-Aa;_iE;!%B03@_VmZOW z4o?GTV0sUu<8l&&n(2>jAq1EL&&infBEm^8Y{bs=j!yE@&>Kl)0NjPjN>xj{z3D_D z(Gg&9$Q|c9s?;jKh_8>Wy>EABsfxQLHtG~@2$!ZYH>G0c^ns}ofy^>o|;_^18tBpKrnBN?Vd0huB*lo1OU@oq7aP-`)i&c@X zx0__xqXg)-9#yc$Q~)ck>@Fw?50l;ZGmYb-FpaD8*tu^A?%mt;f$7>ozGuWzKLpLs zFE_Xh* zKKGRhG*7(gx`4O)ef!5}GSDSdG*sP{N~(u~2seaLvnMSYH`P{dp;$R3{?;WR|2*Hg}|KK&zmW;%E4C-29UcQ+q?J+)^PkxgP2C)lw>^_%Sw{c`x)Aji-SnRWhszRj5k|! zPWBK}Ge(8KihGXc^J}Q)*ceBfsG_t6d9MZ)IDk009C(d*t{bb2gf>p|;Ot&g+#QTL8SJW5|&Gqx1DTYGNc^Srcd^@4fi39dS-Q(9Qrrh)+5_)Q(S}e3d2X; znCe|B*S`Yjp}zK+RJxn+H_C&++tgEh>d|D^h!>6KtA3Ji+}ii*fCibFrkiYsNiZB9 zQMF;ohrOP~Tk_ROtFAzGntb`BF@V+89iKY1kz7EMs!lqq%(i-V)&rd3&ukg4=!G6V zZMOBfr*vFftk%auiKVHWca2FC)*4`R!a$I3UW&)X%^z!>)Y1b%ymkPyGJ$BQ=r5t$-0>fTbiy^`?SbFk<7}=_#c1BbfUGymgu9d{w96#O~|zpH231s27z`^Ogq)esyaF+Hts@jpcoMu+xXX zEUD7L*(MM=9!H||C}V|=Yp&kItsGgSzEdwbDJ|jJEdn$%4$_2&W&~2R3AkC#=NC=w zpq)P`mum(Huo-jZ9}4c;>z`9|UD=guu&arABC+#^A0*5sVraOq?AQM8$OdTuYvKsV*nusIH-1 zbyaU#nSbnZHt1i&@^zNX7UKYcJ@ANQUF+((yl6wzXM{4FmvF?4{I^WFuqWAKv&f1%MHc&Qv z4MbI}x$~TIxgFuca=Deah0WCLJKgAE3ZE}^N}+i5uyT2w7eyH&iEc9W{aR6c|>Zr z(F!l$9qVx3Y}C=J2OhAa+#Efr3oj~h^;uavsnMlxdlDTCKAioyY`c3p{_1r}WoP6M zbbTa{hc^F^IgaTV!N`P{;_#v&AliqryQ4!jABpFheRoaa5Q1E3Et2uTn=CmG^gH$7=6mAae>m zO(aXfHn$C?*S_qYr|p8BtM^Kd+(0_!kMa; z-p_W#mykc|{?tW&!npf#2_`|!bxV_Z#JPdkyz#)cb9M({tk)+d^fLrYZ=m+V9ErAI zXemj+^#=Gr(Rq^ErzBcguoBa~bZYlJwS%Qu_5`c%_qxIH+Gq-A1 zgfB~veikarIC{dyCbm~#6xfg5W1jtA=Vpzf3G@){)*8uVgnrV`1Iwl~(-vqP#K)x;w5-DXFd<|`QoF1!uy>-5&*)cOQ z-ezKEC-Q2lxva6r95^h!u3LCCqv>H}_GT+~?9kS2GZCBiK2so-%Z_+$h+(}AcJ(EN zu8!^%I*4?zN^EmH?t(k07DWo%KuRba*KZx#g(3Q_&clUw40sfd`44Jc1591b4g{o7 z$nzk`5@`V&JI-~2kC0u*Ymh$3h@M3Prjtq$1BCqpR3bu5B{W7Xgybs85{4E>^;O77 zgh-0RUyG{xA{SG{Z?Cfo2T>Y{k}QMTLCgX0HA3(ZiFy{q4n53?;0Y1Q)Ivdw5=iQE z0LpEuQ-q?Wa@Z=@%LzqGBuSF9waw-dCJgA4ICu|0u?K_>lzj)KEbK|eDwHqKq*G8l z2#19l)2nh7(JSN4H@i&|1x7%kp#SM_u;7GEm^Qqtq1_i@)LaPMT$Atx8Z(0+uu9d| zO2$lDV$@9DQtgH?ViXlb6v1MB1_by(0x5jfH?@l$G(y48)0k#mM}3ZVONwwCA%p{C zbwWIxec5Yo8zIP{t-~6pWcO=hx7b{gND=|Ch1Fs`aY$bEip(KlBMcsQc{)KH$G``O zbINw;<&L64k)+cwLCyzMf%Zop(t9T-nUZZDUGSlkh`Z0D@QoZDE7RZ!om38q7uHKK>2SQmj_GB5d4-F~4n3ON(v!s62aFZ zCZnJ1L&@U@rE+WAr@LAR{HS**-8^Pk8wRi);As}vAp5FU=&VZ;UMkzWxmRIT_F}H& zsQ>_wBqAatL_h!pC7?}p7$h{s;W6)TN40B1wyFu?xa$P-)q}Nf3Vn)Y3i`xWfJXsn z<5oR&0S5@PXocZVL1iFKjG#pDRC=2eNh6^V*$EVNb0 zAp#JRoCQ@Bl$A;ZNf9cl(=MczrBDC>1ca$p5+u82RpEY-1}C9DbS(CO?_M}AmQR$) zr4EJxs6j+F7&k&~3k16&ainY{8FU3iv~RAZY-Jl4c|>@NVH8LM>EYq7m>z|GW)aLS z3d~H#q3jR&C&ihw{++nid(qRk1Cvx5-AKAfCM;>L?Ax*xRL3csU{uElq1!Ncg&OR zD0e)mog&HFy@CyRLgk2~LXW)r{N-TdUls@!4ShC>eg()1R)wIVff7^#1>(v94MZ@3 mKMtWq1p|vscfftxrU=Zva_$&> patternsWithMaskComments = new ArrayList>() {{ + private final List> patternsWithMaskComments = new ArrayList>() {{ add(toPatternPair("(pblob|s3.?|swift|wasb.?).*hive-staging.*","### BLOBSTORE_STAGING_PATH ###")); }}; @@ -2318,15 +2324,13 @@ public static void setupMetaStoreTableColumnStatsFor30TBTPCDSWorkload(HiveConf c } br.close(); - File tabColStatsCsv = new File(mdbPath+"csv/TAB_COL_STATS.txt"); - File tabParamsCsv = new File(mdbPath+"csv/TABLE_PARAMS.txt"); + java.nio.file.Path tabColStatsCsv = FileSystems.getDefault().getPath(mdbPath, "csv" ,"TAB_COL_STATS.txt.bz2"); + java.nio.file.Path tabParamsCsv = FileSystems.getDefault().getPath(mdbPath, "csv", "TABLE_PARAMS.txt.bz2"); // Set up the foreign key constraints properly in the TAB_COL_STATS data String tmpBaseDir = System.getProperty(TEST_TMP_DIR_PROPERTY); - File tmpFileLoc1 = new File(tmpBaseDir+"/TAB_COL_STATS.txt"); - File tmpFileLoc2 = new File(tmpBaseDir+"/TABLE_PARAMS.txt"); - FileUtils.copyFile(tabColStatsCsv, tmpFileLoc1); - FileUtils.copyFile(tabParamsCsv, tmpFileLoc2); + java.nio.file.Path tmpFileLoc1 = FileSystems.getDefault().getPath(tmpBaseDir, "TAB_COL_STATS.txt"); + java.nio.file.Path tmpFileLoc2 = FileSystems.getDefault().getPath(tmpBaseDir, "TABLE_PARAMS.txt"); class MyComp implements Comparator { @Override @@ -2338,7 +2342,7 @@ public int compare(String str1, String str2) { } } - SortedMap tableNameToID = new TreeMap(new MyComp()); + final SortedMap tableNameToID = new TreeMap(new MyComp()); rs = s.executeQuery("SELECT * FROM APP.TBLS"); while(rs.next()) { @@ -2351,29 +2355,72 @@ public int compare(String str1, String str2) { } } - for (Map.Entry entry : tableNameToID.entrySet()) { - String toReplace1 = ",_" + entry.getKey() + "_" ; - String replacementString1 = ","+entry.getValue(); - String toReplace2 = "_" + entry.getKey() + "_@" ; - String replacementString2 = ""+entry.getValue()+"@"; - try { - String content1 = FileUtils.readFileToString(tmpFileLoc1, "UTF-8"); - content1 = content1.replaceAll(toReplace1, replacementString1); - FileUtils.writeStringToFile(tmpFileLoc1, content1, "UTF-8"); - String content2 = FileUtils.readFileToString(tmpFileLoc2, "UTF-8"); - content2 = content2.replaceAll(toReplace2, replacementString2); - FileUtils.writeStringToFile(tmpFileLoc2, content2, "UTF-8"); - } catch (IOException e) { - LOG.info("Generating file failed", e); + final Map> data = new HashMap<>(); + rs = s.executeQuery("select TBLS.TBL_NAME, a.COLUMN_NAME, a.TYPE_NAME from " + + "(select COLUMN_NAME, TYPE_NAME, SDS.SD_ID from APP.COLUMNS_V2 join APP.SDS on SDS.CD_ID = COLUMNS_V2.CD_ID) a" + + " join APP.TBLS on TBLS.SD_ID = a.SD_ID"); + while (rs.next()) { + String tblName = rs.getString(1); + String colName = rs.getString(2); + String typeName = rs.getString(3); + Map cols = data.get(tblName); + if (null == cols) { + cols = new HashMap<>(); } + cols.put(colName, typeName); + data.put(tblName, cols); } + BufferedReader reader = new BufferedReader(new InputStreamReader( + new BZip2CompressorInputStream(Files.newInputStream(tabColStatsCsv, StandardOpenOption.READ)))); + + Stream replaced = reader.lines().parallel().map(str-> { + String[] splits = str.split(","); + String tblName = splits[0]; + String colName = splits[1]; + Integer tblID = tableNameToID.get(tblName); + StringBuilder sb = new StringBuilder("default@"+tblName + "@" + colName + "@" + data.get(tblName).get(colName)+"@"); + for (int i = 2; i < splits.length; i++) { + sb.append(splits[i]+"@"); + } + return sb.append(tblID).toString(); + }); + + Files.write(tmpFileLoc1, (Iterable)replaced::iterator); + replaced.close(); + reader.close(); + + BufferedReader reader2 = new BufferedReader(new InputStreamReader( + new BZip2CompressorInputStream(Files.newInputStream(tabParamsCsv, StandardOpenOption.READ)))); + final Map colStats = new ConcurrentHashMap<>(); + Stream replacedStream = reader2.lines().parallel().map(str-> { + String[] splits = str.split("_@"); + String tblName = splits[0]; + Integer tblId = tableNameToID.get(tblName); + Map cols = data.get(tblName); + StringBuilder sb = new StringBuilder(); + sb.append("{\"COLUMN_STATS\":{"); + for (String colName : cols.keySet()) { + sb.append("\""+colName+"\":\"true\","); + } + sb.append("},\"BASIC_STATS\":\"true\"}"); + colStats.put(tblId.toString(), sb.toString()); + + return tblId.toString() + "@" + splits[1]; + }); + + Files.write(tmpFileLoc2, (Iterable)replacedStream::iterator); + Files.write(tmpFileLoc2, (Iterable)colStats.entrySet().stream() + .map(map->map.getKey()+"@COLUMN_STATS_ACCURATE@"+map.getValue())::iterator, StandardOpenOption.APPEND); + + replacedStream.close(); + reader2.close(); // Load the column stats and table params with 30 TB scale String importStatement1 = "CALL SYSCS_UTIL.SYSCS_IMPORT_TABLE(null, '" + "TAB_COL_STATS" + - "', '" + tmpFileLoc1.getAbsolutePath() + - "', ',', null, 'UTF-8', 1)"; + "', '" + tmpFileLoc1.toAbsolutePath().toString() + + "', '@', null, 'UTF-8', 1)"; String importStatement2 = "CALL SYSCS_UTIL.SYSCS_IMPORT_TABLE(null, '" + "TABLE_PARAMS" + - "', '" + tmpFileLoc2.getAbsolutePath() + + "', '" + tmpFileLoc2.toAbsolutePath().toString() + "', '@', null, 'UTF-8', 1)"; try { PreparedStatement psImport1 = conn.prepareStatement(importStatement1); diff --git ql/src/test/results/clientpositive/perf/query14.q.out ql/src/test/results/clientpositive/perf/query14.q.out index 9284353565..d6675bc00f 100644 --- ql/src/test/results/clientpositive/perf/query14.q.out +++ ql/src/test/results/clientpositive/perf/query14.q.out @@ -1,6 +1,6 @@ -Warning: Shuffle Join MERGEJOIN[906][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 5' is a cross product -Warning: Shuffle Join MERGEJOIN[907][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 60' is a cross product Warning: Shuffle Join MERGEJOIN[908][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 112' is a cross product +Warning: Shuffle Join MERGEJOIN[907][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 60' is a cross product +Warning: Shuffle Join MERGEJOIN[906][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 5' is a cross product PREHOOK: query: explain with cross_items as (select i_item_sk ss_item_sk diff --git ql/src/test/results/clientpositive/perf/query23.q.out ql/src/test/results/clientpositive/perf/query23.q.out index a04e5cd139..7a3201e043 100644 --- ql/src/test/results/clientpositive/perf/query23.q.out +++ ql/src/test/results/clientpositive/perf/query23.q.out @@ -1,5 +1,5 @@ -Warning: Shuffle Join MERGEJOIN[367][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 19' is a cross product Warning: Shuffle Join MERGEJOIN[369][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 49' is a cross product +Warning: Shuffle Join MERGEJOIN[367][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 19' is a cross product PREHOOK: query: explain with frequent_ss_items as (select substr(i_item_desc,1,30) itemdesc,i_item_sk item_sk,d_date solddate,count(*) cnt from store_sales diff --git ql/src/test/results/clientpositive/perf/query88.q.out ql/src/test/results/clientpositive/perf/query88.q.out index 5072997e82..fcb4042ade 100644 --- ql/src/test/results/clientpositive/perf/query88.q.out +++ ql/src/test/results/clientpositive/perf/query88.q.out @@ -239,33 +239,33 @@ Stage-0 Group By Operator [GBY_48] (rows=1 width=8) Output:["_col0"],aggregations:["count()"] Merge Join Operator [MERGEJOIN_328] (rows=766650239 width=88) - Conds:RS_44._col1=RS_45._col0(Inner) + Conds:RS_44._col2=RS_45._col0(Inner) <-Map 17 [SIMPLE_EDGE] SHUFFLE [RS_45] PartitionCols:_col0 - Select Operator [SEL_37] (rows=6000 width=107) + Select Operator [SEL_37] (rows=852 width=1910) Output:["_col0"] - Filter Operator [FIL_298] (rows=6000 width=107) - predicate:(((hd_dep_count = 3) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) - TableScan [TS_35] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count","hd_vehicle_count"] + Filter Operator [FIL_298] (rows=852 width=1910) + predicate:((s_store_name = 'ese') and s_store_sk is not null) + TableScan [TS_35] (rows=1704 width=1910) + default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name"] <-Reducer 12 [SIMPLE_EDGE] SHUFFLE [RS_44] - PartitionCols:_col1 + PartitionCols:_col2 Merge Join Operator [MERGEJOIN_327] (rows=696954748 width=88) - Conds:RS_41._col2=RS_42._col0(Inner),Output:["_col1"] + Conds:RS_41._col1=RS_42._col0(Inner),Output:["_col2"] <-Map 16 [SIMPLE_EDGE] SHUFFLE [RS_42] PartitionCols:_col0 - Select Operator [SEL_34] (rows=852 width=1910) + Select Operator [SEL_34] (rows=3600 width=107) Output:["_col0"] - Filter Operator [FIL_297] (rows=852 width=1910) - predicate:((s_store_name = 'ese') and s_store_sk is not null) - TableScan [TS_32] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name"] + Filter Operator [FIL_297] (rows=3600 width=107) + predicate:((((hd_dep_count = 3) and (hd_vehicle_count <= 5)) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) + TableScan [TS_32] (rows=7200 width=107) + default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count","hd_vehicle_count"] <-Reducer 11 [SIMPLE_EDGE] SHUFFLE [RS_41] - PartitionCols:_col2 + PartitionCols:_col1 Merge Join Operator [MERGEJOIN_326] (rows=633595212 width=88) Conds:RS_38._col0=RS_39._col0(Inner),Output:["_col1","_col2"] <-Map 10 [SIMPLE_EDGE] @@ -295,33 +295,33 @@ Stage-0 Group By Operator [GBY_74] (rows=1 width=8) Output:["_col0"],aggregations:["count()"] Merge Join Operator [MERGEJOIN_331] (rows=766650239 width=88) - Conds:RS_70._col1=RS_71._col0(Inner) + Conds:RS_70._col2=RS_71._col0(Inner) <-Map 25 [SIMPLE_EDGE] SHUFFLE [RS_71] PartitionCols:_col0 - Select Operator [SEL_63] (rows=6000 width=107) + Select Operator [SEL_63] (rows=852 width=1910) Output:["_col0"] - Filter Operator [FIL_302] (rows=6000 width=107) - predicate:(((hd_dep_count = 3) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) - TableScan [TS_61] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count","hd_vehicle_count"] + Filter Operator [FIL_302] (rows=852 width=1910) + predicate:((s_store_name = 'ese') and s_store_sk is not null) + TableScan [TS_61] (rows=1704 width=1910) + default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name"] <-Reducer 20 [SIMPLE_EDGE] SHUFFLE [RS_70] - PartitionCols:_col1 + PartitionCols:_col2 Merge Join Operator [MERGEJOIN_330] (rows=696954748 width=88) - Conds:RS_67._col2=RS_68._col0(Inner),Output:["_col1"] + Conds:RS_67._col1=RS_68._col0(Inner),Output:["_col2"] <-Map 24 [SIMPLE_EDGE] SHUFFLE [RS_68] PartitionCols:_col0 - Select Operator [SEL_60] (rows=852 width=1910) + Select Operator [SEL_60] (rows=3600 width=107) Output:["_col0"] - Filter Operator [FIL_301] (rows=852 width=1910) - predicate:((s_store_name = 'ese') and s_store_sk is not null) - TableScan [TS_58] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name"] + Filter Operator [FIL_301] (rows=3600 width=107) + predicate:((((hd_dep_count = 3) and (hd_vehicle_count <= 5)) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) + TableScan [TS_58] (rows=7200 width=107) + default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count","hd_vehicle_count"] <-Reducer 19 [SIMPLE_EDGE] SHUFFLE [RS_67] - PartitionCols:_col2 + PartitionCols:_col1 Merge Join Operator [MERGEJOIN_329] (rows=633595212 width=88) Conds:RS_64._col0=RS_65._col0(Inner),Output:["_col1","_col2"] <-Map 18 [SIMPLE_EDGE] @@ -351,33 +351,33 @@ Stage-0 Group By Operator [GBY_100] (rows=1 width=8) Output:["_col0"],aggregations:["count()"] Merge Join Operator [MERGEJOIN_334] (rows=766650239 width=88) - Conds:RS_96._col1=RS_97._col0(Inner) + Conds:RS_96._col2=RS_97._col0(Inner) <-Map 33 [SIMPLE_EDGE] SHUFFLE [RS_97] PartitionCols:_col0 - Select Operator [SEL_89] (rows=6000 width=107) + Select Operator [SEL_89] (rows=852 width=1910) Output:["_col0"] - Filter Operator [FIL_306] (rows=6000 width=107) - predicate:(((hd_dep_count = 3) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) - TableScan [TS_87] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count","hd_vehicle_count"] + Filter Operator [FIL_306] (rows=852 width=1910) + predicate:((s_store_name = 'ese') and s_store_sk is not null) + TableScan [TS_87] (rows=1704 width=1910) + default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name"] <-Reducer 28 [SIMPLE_EDGE] SHUFFLE [RS_96] - PartitionCols:_col1 + PartitionCols:_col2 Merge Join Operator [MERGEJOIN_333] (rows=696954748 width=88) - Conds:RS_93._col2=RS_94._col0(Inner),Output:["_col1"] + Conds:RS_93._col1=RS_94._col0(Inner),Output:["_col2"] <-Map 32 [SIMPLE_EDGE] SHUFFLE [RS_94] PartitionCols:_col0 - Select Operator [SEL_86] (rows=852 width=1910) + Select Operator [SEL_86] (rows=3600 width=107) Output:["_col0"] - Filter Operator [FIL_305] (rows=852 width=1910) - predicate:((s_store_name = 'ese') and s_store_sk is not null) - TableScan [TS_84] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name"] + Filter Operator [FIL_305] (rows=3600 width=107) + predicate:((((hd_dep_count = 3) and (hd_vehicle_count <= 5)) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) + TableScan [TS_84] (rows=7200 width=107) + default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count","hd_vehicle_count"] <-Reducer 27 [SIMPLE_EDGE] SHUFFLE [RS_93] - PartitionCols:_col2 + PartitionCols:_col1 Merge Join Operator [MERGEJOIN_332] (rows=633595212 width=88) Conds:RS_90._col0=RS_91._col0(Inner),Output:["_col1","_col2"] <-Map 26 [SIMPLE_EDGE] @@ -407,33 +407,33 @@ Stage-0 Group By Operator [GBY_126] (rows=1 width=8) Output:["_col0"],aggregations:["count()"] Merge Join Operator [MERGEJOIN_337] (rows=766650239 width=88) - Conds:RS_122._col1=RS_123._col0(Inner) + Conds:RS_122._col2=RS_123._col0(Inner) <-Map 41 [SIMPLE_EDGE] SHUFFLE [RS_123] PartitionCols:_col0 - Select Operator [SEL_115] (rows=6000 width=107) + Select Operator [SEL_115] (rows=852 width=1910) Output:["_col0"] - Filter Operator [FIL_310] (rows=6000 width=107) - predicate:(((hd_dep_count = 3) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) - TableScan [TS_113] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count","hd_vehicle_count"] + Filter Operator [FIL_310] (rows=852 width=1910) + predicate:((s_store_name = 'ese') and s_store_sk is not null) + TableScan [TS_113] (rows=1704 width=1910) + default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name"] <-Reducer 36 [SIMPLE_EDGE] SHUFFLE [RS_122] - PartitionCols:_col1 + PartitionCols:_col2 Merge Join Operator [MERGEJOIN_336] (rows=696954748 width=88) - Conds:RS_119._col2=RS_120._col0(Inner),Output:["_col1"] + Conds:RS_119._col1=RS_120._col0(Inner),Output:["_col2"] <-Map 40 [SIMPLE_EDGE] SHUFFLE [RS_120] PartitionCols:_col0 - Select Operator [SEL_112] (rows=852 width=1910) + Select Operator [SEL_112] (rows=3600 width=107) Output:["_col0"] - Filter Operator [FIL_309] (rows=852 width=1910) - predicate:((s_store_name = 'ese') and s_store_sk is not null) - TableScan [TS_110] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name"] + Filter Operator [FIL_309] (rows=3600 width=107) + predicate:((((hd_dep_count = 3) and (hd_vehicle_count <= 5)) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) + TableScan [TS_110] (rows=7200 width=107) + default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count","hd_vehicle_count"] <-Reducer 35 [SIMPLE_EDGE] SHUFFLE [RS_119] - PartitionCols:_col2 + PartitionCols:_col1 Merge Join Operator [MERGEJOIN_335] (rows=633595212 width=88) Conds:RS_116._col0=RS_117._col0(Inner),Output:["_col1","_col2"] <-Map 34 [SIMPLE_EDGE] @@ -463,33 +463,33 @@ Stage-0 Group By Operator [GBY_152] (rows=1 width=8) Output:["_col0"],aggregations:["count()"] Merge Join Operator [MERGEJOIN_340] (rows=766650239 width=88) - Conds:RS_148._col1=RS_149._col0(Inner) + Conds:RS_148._col2=RS_149._col0(Inner) <-Map 49 [SIMPLE_EDGE] SHUFFLE [RS_149] PartitionCols:_col0 - Select Operator [SEL_141] (rows=6000 width=107) + Select Operator [SEL_141] (rows=852 width=1910) Output:["_col0"] - Filter Operator [FIL_314] (rows=6000 width=107) - predicate:(((hd_dep_count = 3) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) - TableScan [TS_139] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count","hd_vehicle_count"] + Filter Operator [FIL_314] (rows=852 width=1910) + predicate:((s_store_name = 'ese') and s_store_sk is not null) + TableScan [TS_139] (rows=1704 width=1910) + default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name"] <-Reducer 44 [SIMPLE_EDGE] SHUFFLE [RS_148] - PartitionCols:_col1 + PartitionCols:_col2 Merge Join Operator [MERGEJOIN_339] (rows=696954748 width=88) - Conds:RS_145._col2=RS_146._col0(Inner),Output:["_col1"] + Conds:RS_145._col1=RS_146._col0(Inner),Output:["_col2"] <-Map 48 [SIMPLE_EDGE] SHUFFLE [RS_146] PartitionCols:_col0 - Select Operator [SEL_138] (rows=852 width=1910) + Select Operator [SEL_138] (rows=3600 width=107) Output:["_col0"] - Filter Operator [FIL_313] (rows=852 width=1910) - predicate:((s_store_name = 'ese') and s_store_sk is not null) - TableScan [TS_136] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name"] + Filter Operator [FIL_313] (rows=3600 width=107) + predicate:((((hd_dep_count = 3) and (hd_vehicle_count <= 5)) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) + TableScan [TS_136] (rows=7200 width=107) + default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count","hd_vehicle_count"] <-Reducer 43 [SIMPLE_EDGE] SHUFFLE [RS_145] - PartitionCols:_col2 + PartitionCols:_col1 Merge Join Operator [MERGEJOIN_338] (rows=633595212 width=88) Conds:RS_142._col0=RS_143._col0(Inner),Output:["_col1","_col2"] <-Map 42 [SIMPLE_EDGE] @@ -519,33 +519,33 @@ Stage-0 Group By Operator [GBY_22] (rows=1 width=8) Output:["_col0"],aggregations:["count()"] Merge Join Operator [MERGEJOIN_325] (rows=766650239 width=88) - Conds:RS_18._col1=RS_19._col0(Inner) + Conds:RS_18._col2=RS_19._col0(Inner) <-Map 9 [SIMPLE_EDGE] SHUFFLE [RS_19] PartitionCols:_col0 - Select Operator [SEL_11] (rows=6000 width=107) + Select Operator [SEL_11] (rows=852 width=1910) Output:["_col0"] - Filter Operator [FIL_294] (rows=6000 width=107) - predicate:(((hd_dep_count = 3) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) - TableScan [TS_9] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count","hd_vehicle_count"] + Filter Operator [FIL_294] (rows=852 width=1910) + predicate:((s_store_name = 'ese') and s_store_sk is not null) + TableScan [TS_9] (rows=1704 width=1910) + default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name"] <-Reducer 3 [SIMPLE_EDGE] SHUFFLE [RS_18] - PartitionCols:_col1 + PartitionCols:_col2 Merge Join Operator [MERGEJOIN_324] (rows=696954748 width=88) - Conds:RS_15._col2=RS_16._col0(Inner),Output:["_col1"] + Conds:RS_15._col1=RS_16._col0(Inner),Output:["_col2"] <-Map 8 [SIMPLE_EDGE] SHUFFLE [RS_16] PartitionCols:_col0 - Select Operator [SEL_8] (rows=852 width=1910) + Select Operator [SEL_8] (rows=3600 width=107) Output:["_col0"] - Filter Operator [FIL_293] (rows=852 width=1910) - predicate:((s_store_name = 'ese') and s_store_sk is not null) - TableScan [TS_6] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name"] + Filter Operator [FIL_293] (rows=3600 width=107) + predicate:((((hd_dep_count = 3) and (hd_vehicle_count <= 5)) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) + TableScan [TS_6] (rows=7200 width=107) + default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count","hd_vehicle_count"] <-Reducer 2 [SIMPLE_EDGE] SHUFFLE [RS_15] - PartitionCols:_col2 + PartitionCols:_col1 Merge Join Operator [MERGEJOIN_323] (rows=633595212 width=88) Conds:RS_12._col0=RS_13._col0(Inner),Output:["_col1","_col2"] <-Map 1 [SIMPLE_EDGE] @@ -575,33 +575,33 @@ Stage-0 Group By Operator [GBY_178] (rows=1 width=8) Output:["_col0"],aggregations:["count()"] Merge Join Operator [MERGEJOIN_343] (rows=766650239 width=88) - Conds:RS_174._col1=RS_175._col0(Inner) + Conds:RS_174._col2=RS_175._col0(Inner) <-Map 57 [SIMPLE_EDGE] SHUFFLE [RS_175] PartitionCols:_col0 - Select Operator [SEL_167] (rows=6000 width=107) + Select Operator [SEL_167] (rows=852 width=1910) Output:["_col0"] - Filter Operator [FIL_318] (rows=6000 width=107) - predicate:(((hd_dep_count = 3) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) - TableScan [TS_165] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count","hd_vehicle_count"] + Filter Operator [FIL_318] (rows=852 width=1910) + predicate:((s_store_name = 'ese') and s_store_sk is not null) + TableScan [TS_165] (rows=1704 width=1910) + default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name"] <-Reducer 52 [SIMPLE_EDGE] SHUFFLE [RS_174] - PartitionCols:_col1 + PartitionCols:_col2 Merge Join Operator [MERGEJOIN_342] (rows=696954748 width=88) - Conds:RS_171._col2=RS_172._col0(Inner),Output:["_col1"] + Conds:RS_171._col1=RS_172._col0(Inner),Output:["_col2"] <-Map 56 [SIMPLE_EDGE] SHUFFLE [RS_172] PartitionCols:_col0 - Select Operator [SEL_164] (rows=852 width=1910) + Select Operator [SEL_164] (rows=3600 width=107) Output:["_col0"] - Filter Operator [FIL_317] (rows=852 width=1910) - predicate:((s_store_name = 'ese') and s_store_sk is not null) - TableScan [TS_162] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name"] + Filter Operator [FIL_317] (rows=3600 width=107) + predicate:((((hd_dep_count = 3) and (hd_vehicle_count <= 5)) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) + TableScan [TS_162] (rows=7200 width=107) + default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count","hd_vehicle_count"] <-Reducer 51 [SIMPLE_EDGE] SHUFFLE [RS_171] - PartitionCols:_col2 + PartitionCols:_col1 Merge Join Operator [MERGEJOIN_341] (rows=633595212 width=88) Conds:RS_168._col0=RS_169._col0(Inner),Output:["_col1","_col2"] <-Map 50 [SIMPLE_EDGE] @@ -631,33 +631,33 @@ Stage-0 Group By Operator [GBY_204] (rows=1 width=8) Output:["_col0"],aggregations:["count()"] Merge Join Operator [MERGEJOIN_346] (rows=766650239 width=88) - Conds:RS_200._col1=RS_201._col0(Inner) + Conds:RS_200._col2=RS_201._col0(Inner) <-Map 65 [SIMPLE_EDGE] SHUFFLE [RS_201] PartitionCols:_col0 - Select Operator [SEL_193] (rows=6000 width=107) + Select Operator [SEL_193] (rows=852 width=1910) Output:["_col0"] - Filter Operator [FIL_322] (rows=6000 width=107) - predicate:(((hd_dep_count = 3) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) - TableScan [TS_191] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count","hd_vehicle_count"] + Filter Operator [FIL_322] (rows=852 width=1910) + predicate:((s_store_name = 'ese') and s_store_sk is not null) + TableScan [TS_191] (rows=1704 width=1910) + default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name"] <-Reducer 60 [SIMPLE_EDGE] SHUFFLE [RS_200] - PartitionCols:_col1 + PartitionCols:_col2 Merge Join Operator [MERGEJOIN_345] (rows=696954748 width=88) - Conds:RS_197._col2=RS_198._col0(Inner),Output:["_col1"] + Conds:RS_197._col1=RS_198._col0(Inner),Output:["_col2"] <-Map 64 [SIMPLE_EDGE] SHUFFLE [RS_198] PartitionCols:_col0 - Select Operator [SEL_190] (rows=852 width=1910) + Select Operator [SEL_190] (rows=3600 width=107) Output:["_col0"] - Filter Operator [FIL_321] (rows=852 width=1910) - predicate:((s_store_name = 'ese') and s_store_sk is not null) - TableScan [TS_188] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name"] + Filter Operator [FIL_321] (rows=3600 width=107) + predicate:((((hd_dep_count = 3) and (hd_vehicle_count <= 5)) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) + TableScan [TS_188] (rows=7200 width=107) + default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count","hd_vehicle_count"] <-Reducer 59 [SIMPLE_EDGE] SHUFFLE [RS_197] - PartitionCols:_col2 + PartitionCols:_col1 Merge Join Operator [MERGEJOIN_344] (rows=633595212 width=88) Conds:RS_194._col0=RS_195._col0(Inner),Output:["_col1","_col2"] <-Map 58 [SIMPLE_EDGE] -- 2.11.0 (Apple Git-81)