[IMPALA-3079] Fix Sequence file writer (crashes or produces invalid files)

• Type: Bug
• Status: Resolved
• Priority: Major
• Resolution: Fixed
• Affects Version/s: Impala 2.5.0
• Fix Version/s: Impala 2.9.0
• Component/s: Backend
• Target Version: Product Backlog
• Assignee: Attila Jeges
• Reporter: Mostafa Mokhtar
• Created: 25/Feb/16 01:14
• Updated: 20/Jul/17 18:33
• Resolved: 27/Apr/17 12:26

Description

Stack

#0  0x0000003f38832625 in raise () from /lib64/libc.so.6
#1  0x0000003f38833e05 in abort () from /lib64/libc.so.6
#2  0x00007f956d814c55 in os::abort(bool) () from /opt/toolchain/sun-jdk-64bit-1.7.0.75/jre/lib/amd64/server/libjvm.so
#3  0x00007f956d996cd7 in VMError::report_and_die() () from /opt/toolchain/sun-jdk-64bit-1.7.0.75/jre/lib/amd64/server/libjvm.so
#4  0x00007f956d819b6f in JVM_handle_linux_signal () from /opt/toolchain/sun-jdk-64bit-1.7.0.75/jre/lib/amd64/server/libjvm.so
#5  <signal handler called>
#6  0x0000000000cf1992 in snappy::internal::CompressFragment(char const*, unsigned long, char*, unsigned short*, int) ()
#7  0x0000000000cf2087 in snappy::Compress(snappy::Source*, snappy::Sink*) ()
#8  0x0000000000cf3051 in snappy::RawCompress(char const*, unsigned long, char*, unsigned long*) ()
#9  0x0000000000a8c158 in impala::SnappyBlockCompressor::ProcessBlock(bool, long, unsigned char const*, long*, unsigned char**) ()
#10 0x0000000000cc9177 in impala::HdfsSequenceTableWriter::WriteCompressedBlock() ()
#11 0x0000000000cc97d0 in impala::HdfsSequenceTableWriter::Flush() ()
#12 0x0000000000cc9ac3 in impala::HdfsSequenceTableWriter::AppendRowBatch(impala::RowBatch*, std::vector<int, std::allocator<int> > const&, bool*) ()
#13 0x0000000000cc37e9 in impala::HdfsTableSink::Send(impala::RuntimeState*, impala::RowBatch*, bool) ()
#14 0x0000000000ca9364 in impala::PlanFragmentExecutor::OpenInternal() ()
#15 0x0000000000ca98e7 in impala::PlanFragmentExecutor::Open() ()
#16 0x0000000000a60df8 in impala::FragmentMgr::FragmentExecState::Exec() ()
#17 0x0000000000a589d2 in impala::FragmentMgr::FragmentThread(impala::TUniqueId) ()
#18 0x0000000000a59bea in boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf1<void, impala::FragmentMgr, impala::TUniqueId>, boost::_bi::list2<boost::_bi::value<impala::FragmentMgr*>, boost::_bi::value<impala::TUniqueId> > >, void>::invoke(boost::detail::function::function_buffer&) ()
#19 0x0000000000af7da7 in impala::Thread::SuperviseThread(std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, boost::function<void ()()>, impala::Promise<long>*) ()
#20 0x0000000000af86c4 in boost::detail::thread_data<boost::_bi::bind_t<void, void (*)(std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, boost::function<void ()()>, impala::Promise<long>*), boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void ()()> >, boost::_bi::value<impala::Promise<long>*> > > >::run() ()
#21 0x0000000000d31c4a in ?? ()
#22 0x0000003f38c079d1 in start_thread () from /lib64/libpthread.so.0
#23 0x0000003f388e88fd in clone () from /lib64/libc.so.6

DDL

set ALLOW_UNSUPPORTED_FORMATS=1;
CREATE TABLE tpcds_1000_parquet.store_sales_sequencefile (                                                                  |
  ss_sold_time_sk INT,                                                                                                      |
  ss_item_sk BIGINT,                                                                                                        |
  ss_customer_sk INT,                                                                                                       |
  ss_cdemo_sk INT,                                                                                                          |
  ss_hdemo_sk INT,                                                                                                          |
  ss_addr_sk INT,                                                                                                           |
  ss_store_sk INT,                                                                                                          |
  ss_promo_sk INT,                                                                                                          |
  ss_ticket_number BIGINT,                                                                                                  |
  ss_quantity INT,                                                                                                          |
  ss_wholesale_cost DOUBLE,                                                                                                 |
  ss_list_price DOUBLE,                                                                                                     |
  ss_sales_price DOUBLE,                                                                                                    |
  ss_ext_discount_amt DOUBLE,                                                                                               |
  ss_ext_sales_price DOUBLE,                                                                                                |
  ss_ext_wholesale_cost DOUBLE,                                                                                             |
  ss_ext_list_price DOUBLE,                                                                                                 |
  ss_ext_tax DOUBLE,                                                                                                        |
  ss_coupon_amt DOUBLE,                                                                                                     |
  ss_net_paid DOUBLE,                                                                                                       |
  ss_net_paid_inc_tax DOUBLE,                                                                                               |
  ss_net_profit DOUBLE                                                                                                      |
)                                                                                                                           |
PARTITIONED BY (                                                                                                            |
  ss_sold_date_sk INT                                                                                                       |
)                                                                                                                           |
STORED AS SEQUENCEFILE                                                                                                      |

Query

insert into store_sales_sequencefile partition(ss_sold_date_sk) select * from store_sales where ss_sold_date_sk between 2450816 and 2451200;

Issue Links

• is broken by: IMPALA-5257 TestTableWriters.test_seq_writer_hive_compatibility fails in local file system build
• is broken by: IMPALA-5261 Heap use-after-free in HdfsSequenceTableWriter::ConsumeRow()
• is broken by: IMPALA-5407 Crash SIGSEGV in DeepCopyVarlenData while inserting into sequencefile
• is related to: IMPALA-5261 Heap use-after-free in HdfsSequenceTableWriter::ConsumeRow()