There is a bug triggered in APR/apr_snprintf when mod_deflate is enabled resulting in an instant SIGSEGV. Apache 2.2.0 segfaults as soon as the response is sent back to the client. The configuration is pretty basic; mod_deflate is simply enabled with SetOutputFilter DEFLATE in a <VirtualHost></VirtualHost> This is on OpenBSD 3.8-i386. ssehic@build-2-i386:/apache/core/bin$ sudo gdb httpd GNU gdb 6.3 Copyright 2004 Free Software Foundation, Inc. GDB is free software, covered by the GNU General Public License, and you are welcome to change it and/or distribute copies of it under certain conditions. Type "show copying" to see the conditions. There is absolutely no warranty for GDB. Type "show warranty" for details. This GDB was configured as "i386-unknown-openbsd3.8"... (gdb) run -X -d /apache/core Starting program: /apache/core/bin/httpd -X -d /apache/core Program received signal SIGSEGV, Segmentation fault. 0x00407206 in apr_vformatter (flush_func=0x407544 <snprintf_flush>, vbuff=0xcfbf2b98, fmt=0x3c009cc3 "s", ap=0xcfbf6c84 "") at /apache/source/httpd-2.2.0/srclib/apr/strings/apr_snprintf.c:968 968 s_len = strlen(s); (gdb) bt #0 0x00407206 in apr_vformatter (flush_func=0x407544 <snprintf_flush>, vbuff=0xcfbf2b98, fmt=0x3c009cc3 "s", ap=0xcfbf6c84 "") at /apache/source/httpd-2.2.0/srclib/apr/strings/apr_snprintf.c:968 #1 0x00407606 in apr_vsnprintf (buf=0x0, len=8115, format=0x3c009ca0 "Zlib: Compressed %ld to %ld : URL %s", ap=0xcfbf6c78 "") at /apache/source/httpd-2.2.0/srclib/apr/strings/apr_snprintf.c:1353 #2 0x1c031bc1 in log_error_core (file=0x3c009eea "mod_deflate.c", line=447, level=7, status=0, s=0x801f79c0, c=0x1fb3, r=0x87a24050, pool=0x0, fmt=0x3c009ca0 "Zlib: Compressed %ld to %ld : URL %s", args=0xcfbf6c78 "") at log.c:562 #3 0x1c031ee1 in ap_log_rerror (file=0x3c009eea "mod_deflate.c", line=447, level=7, status=0, r=0x87a24050, fmt=0x3c009ca0 "Zlib: Compressed %ld to %ld : URL %s") at log.c:633 #4 0x1c03df71 in deflate_out_filter (f=0x87a25be8, bb=0x87a25ef8) at mod_deflate.c:447 #5 0x1c036cfb in ap_pass_brigade (next=0x73, bb=0xffffffff) at util_filter.c:526 #6 0x1c029be5 in default_handler (r=0x87a24050) at core.c:3701 #7 0x1c02e7ca in ap_run_handler (r=0x87a24050) at config.c:157 #8 0x1c02ec8e in ap_invoke_handler (r=0x87a24050) at config.c:371 #9 0x1c064ef7 in ap_process_request (r=0x87a24050) at http_request.c:258 #10 0x1c062f79 in ap_process_http_connection (c=0x8a58f128) at http_core.c:171 #11 0x1c0340e2 in ap_run_process_connection (c=0x8a58f128) at connection.c:43 #12 0x1c06a2b4 in child_main (child_num_arg=0) at prefork.c:640 #13 0x1c06a398 in make_child (s=0x89c17d70, slot=0) at prefork.c:680 #14 0x1c06ade1 in ap_mpm_run (_pconf=0x7e846018, plog=0x7ff52018, s=0x89c17d70) at prefork.c:956 #15 0x1c01ef44 in main (argc=4, argv=0xcfbf7004) at main.c:712 (gdb) bt full #0 0x00407206 in apr_vformatter (flush_func=0x407544 <snprintf_flush>, vbuff=0xcfbf2b98, fmt=0x3c009cc3 "s", ap=0xcfbf6c84 "") at /apache/source/httpd-2.2.0/srclib/apr/strings/apr_snprintf.c:968 print_something = YES sp = 0xcfbf2bfe "p\n" bep = 0xcfbf4b92 "@ \022" cc = 30 i = 2 s = 0x2 <Address 0x2 out of bounds> q = 0x0 s_len = 1 min_width = 0 precision = 0 adjust = RIGHT pad_char = 32 ' ' prefix_char = 0 '\0' fp_num = 4.9406564584124654e-324 i_num = 0 ui_num = 115 num_buf = "\177\f\177\022Õ\a\000\000Õ\a\000\000\004<u. ìv.Pôv.\230)¿Ï\222\227z\016\210)¿Ï\020\016\000\000 ìv.ð)¿Ï¥ìv.ð)¿Ï¨$\224CÖÞo\n\004<u.h+w.È)¿Ï=\230z\016h+w.\000\000\000\000ð)¿Ï\000\000\000\000¸)¿Ï\000\000\000\000\a\230z\016è\b@ ð)¿Ï+\a\004\0008*¿ÏP\206A\000ð)¿Ïð)¿Ï@B\017\000\000\000\000\000|*¿Ï\004m4\211oãz\024¨$\224C,\000\000\000\035\000\000\000\f\000\000\000\005\000\000\000\v\000\000\000i\000\000\000\001\000\000\000R\001\000\000\000\000\000\000\020\016\000\000"... char_buf = "¿Ï" var_type = IS_SHORT alternate_form = NO print_sign = NO print_blank = NO adjust_precision = NO adjust_width = NO is_negative = 0 #1 0x00407606 in apr_vsnprintf (buf=0x0, len=8115, format=0x3c009ca0 "Zlib: Compressed %ld to %ld : URL %s", ap=0xcfbf6c78 "") at /apache/source/httpd-2.2.0/srclib/apr/strings/apr_snprintf.c:1353 cc = 115 vbuff = {curpos = 0xcfbf2be0 "Zlib: Compressed 0 to 0 : URL p\n", endpos = 0xcfbf4b92 "@ \022"} #2 0x1c031bc1 in log_error_core (file=0x3c009eea "mod_deflate.c", line=447, level=7, status=0, s=0x801f79c0, c=0x1fb3, r=0x87a24050, pool=0x0, fmt=0x3c009ca0 "Zlib: Compressed %ld to %ld : URL %s", args=0xcfbf6c78 "") at log.c:562 errstr = "[Mon Dec 05 12:29:44 2005] [debug] mod_deflate.c(447): [client 192.168.0.12] \000p\n \005\000&\200aþ%@L¿Ï\000\220Î\203hL¿ÏO\001\000\000O\001@\000\206\001\000\000pL¿Ï\001\000\000\000,Ýy\016\004<u.À\r\225~\\é\a\000\230L¿Ï\000ïy\016À\r\225~\\é\a\000 @\221\177L\001\000\000v\232Û\033\000\220¥\212ð\237¥\212\004<u.À\r\225~\020!u.ÈL¿Ï\020üy\016··É¡\000\000\000\000"... scratch = "Zlib: Compressed 0 to 0 : URL p\n\000\220Î\203Ð\223o*\030,¿ÏÂ|p\n\000\220Î\203Ð\223o*(,¿ÏB}p\n\000\000\000\000Ð\223o*H,¿Ï\027\016p\n\001\000\000\000{\000\000\000\000\220Î\203è\b@ ¨,¿Ï\020M¿Ï\210,¿ÏïÎ@\000\f\000\000\000\020M¿Ï{\000\000\000\220 õ\177\220,¿Ïöb\036<+\a\004\0001681··É¡{\000\000\000³\004\000\000è\b@ {\000\000\000\020M¿Ï¸,¿ÏØ«@\000\220 õ\177\020M¿Ï¨,¿Ï¨,¿Ï¨,¿Ï{\000\000\000"... len = 77 errstrlen = 3485428856 logf = (apr_file_t *) 0x7ff52090 referer = 0x0 level_and_mask = 7 #3 0x1c031ee1 in ap_log_rerror (file=0x3c009eea "mod_deflate.c", line=447, level=7, status=0, r=0x87a24050, fmt=0x3c009ca0 "Zlib: Compressed %ld to %ld : URL %s") at log.c:633 No locals. #4 0x1c03df71 in deflate_out_filter (f=0x87a25be8, bb=0x87a25ef8) at mod_deflate.c:447 buf = 0x7d05cfb8 "" deflate_len = 2 e = (apr_bucket *) 0x0 r = (request_rec *) 0x87a24050 ctx = (deflate_ctx *) 0x87a25f80 zRC = 115 c = (deflate_filter_config *) 0x81891600 #5 0x1c036cfb in ap_pass_brigade (next=0x73, bb=0xffffffff) at util_filter.c:526 e = (apr_bucket *) 0x0 #6 0x1c029be5 in default_handler (r=0x87a24050) at core.c:3701 fsize = 0 c = (conn_rec *) 0x8a58f128 bb = (apr_bucket_brigade *) 0x87a25ef8 e = (apr_bucket *) 0x7caa81f0 d = (core_dir_config *) 0x87a25580 errstatus = 0 fd = (apr_file_t *) 0x87a25dc8 status = 0 bld_content_md5 = 0 #7 0x1c02e7ca in ap_run_handler (r=0x87a24050) at config.c:157 pHook = (ap_LINK_handler_t *) 0x0 n = 2 rv = 0 #8 0x1c02ec8e in ap_invoke_handler (r=0x87a24050) at config.c:371 handler = 0x7f673df0 "text/html" result = -2019409840 old_handler = 0x0 #9 0x1c064ef7 in ap_process_request (r=0x87a24050) at http_request.c:258 access_status = 115 #10 0x1c062f79 in ap_process_http_connection (c=0x8a58f128) at http_core.c:171 r = (request_rec *) 0x87a24050 csd = (apr_socket_t *) 0x0 #11 0x1c0340e2 in ap_run_process_connection (c=0x8a58f128) at connection.c:43 pHook = (ap_LINK_process_connection_t *) 0x0 n = 0 rv = 0 #12 0x1c06a2b4 in child_main (child_num_arg=0) at prefork.c:640 current_conn = (conn_rec *) 0x8a58f128 csd = (void *) 0x8a58f050 ptrans = (apr_pool_t *) 0x8a58f018 allocator = (apr_allocator_t *) 0x8478bb80 status = 0 i = -1 lr = (ap_listen_rec *) 0x8a58f128 pollset = (apr_pollset_t *) 0x8a58d0e8 sbh = (ap_sb_handle_t *) 0x8a58d0e0 bucket_alloc = (apr_bucket_alloc_t *) 0x7caa8018 last_poll_idx = 0 #13 0x1c06a398 in make_child (s=0x89c17d70, slot=0) at prefork.c:680 pid = 470195400 #14 0x1c06ade1 in ap_mpm_run (_pconf=0x7e846018, plog=0x7ff52018, s=0x89c17d70) at prefork.c:956 index = -1983808144 remaining_children_to_start = -1983808144 rv = 0 #15 0x1c01ef44 in main (argc=4, argv=0xcfbf7004) at main.c:712 exit_status = -809537608 c = 100 'd' configtestonly = 0 confname = 0x3c00035e "conf/httpd.conf" def_server_root = 0xcfbf7183 "/apache/core" temp_error_log = 0x0 error = 0x0 process = (process_rec *) 0x84e7a098 server_conf = (server_rec *) 0x89c17d70 pglobal = (apr_pool_t *) 0x84e7a018 pconf = (apr_pool_t *) 0x7e846018 plog = (apr_pool_t *) 0x7ff52018 ptemp = (apr_pool_t *) 0x7d173018 pcommands = (apr_pool_t *) 0x81737018 opt = (apr_getopt_t *) 0x817370b0 rv = -809537680 mod = (module **) 0x89c17d70 optarg = 0xcfbf7183 "/apache/core" signal_server = (apr_OFN_ap_signal_server_t *) 0x73 (gdb) quit The program is running. Exit anyway? (y or n) y ssehic@build-2-i386:/apache/core/bin$ ldd httpd: Start End Type Ref Name 00000000 00000000 exe 1 httpd 04a9d000 24aa5000 rlib 1 /usr/lib/libz.so.4.1 00243000 2024e000 rlib 1 /usr/lib/libssl.so.10.0 04b81000 24baf000 rlib 1 /usr/lib/libcrypto.so.12.0 0334e000 23355000 rlib 1 /usr/lib/libm.so.2.0 0af7a000 2af7e000 rlib 1 /apache/core/lib/libaprutil-1.so.2.2 0dbd4000 2dbde000 rlib 2 /usr/local/lib/libexpat.so.4.0 0c9ba000 2c9bf000 rlib 2 /apache/core/lib/libapr-1.so.2.2 0af8b000 2af94000 rlib 3 /usr/lib/libpthread.so.6.1 01283000 212b4000 rlib 1 /usr/lib/libc.so.38.2 0057f000 0057f000 rtld 1 /usr/libexec/ld.so ssehic@build-2-i386:/apache/core/bin$ ./httpd -lCompiled in modules: core.c mod_authn_file.c mod_authn_default.c mod_authz_host.c mod_authz_groupfile.c mod_authz_user.c mod_authz_default.c mod_auth_basic.c mod_cache.c mod_disk_cache.c mod_filter.c mod_deflate.c mod_log_config.c mod_env.c mod_setenvif.c mod_proxy.c mod_proxy_connect.c mod_proxy_ftp.c mod_proxy_http.c mod_proxy_ajp.c mod_proxy_balancer.c mod_ssl.c prefork.c http_core.c mod_mime.c mod_dir.c mod_so.c
After some additional testing, it seems like that bug is only triggered when running with "LogLevel debug"
Can you print r->uri in the deflate_out_filter context? I can't see why it would be invalid. Can you also do: (gdb) ptype z_stream (gdb) ptype uLong to check the types exposed by zlib.
Sure, here we go: ssehic@build-2-i386:/apache/core/bin$ sudo gdb httpd GNU gdb 6.3 Copyright 2004 Free Software Foundation, Inc. GDB is free software, covered by the GNU General Public License, and you are welcome to change it and/or distribute copies of it under certain conditions. Type "show copying" to see the conditions. There is absolutely no warranty for GDB. Type "show warranty" for details. This GDB was configured as "i386-unknown-openbsd3.8"... (gdb) b deflate_out_filter Breakpoint 1 at 0x1c03dc2d: file mod_deflate.c, line 221. (gdb) run -X -d /apache/core Starting program: /apache/core/bin/httpd -X -d /apache/core Breakpoint 1, deflate_out_filter (f=0x7c593340, bb=0x84459eb0) at mod_deflate.c:221 221 request_rec *r = f->r; (gdb) p r->uri $1 = 0x0 (gdb) c Continuing. Program received signal SIGSEGV, Segmentation fault. 0x08793206 in apr_vformatter (flush_func=0x8793544 <snprintf_flush>, vbuff=0xcfbcded8, fmt=0x3c009cc3 "s", ap=0xcfbd1fc4 "") at /apache/source/httpd-2.2.0/srclib/apr/strings/apr_snprintf.c:968 968 s_len = strlen(s); (gdb) ptype z_stream type = struct z_stream_s { Bytef *next_in; uInt avail_in; off_t total_in; Bytef *next_out; uInt avail_out; off_t total_out; char *msg; struct internal_state *state; alloc_func zalloc; free_func zfree; voidpf opaque; int data_type; uLong adler; uLong reserved; } (gdb) ptype uLong type = long unsigned int
I'm afraid that someone has changed the zlib API for the copy of zlib you have: off_t total_in; ... off_t total_out; both those fields should be uLong. So the varargs offsets will be broken and the %s will hit garbarge. If you compile with -Wall you should see a warning. If this is as-packaged by OpenBSD that's pretty bad - upstream own the API not the packager, please complain loudly to them.