I1009 09:23:11.882787 3153 fetcher.cpp:560] Fetcher Info: {"cache_directory":"/tmp/mesos/fetch/nobody","items":[{"action":"BYPASS_CACHE","uri":{"extract":true,"value":"https://downloads.mesosphere.com/kafka/assets/kafka_2.12-2.1.0.tgz"}},{"action":"BYPASS_CACHE","uri":{"extract":true,"value":"https://downloads.mesosphere.com/java/server-jre-8u192-linux-x64.tar.gz"}},{"action":"BYPASS_CACHE","uri":{"extract":true,"value":"http://downloads.mesosphere.com/kafka/assets/zookeeper-3.4.13.jar"}},{"action":"BYPASS_CACHE","uri":{"extract":true,"value":"https://downloads.mesosphere.com/dcos-commons/artifacts/0.55.2/bootstrap.zip"}},{"action":"BYPASS_CACHE","uri":{"extract":true,"value":"https://downloads.mesosphere.com/libmesos-bundle/libmesos-bundle-1.12.0.tar.gz"}},{"action":"BYPASS_CACHE","uri":{"extract":true,"value":"http://downloads.mesosphere.com/kafka/assets/kafka-statsd-metrics2-0.5.3.jar"}},{"action":"BYPASS_CACHE","uri":{"extract":true,"value":"http://downloads.mesosphere.com/kafka/assets/java-dogstatsd-client-2.3.jar"}},{"action":"BYPASS_CACHE","uri":{"extract":true,"value":"https://downloads.mesosphere.com/kafka/assets/2.5.0-2.1.0/setup-helper.zip"}},{"action":"BYPASS_CACHE","uri":{"extract":true,"value":"http://downloads.mesosphere.com/kafka/assets/kafka-custom-principal-builder-1.0.0.jar"}},{"action":"BYPASS_CACHE","uri":{"extract":false,"output_file":"config-templates/server-properties","value":"http://kafka01-dcp-xg-prod.marathon.autoip.dcos.thisdcos.directory:31884/v1/artifacts/template/bd744b9b-926f-468a-a615-ad6445460bb4/kafka/broker/server-properties"}}],"sandbox_directory":"/var/lib/mesos/slave/slaves/e67d2813-79ab-4885-8bd0-3ca45a8960bc-S7/frameworks/ef477552-90f0-4a11-979d-70a7659eecae-0000/executors/prod.xg.dcp.kafka01__kafka__c0e289ae-9a26-4585-bb4d-dd3a28a346fd/runs/3797d666-632e-44ce-9b8f-09e2c226a11f/containers/714ff76b-a31b-4c23-8ce4-664c685715d8","stall_timeout":{"nanoseconds":60000000000},"user":"nobody"} I1009 09:23:11.903434 3153 fetcher.cpp:457] Fetching URI 'https://downloads.mesosphere.com/kafka/assets/kafka_2.12-2.1.0.tgz' I1009 09:23:11.903483 3153 fetcher.cpp:292] Fetching directly into the sandbox directory I1009 09:23:11.903523 3153 fetcher.cpp:225] Fetching URI 'https://downloads.mesosphere.com/kafka/assets/kafka_2.12-2.1.0.tgz' I1009 09:23:11.903546 3153 fetcher.cpp:175] Downloading resource from 'https://downloads.mesosphere.com/kafka/assets/kafka_2.12-2.1.0.tgz' to '/var/lib/mesos/slave/slaves/e67d2813-79ab-4885-8bd0-3ca45a8960bc-S7/frameworks/ef477552-90f0-4a11-979d-70a7659eecae-0000/executors/prod.xg.dcp.kafka01__kafka__c0e289ae-9a26-4585-bb4d-dd3a28a346fd/runs/3797d666-632e-44ce-9b8f-09e2c226a11f/containers/714ff76b-a31b-4c23-8ce4-664c685715d8/kafka_2.12-2.1.0.tgz' I1009 09:23:15.512791 3153 fetcher.cpp:616] Fetched 'https://downloads.mesosphere.com/kafka/assets/kafka_2.12-2.1.0.tgz' to '/var/lib/mesos/slave/slaves/e67d2813-79ab-4885-8bd0-3ca45a8960bc-S7/frameworks/ef477552-90f0-4a11-979d-70a7659eecae-0000/executors/prod.xg.dcp.kafka01__kafka__c0e289ae-9a26-4585-bb4d-dd3a28a346fd/runs/3797d666-632e-44ce-9b8f-09e2c226a11f/containers/714ff76b-a31b-4c23-8ce4-664c685715d8/kafka_2.12-2.1.0.tgz' I1009 09:23:15.512850 3153 fetcher.cpp:457] Fetching URI 'https://downloads.mesosphere.com/java/server-jre-8u192-linux-x64.tar.gz' I1009 09:23:15.512858 3153 fetcher.cpp:292] Fetching directly into the sandbox directory I1009 09:23:15.512876 3153 fetcher.cpp:225] Fetching URI 'https://downloads.mesosphere.com/java/server-jre-8u192-linux-x64.tar.gz' I1009 09:23:15.512888 3153 fetcher.cpp:175] Downloading resource from 'https://downloads.mesosphere.com/java/server-jre-8u192-linux-x64.tar.gz' to '/var/lib/mesos/slave/slaves/e67d2813-79ab-4885-8bd0-3ca45a8960bc-S7/frameworks/ef477552-90f0-4a11-979d-70a7659eecae-0000/executors/prod.xg.dcp.kafka01__kafka__c0e289ae-9a26-4585-bb4d-dd3a28a346fd/runs/3797d666-632e-44ce-9b8f-09e2c226a11f/containers/714ff76b-a31b-4c23-8ce4-664c685715d8/server-jre-8u192-linux-x64.tar.gz' I1009 09:23:22.843642 3153 fetcher.cpp:616] Fetched 'https://downloads.mesosphere.com/java/server-jre-8u192-linux-x64.tar.gz' to '/var/lib/mesos/slave/slaves/e67d2813-79ab-4885-8bd0-3ca45a8960bc-S7/frameworks/ef477552-90f0-4a11-979d-70a7659eecae-0000/executors/prod.xg.dcp.kafka01__kafka__c0e289ae-9a26-4585-bb4d-dd3a28a346fd/runs/3797d666-632e-44ce-9b8f-09e2c226a11f/containers/714ff76b-a31b-4c23-8ce4-664c685715d8/server-jre-8u192-linux-x64.tar.gz' I1009 09:23:22.843699 3153 fetcher.cpp:457] Fetching URI 'http://downloads.mesosphere.com/kafka/assets/zookeeper-3.4.13.jar' I1009 09:23:22.843705 3153 fetcher.cpp:292] Fetching directly into the sandbox directory I1009 09:23:22.843724 3153 fetcher.cpp:225] Fetching URI 'http://downloads.mesosphere.com/kafka/assets/zookeeper-3.4.13.jar' I1009 09:23:22.843734 3153 fetcher.cpp:175] Downloading resource from 'http://downloads.mesosphere.com/kafka/assets/zookeeper-3.4.13.jar' to '/var/lib/mesos/slave/slaves/e67d2813-79ab-4885-8bd0-3ca45a8960bc-S7/frameworks/ef477552-90f0-4a11-979d-70a7659eecae-0000/executors/prod.xg.dcp.kafka01__kafka__c0e289ae-9a26-4585-bb4d-dd3a28a346fd/runs/3797d666-632e-44ce-9b8f-09e2c226a11f/containers/714ff76b-a31b-4c23-8ce4-664c685715d8/zookeeper-3.4.13.jar' W1009 09:23:23.173710 3153 fetcher.cpp:333] Copying instead of extracting resource from URI with 'extract' flag, because it does not seem to be an archive: http://downloads.mesosphere.com/kafka/assets/zookeeper-3.4.13.jar I1009 09:23:23.173739 3153 fetcher.cpp:616] Fetched 'http://downloads.mesosphere.com/kafka/assets/zookeeper-3.4.13.jar' to '/var/lib/mesos/slave/slaves/e67d2813-79ab-4885-8bd0-3ca45a8960bc-S7/frameworks/ef477552-90f0-4a11-979d-70a7659eecae-0000/executors/prod.xg.dcp.kafka01__kafka__c0e289ae-9a26-4585-bb4d-dd3a28a346fd/runs/3797d666-632e-44ce-9b8f-09e2c226a11f/containers/714ff76b-a31b-4c23-8ce4-664c685715d8/zookeeper-3.4.13.jar' I1009 09:23:23.173745 3153 fetcher.cpp:457] Fetching URI 'https://downloads.mesosphere.com/dcos-commons/artifacts/0.55.2/bootstrap.zip' I1009 09:23:23.173750 3153 fetcher.cpp:292] Fetching directly into the sandbox directory I1009 09:23:23.173763 3153 fetcher.cpp:225] Fetching URI 'https://downloads.mesosphere.com/dcos-commons/artifacts/0.55.2/bootstrap.zip' I1009 09:23:23.173771 3153 fetcher.cpp:175] Downloading resource from 'https://downloads.mesosphere.com/dcos-commons/artifacts/0.55.2/bootstrap.zip' to '/var/lib/mesos/slave/slaves/e67d2813-79ab-4885-8bd0-3ca45a8960bc-S7/frameworks/ef477552-90f0-4a11-979d-70a7659eecae-0000/executors/prod.xg.dcp.kafka01__kafka__c0e289ae-9a26-4585-bb4d-dd3a28a346fd/runs/3797d666-632e-44ce-9b8f-09e2c226a11f/containers/714ff76b-a31b-4c23-8ce4-664c685715d8/bootstrap.zip' I1009 09:23:23.571940 3153 fetcher.cpp:616] Fetched 'https://downloads.mesosphere.com/dcos-commons/artifacts/0.55.2/bootstrap.zip' to '/var/lib/mesos/slave/slaves/e67d2813-79ab-4885-8bd0-3ca45a8960bc-S7/frameworks/ef477552-90f0-4a11-979d-70a7659eecae-0000/executors/prod.xg.dcp.kafka01__kafka__c0e289ae-9a26-4585-bb4d-dd3a28a346fd/runs/3797d666-632e-44ce-9b8f-09e2c226a11f/containers/714ff76b-a31b-4c23-8ce4-664c685715d8/bootstrap.zip' I1009 09:23:23.571982 3153 fetcher.cpp:457] Fetching URI 'https://downloads.mesosphere.com/libmesos-bundle/libmesos-bundle-1.12.0.tar.gz' I1009 09:23:23.571991 3153 fetcher.cpp:292] Fetching directly into the sandbox directory I1009 09:23:23.572016 3153 fetcher.cpp:225] Fetching URI 'https://downloads.mesosphere.com/libmesos-bundle/libmesos-bundle-1.12.0.tar.gz' I1009 09:23:23.573675 3153 fetcher.cpp:175] Downloading resource from 'https://downloads.mesosphere.com/libmesos-bundle/libmesos-bundle-1.12.0.tar.gz' to '/var/lib/mesos/slave/slaves/e67d2813-79ab-4885-8bd0-3ca45a8960bc-S7/frameworks/ef477552-90f0-4a11-979d-70a7659eecae-0000/executors/prod.xg.dcp.kafka01__kafka__c0e289ae-9a26-4585-bb4d-dd3a28a346fd/runs/3797d666-632e-44ce-9b8f-09e2c226a11f/containers/714ff76b-a31b-4c23-8ce4-664c685715d8/libmesos-bundle-1.12.0.tar.gz' I1009 09:23:30.461477 3153 fetcher.cpp:616] Fetched 'https://downloads.mesosphere.com/libmesos-bundle/libmesos-bundle-1.12.0.tar.gz' to '/var/lib/mesos/slave/slaves/e67d2813-79ab-4885-8bd0-3ca45a8960bc-S7/frameworks/ef477552-90f0-4a11-979d-70a7659eecae-0000/executors/prod.xg.dcp.kafka01__kafka__c0e289ae-9a26-4585-bb4d-dd3a28a346fd/runs/3797d666-632e-44ce-9b8f-09e2c226a11f/containers/714ff76b-a31b-4c23-8ce4-664c685715d8/libmesos-bundle-1.12.0.tar.gz' I1009 09:23:30.461534 3153 fetcher.cpp:457] Fetching URI 'http://downloads.mesosphere.com/kafka/assets/kafka-statsd-metrics2-0.5.3.jar' I1009 09:23:30.461540 3153 fetcher.cpp:292] Fetching directly into the sandbox directory I1009 09:23:30.461557 3153 fetcher.cpp:225] Fetching URI 'http://downloads.mesosphere.com/kafka/assets/kafka-statsd-metrics2-0.5.3.jar' I1009 09:23:30.461567 3153 fetcher.cpp:175] Downloading resource from 'http://downloads.mesosphere.com/kafka/assets/kafka-statsd-metrics2-0.5.3.jar' to '/var/lib/mesos/slave/slaves/e67d2813-79ab-4885-8bd0-3ca45a8960bc-S7/frameworks/ef477552-90f0-4a11-979d-70a7659eecae-0000/executors/prod.xg.dcp.kafka01__kafka__c0e289ae-9a26-4585-bb4d-dd3a28a346fd/runs/3797d666-632e-44ce-9b8f-09e2c226a11f/containers/714ff76b-a31b-4c23-8ce4-664c685715d8/kafka-statsd-metrics2-0.5.3.jar' W1009 09:23:30.582538 3153 fetcher.cpp:333] Copying instead of extracting resource from URI with 'extract' flag, because it does not seem to be an archive: http://downloads.mesosphere.com/kafka/assets/kafka-statsd-metrics2-0.5.3.jar I1009 09:23:30.582581 3153 fetcher.cpp:616] Fetched 'http://downloads.mesosphere.com/kafka/assets/kafka-statsd-metrics2-0.5.3.jar' to '/var/lib/mesos/slave/slaves/e67d2813-79ab-4885-8bd0-3ca45a8960bc-S7/frameworks/ef477552-90f0-4a11-979d-70a7659eecae-0000/executors/prod.xg.dcp.kafka01__kafka__c0e289ae-9a26-4585-bb4d-dd3a28a346fd/runs/3797d666-632e-44ce-9b8f-09e2c226a11f/containers/714ff76b-a31b-4c23-8ce4-664c685715d8/kafka-statsd-metrics2-0.5.3.jar' I1009 09:23:30.582589 3153 fetcher.cpp:457] Fetching URI 'http://downloads.mesosphere.com/kafka/assets/java-dogstatsd-client-2.3.jar' I1009 09:23:30.582593 3153 fetcher.cpp:292] Fetching directly into the sandbox directory I1009 09:23:30.582610 3153 fetcher.cpp:225] Fetching URI 'http://downloads.mesosphere.com/kafka/assets/java-dogstatsd-client-2.3.jar' I1009 09:23:30.582618 3153 fetcher.cpp:175] Downloading resource from 'http://downloads.mesosphere.com/kafka/assets/java-dogstatsd-client-2.3.jar' to '/var/lib/mesos/slave/slaves/e67d2813-79ab-4885-8bd0-3ca45a8960bc-S7/frameworks/ef477552-90f0-4a11-979d-70a7659eecae-0000/executors/prod.xg.dcp.kafka01__kafka__c0e289ae-9a26-4585-bb4d-dd3a28a346fd/runs/3797d666-632e-44ce-9b8f-09e2c226a11f/containers/714ff76b-a31b-4c23-8ce4-664c685715d8/java-dogstatsd-client-2.3.jar' W1009 09:23:30.652798 3153 fetcher.cpp:333] Copying instead of extracting resource from URI with 'extract' flag, because it does not seem to be an archive: http://downloads.mesosphere.com/kafka/assets/java-dogstatsd-client-2.3.jar I1009 09:23:30.652827 3153 fetcher.cpp:616] Fetched 'http://downloads.mesosphere.com/kafka/assets/java-dogstatsd-client-2.3.jar' to '/var/lib/mesos/slave/slaves/e67d2813-79ab-4885-8bd0-3ca45a8960bc-S7/frameworks/ef477552-90f0-4a11-979d-70a7659eecae-0000/executors/prod.xg.dcp.kafka01__kafka__c0e289ae-9a26-4585-bb4d-dd3a28a346fd/runs/3797d666-632e-44ce-9b8f-09e2c226a11f/containers/714ff76b-a31b-4c23-8ce4-664c685715d8/java-dogstatsd-client-2.3.jar' I1009 09:23:30.652832 3153 fetcher.cpp:457] Fetching URI 'https://downloads.mesosphere.com/kafka/assets/2.5.0-2.1.0/setup-helper.zip' I1009 09:23:30.652837 3153 fetcher.cpp:292] Fetching directly into the sandbox directory I1009 09:23:30.652853 3153 fetcher.cpp:225] Fetching URI 'https://downloads.mesosphere.com/kafka/assets/2.5.0-2.1.0/setup-helper.zip' I1009 09:23:30.652861 3153 fetcher.cpp:175] Downloading resource from 'https://downloads.mesosphere.com/kafka/assets/2.5.0-2.1.0/setup-helper.zip' to '/var/lib/mesos/slave/slaves/e67d2813-79ab-4885-8bd0-3ca45a8960bc-S7/frameworks/ef477552-90f0-4a11-979d-70a7659eecae-0000/executors/prod.xg.dcp.kafka01__kafka__c0e289ae-9a26-4585-bb4d-dd3a28a346fd/runs/3797d666-632e-44ce-9b8f-09e2c226a11f/containers/714ff76b-a31b-4c23-8ce4-664c685715d8/setup-helper.zip' I1009 09:23:30.993618 3153 fetcher.cpp:616] Fetched 'https://downloads.mesosphere.com/kafka/assets/2.5.0-2.1.0/setup-helper.zip' to '/var/lib/mesos/slave/slaves/e67d2813-79ab-4885-8bd0-3ca45a8960bc-S7/frameworks/ef477552-90f0-4a11-979d-70a7659eecae-0000/executors/prod.xg.dcp.kafka01__kafka__c0e289ae-9a26-4585-bb4d-dd3a28a346fd/runs/3797d666-632e-44ce-9b8f-09e2c226a11f/containers/714ff76b-a31b-4c23-8ce4-664c685715d8/setup-helper.zip' I1009 09:23:30.993646 3153 fetcher.cpp:457] Fetching URI 'http://downloads.mesosphere.com/kafka/assets/kafka-custom-principal-builder-1.0.0.jar' I1009 09:23:30.993662 3153 fetcher.cpp:292] Fetching directly into the sandbox directory I1009 09:23:30.993676 3153 fetcher.cpp:225] Fetching URI 'http://downloads.mesosphere.com/kafka/assets/kafka-custom-principal-builder-1.0.0.jar' I1009 09:23:30.993685 3153 fetcher.cpp:175] Downloading resource from 'http://downloads.mesosphere.com/kafka/assets/kafka-custom-principal-builder-1.0.0.jar' to '/var/lib/mesos/slave/slaves/e67d2813-79ab-4885-8bd0-3ca45a8960bc-S7/frameworks/ef477552-90f0-4a11-979d-70a7659eecae-0000/executors/prod.xg.dcp.kafka01__kafka__c0e289ae-9a26-4585-bb4d-dd3a28a346fd/runs/3797d666-632e-44ce-9b8f-09e2c226a11f/containers/714ff76b-a31b-4c23-8ce4-664c685715d8/kafka-custom-principal-builder-1.0.0.jar' W1009 09:23:31.043010 3153 fetcher.cpp:333] Copying instead of extracting resource from URI with 'extract' flag, because it does not seem to be an archive: http://downloads.mesosphere.com/kafka/assets/kafka-custom-principal-builder-1.0.0.jar I1009 09:23:31.043038 3153 fetcher.cpp:616] Fetched 'http://downloads.mesosphere.com/kafka/assets/kafka-custom-principal-builder-1.0.0.jar' to '/var/lib/mesos/slave/slaves/e67d2813-79ab-4885-8bd0-3ca45a8960bc-S7/frameworks/ef477552-90f0-4a11-979d-70a7659eecae-0000/executors/prod.xg.dcp.kafka01__kafka__c0e289ae-9a26-4585-bb4d-dd3a28a346fd/runs/3797d666-632e-44ce-9b8f-09e2c226a11f/containers/714ff76b-a31b-4c23-8ce4-664c685715d8/kafka-custom-principal-builder-1.0.0.jar' I1009 09:23:31.043045 3153 fetcher.cpp:457] Fetching URI 'http://kafka01-dcp-xg-prod.marathon.autoip.dcos.thisdcos.directory:31884/v1/artifacts/template/bd744b9b-926f-468a-a615-ad6445460bb4/kafka/broker/server-properties' I1009 09:23:31.043049 3153 fetcher.cpp:292] Fetching directly into the sandbox directory I1009 09:23:31.043196 3153 fetcher.cpp:225] Fetching URI 'http://kafka01-dcp-xg-prod.marathon.autoip.dcos.thisdcos.directory:31884/v1/artifacts/template/bd744b9b-926f-468a-a615-ad6445460bb4/kafka/broker/server-properties' I1009 09:23:31.043208 3153 fetcher.cpp:175] Downloading resource from 'http://kafka01-dcp-xg-prod.marathon.autoip.dcos.thisdcos.directory:31884/v1/artifacts/template/bd744b9b-926f-468a-a615-ad6445460bb4/kafka/broker/server-properties' to '/var/lib/mesos/slave/slaves/e67d2813-79ab-4885-8bd0-3ca45a8960bc-S7/frameworks/ef477552-90f0-4a11-979d-70a7659eecae-0000/executors/prod.xg.dcp.kafka01__kafka__c0e289ae-9a26-4585-bb4d-dd3a28a346fd/runs/3797d666-632e-44ce-9b8f-09e2c226a11f/containers/714ff76b-a31b-4c23-8ce4-664c685715d8/config-templates/server-properties' I1009 09:23:31.093873 3153 fetcher.cpp:616] Fetched 'http://kafka01-dcp-xg-prod.marathon.autoip.dcos.thisdcos.directory:31884/v1/artifacts/template/bd744b9b-926f-468a-a615-ad6445460bb4/kafka/broker/server-properties' to '/var/lib/mesos/slave/slaves/e67d2813-79ab-4885-8bd0-3ca45a8960bc-S7/frameworks/ef477552-90f0-4a11-979d-70a7659eecae-0000/executors/prod.xg.dcp.kafka01__kafka__c0e289ae-9a26-4585-bb4d-dd3a28a346fd/runs/3797d666-632e-44ce-9b8f-09e2c226a11f/containers/714ff76b-a31b-4c23-8ce4-664c685715d8/config-templates/server-properties' I1009 09:23:31.093902 3153 fetcher.cpp:621] Successfully fetched all URIs into '/var/lib/mesos/slave/slaves/e67d2813-79ab-4885-8bd0-3ca45a8960bc-S7/frameworks/ef477552-90f0-4a11-979d-70a7659eecae-0000/executors/prod.xg.dcp.kafka01__kafka__c0e289ae-9a26-4585-bb4d-dd3a28a346fd/runs/3797d666-632e-44ce-9b8f-09e2c226a11f/containers/714ff76b-a31b-4c23-8ce4-664c685715d8' Marked '/' as rslave Prepared mount '{"flags":14,"source":"proc","target":"/proc","type":"proc"}' Prepared mount '{"flags":20480,"source":"/var/lib/mesos/slave/slaves/e67d2813-79ab-4885-8bd0-3ca45a8960bc-S7/frameworks/ef477552-90f0-4a11-979d-70a7659eecae-0000/executors/prod.xg.dcp.kafka01__kafka__c0e289ae-9a26-4585-bb4d-dd3a28a346fd/runs/3797d666-632e-44ce-9b8f-09e2c226a11f/containers/714ff76b-a31b-4c23-8ce4-664c685715d8/tmp","target":"/tmp"}' Prepared mount '{"flags":20480,"source":"/var/lib/mesos/slave/slaves/e67d2813-79ab-4885-8bd0-3ca45a8960bc-S7/frameworks/ef477552-90f0-4a11-979d-70a7659eecae-0000/executors/prod.xg.dcp.kafka01__kafka__c0e289ae-9a26-4585-bb4d-dd3a28a346fd/runs/3797d666-632e-44ce-9b8f-09e2c226a11f/kafka-broker-data","target":"/var/lib/mesos/slave/slaves/e67d2813-79ab-4885-8bd0-3ca45a8960bc-S7/frameworks/ef477552-90f0-4a11-979d-70a7659eecae-0000/executors/prod.xg.dcp.kafka01__kafka__c0e289ae-9a26-4585-bb4d-dd3a28a346fd/runs/3797d666-632e-44ce-9b8f-09e2c226a11f/containers/714ff76b-a31b-4c23-8ce4-664c685715d8/kafka-broker-data"}' Executing pre-exec command '{"arguments":["mount","-n","-t","ramfs","ramfs","/var/lib/mesos/slave/slaves/e67d2813-79ab-4885-8bd0-3ca45a8960bc-S7/frameworks/ef477552-90f0-4a11-979d-70a7659eecae-0000/executors/prod.xg.dcp.kafka01__kafka__c0e289ae-9a26-4585-bb4d-dd3a28a346fd/runs/3797d666-632e-44ce-9b8f-09e2c226a11f/containers/714ff76b-a31b-4c23-8ce4-664c685715d8/.secret-ccd7c678-7b35-4e0b-8bd7-da219df9b329"],"shell":false,"value":"mount"}' 2019/10/09 09:23:31 Starting setup-helper... 2019/10/09 09:23:31 Calculating security and listener settings... 2019/10/09 09:23:31 Setting listeners... 2019/10/09 09:23:31 Attempting to write to listeners: listeners=PLAINTEXT://10.75.148.1:9610 2019/10/09 09:23:31 Calculated working directory as: /var/lib/mesos/slave/slaves/e67d2813-79ab-4885-8bd0-3ca45a8960bc-S7/frameworks/ef477552-90f0-4a11-979d-70a7659eecae-0000/executors/prod.xg.dcp.kafka01__kafka__c0e289ae-9a26-4585-bb4d-dd3a28a346fd/runs/3797d666-632e-44ce-9b8f-09e2c226a11f/containers/714ff76b-a31b-4c23-8ce4-664c685715d8 2019/10/09 09:23:31 Attempting to write to advertised.listeners: advertised.listeners=PLAINTEXT://10.75.148.1:9610 2019/10/09 09:23:31 Calculated working directory as: /var/lib/mesos/slave/slaves/e67d2813-79ab-4885-8bd0-3ca45a8960bc-S7/frameworks/ef477552-90f0-4a11-979d-70a7659eecae-0000/executors/prod.xg.dcp.kafka01__kafka__c0e289ae-9a26-4585-bb4d-dd3a28a346fd/runs/3797d666-632e-44ce-9b8f-09e2c226a11f/containers/714ff76b-a31b-4c23-8ce4-664c685715d8 2019/10/09 09:23:31 Set listeners 2019/10/09 09:23:31 Setting security.inter.broker.protocol... 2019/10/09 09:23:31 Attempting to write to security.inter.broker.protocol: security.inter.broker.protocol=PLAINTEXT 2019/10/09 09:23:31 Calculated working directory as: /var/lib/mesos/slave/slaves/e67d2813-79ab-4885-8bd0-3ca45a8960bc-S7/frameworks/ef477552-90f0-4a11-979d-70a7659eecae-0000/executors/prod.xg.dcp.kafka01__kafka__c0e289ae-9a26-4585-bb4d-dd3a28a346fd/runs/3797d666-632e-44ce-9b8f-09e2c226a11f/containers/714ff76b-a31b-4c23-8ce4-664c685715d8 2019/10/09 09:23:31 Set security.inter.broker.protocol 2019/10/09 09:23:31 Setting super.users 2019/10/09 09:23:31 Attempting to write to super.users: 2019/10/09 09:23:31 Calculated working directory as: /var/lib/mesos/slave/slaves/e67d2813-79ab-4885-8bd0-3ca45a8960bc-S7/frameworks/ef477552-90f0-4a11-979d-70a7659eecae-0000/executors/prod.xg.dcp.kafka01__kafka__c0e289ae-9a26-4585-bb4d-dd3a28a346fd/runs/3797d666-632e-44ce-9b8f-09e2c226a11f/containers/714ff76b-a31b-4c23-8ce4-664c685715d8 2019/10/09 09:23:31 Set super.users 2019/10/09 09:23:31 Calculated security and listener settings 2019/10/09 09:23:31 setup-helper complete 2019/10/09 09:23:31 Bootstrapping with environment: =/var/lib/mesos/slave/slaves/e67d2813-79ab-4885-8bd0-3ca45a8960bc-S7/frameworks/ef477552-90f0-4a11-979d-70a7659eecae-0000/executors/prod.xg.dcp.kafka01__kafka__c0e289ae-9a26-4585-bb4d-dd3a28a346fd/runs/3797d666-632e-44ce-9b8f-09e2c226a11f/containers/714ff76b-a31b-4c23-8ce4-664c685715d8/bootstrap CONFIG_TEMPLATE_SERVER_PROPERTIES=config-templates/server-properties,kafka_2.12-2.1.0/config/server.properties FRAMEWORK_HOST=prodxgdcpkafka01.autoip.dcos.thisdcos.directory FRAMEWORK_NAME=/prod/xg/dcp/kafka01 FRAMEWORK_VIP_HOST=prodxgdcpkafka01.l4lb.thisdcos.directory JAVA_HOME=/var/lib/mesos/slave/slaves/e67d2813-79ab-4885-8bd0-3ca45a8960bc-S7/frameworks/ef477552-90f0-4a11-979d-70a7659eecae-0000/executors/prod.xg.dcp.kafka01__kafka__c0e289ae-9a26-4585-bb4d-dd3a28a346fd/runs/3797d666-632e-44ce-9b8f-09e2c226a11f/containers/714ff76b-a31b-4c23-8ce4-664c685715d8/jdk1.8.0_192/jre/ KAFKA_ADVERTISE_HOST=true KAFKA_AUTO_CREATE_TOPICS_ENABLE=false KAFKA_AUTO_LEADER_REBALANCE_ENABLE=true KAFKA_BACKGROUND_THREADS=10 KAFKA_BROKER_PORT=9610 KAFKA_COMPRESSION_TYPE=producer KAFKA_CONNECTIONS_MAX_IDLE_MS=600000 KAFKA_CONTROLLED_SHUTDOWN_ENABLE=true KAFKA_CONTROLLED_SHUTDOWN_MAX_RETRIES=3 KAFKA_CONTROLLED_SHUTDOWN_RETRY_BACKOFF_MS=5000 KAFKA_CONTROLLER_SOCKET_TIMEOUT_MS=30000 KAFKA_DEFAULT_REPLICATION_FACTOR=3 KAFKA_DELETE_RECORDS_PURGATORY_PURGE_INTERVAL_REQUESTS=1 KAFKA_DELETE_TOPIC_ENABLE=true KAFKA_DISK_PATH=kafka-broker-data KAFKA_FETCH_PURGATORY_PURGE_INTERVAL_REQUESTS=1000 KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=3000 KAFKA_GROUP_MAX_SESSION_TIMEOUT_MS=300000 KAFKA_GROUP_MIN_SESSION_TIMEOUT_MS=6000 KAFKA_HEAP_OPTS=-Xms16192M -Xmx16192M KAFKA_INTER_BROKER_PROTOCOL_VERSION=2.1 KAFKA_LEADER_IMBALANCE_CHECK_INTERVAL_SECONDS=300 KAFKA_LEADER_IMBALANCE_PER_BROKER_PERCENTAGE=10 KAFKA_LOG_CLEANER_BACKOFF_MS=15000 KAFKA_LOG_CLEANER_DEDUPE_BUFFER_SIZE=134217728 KAFKA_LOG_CLEANER_DELETE_RETENTION_MS=86400000 KAFKA_LOG_CLEANER_ENABLE=true KAFKA_LOG_CLEANER_IO_BUFFER_LOAD_FACTOR=0.9 KAFKA_LOG_CLEANER_IO_BUFFER_SIZE=524288 KAFKA_LOG_CLEANER_IO_MAX_BYTES_PER_SECOND=179769313486231570000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 KAFKA_LOG_CLEANER_MIN_CLEANABLE_RATIO=0.5 KAFKA_LOG_CLEANER_MIN_COMPACTION_LAG_MS=0 KAFKA_LOG_CLEANER_THREADS=1 KAFKA_LOG_CLEANUP_POLICY=delete KAFKA_LOG_FLUSH_INTERVAL_MESSAGES=9223372036854775807 KAFKA_LOG_FLUSH_OFFSET_CHECKPOINT_INTERVAL_MS=60000 KAFKA_LOG_FLUSH_SCHEDULER_INTERVAL_MS=9223372036854775807 KAFKA_LOG_FLUSH_START_OFFSET_CHECKPOINT_INTERVAL_MS=60000 KAFKA_LOG_INDEX_INTERVAL_BYTES=4096 KAFKA_LOG_INDEX_SIZE_MAX_BYTES=10485760 KAFKA_LOG_MESSAGE_FORMAT_VERSION=2.1 KAFKA_LOG_PREALLOCATE=false KAFKA_LOG_RETENTION_BYTES=-1 KAFKA_LOG_RETENTION_CHECK_INTERVAL_MS=300000 KAFKA_LOG_RETENTION_HOURS=168 KAFKA_LOG_ROLL_HOURS=168 KAFKA_LOG_ROLL_JITTER_HOURS=0 KAFKA_LOG_SEGMENT_BYTES=1073741824 KAFKA_LOG_SEGMENT_DELETE_DELAY_MS=60000 KAFKA_MAX_CONNECTIONS_PER_IP=2147483647 KAFKA_MAX_CONNECTIONS_PER_IP_OVERRIDES= KAFKA_MESSAGE_MAX_BYTES=1000012 KAFKA_METRICS_NUM_SAMPLES=2 KAFKA_METRICS_REPORTERS=com.airbnb.kafka.kafka08.StatsdMetricsReporter KAFKA_METRICS_SAMPLE_WINDOW_MS=30000 KAFKA_MIN_INSYNC_REPLICAS=1 KAFKA_NUM_IO_THREADS=8 KAFKA_NUM_NETWORK_THREADS=3 KAFKA_NUM_PARTITIONS=1 KAFKA_NUM_RECOVERY_THREADS_PER_DATA_DIR=1 KAFKA_NUM_REPLICA_FETCHERS=1 KAFKA_OFFSETS_COMMIT_REQUIRED_ACKS=-1 KAFKA_OFFSETS_COMMIT_TIMEOUT_MS=5000 KAFKA_OFFSETS_LOAD_BUFFER_SIZE=5242880 KAFKA_OFFSETS_RETENTION_CHECK_INTERVAL_MS=600000 KAFKA_OFFSETS_RETENTION_MINUTES=1440 KAFKA_OFFSETS_TOPIC_COMPRESSION_CODEC=0 KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS=50 KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=3 KAFKA_OFFSETS_TOPIC_SEGMENT_BYTES=104857600 KAFKA_OFFSET_METADATA_MAX_BYTES=4096 KAFKA_PRODUCER_PURGATORY_PURGE_INTERVAL_REQUESTS=1000 KAFKA_QUEUED_MAX_REQUESTS=500 KAFKA_QUEUED_MAX_REQUEST_BYTES=-1 KAFKA_QUOTA_CONSUMER_DEFAULT=9223372036854775807 KAFKA_QUOTA_PRODUCER_DEFAULT=9223372036854775807 KAFKA_QUOTA_WINDOW_NUM=11 KAFKA_QUOTA_WINDOW_SIZE_SECONDS=1 KAFKA_REPLICATION_QUOTA_WINDOW_NUM=11 KAFKA_REPLICATION_QUOTA_WINDOW_SIZE_SECONDS=1 KAFKA_REPLICA_FETCH_BACKOFF_MS=1000 KAFKA_REPLICA_FETCH_MAX_BYTES=1048576 KAFKA_REPLICA_FETCH_MIN_BYTES=1 KAFKA_REPLICA_FETCH_RESPONSE_MAX_BYTES=10485760 KAFKA_REPLICA_FETCH_WAIT_MAX_MS=500 KAFKA_REPLICA_HIGH_WATERMARK_CHECKPOINT_INTERVAL_MS=5000 KAFKA_REPLICA_LAG_TIME_MAX_MS=10000 KAFKA_REPLICA_SOCKET_RECEIVE_BUFFER_BYTES=65536 KAFKA_REPLICA_SOCKET_TIMEOUT_MS=30000 KAFKA_REQUEST_TIMEOUT_MS=120000 KAFKA_RESERVED_BROKER_MAX_ID=1000 KAFKA_SOCKET_RECEIVE_BUFFER_BYTES=102400 KAFKA_SOCKET_REQUEST_MAX_BYTES=104857600 KAFKA_SOCKET_SEND_BUFFER_BYTES=102400 KAFKA_SSL_ENDPOINT_IDENTIFICATION_ENABLED=true KAFKA_TRANSACTIONAL_ID_EXPIRATION_MS=604800000 KAFKA_TRANSACTION_ABORT_TIMED_OUT_TRANSACTION_CLEANUP_INTERVAL_MS=60000 KAFKA_TRANSACTION_MAX_TIMEOUT_MS=900000 KAFKA_TRANSACTION_REMOVE_EXPIRED_TRANSACTION_CLEANUP_INTERVAL_MS=3600000 KAFKA_TRANSACTION_STATE_LOG_LOAD_BUFFER_SIZE=5242880 KAFKA_TRANSACTION_STATE_LOG_MIN_ISR=2 KAFKA_TRANSACTION_STATE_LOG_NUM_PARTITIONS=50 KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=3 KAFKA_TRANSACTION_STATE_LOG_SEGMENT_BYTES=104857600 KAFKA_UNCLEAN_LEADER_ELECTION_ENABLE=false KAFKA_ZOOKEEPER_SESSION_TIMEOUT_MS=12000 KAFKA_ZOOKEEPER_SYNC_TIME_MS=2000 KAFKA_ZOOKEEPER_URI=zookeeper-0-server.prodkafkazk01.autoip.dcos.thisdcos.directory:1150,zookeeper-1-server.prodkafkazk01.autoip.dcos.thisdcos.directory:1150,zookeeper-2-server.prodkafkazk01.autoip.dcos.thisdcos.directory:1150,zookeeper-3-server.prodkafkazk01.autoip.dcos.thisdcos.directory:1150,zookeeper-4-server.prodkafkazk01.autoip.dcos.thisdcos.directory:1150/prod/xg/dcp/kafka01 LIBPROCESS_IP=10.75.148.1 MESOS_CONTAINER_IP=10.75.148.1 MESOS_SANDBOX=/var/lib/mesos/slave/slaves/e67d2813-79ab-4885-8bd0-3ca45a8960bc-S7/frameworks/ef477552-90f0-4a11-979d-70a7659eecae-0000/executors/prod.xg.dcp.kafka01__kafka__c0e289ae-9a26-4585-bb4d-dd3a28a346fd/runs/3797d666-632e-44ce-9b8f-09e2c226a11f/containers/714ff76b-a31b-4c23-8ce4-664c685715d8 METRIC_REPORTERS=com.airbnb.kafka.kafka09.StatsdMetricsReporter PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin PLACEMENT_REFERENCED_REGION=true PLACEMENT_REFERENCED_ZONE=false POD_INSTANCE_INDEX=4 PWD=/var/lib/mesos/slave/slaves/e67d2813-79ab-4885-8bd0-3ca45a8960bc-S7/frameworks/ef477552-90f0-4a11-979d-70a7659eecae-0000/executors/prod.xg.dcp.kafka01__kafka__c0e289ae-9a26-4585-bb4d-dd3a28a346fd/runs/3797d666-632e-44ce-9b8f-09e2c226a11f/containers/714ff76b-a31b-4c23-8ce4-664c685715d8 SCHEDULER_API_HOSTNAME=kafka01-dcp-xg-prod.marathon.autoip.dcos.thisdcos.directory SCHEDULER_API_PORT=31884 SETUP_HELPER_ADVERTISED_LISTENERS=advertised.listeners=PLAINTEXT://10.75.148.1:9610 SETUP_HELPER_LISTENERS=listeners=PLAINTEXT://10.75.148.1:9610 SETUP_HELPER_SECURITY_INTER_BROKER_PROTOCOL=security.inter.broker.protocol=PLAINTEXT SETUP_HELPER_SUPER_USERS= SHLVL=1 STATSD_UDP_HOST=198.51.100.1 STATSD_UDP_PORT=59954 TASK_NAME=kafka-4-broker _=./bootstrap kafka-4-broker=true 2019/10/09 09:23:31 Resolve disabled via -resolve=false: Skipping host resolution 2019/10/09 09:23:31 Writing rendered 'kafka_2.12-2.1.0/config/server.properties' from envvar 'CONFIG_TEMPLATE_SERVER_PROPERTIES' with the following changes (14749 bytes -> 9601 bytes): # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # see kafka.server.KafkaConfig for additional details and defaults ############################# Server Basics ############################# # The id of the broker. This must be set to a unique integer for each broker. - broker.id={{POD_INSTANCE_INDEX}} + broker.id=4 - {{#PLACEMENT_REFERENCED_ZONE}} - {{#ZONE}} - broker.rack={{ZONE}} - {{/ZONE}} - {{/PLACEMENT_REFERENCED_ZONE}} ############################# Socket Server Settings ############################# # The address the socket server listens on. It will get the value returned from # java.net.InetAddress.getCanonicalHostName() if not configured. # FORMAT: # listeners = security_protocol://host_name:port # EXAMPLE: # listeners = PLAINTEXT://your.host.name:9092 # # Hostname and port the broker will advertise to producers and consumers. If not set, # it uses the value for "listeners" if configured. Otherwise, it will use the value # returned from java.net.InetAddress.getCanonicalHostName(). # # advertised.listeners=PLAINTEXT://your.host.name:9092 - {{SETUP_HELPER_ADVERTISED_LISTENERS}} - {{SETUP_HELPER_LISTENERS}} - - {{#SECURITY_TRANSPORT_ENCRYPTION_ENABLED}} - ############################# TLS Settings ############################# - ssl.keystore.location={{MESOS_SANDBOX}}/broker.keystore - ssl.keystore.password=notsecure - ssl.key.password=notsecure - - ssl.truststore.location={{MESOS_SANDBOX}}/broker.truststore - ssl.truststore.password=notsecure - - ssl.enabled.protocols=TLSv1.2 - - {{#SECURITY_TRANSPORT_ENCRYPTION_CIPHERS}} - ssl.cipher.suites={{SECURITY_TRANSPORT_ENCRYPTION_CIPHERS}} - {{/SECURITY_TRANSPORT_ENCRYPTION_CIPHERS}} - - # If Kerberos is NOT enabled, then SSL authentication can be turned on. - {{^SECURITY_KERBEROS_ENABLED}} - {{#SECURITY_SSL_AUTHENTICATION_ENABLED}} - ssl.client.auth=required - {{/SECURITY_SSL_AUTHENTICATION_ENABLED}} - {{/SECURITY_KERBEROS_ENABLED}} - {{/SECURITY_TRANSPORT_ENCRYPTION_ENABLED}} + advertised.listeners=PLAINTEXT://10.75.148.1:9610 + listeners=PLAINTEXT://10.75.148.1:9610 - {{#SECURITY_KERBEROS_ENABLED}} - ############################# Kerberos Settings ############################# - sasl.mechanism.inter.broker.protocol=GSSAPI - sasl.enabled.mechanisms=GSSAPI - sasl.kerberos.service.name={{SECURITY_KERBEROS_PRIMARY}} - {{/SECURITY_KERBEROS_ENABLED}} ## Inter Broker Protocol - {{SETUP_HELPER_SECURITY_INTER_BROKER_PROTOCOL}} + security.inter.broker.protocol=PLAINTEXT # The number of threads handling network requests - num.network.threads={{KAFKA_NUM_NETWORK_THREADS}} + num.network.threads=3 # The number of threads doing disk I/O - num.io.threads={{KAFKA_NUM_IO_THREADS}} + num.io.threads=8 # The send buffer (SO_SNDBUF) used by the socket server - socket.send.buffer.bytes={{KAFKA_SOCKET_SEND_BUFFER_BYTES}} + socket.send.buffer.bytes=102400 # The receive buffer (SO_RCVBUF) used by the socket server - socket.receive.buffer.bytes={{KAFKA_SOCKET_RECEIVE_BUFFER_BYTES}} + socket.receive.buffer.bytes=102400 # The maximum size of a request that the socket server will accept (protection against OOM) - socket.request.max.bytes={{KAFKA_SOCKET_REQUEST_MAX_BYTES}} + socket.request.max.bytes=104857600 - {{#SECURITY_AUTHORIZATION_ENABLED}} - ############################# Authorization Settings ############################# - authorizer.class.name=kafka.security.auth.SimpleAclAuthorizer - super.users={{SETUP_HELPER_SUPER_USERS}} - allow.everyone.if.no.acl.found={{SECURITY_AUTHORIZATION_ALLOW_EVERYONE_IF_NO_ACL_FOUND}} - {{^SECURITY_KERBEROS_ENABLED}} - {{#SECURITY_SSL_AUTHENTICATION_ENABLED}} - principal.builder.class=de.thmshmm.kafka.CustomPrincipalBuilder - {{/SECURITY_SSL_AUTHENTICATION_ENABLED}} - {{/SECURITY_KERBEROS_ENABLED}} - {{/SECURITY_AUTHORIZATION_ENABLED}} ############################# Log Basics ############################# # A comma separated list of directories under which to store log files - log.dirs={{KAFKA_DISK_PATH}}/broker-{{POD_INSTANCE_INDEX}} + log.dirs=kafka-broker-data/broker-4 # The default number of log partitions per topic. More partitions allow greater # parallelism for consumption, but this will also result in more files across # the brokers. - num.partitions={{KAFKA_NUM_PARTITIONS}} + num.partitions=1 # The number of threads per data directory to be used for log recovery at startup and flushing at shutdown. # This value is recommended to be increased for installations with data dirs located in RAID array. - num.recovery.threads.per.data.dir={{KAFKA_NUM_RECOVERY_THREADS_PER_DATA_DIR}} + num.recovery.threads.per.data.dir=1 ############################# Log Flush Policy ############################# # Messages are immediately written to the filesystem but by default we only fsync() to sync # the OS cache lazily. The following configurations control the flush of data to disk. # There are a few important trade-offs here: # 1. Durability: Unflushed data may be lost if you are not using replication. # 2. Latency: Very large flush intervals may lead to latency spikes when the flush does occur as there will be a lot of data to flush. # 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to exceessive seeks. # The settings below allow one to configure the flush policy to flush data after a period of time or # every N messages (or both). This can be done globally and overridden on a per-topic basis. # The number of messages to accept before forcing a flush of data to disk - log.flush.interval.messages={{KAFKA_LOG_FLUSH_INTERVAL_MESSAGES}} + log.flush.interval.messages=9223372036854775807 - {{#KAFKA_LOG_FLUSH_INTERVAL_MS}} - log.flush.interval.ms={{KAFKA_LOG_FLUSH_INTERVAL_MS}} - {{/KAFKA_LOG_FLUSH_INTERVAL_MS}} ############################# Log Retention Policy ############################# # The following configurations control the disposal of log segments. The policy can # be set to delete segments after a period of time, or after a given size has accumulated. # A segment will be deleted whenever *either* of these criteria are met. Deletion always happens # from the end of the log. # The minimum age of a log file to be eligible for deletion - {{#KAFKA_LOG_RETENTION_MS}} - log.retention.ms={{KAFKA_LOG_RETENTION_MS}} - {{/KAFKA_LOG_RETENTION_MS}} - {{#KAFKA_LOG_RETENTION_MINUTES}} - log.retention.minutes={{KAFKA_LOG_RETENTION_MINUTES}} - {{/KAFKA_LOG_RETENTION_MINUTES}} - log.retention.hours={{KAFKA_LOG_RETENTION_HOURS}} + log.retention.hours=168 # A size-based retention policy for logs. Segments are pruned from the log as long as the remaining # segments don't drop below log.retention.bytes. - log.retention.bytes={{KAFKA_LOG_RETENTION_BYTES}} + log.retention.bytes=-1 # The maximum size of a log segment file. When this size is reached a new log segment will be created. - log.segment.bytes={{KAFKA_LOG_SEGMENT_BYTES}} + log.segment.bytes=1073741824 # The interval at which log segments are checked to see if they can be deleted according # to the retention policies - log.retention.check.interval.ms={{KAFKA_LOG_RETENTION_CHECK_INTERVAL_MS}} + log.retention.check.interval.ms=300000 ############################# Zookeeper ############################# # Zookeeper connection string (see zookeeper docs for details). # This is a comma separated host:port pairs, each corresponding to a zk # server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002". # You can also append an optional chroot string to the urls to specify the # root directory for all kafka znodes. - zookeeper.connect={{KAFKA_ZOOKEEPER_URI}} + zookeeper.connect=zookeeper-0-server.prodkafkazk01.autoip.dcos.thisdcos.directory:1150,zookeeper-1-server.prodkafkazk01.autoip.dcos.thisdcos.directory:1150,zookeeper-2-server.prodkafkazk01.autoip.dcos.thisdcos.directory:1150,zookeeper-3-server.prodkafkazk01.autoip.dcos.thisdcos.directory:1150,zookeeper-4-server.prodkafkazk01.autoip.dcos.thisdcos.directory:1150/prod/xg/dcp/kafka01 # Timeout in ms for connecting to zookeeper zookeeper.connection.timeout.ms=6000 ########################### Addition Parameters ######################## - external.kafka.statsd.port={{STATSD_UDP_PORT}} - external.kafka.statsd.host={{STATSD_UDP_HOST}} + external.kafka.statsd.port=59954 + external.kafka.statsd.host=198.51.100.1 external.kafka.statsd.reporter.enabled=true external.kafka.statsd.tag.enabled=true external.kafka.statsd.metrics.exclude_regex= - auto.create.topics.enable={{KAFKA_AUTO_CREATE_TOPICS_ENABLE}} - auto.leader.rebalance.enable={{KAFKA_AUTO_LEADER_REBALANCE_ENABLE}} + auto.create.topics.enable=false + auto.leader.rebalance.enable=true - background.threads={{KAFKA_BACKGROUND_THREADS}} + background.threads=10 - compression.type={{KAFKA_COMPRESSION_TYPE}} + compression.type=producer - connections.max.idle.ms={{KAFKA_CONNECTIONS_MAX_IDLE_MS}} + connections.max.idle.ms=600000 - controlled.shutdown.enable={{KAFKA_CONTROLLED_SHUTDOWN_ENABLE}} - controlled.shutdown.max.retries={{KAFKA_CONTROLLED_SHUTDOWN_MAX_RETRIES}} - controlled.shutdown.retry.backoff.ms={{KAFKA_CONTROLLED_SHUTDOWN_RETRY_BACKOFF_MS}} - controller.socket.timeout.ms={{KAFKA_CONTROLLER_SOCKET_TIMEOUT_MS}} + controlled.shutdown.enable=true + controlled.shutdown.max.retries=3 + controlled.shutdown.retry.backoff.ms=5000 + controller.socket.timeout.ms=30000 - default.replication.factor={{KAFKA_DEFAULT_REPLICATION_FACTOR}} + default.replication.factor=3 - delete.topic.enable={{KAFKA_DELETE_TOPIC_ENABLE}} + delete.topic.enable=true - delete.records.purgatory.purge.interval.requests={{KAFKA_DELETE_RECORDS_PURGATORY_PURGE_INTERVAL_REQUESTS}} + delete.records.purgatory.purge.interval.requests=1 - fetch.purgatory.purge.interval.requests={{KAFKA_FETCH_PURGATORY_PURGE_INTERVAL_REQUESTS}} + fetch.purgatory.purge.interval.requests=1000 - group.max.session.timeout.ms={{KAFKA_GROUP_MAX_SESSION_TIMEOUT_MS}} - group.min.session.timeout.ms={{KAFKA_GROUP_MIN_SESSION_TIMEOUT_MS}} - group.initial.rebalance.delay.ms={{KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS}} + group.max.session.timeout.ms=300000 + group.min.session.timeout.ms=6000 + group.initial.rebalance.delay.ms=3000 - inter.broker.protocol.version={{KAFKA_INTER_BROKER_PROTOCOL_VERSION}} + inter.broker.protocol.version=2.1 - leader.imbalance.check.interval.seconds={{KAFKA_LEADER_IMBALANCE_CHECK_INTERVAL_SECONDS}} - leader.imbalance.per.broker.percentage={{KAFKA_LEADER_IMBALANCE_PER_BROKER_PERCENTAGE}} + leader.imbalance.check.interval.seconds=300 + leader.imbalance.per.broker.percentage=10 - log.cleaner.backoff.ms={{KAFKA_LOG_CLEANER_BACKOFF_MS}} - log.cleaner.dedupe.buffer.size={{KAFKA_LOG_CLEANER_DEDUPE_BUFFER_SIZE}} - log.cleaner.delete.retention.ms={{KAFKA_LOG_CLEANER_DELETE_RETENTION_MS}} - log.cleaner.enable={{KAFKA_LOG_CLEANER_ENABLE}} - log.cleaner.io.buffer.load.factor={{KAFKA_LOG_CLEANER_IO_BUFFER_LOAD_FACTOR}} - log.cleaner.io.buffer.size={{KAFKA_LOG_CLEANER_IO_BUFFER_SIZE}} - log.cleaner.io.max.bytes.per.second={{KAFKA_LOG_CLEANER_IO_MAX_BYTES_PER_SECOND}} - log.cleaner.min.cleanable.ratio={{KAFKA_LOG_CLEANER_MIN_CLEANABLE_RATIO}} - log.cleaner.min.compaction.lag.ms={{KAFKA_LOG_CLEANER_MIN_COMPACTION_LAG_MS}} - log.cleaner.threads={{KAFKA_LOG_CLEANER_THREADS}} - log.cleanup.policy={{KAFKA_LOG_CLEANUP_POLICY}} + log.cleaner.backoff.ms=15000 + log.cleaner.dedupe.buffer.size=134217728 + log.cleaner.delete.retention.ms=86400000 + log.cleaner.enable=true + log.cleaner.io.buffer.load.factor=0.9 + log.cleaner.io.buffer.size=524288 + log.cleaner.io.max.bytes.per.second=179769313486231570000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 + log.cleaner.min.cleanable.ratio=0.5 + log.cleaner.min.compaction.lag.ms=0 + log.cleaner.threads=1 + log.cleanup.policy=delete - log.flush.offset.checkpoint.interval.ms={{KAFKA_LOG_FLUSH_OFFSET_CHECKPOINT_INTERVAL_MS}} - log.flush.scheduler.interval.ms={{KAFKA_LOG_FLUSH_SCHEDULER_INTERVAL_MS}} - log.flush.start.offset.checkpoint.interval.ms={{KAFKA_LOG_FLUSH_START_OFFSET_CHECKPOINT_INTERVAL_MS}} + log.flush.offset.checkpoint.interval.ms=60000 + log.flush.scheduler.interval.ms=9223372036854775807 + log.flush.start.offset.checkpoint.interval.ms=60000 - log.index.interval.bytes={{KAFKA_LOG_INDEX_INTERVAL_BYTES}} - log.index.size.max.bytes={{KAFKA_LOG_INDEX_SIZE_MAX_BYTES}} + log.index.interval.bytes=4096 + log.index.size.max.bytes=10485760 - log.message.format.version={{KAFKA_LOG_MESSAGE_FORMAT_VERSION}} + log.message.format.version=2.1 - log.preallocate={{KAFKA_LOG_PREALLOCATE}} + log.preallocate=false - {{#KAFKA_LOG_ROLL_MS}} - log.roll.ms={{KAFKA_LOG_ROLL_MS}} - {{/KAFKA_LOG_ROLL_MS}} - log.roll.hours={{KAFKA_LOG_ROLL_HOURS}} - {{#KAFKA_LOG_ROLL_JITTER_MS}} - log.roll.jitter.ms={{KAFKA_LOG_ROLL_JITTER_MS}} - {{/KAFKA_LOG_ROLL_JITTER_MS}} - log.roll.jitter.hours={{KAFKA_LOG_ROLL_JITTER_HOURS}} + log.roll.hours=168 + log.roll.jitter.hours=0 - log.segment.delete.delay.ms={{KAFKA_LOG_SEGMENT_DELETE_DELAY_MS}} + log.segment.delete.delay.ms=60000 - max.connections.per.ip.overrides={{KAFKA_MAX_CONNECTIONS_PER_IP_OVERRIDES}} - max.connections.per.ip={{KAFKA_MAX_CONNECTIONS_PER_IP}} + max.connections.per.ip.overrides= + max.connections.per.ip=2147483647 - message.max.bytes={{KAFKA_MESSAGE_MAX_BYTES}} + message.max.bytes=1000012 - kafka.metrics.reporters={{KAFKA_METRICS_REPORTERS}} - metric.reporters={{METRIC_REPORTERS}} - metrics.num.samples={{KAFKA_METRICS_NUM_SAMPLES}} - metrics.sample.window.ms={{KAFKA_METRICS_SAMPLE_WINDOW_MS}} + kafka.metrics.reporters=com.airbnb.kafka.kafka08.StatsdMetricsReporter + metric.reporters=com.airbnb.kafka.kafka09.StatsdMetricsReporter + metrics.num.samples=2 + metrics.sample.window.ms=30000 - min.insync.replicas={{KAFKA_MIN_INSYNC_REPLICAS}} + min.insync.replicas=1 - num.replica.fetchers={{KAFKA_NUM_REPLICA_FETCHERS}} + num.replica.fetchers=1 - offset.metadata.max.bytes={{KAFKA_OFFSET_METADATA_MAX_BYTES}} - offsets.commit.required.acks={{KAFKA_OFFSETS_COMMIT_REQUIRED_ACKS}} - offsets.commit.timeout.ms={{KAFKA_OFFSETS_COMMIT_TIMEOUT_MS}} - offsets.load.buffer.size={{KAFKA_OFFSETS_LOAD_BUFFER_SIZE}} - offsets.retention.check.interval.ms={{KAFKA_OFFSETS_RETENTION_CHECK_INTERVAL_MS}} - offsets.retention.minutes={{KAFKA_OFFSETS_RETENTION_MINUTES}} - offsets.topic.compression.codec={{KAFKA_OFFSETS_TOPIC_COMPRESSION_CODEC}} - offsets.topic.num.partitions={{KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS}} - offsets.topic.replication.factor={{KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR}} - offsets.topic.segment.bytes={{KAFKA_OFFSETS_TOPIC_SEGMENT_BYTES}} + offset.metadata.max.bytes=4096 + offsets.commit.required.acks=-1 + offsets.commit.timeout.ms=5000 + offsets.load.buffer.size=5242880 + offsets.retention.check.interval.ms=600000 + offsets.retention.minutes=1440 + offsets.topic.compression.codec=0 + offsets.topic.num.partitions=50 + offsets.topic.replication.factor=3 + offsets.topic.segment.bytes=104857600 - producer.purgatory.purge.interval.requests={{KAFKA_PRODUCER_PURGATORY_PURGE_INTERVAL_REQUESTS}} + producer.purgatory.purge.interval.requests=1000 - queued.max.requests={{KAFKA_QUEUED_MAX_REQUESTS}} - queued.max.request.bytes={{KAFKA_QUEUED_MAX_REQUEST_BYTES}} - quota.consumer.default={{KAFKA_QUOTA_CONSUMER_DEFAULT}} - quota.producer.default={{KAFKA_QUOTA_PRODUCER_DEFAULT}} - quota.window.num={{KAFKA_QUOTA_WINDOW_NUM}} - quota.window.size.seconds={{KAFKA_QUOTA_WINDOW_SIZE_SECONDS}} + queued.max.requests=500 + queued.max.request.bytes=-1 + quota.consumer.default=9223372036854775807 + quota.producer.default=9223372036854775807 + quota.window.num=11 + quota.window.size.seconds=1 - replica.fetch.backoff.ms={{KAFKA_REPLICA_FETCH_BACKOFF_MS}} - replica.fetch.max.bytes={{KAFKA_REPLICA_FETCH_MAX_BYTES}} - replica.fetch.min.bytes={{KAFKA_REPLICA_FETCH_MIN_BYTES}} - replica.fetch.response.max.bytes={{KAFKA_REPLICA_FETCH_RESPONSE_MAX_BYTES}} - replica.fetch.wait.max.ms={{KAFKA_REPLICA_FETCH_WAIT_MAX_MS}} - replica.high.watermark.checkpoint.interval.ms={{KAFKA_REPLICA_HIGH_WATERMARK_CHECKPOINT_INTERVAL_MS}} - replica.lag.time.max.ms={{KAFKA_REPLICA_LAG_TIME_MAX_MS}} - replica.socket.receive.buffer.bytes={{KAFKA_REPLICA_SOCKET_RECEIVE_BUFFER_BYTES}} - replica.socket.timeout.ms={{KAFKA_REPLICA_SOCKET_TIMEOUT_MS}} + replica.fetch.backoff.ms=1000 + replica.fetch.max.bytes=1048576 + replica.fetch.min.bytes=1 + replica.fetch.response.max.bytes=10485760 + replica.fetch.wait.max.ms=500 + replica.high.watermark.checkpoint.interval.ms=5000 + replica.lag.time.max.ms=10000 + replica.socket.receive.buffer.bytes=65536 + replica.socket.timeout.ms=30000 - replication.quota.window.num={{KAFKA_REPLICATION_QUOTA_WINDOW_NUM}} - replication.quota.window.size.seconds={{KAFKA_REPLICATION_QUOTA_WINDOW_SIZE_SECONDS}} + replication.quota.window.num=11 + replication.quota.window.size.seconds=1 - request.timeout.ms={{KAFKA_REQUEST_TIMEOUT_MS}} + request.timeout.ms=120000 - reserved.broker.max.id={{KAFKA_RESERVED_BROKER_MAX_ID}} + reserved.broker.max.id=1000 - {{#KAFKA_SSL_ENDPOINT_IDENTIFICATION_ENABLED}} ssl.endpoint.identification.algorithm=HTTPS - {{/KAFKA_SSL_ENDPOINT_IDENTIFICATION_ENABLED}} - {{^KAFKA_SSL_ENDPOINT_IDENTIFICATION_ENABLED}} - ssl.endpoint.identification.algorithm= - {{/KAFKA_SSL_ENDPOINT_IDENTIFICATION_ENABLED}} - transaction.state.log.segment.bytes={{KAFKA_TRANSACTION_STATE_LOG_SEGMENT_BYTES}} - transaction.remove.expired.transaction.cleanup.interval.ms={{KAFKA_TRANSACTION_REMOVE_EXPIRED_TRANSACTION_CLEANUP_INTERVAL_MS}} - transaction.max.timeout.ms={{KAFKA_TRANSACTION_MAX_TIMEOUT_MS}} - transaction.state.log.num.partitions={{KAFKA_TRANSACTION_STATE_LOG_NUM_PARTITIONS}} - transaction.abort.timed.out.transaction.cleanup.interval.ms={{KAFKA_TRANSACTION_ABORT_TIMED_OUT_TRANSACTION_CLEANUP_INTERVAL_MS}} - transaction.state.log.load.buffer.size={{KAFKA_TRANSACTION_STATE_LOG_LOAD_BUFFER_SIZE}} - transactional.id.expiration.ms={{KAFKA_TRANSACTIONAL_ID_EXPIRATION_MS}} - transaction.state.log.replication.factor={{KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR}} - transaction.state.log.min.isr={{KAFKA_TRANSACTION_STATE_LOG_MIN_ISR}} + transaction.state.log.segment.bytes=104857600 + transaction.remove.expired.transaction.cleanup.interval.ms=3600000 + transaction.max.timeout.ms=900000 + transaction.state.log.num.partitions=50 + transaction.abort.timed.out.transaction.cleanup.interval.ms=60000 + transaction.state.log.load.buffer.size=5242880 + transactional.id.expiration.ms=604800000 + transaction.state.log.replication.factor=3 + transaction.state.log.min.isr=2 - unclean.leader.election.enable={{KAFKA_UNCLEAN_LEADER_ELECTION_ENABLE}} + unclean.leader.election.enable=false - zookeeper.session.timeout.ms={{KAFKA_ZOOKEEPER_SESSION_TIMEOUT_MS}} - zookeeper.sync.time.ms={{KAFKA_ZOOKEEPER_SYNC_TIME_MS}} + zookeeper.session.timeout.ms=12000 + zookeeper.sync.time.ms=2000 ######################################################################## 2019/10/09 09:23:33 Successfully installed the certificate. 2019/10/09 09:23:33 SDK Bootstrap successful.