Minor Description
Using kafka-configs should validate dynamic configurations before applying. It is possible to send a file with invalid configurations.
For example a file containing the following:
{
"routes": {
"crn:///kafka=*": {
"management": {
"allowed": "confluent-audit-log-events_audit",
"denied": "confluent-audit-log-events-denied"
},
"describe": {
"allowed": "",
"denied": "confluent-audit-log-events-denied"
},
"authentication": {
"allowed": "confluent-audit-log-events_audit",
"denied": "confluent-audit-log-events-denied-authn"
},
"authorize": {
"allowed": "confluent-audit-log-events_audit",
"denied": "confluent-audit-log-events-denied-authz"
},
"interbroker": {
"allowed": "",
"denied": ""
}
},
"crn:///kafka=*/group=*": {
"consume": {
"allowed": "confluent-audit-log-events_audit",
"denied": "confluent-audit-log-events"
}
},
"crn:///kafka=*/topic=*": {
"produce": {
"allowed": "confluent-audit-log-events_audit",
"denied": "confluent-audit-log-events"
},
"consume": {
"allowed": "confluent-audit-log-events_audit",
"denied": "confluent-audit-log-events"
}
}
},
"destinations": {
"topics": {
"confluent-audit-log-events": {
"retention_ms": 7776000000
},
"confluent-audit-log-events-denied": {
"retention_ms": 7776000000
},
"confluent-audit-log-events-denied-authn": {
"retention_ms": 7776000000
},
"confluent-audit-log-events-denied-authz": {
"retention_ms": 7776000000
},
"confluent-audit-log-events_audit": {
"retention_ms": 7776000000
}
}
},
"default_topics": {
"allowed": "confluent-audit-log-events_audit",
"denied": "confluent-audit-log-events"
},
"excluded_principals": [
"User:schemaregistryUser",
"User:ANONYMOUS",
"User:appSA",
"User:admin",
"User:connectAdmin",
"User:connectorSubmitter",
"User:connectorSA",
"User:schemaregistryUser",
"User:ksqlDBAdmin",
"User:ksqlDBUser",
"User:controlCenterAndKsqlDBServer",
"User:controlcenterAdmin",
"User:restAdmin",
"User:appSA",
"User:clientListen",
"User:superUser"
]
}
kafka-configs --bootstrap-server $KAFKA_BOOTSTRAP --entity-type brokers --entity-default --alter --add-config-file audit-log.json
Yields the following dynamic configs:
Default configs for brokers in the cluster are: "destinations"=null sensitive=true synonyms={DYNAMIC_DEFAULT_BROKER_CONFIG:"destinations"=null} "confluent-audit-log-events-denied-authn"=null sensitive=true synonyms={DYNAMIC_DEFAULT_BROKER_CONFIG:"confluent-audit-log-events-denied-authn"=null} "routes"=null sensitive=true synonyms={DYNAMIC_DEFAULT_BROKER_CONFIG:"routes"=null} "User=null sensitive=true synonyms={DYNAMIC_DEFAULT_BROKER_CONFIG:"User=null} },=null sensitive=true synonyms={DYNAMIC_DEFAULT_BROKER_CONFIG:},=null} "excluded_principals"=null sensitive=true synonyms={DYNAMIC_DEFAULT_BROKER_CONFIG:"excluded_principals"=null} "confluent-audit-log-events_audit"=null sensitive=true synonyms={DYNAMIC_DEFAULT_BROKER_CONFIG:"confluent-audit-log-events_audit"=null} "authorize"=null sensitive=true synonyms={DYNAMIC_DEFAULT_BROKER_CONFIG:"authorize"=null} "default_topics"=null sensitive=true synonyms={DYNAMIC_DEFAULT_BROKER_CONFIG:"default_topics"=null} "topics"=null sensitive=true synonyms={DYNAMIC_DEFAULT_BROKER_CONFIG:"topics"=null} ]=null sensitive=true synonyms={DYNAMIC_DEFAULT_BROKER_CONFIG:]=null} "interbroker"=null sensitive=true synonyms={DYNAMIC_DEFAULT_BROKER_CONFIG:"interbroker"=null} "produce"=null sensitive=true synonyms={DYNAMIC_DEFAULT_BROKER_CONFIG:"produce"=null} "denied"=null sensitive=true synonyms={DYNAMIC_DEFAULT_BROKER_CONFIG:"denied"=null} "confluent-audit-log-events-denied"=null sensitive=true synonyms={DYNAMIC_DEFAULT_BROKER_CONFIG:"confluent-audit-log-events-denied"=null} "confluent-audit-log-events"=null sensitive=true synonyms={DYNAMIC_DEFAULT_BROKER_CONFIG:"confluent-audit-log-events"=null} "crn=null sensitive=true synonyms={DYNAMIC_DEFAULT_BROKER_CONFIG:"crn=null} "management"=null sensitive=true synonyms={DYNAMIC_DEFAULT_BROKER_CONFIG:"management"=null} "describe"=null sensitive=true synonyms={DYNAMIC_DEFAULT_BROKER_CONFIG:"describe"=null} "allowed"=null sensitive=true synonyms={DYNAMIC_DEFAULT_BROKER_CONFIG:"allowed"=null} "consume"=null sensitive=true synonyms={DYNAMIC_DEFAULT_BROKER_CONFIG:"consume"=null} "confluent-audit-log-events-denied-authz"=null sensitive=true synonyms={DYNAMIC_DEFAULT_BROKER_CONFIG:"confluent-audit-log-events-denied-authz"=null} "retention_ms"=null sensitive=true synonyms={DYNAMIC_DEFAULT_BROKER_CONFIG:"retention_ms"=null} {=null sensitive=true synonyms={DYNAMIC_DEFAULT_BROKER_CONFIG:{=null} "authentication"=null sensitive=true synonyms={DYNAMIC_DEFAULT_BROKER_CONFIG:"authentication"=null} }=null sensitive=true synonyms={DYNAMIC_DEFAULT_BROKER_CONFIG:}=null}
Attempting to remove the dynamic configs via kafka-configs will not allow removal of entries with a comma.
kafka-configs --bootstrap-server $KAFKA_BOOTSTRAP --entity-type brokers --alter --entity-default \ --delete-config '"User' \ --delete-config '"destinations"' \ --delete-config '"confluent-audit-log-events_audit"' \ --delete-config '"authorize"' \ --delete-config '"authentication"' \ --delete-config '"topics"' \ --delete-config '"interbroker"' \ --delete-config '"produce"' \ --delete-config '"allowed"' \ --delete-config '"confluent-audit-log-events_audit"' \ --delete-config '"confluent-audit-log-events-denied-authn"' \ --delete-config '"routes"' \ --delete-config '"excluded_principals"' \ --delete-config '"default_topics"' \ --delete-config '"denied"' \ --delete-config '"confluent-audit-log-events"' \ --delete-config '"confluent-audit-log-events"' \ --delete-config '"confluent-audit-log-events-denied"' \ --delete-config '"management"' \ --delete-config '"describe"' \ --delete-config '"consume"' \ --delete-config '"confluent-audit-log-events-denied-authz"' \ --delete-config '"retention_ms"' \ --delete-config '"crn' \ --delete-config ']' \ --delete-config '{' \ --delete-config '}' \ --delete-config '},' All sensitive broker config entries must be specified for --alter, missing entries: Set(},)
ConfigCommand.scala removes the comma, which blocks the config from removal:
Current workaround is to reset all dynamic configurations with zookeeper-shell:
get /config/brokers/<default> {"version":1,"config":{"\"destinations\"":"{","\"User":"superUser\"","\"confluent-audit-log-events_audit\"":"{","\"authorize\"":"{","\"topics\"":"{","\"interbroker\"":"{","\"produce\"":"{","\"allowed\"":"\"confluent-audit-log-events_audit\",","\"retention_ms\"":"7776000000","\"confluent-audit-log-events-denied-authn\"":"{","\"routes\"":" {","},":"","\"excluded_principals\"":"[","\"default_topics\"":"\{","]":"","\"denied\"":"\"confluent-audit-log-events\"","\"confluent-audit-log-events\"":"{","\"confluent-audit-log-events-denied\"":"{","\"management\"":"{","\"crn":"///kafka=/topic=\": {","\"describe\"":"{","\"consume\"":"{","\"confluent-audit-log-events-denied-authz\"":"{","{":"","\"authentication\"":"{","} ":""}} set /config/brokers/<default> {"version":1,"config":{}}
Since workaround relies on ZooKeeper the workaround would not be an option when using KRaft mode.
Attachments
Issue Links
- links to