Type:
Bug
Status:
Patch Available
Priority:
Major
Resolution:
Unresolved
Affects Version/s:
None
Fix Version/s:
None
Component/s:
None
To decommission a dead DN, the complete logic should be
Dead, NORMAL -> Dead, DECOMMISSION_INPROGRESS -> Dead, DECOMMISSIONED
Currently logic:
If a DN is already dead when DECOMMISSIONING starts, it becomes DECOMMISSIONED immediately. DECOMMISSION_INPROGRESS is skipped.
This logic is introduced by HDFS-7374 which is made because of HDFS-6791 .
HDFS-6791 keeps the node in DECOMMISSION_INPROGRESS state if the node becomes dead during decommission, which could possibly make a dead DN in DECOMMISSION_INPROGRESS forever, if the DN could never be alive.
However, putting a dead DN to DECOMMISSIONED directly is not secure. For example, 3 DN of the same block are dead at the same time, then the administrator wants to decommission them unintentionally. Namenode should check first before transit them to DECOMMISSIONED. Otherwise, it would be a data loss.
In this case, all 3 DNs can't become DECOMMISSIONED which is by design. The administrator needs to do some manual intervention, either repair the dead machine or service or recover the data before take action on them.
This change is to add Dead, DECOMMISSION_INPROGRESS back.
1. Dead normal DN is in DECOMMISSION_INPROGRESS first.
2. NN checks pendingReplicationBlocksCount and underReplicatedBlocksCount are both 0.
3. Transit the dead DN to DECOMMISSIONED.
2 is implemented by HDFS-7409 , which adds a check to allow dead nodes in DECOMMISSION_IN_PROGRESS to progress to DECOMMISSIONED state if all files on the filesystem are fully-replicated.
{"report":{"fcp":3122.1999999284744,"ttfb":481.7999999523163,"pageVisibility":"visible","entityId":13348994,"key":"jira.project.issue.view-issue","isInitial":true,"threshold":1000,"elementTimings":{},"userDeviceMemory":8,"userDeviceProcessors":16,"apdex":0.5,"journeyId":"6e1e9718-52e4-4f2e-bb75-0048ea10e567","navigationType":0,"readyForUser":3258.2999999523163,"redirectCount":0,"resourceLoadedEnd":2177.6999999284744,"resourceLoadedStart":486.7999999523163,"resourceTiming":[{"duration":540.3999999761581,"initiatorType":"link","name":"https://issues.apache.org/jira/s/b62489a2eaac59d9b8a093c1a51d034f-CDN/xd97tr/820010/13pdxe5/49fa3aa3d35a2cc689cbf274e66cc41a/_/download/contextbatch/css/_super/batch.css","startTime":486.7999999523163,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":486.7999999523163,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":1027.1999999284744,"responseStart":0,"secureConnectionStart":0},{"duration":540.3999999761581,"initiatorType":"link","name":"https://issues.apache.org/jira/s/56490edcf9d54e35149505f78cca6a47-CDN/xd97tr/820010/13pdxe5/72cb823bcc50211a60c1ebe830467cae/_/download/contextbatch/css/jira.browse.project,jira.view.issue,project.issue.navigator,atl.general,atl.global,jira.global,jira.general,-_super/batch.css?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&richediton=true&slack-enabled=true","startTime":487.10000002384186,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":487.10000002384186,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":1027.5,"responseStart":0,"secureConnectionStart":0},{"duration":615.8999999761581,"initiatorType":"script","name":"https://issues.apache.org/jira/s/5263129088916436ab9aeb2417075b3f-CDN/xd97tr/820010/13pdxe5/49fa3aa3d35a2cc689cbf274e66cc41a/_/download/contextbatch/js/_super/batch.js?locale=en-UK","startTime":487.2999999523163,"connectEnd":487.2999999523163,"connectStart":487.2999999523163,"domainLookupEnd":487.2999999523163,"domainLookupStart":487.2999999523163,"fetchStart":487.2999999523163,"redirectEnd":0,"redirectStart":0,"requestStart":487.2999999523163,"responseEnd":1103.1999999284744,"responseStart":1103.1999999284744,"secureConnectionStart":487.2999999523163},{"duration":1406.8999999761581,"initiatorType":"script","name":"https://issues.apache.org/jira/s/611c208bd094adb71a6f4f3e7f6fff3d-CDN/xd97tr/820010/13pdxe5/72cb823bcc50211a60c1ebe830467cae/_/download/contextbatch/js/jira.browse.project,jira.view.issue,project.issue.navigator,atl.general,atl.global,jira.global,jira.general,-_super/batch.js?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&locale=en-UK&richediton=true&slack-enabled=true","startTime":487.39999997615814,"connectEnd":487.39999997615814,"connectStart":487.39999997615814,"domainLookupEnd":487.39999997615814,"domainLookupStart":487.39999997615814,"fetchStart":487.39999997615814,"redirectEnd":0,"redirectStart":0,"requestStart":487.39999997615814,"responseEnd":1894.2999999523163,"responseStart":1894.2999999523163,"secureConnectionStart":487.39999997615814},{"duration":619.1999999284744,"initiatorType":"script","name":"https://issues.apache.org/jira/s/d41d8cd98f00b204e9800998ecf8427e-CDN/xd97tr/820010/13pdxe5/1.0/_/download/batch/jira.webresources:calendar-en/jira.webresources:calendar-en.js","startTime":487.60000002384186,"connectEnd":487.60000002384186,"connectStart":487.60000002384186,"domainLookupEnd":487.60000002384186,"domainLookupStart":487.60000002384186,"fetchStart":487.60000002384186,"redirectEnd":0,"redirectStart":0,"requestStart":487.60000002384186,"responseEnd":1106.7999999523163,"responseStart":1106.7999999523163,"secureConnectionStart":487.60000002384186},{"duration":619.9000000953674,"initiatorType":"script","name":"https://issues.apache.org/jira/s/d41d8cd98f00b204e9800998ecf8427e-CDN/xd97tr/820010/13pdxe5/1.0/_/download/batch/jira.webresources:calendar-localisation-moment/jira.webresources:calendar-localisation-moment.js","startTime":487.6999999284744,"connectEnd":487.6999999284744,"connectStart":487.6999999284744,"domainLookupEnd":487.6999999284744,"domainLookupStart":487.6999999284744,"fetchStart":487.6999999284744,"redirectEnd":0,"redirectStart":0,"requestStart":487.6999999284744,"responseEnd":1107.6000000238419,"responseStart":1107.6000000238419,"secureConnectionStart":487.6999999284744},{"duration":625.3999999761581,"initiatorType":"link","name":"https://issues.apache.org/jira/s/981f587853769311cda7c3b845131a06-CDN/xd97tr/820010/13pdxe5/cb5a5495a038c0744457f25821ba9ee8/_/download/contextbatch/css/jira.global.look-and-feel,-_super/batch.css","startTime":487.89999997615814,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":487.89999997615814,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":1113.2999999523163,"responseStart":0,"secureConnectionStart":0},{"duration":620.6999999284744,"initiatorType":"script","name":"https://issues.apache.org/jira/rest/api/1.0/shortcuts/820010/5840efff50357da9055d4714dc0713f/shortcuts.js?context=issuenavigation&context=issueaction","startTime":488,"connectEnd":488,"connectStart":488,"domainLookupEnd":488,"domainLookupStart":488,"fetchStart":488,"redirectEnd":0,"redirectStart":0,"requestStart":488,"responseEnd":1108.6999999284744,"responseStart":1108.6999999284744,"secureConnectionStart":488},{"duration":609.8999999761581,"initiatorType":"link","name":"https://issues.apache.org/jira/s/3ac36323ba5e4eb0af2aa7ac7211b4bb-CDN/xd97tr/820010/13pdxe5/efa42a25652b26dfd802540c024826b3/_/download/contextbatch/css/com.atlassian.jira.projects.sidebar.init,-_super,-jira.view.issue,-project.issue.navigator/batch.css?jira.create.linked.issue=true&richediton=true","startTime":503.39999997615814,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":503.39999997615814,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":1113.2999999523163,"responseStart":0,"secureConnectionStart":0},{"duration":604.1000000238419,"initiatorType":"script","name":"https://issues.apache.org/jira/s/efa8931cd5ac13ed95c56ca8a1dc1967-CDN/xd97tr/820010/13pdxe5/efa42a25652b26dfd802540c024826b3/_/download/contextbatch/js/com.atlassian.jira.projects.sidebar.init,-_super,-jira.view.issue,-project.issue.navigator/batch.js?jira.create.linked.issue=true&locale=en-UK&richediton=true","startTime":506.5,"connectEnd":506.5,"connectStart":506.5,"domainLookupEnd":506.5,"domainLookupStart":506.5,"fetchStart":506.5,"redirectEnd":0,"redirectStart":0,"requestStart":506.5,"responseEnd":1110.6000000238419,"responseStart":1110.6000000238419,"secureConnectionStart":506.5},{"duration":1465.3999999761581,"initiatorType":"script","name":"https://issues.apache.org/jira/s/d41d8cd98f00b204e9800998ecf8427e-CDN/xd97tr/820010/13pdxe5/1.0/_/download/batch/jira.webresources:bigpipe-js/jira.webresources:bigpipe-js.js","startTime":707.5,"connectEnd":707.5,"connectStart":707.5,"domainLookupEnd":707.5,"domainLookupStart":707.5,"fetchStart":707.5,"redirectEnd":0,"redirectStart":0,"requestStart":707.5,"responseEnd":2172.899999976158,"responseStart":2172.899999976158,"secureConnectionStart":707.5},{"duration":1470.0999999046326,"initiatorType":"script","name":"https://issues.apache.org/jira/s/d41d8cd98f00b204e9800998ecf8427e-CDN/xd97tr/820010/13pdxe5/1.0/_/download/batch/jira.webresources:bigpipe-init/jira.webresources:bigpipe-init.js","startTime":707.6000000238419,"connectEnd":707.6000000238419,"connectStart":707.6000000238419,"domainLookupEnd":707.6000000238419,"domainLookupStart":707.6000000238419,"fetchStart":707.6000000238419,"redirectEnd":0,"redirectStart":0,"requestStart":707.6000000238419,"responseEnd":2177.6999999284744,"responseStart":2177.6999999284744,"secureConnectionStart":707.6000000238419},{"duration":545,"initiatorType":"xmlhttprequest","name":"https://issues.apache.org/jira/rest/webResources/1.0/resources","startTime":1361.6999999284744,"connectEnd":1361.6999999284744,"connectStart":1361.6999999284744,"domainLookupEnd":1361.6999999284744,"domainLookupStart":1361.6999999284744,"fetchStart":1361.6999999284744,"redirectEnd":0,"redirectStart":0,"requestStart":1361.6999999284744,"responseEnd":1906.6999999284744,"responseStart":1906.6999999284744,"secureConnectionStart":1361.6999999284744}],"fetchStart":0,"domainLookupStart":0,"domainLookupEnd":0,"connectStart":0,"connectEnd":0,"requestStart":321,"responseStart":482,"responseEnd":701,"domLoading":486,"domInteractive":3514,"domContentLoadedEventStart":3514,"domContentLoadedEventEnd":3580,"domComplete":3994,"loadEventStart":3994,"loadEventEnd":3998,"userAgent":"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)","marks":[{"name":"bigPipe.sidebar-id.start","time":3313.5},{"name":"bigPipe.sidebar-id.end","time":3314.2999999523163},{"name":"bigPipe.activity-panel-pipe-id.start","time":3314.399999976158},{"name":"bigPipe.activity-panel-pipe-id.end","time":3315.6999999284744},{"name":"activityTabFullyLoaded","time":3601.399999976158}],"measures":[],"correlationId":"c2646e955cc396","effectiveType":"4g","downlink":10,"rtt":0,"serverDuration":90,"dbReadsTimeInMs":4,"dbConnsTimeInMs":12,"applicationHash":"ace47f9899e9ee25d7157d59aa17ab06aee30d3d","experiments":[]}}
cc mingma, andrew.wang, zhz ,elgoiri