Major Description
Under load (up to 24 clients trying to append to a non-existent file in separate encryption zones simultaneously), the KMS returns a 400 for a character that is not present in the path or key name. For example,
IOException: ERROR_APPLICATION: HTTP status [400], message [Illegal character 0xA] at org.apache.hadoop.util.HttpExceptionUtils.validateResponse(HttpExceptionUtils.java:174) at org.apache.hadoop.crypto.key.kms.KMSClientProvider.call(KMSClientProvider.java:540) at org.apache.hadoop.crypto.key.kms.KMSClientProvider.call(KMSClientProvider.java:536) at org.apache.hadoop.crypto.key.kms.KMSClientProvider.call(KMSClientProvider.java:501) at org.apache.hadoop.crypto.key.kms.KMSClientProvider.getMetadata(KMSClientProvider.java:877) at org.apache.hadoop.crypto.key.kms.LoadBalancingKMSClientProvider$13.call(LoadBalancingKMSClientProvider.java:393) at org.apache.hadoop.crypto.key.kms.LoadBalancingKMSClientProvider$13.call(LoadBalancingKMSClientProvider.java:390) at org.apache.hadoop.crypto.key.kms.LoadBalancingKMSClientProvider.doOp(LoadBalancingKMSClientProvider.java:123) at org.apache.hadoop.crypto.key.kms.LoadBalancingKMSClientProvider.getMetadata(LoadBalancingKMSClientProvider.java:390) at org.apache.hadoop.crypto.key.KeyProviderExtension.getMetadata(KeyProviderExtension.java:100) at org.apache.hadoop.hdfs.server.namenode.FSDirEncryptionZoneOp.ensureKeyIsInitialized(FSDirEncryptionZoneOp.java:124) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.createEncryptionZone(FSNamesystem.java:7002) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.createEncryptionZone(NameNodeRpcServer.java:2036) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.createEncryptionZone(ClientNamenodeProtocolServerSideTranslatorPB.java:1448) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:523) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:991) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:869) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:815) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1962) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2675)
I have a number of path/key pairs that were used to trigger this, but I've tried hard-coding one of them, and it seems to work fine:
('Path:', u'/chai_test37791178-e643-4ade-bbea-3a46a368eb01-\u0531\u308b\ufea1\u10d6\ufe8f\uc738\u2116\u05e1\u0562\u0be9\u050e/chai_testdc99561d-c64b-48c2-a1b2-203eecd424b0-\u0718\u05f1\u0133\u06f3\u2083\u316a\u0ec4\u07cb\u22e9\xa5\u07d0\u318d\xbc\u0e01\u02e8\u0277\u316a\u2030\u3113\u0e81\u9fa5/chai_test8f897e3a-fed6-4e5a-ae3e-524c73b760cc-\u05d0\u10fb\u3405\u0105\u2100\u0e0d\u0256\u9fa5\u0baa\u10f5\u0429\u2122\u05d0\u3106\u02b1\u3113\u2014\u2122\u05d0\ud7a3') ('Key:', u'\u2122\u0903\u0501\u308b\u06f3\u2122\u3485\u30c5\u3042\u10fb\u208c\u03b4\u3131\u207e\u1d3d\u0507\ufb17\u30e8\u3129\u3485\u0561\u0561\u02d5\uc738a\u0509\u06f3\u3023\u207e\u20a3\u3042\u07f9\u0256\u0ec4\u2122\u0133\u05f1\u0509\u308b\u05d1\u311d\u02d5\u10d0\u20aa\xa5\u316a\u0718\u0920\ud7a3\u30fe\u3138\u0969\u05e1\u0bc2\u0ed3\u2030\ufea1\u1d02\u309e\u0920\u201c\u20ac\u3405\u30a2\u025e\u20a3\u0e0d\u22e9\u0133\u0be9\u2287\u30fe\u7530\u2122\u07c4\u0baa\u22e9\u208c\u0507\u0e01\u10f5\u2230\u3485\ubb80\xf1\u0449\u02d5\u0133\u311d\u0133\u3485\uac00\u2116\u3113\u0259\u0718\u03e1\u0277\u0ed3\u10fb\u2100\u0e81\u02b1\u2030\u2103\xa5\u10fb\uac00\u311d\u30a2\u3042\u0720\xa5\u02ac\u3023\u050f\u4e00\u061f\u0133\u050fa\u0ec4\u61a8\u30c5\u0969\u0449\u2083\u20aa\u20ac\u061f\u20a3\u201c\u2100\u208c\u9fa5\u01055\u0be9\u2100\xbc\u3113\u0562\u050f\u0920\xf1\u03cd\u0718\u9fa5\u2230\u0459\u0969\u0133\u20a3\u316a\u0903\u3138\u10d6\u2100\u0720\u0507\u03b4\u0718\u2083\u0903\ud7a3\u05e1\ufe8f\u2200\u3129\u0e81\u30c5\u201c\u0259\u311d\u3113\u1d6b\u05e1\u3485\u0449\u2074\u0133\u025e\u1d6b\u02ac\u20a3\u3106\u0718\u1d25\u07cb\xf1\u0256a\u318d\u07cb\u203b\u0449\u0e0d\u0195\u22e9\u0969\u3129\u316a\u0277\u0259\u0e01\u0baa\u1d25\u0195\u0700\u2103\u0e01\ufb01\xa5\u0baa\ufb01\u3485\u0133\u2116\u3131\u3106\u4e00\u0459\u20a2\u07cb\u9fa5\u0459\u03cd\u9fa5\u3106\u10d6\u05e1\u311d\u0259\u2014\u10d0\u0920\u0e0d\u0259\u0920\u20a3\ufb17\u03b4\u61a8\u1d3d\u7530\u0b85\u0ed3\u2230\ufb00\u2074\u0133\ufb4f\u02d5\u3485\ufb17\u30a2\u2116\u1d6b\u0133\ufb00\u30e8\u0be9\xf1\u10d6\u0920\u0700\u0940\u0b85\u0561\u311d\u3005\u0be9\ufe8f\u0940\u0e01\u2116\u05e1\u2122\ufb00\u203b\u026b\u0133\u0e53\u02ac\u9fa5\u0903\u03b4\u0903\u3050\u09035\u61a8!\u10d6\u2014\u2074\u0903\u0449\u025e\u0449!\u30e8\xf1\u0ed3\u2200\u0b85\ufb17\u2083\u2200\u316a\u0449\u3106\u3050\u2103\uac00\u3131\u2030\u0baa\u0449\u20aa\u203b\ufea1\xbc\u0718\u07f9\u0e9c\u0e01\u05e1\u0133\u309e\u05d1\u20a2\ufea1\xa5\u07d0\u0133\u3138\u3131\u20ac\u0969\u0be9\u0105\u0718\u0710\u0e53\u3138\u0459\u0920\u03b4\u0be9\u2030\u0449\u025e\u316a\u0bc2\u0449\ufea1\ufe8f\u4e00\u203b\uac00\u2122\u025b\u2230\u316a\u0920\u04c4\u0562\u0be9\u0baa\u30a2\u02ac\ufea1a\ufb4f\u3405\u03cd') ('Path:', u'/chai_testecfb7a18-904a-4b8d-9e05-77b34c7a3a68-\u0532\ufe8f\u61a8\u0e53\u0195\u30fe\u3fdc\u3485/chai_test12a013d3-78db-454b-a4aa-c09bc9ab1f93-\ufb01\u0ec4\u20aa\u04c3\u0969\u02e8\u2074\ufb01\u0104\u3023\u05e1\u0104\u026b\u2116\u308b\u1d6b\ufb17\u10f5\u050e\u20a2\u3050') ('Key:', u'\u1d6b\u0259\u03cd\u0940\u0e9ca\u06f3a\ufb17\u0903\u2116\u0195\u10f5\u07cb\u3023\u20aaa\ubb80\u0903\u0277\u3005\u30a2\u2122\u3113\u3131\u0baa\u4e00\u05f1\u0561\u30e8\u311d\u0e01\u0ec4\u0562\u309e\u0449\u0501\ud7a3\u1d3d\u0940\u025e\u207e\u04c4\u10d0\u309e\u2014\u026b\u061f\u0509\u0e0d\ufb17\u1d3d\ufb4f\u0459\u02d5\u208c\u30fe\u7530\u07d0\u0ec4\u05f1\u2200\u2103\u05f1\u0b85\u04c4\u208c\u10d6\u07c4\u0718\u07d0\u0562\u7530\u0b85\u30c5\u06f3\u201c\u309e\u0e53\u2200\u02e8\u03e1\u0903\u201c\u20ac\u2287\u026b\u3005\ud7a3\u2200\ufe8f\u10f5\u0710\u0ec4\u0969\u208ca\u04c4\u0509\ud7a3\u0256\u10d0\u02d5\u0bc2\u3023\u2074\u0501\u0905\uac00\u02d5\u2200\uc738\u0562\u0720\u3113\u0259\u03b4\u0e815\u3042\u05e1\u20a2\u1d3d\u203b\u22e9\u0be9\xf1\u10f5\u2014\xf1\u0105\u316a\u025e\u0710\u30a2\u0718\u0710\u026b\u0105\xa5\u026b\u04c4\u2083\u30e8\u0969\u9fa5\u3fdc\u0ed3\u3005\u7530\u06f3\u0baa\ufb4f\ufb01\u0b85\u061f\u07f9\u0256\u2287\u0133\u01ba\u22e9\u0562\u0133\u1d3d\u026b\u0720\u2083\u30a2\u3113\u2083\u0562\u20aa\u0be9\u0b85\u05d1\u0507\u0b85\u0718\u0baa\u2122\u0562\u3005\u3012\u318d\u2116\u2014\ufb01\u03e1\ufea1\uac00\u0710\u1d3d\u61a8\u0e9c\u04c4\u0710\u309e\ubb80\u03b4\u2103\u025b\u03e1\u0905\u3042\u0ed3\u03b4\u0562\u3485\u2200\u05f1\u05e1\u02e8\u1d6b\u10fb\ufea1\u1d3d\u3485\ufe8f!\u208c\u04c4\u0195\u2030\u2116\u2083\u02d5\u20ac\u0133\u01ba\u311d\u10fb\ud7a3\u3485\u025e\u61a8\u2074\u2116\u3023\u3042\u20ac\u07cb\u3050\u07cb5\u2116\u3106\ud7a3\u0969\u05e1\u03b4\u2074\u05d1\u9fa5\u0e0d\u0e01\u2014\u203b\u0449\u2287\u61a8\u04c4\u3129\u05e1\u07d0\u3012\u20ac\u0562\u0256\u05f1\u20a2\u01ba\u07c4\u0e01\u03cd\u10d0\u207e\u0195\u61a8\u3023\u208c\u0133\u7530\uac00\u0256\u03b4\u2122\u02b1\ufb4f\u2230\u2103\u308b\u0449') ('Path:', u'/chai_testfdf646a0-0641-46b2-99bf-bccae76af2df-\u10fb\u0532\u3129A\u0718\u05d1\xbc\u0ed3\u0277\u2030/chai_test03cfc256-f035-4e38-93be-5e1480566856-\u201c\u0133\u1d6b\u07d0\u30fe\u0baa\u0baa\u03b4\u0105\u05d0\u0710\u2122\u207e\u06f3\u61a8\ufea1') ('Key:', u'\u0baa\u07f9\u311d\u0133\u1d6b\u3485\u3138\u07f9\u1d02\u0ed3\u0710\u309e\u9fa5\u04c4\u0133\u3113\u02d5\ufea1\u07c4a\u0501\u22e9\ufea1\u061f\u0105\u309e\u2122\u0be9\u20ac\u203b\u207e\u0baa\u208c\u0105\u0561\ufb17\xf1\u2200\u0459\u1d6b\u20ac\u2083\u1d02\u025e\u2230\u3050\u20ac\uac00\u0562\u0903\u9fa5\ufea1\u3129\u02e8\u2014\u2230\u0256\u3405\u2100\u3fdc\u30c5\uc738\u0e0d\u3fdc\u0259\u03b4\u07f9\u2287\u07f9\u3042\u0905\u201c\u3fdc\u0ed3\u0b85\u0562\u2287\u0259\u3106\u3106\u0561\u20a3\u026b\u025b\u2122\u2287\u207e\u3042\u0920\u07f9\u0ec4\u02e8\u02e8\u0561\uc738\u0bc2\u0baa\xf1\u376c\u05d0\u30fe\u30c5\u3023\u0969\u0133\u0e01\u61a8\u0259\u0ed3\u0133\u30c5\u0259\u02e8\u0256\u0be9\u05d0\u0133\u03cd\u0e01\u0562\u0449\u0920\u0b85\u2014\ufb01\u3050\u376c\u0940\u311d\u1d02\u05f1\u0baa\ufb17\u7530\u061f\u0e0d\u311d\u10d0\u02e8\u3023\u0133\u3405\u0561\u0720\u20a3\u02d5\u316a\ubb80\u3fdc\u0133\u05e1\u3fdc\u2100\u2100\u03cd\u7530\u3405\ufe8f\u2074\u3129\u03cd\u0bc2\u0259\u0509\u376c\u06f3a\u0b85\u20a3\u02b1\u05e1\u30e8\u0562\u30fe\u05d1\u4e00\u0259\u3113\u025b\u10d0\u3405\u0be9\u02ac\u3023\u311d\u0905\u61a8\u2014\ufb17\u0459\u3138\ubb80\u02ac\u308b\ufb01\u10f5\u0e81\u208c\u2103\u1d3d\u02e8\u3050\u3485\u0256\u308b\u0e01\u10d0\u201c\xa5\u2074\u04c4\u0e9c\u0449\u02e8\u309e\xf1\u3129\xa5\u3fdc\u0718\u0562\ufea1\u0ed3\u311d\u0700\u0e01\u03b4\u3405\u316a\xf1\u0259\u3485\u0195\u3485\u0700\u9fa5\u0105\u05d0\u3129\u0259\u9fa5\u0b85\u07c4\u10fb\u308b\u0ed3\u0ed3\ud7a3\u30a2\u0ed3\u0e01\u3012\ufb17\u0562\ufea1a\u2230\u02e8\ufb01\u026b!\u2122\u0507\u30e8\u3012\u0bc2\ubb80\u0700\xf1\u02d5\xa5\u04c4\u03b4\u3042\u3106\u10f5\u0562\u05e1\u316a\u0509\u0ec4\u0920\u07cb\u03b4\u0105\ubb80\u0459\ufb00\u1d25\u06f3\u0449\u050f\u0720\u0700\u20aaa\u3005!\u0277\u309e\u4e00\xbc\u3485\u10d6\u61a8\u0449\u01ba\u03b4\u0459\u025e\u07f9\u1d25\u311d\u05d1\xa5\u22e9\u376c\xa5\u0710\u20a2\u050f\u0449\u0133\xf1\u02d5\u0562\u2014\uc738\u3fdc\u20a3!\u20a2\u61a8\u061f\u0105\u3131\u0277\u3113\u07205\u0be9\u0710\u2074\u3106\u0507\ufb015\u0507\u03b4\u3129\u0e01\u61a8\uc738\u03b4\u1d02\u3005\u3023\xa5\u0562\xf1\u0920\u2100\u2200\u061f\u20a3\u2230\u0e9c\u2200\xf1\u3005\u0ed3\u01ba\u05d0\uac00\u2116\u0105\xf1\u03e1\u0700\u0bc2\u9fa5\u309e\u0969\u3012\u10fb\u03cd\u02d5\u309e5\u01ba\u0940\ufb4f\u025e\u2074\u3050\u02ac\u0e0d\u050f\ufe8f')
These are pretty hard to reproduce, and the only commonality I've found among repros is the presence of ! in the key names. The character/byte the KMS complains about never seems to be present. Also, there are other exceptions present in the NameNode logs, but I'm not sure they're relevant. LMK if seeing those would help.