Added patch for SCRAM-SHA-256/512 support.
CHANGES
- SCRAM-SHA-256/512 support was added in forked branch https://github.com/SOHU-Co/kafka-node/pull/1392/commits/b4935067709144d6822765822ef21be95d72f635#diff-053150b640a7ce75eff69d1a22cae7f0f94ad64ce9a855db544dda0929316519
- boosted kafka-node version to latest 5.0.0 and fixed the version in package.json file
- created a patch file to include diff between fcc8aef(v5.0.0) af8c7c6(last commit on forked feat/sasl_scram branch)
bota@LAPTOP-0GU1T19V:~/IAP/apps/kafka-node-scram$ git status
On branch feat/sasl_scram
Your branch is up to date with 'origin/feat/sasl_scram'.
Changes not staged for commit:
(use "git add <file>..." to update what will be committed)
(use "git restore <file>..." to discard changes in working directory)
modified: package-lock.json
no changes added to commit (use "git add" and/or "git commit -a")
bota@LAPTOP-0GU1T19V:~/IAP/apps/kafka-node-scram$ git log -10 --oneline
af8c7c6 (HEAD -> feat/sasl_scram, origin/feat/sasl_scram) chore: updated README
d718c94 feat: handshake version 0+, fixed issue with plain protocol
486a512 fix: PLAIN auth timeout issue, calls callback
ed949c5 fix: reduced risk of collision by using uuid.v4() instead of Date.now()
b493506 feat: added sasl scram 256 and 512 functionality
fcc8aef (origin/master, origin/HEAD, master) Fix ready event emitting for producers (#1349)
6d58279 V5.0.0 (#1347)
127d437 V4.1.3 (#1346)
26fac1c Improve recovering from broker deploys (#1345)
51b6292 Fix describe configs for multiple brokers (#1772) (#1280)
bota@LAPTOP-0GU1T19V:~/IAP/apps/kafka-node-scram$ git diff --name-only fcc8aef af8c7c6
README.md
docker/docker-compose.0.11.yml
docker/docker-compose.1.0.yml
docker/docker-compose.1.1.yml
docker/docker-compose.2.0.yml
docker/sasl/sasl.conf
docker/start-kafka.sh
lib/baseClient.js
lib/kafkaClient.js
lib/protocol/protocol.js
lib/protocol/scram.js
package-lock.json
test/test.kafkaClient.js
test/test.producer.js
bota@LAPTOP-0GU1T19V:~/IAP/apps/kafka-node-scram$
TESTS:
- TEST1: Adapter startup config:
{
"name": "kafka-2",
"model": "@itentialopensource/adapter-kafka",
"type": "Adapter",
"properties": {
"id": "kafka-2",
"type": "Kafka",
"brokers": [],
"groups": [],
"properties": {
"registry_url": "",
"writeTime": 30000,
"stub": false,
"host": "kafka_sasl",
"port": 9092,
"client": {
"connectTimeout": 10000,
"requestTimeout": 3000,
"autoConnect": true,
"connectRetryOptions": {},
"idleConnection": 300000,
"reconnectOnIdle": true,
"maxAsyncRequests": 10,
"sslOptions": {
"rejectUnauthorized": false,
"minVersion": "TLSv1.2"
},
"sasl": {
"mechanism": "SCRAM-SHA-512",
"username": "foo",
"password": "bar"
}
},
"producer": {
"requireAcks": 1,
"ackTimeoutMs": 100,
"partitionerType": 0
},
"consumer": {
"groupId": "kafka-node-group",
"autoCommit": false,
"autoCommitIntervalMs": 5000,
"fetchMaxWaitMs": 100,
"fetchMinBytes": 1,
"fetchMaxBytes": 1048576,
"fromOffset": true,
"encoding": "utf8",
"keyEncoding": "utf8"
}
}
},
"isEncrypted": true,
"loggerProps": {
"description": "Logging",
"log_directory": "/opt/itential/logs",
"log_filename": "kafka-2.log",
"log_max_file_size": 10485760,
"log_max_files": 10,
"log_timezone_offset": 0,
"syslog": {
"level": "warning",
"host": "127.0.0.1",
"port": 514,
"protocol": "udp4",
"facility": "local0",
"type": "BSD",
"path": "",
"pid": "process.pid",
"localhost": "",
"app_name": "",
"eol": ""
},
"log_level": "info",
"console_level": "info"
},
"virtual": false
}
logs:
automation-platform | kafka-2 ready to work.
automation-platform | 2022-09-21T11:48:18.870Z - info: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itential/pronghorn-core/core/Sheriff.bin:934, message=[ 'kafka-2 is OFFLINE!' ]
automation-platform | 2022-09-21T11:48:18.871Z - info: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itential/pronghorn-core/core/Alarm.bin:255, message=[
automation-platform | 'Alarms module sending AdapterUnreachable adapter = kafka-2 host = null port = null'
automation-platform | ]
automation-platform | 2022-09-21T11:48:18.873Z - debug: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itentialopensource/adapter-kafka/node_modules/kafka-node/lib/kafkaClient.js:511, message=[ 'kafka-node-client sending versions request to kafka_sasl:9092' ]
automation-platform | 2022-09-21T11:48:18.877Z - debug: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itentialopensource/adapter-kafka/node_modules/kafka-node/lib/kafkaClient.js:242, message=[ 'broker socket connected %j', { host: 'kafka_sasl', port: 9092 } ]
automation-platform | 2022-09-21T11:48:18.878Z - debug: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itentialopensource/adapter-kafka/node_modules/kafka-node/lib/kafkaClient.js:146, message=[ 'connected to socket, trying to load initial metadata' ]
automation-platform | 2022-09-21T11:48:18.879Z - debug: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itentialopensource/adapter-kafka/node_modules/kafka-node/lib/kafkaClient.js:930, message=[
automation-platform | 'missing apiSupport waiting until broker is ready...(loadMetadataForTopics)'
automation-platform | ]
automation-platform | 2022-09-21T11:48:18.879Z - debug: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itentialopensource/adapter-kafka/node_modules/kafka-node/lib/kafkaClient.js:1025, message=[
automation-platform | 'waitUntilReady [BrokerWrapper kafka_sasl:9092 (connected: true) (ready: false) (idle: false) (needAuthentication: true) (authenticated: false)]'
automation-platform | ]
automation-platform | 2022-09-21T11:48:18.891Z - debug: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itentialopensource/adapter-kafka/node_modules/kafka-node/lib/kafkaClient.js:644, message=[ 'Received versions response from kafka_sasl:9092' ]
automation-platform | 2022-09-21T11:48:18.891Z - debug: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itentialopensource/adapter-kafka/node_modules/kafka-node/lib/kafkaClient.js:651, message=[
automation-platform | 'setting api support to %j',
automation-platform | [Object: null prototype] {
automation-platform | '21': { min: 0, max: 2, usable: false },
automation-platform | '22': { min: 0, max: 4, usable: false },
automation-platform | '23': { min: 0, max: 4, usable: false },
automation-platform | '24': { min: 0, max: 3, usable: false },
automation-platform | '25': { min: 0, max: 3, usable: false },
automation-platform | '26': { min: 0, max: 3, usable: false },
automation-platform | '27': { min: 0, max: 1, usable: false },
automation-platform | '28': { min: 0, max: 3, usable: false },
automation-platform | '29': { min: 0, max: 2, usable: false },
automation-platform | '30': { min: 0, max: 2, usable: false },
automation-platform | '31': { min: 0, max: 2, usable: false },
automation-platform | '33': { min: 0, max: 2, usable: false },
automation-platform | '34': { min: 0, max: 2, usable: false },
automation-platform | '35': { min: 0, max: 2, usable: false },
automation-platform | '37': { min: 0, max: 3, usable: false },
automation-platform | '38': { min: 0, max: 2, usable: false },
automation-platform | '39': { min: 0, max: 2, usable: false },
automation-platform | '40': { min: 0, max: 2, usable: false },
automation-platform | '41': { min: 0, max: 2, usable: false },
automation-platform | '42': { min: 0, max: 2, usable: false },
automation-platform | '43': { min: 0, max: 2, usable: false },
automation-platform | '44': { min: 0, max: 1, usable: false },
automation-platform | '45': { min: 0, max: 0, usable: false },
automation-platform | '46': { min: 0, max: 0, usable: false },
automation-platform | '47': { min: 0, max: 0, usable: false },
automation-platform | '48': { min: 0, max: 1, usable: false },
automation-platform | '49': { min: 0, max: 1, usable: false },
automation-platform | '50': { min: 0, max: 0, usable: false },
automation-platform | '51': { min: 0, max: 0, usable: false },
automation-platform | '56': { min: 0, max: 0, usable: false },
automation-platform | '57': { min: 0, max: 0, usable: false },
automation-platform | '60': { min: 0, max: 0, usable: false },
automation-platform | '61': { min: 0, max: 0, usable: false },
automation-platform | '65': { min: 0, max: 0, usable: false },
automation-platform | '66': { min: 0, max: 0, usable: false },
automation-platform | '67': { min: 0, max: 0, usable: false },
automation-platform | produce: { min: 0, max: 9, usable: 2 },
automation-platform | fetch: { min: 0, max: 13, usable: 2 },
automation-platform | offset: { min: 0, max: 7, usable: 0 },
automation-platform | metadata: { min: 0, max: 12, usable: 1 },
automation-platform | leader: { min: 0, max: 5, usable: false },
automation-platform | stopReplica: { min: 0, max: 3, usable: false },
automation-platform | updateMetadata: { min: 0, max: 7, usable: false },
automation-platform | controlledShutdown: { min: 0, max: 3, usable: false },
automation-platform | offsetCommit: { min: 0, max: 8, usable: 2 },
automation-platform | offsetFetch: { min: 0, max: 8, usable: 1 },
automation-platform | groupCoordinator: { min: 0, max: 4, usable: 0 },
automation-platform | joinGroup: { min: 0, max: 7, usable: 0 },
automation-platform | heartbeat: { min: 0, max: 4, usable: 0 },
automation-platform | leaveGroup: { min: 0, max: 4, usable: 0 },
automation-platform | syncGroup: { min: 0, max: 5, usable: 0 },
automation-platform | describeGroups: { min: 0, max: 5, usable: 0 },
automation-platform | listGroups: { min: 0, max: 4, usable: 0 },
automation-platform | saslHandshake: { min: 0, max: 1, usable: 1 },
automation-platform | apiVersions: { min: 0, max: 3, usable: 0 },
automation-platform | createTopics: { min: 0, max: 7, usable: 1 },
automation-platform | deleteTopics: { min: 0, max: 6, usable: false },
automation-platform | describeConfigs: { min: 0, max: 4, usable: 2 },
automation-platform | saslAuthenticate: { min: 0, max: 2, usable: 0 }
automation-platform | }
automation-platform | ]
automation-platform | 2022-09-21T11:48:18.892Z - debug: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itentialopensource/adapter-kafka/node_modules/kafka-node/lib/kafkaClient.js:685, message=[
automation-platform | 'Sending SASL/SCRAM-SHA-512 handshake request to [BrokerWrapper kafka_sasl:9092 (connected: true) (ready: false) (idle: false) (needAuthentication: true) (authenticated: false)]'
automation-platform | ]
automation-platform | 2022-09-21T11:48:18.899Z - debug: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itentialopensource/adapter-kafka/node_modules/kafka-node/lib/kafkaClient.js:693, message=[
automation-platform | 'Sending SASL/SCRAM-SHA-512 authentication request to kafka_sasl:9092'
automation-platform | ]
automation-platform | 2022-09-21T11:48:18.909Z - debug: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itentialopensource/adapter-kafka/node_modules/kafka-node/lib/kafkaClient.js:1029, message=[ 'broker is now ready' ]
automation-platform | 2022-09-21T11:48:18.917Z - debug: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itentialopensource/adapter-kafka/node_modules/kafka-node/lib/kafkaClient.js:435, message=[ 'kafka-node-client updated internal metadata' ]
automation-platform | 2022-09-21T11:48:18.919Z - info: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itentialopensource/adapter-kafka/adapter.js:419, message=[ 'EMITTED ONLINE' ]
automation-platform | 2022-09-21T11:48:18.920Z - debug: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itentialopensource/adapter-kafka/adapter.js:437, message=[
automation-platform | 'kafka-2-adapter-connect',
automation-platform | 'consumerPayloads',
automation-platform | [
automation-platform | { topic: 't1', partition: 0, offset: 887 },
automation-platform | { topic: 'travis', partition: 0, offset: 25 },
automation-platform | { topic: 'winter', partition: 0, offset: 7 },
automation-platform | { topic: 'kafka', partition: 0, offset: 11 },
automation-platform | { topic: 'traivs', partition: 0, offset: 36 }
automation-platform | ],
automation-platform | {
automation-platform | groupId: 'kafka-node-group',
automation-platform | autoCommit: false,
automation-platform | autoCommitIntervalMs: 5000,
automation-platform | fetchMaxWaitMs: 100,
automation-platform | fetchMinBytes: 1,
automation-platform | fetchMaxBytes: 1048576,
automation-platform | fromOffset: true,
automation-platform | encoding: 'utf8',
automation-platform | keyEncoding: 'utf8'
automation-platform | }
automation-platform | ]
automation-platform | 2022-09-21T11:48:18.925Z - debug: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itentialopensource/adapter-kafka/node_modules/kafka-node/lib/kafkaClient.js:435, message=[ 'kafka-node-client updated internal metadata' ]
automation-platform | 2022-09-21T11:48:18.929Z - error: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itentialopensource/adapter-kafka/adapter.js:442, message=[
automation-platform | 'Consumer Error: LeaderNotAvailable,LeaderNotAvailable,LeaderNotAvailable,LeaderNotAvailable,LeaderNotAvailable'
automation-platform | ]
automation-platform | 2022-09-21T11:48:19.052Z - warn: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itential/pronghorn-core/core/WebServer.bin:609, message=[
automation-platform | 'Failed to load schema for NSOManager getQueueItemDetails queueItemId: Error: schema is invalid: data/properties/queueItemId/examples should be array'
automation-platform | ]
automation-platform | 2022-09-21T11:48:19.360Z - info: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itential/pronghorn-core/core/Alarm.bin:289, message=[
automation-platform | 'Alarms module sending AdapterConnected adapter = kafka-2 host = null port = null'
automation-platform | ]
automation-platform | 2022-09-21T11:48:48.844Z - debug: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itentialopensource/adapter-kafka/adapter.js:387, message=[ 'Update .topics.json file' ]
automation-platform | 2022-09-21T11:49:18.842Z - debug: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itentialopensource/adapter-kafka/adapter.js:387, message=[ 'Update .topics.json file' ]
automation-platform | 2022-09-21T11:49:48.844Z - debug: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itentialopensource/adapter-kafka/adapter.js:387, message=[ 'Update .topics.json file' ]
automation-platform | 2022-09-21T11:50:18.843Z - debug: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itentialopensource/adapter-kafka/adapter.js:387, message=[ 'Update .topics.json file' ]
automation-platform | 2022-09-21T11:50:48.845Z - debug: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itentialopensource/adapter-kafka/adapter.js:387, message=[ 'Update .topics.json file' ]
automation-platform | 2022-09-21T11:51:18.844Z - debug: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itentialopensource/adapter-kafka/adapter.js:387, message=[ 'Update .topics.json file' ]
automation-platform | 2022-09-21T11:51:48.845Z - debug: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itentialopensource/adapter-kafka/adapter.js:387, message=[ 'Update .topics.json file' ]
automation-platform | 2022-09-21T11:52:18.845Z - debug: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itentialopensource/adapter-kafka/adapter.js:387, message=[ 'Update .topics.json file' ]
automation-platform | 2022-09-21T11:52:48.845Z - debug: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itentialopensource/adapter-kafka/adapter.js:387, message=[ 'Update .topics.json file' ]
automation-platform | 2022-09-21T11:53:18.845Z - debug: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itentialopensource/adapter-kafka/adapter.js:387, message=[ 'Update .topics.json file' ]
automation-platform | 2022-09-21T11:53:48.846Z - debug: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itentialopensource/adapter-kafka/adapter.js:387, message=[ 'Update .topics.json file' ]
automation-platform | 2022-09-21T11:54:18.846Z - debug: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itentialopensource/adapter-kafka/adapter.js:387, message=[ 'Update .topics.json file' ]
automation-platform | 2022-09-21T11:54:48.846Z - debug: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itentialopensource/adapter-kafka/adapter.js:387, message=[ 'Update .topics.json file' ]
automation-platform | 2022-09-21T11:55:18.846Z - debug: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itentialopensource/adapter-kafka/adapter.js:387, message=[ 'Update .topics.json file' ]
Edited by Bogdan Tabor