Added patch for SCRAM-SHA-256/512 support. (!20) · Merge requests · Itential Open Source / adapters / Notification Messaging / adapter-kafka

Bogdan Tabor requested to merge patch/ADAPT-2369 into master Sep 21, 2022

CHANGES

SCRAM-SHA-256/512 support was added in forked branch https://github.com/SOHU-Co/kafka-node/pull/1392/commits/b4935067709144d6822765822ef21be95d72f635#diff-053150b640a7ce75eff69d1a22cae7f0f94ad64ce9a855db544dda0929316519
boosted kafka-node version to latest 5.0.0 and fixed the version in package.json file
created a patch file to include diff between fcc8aef(v5.0.0) af8c7c6(last commit on forked feat/sasl_scram branch)

bota@LAPTOP-0GU1T19V:~/IAP/apps/kafka-node-scram$ git status
On branch feat/sasl_scram
Your branch is up to date with 'origin/feat/sasl_scram'.

Changes not staged for commit:
  (use "git add <file>..." to update what will be committed)
  (use "git restore <file>..." to discard changes in working directory)
        modified:   package-lock.json

no changes added to commit (use "git add" and/or "git commit -a")

bota@LAPTOP-0GU1T19V:~/IAP/apps/kafka-node-scram$ git log -10 --oneline
af8c7c6 (HEAD -> feat/sasl_scram, origin/feat/sasl_scram) chore: updated README
d718c94 feat: handshake version 0+, fixed issue with plain protocol
486a512 fix: PLAIN auth timeout issue, calls callback
ed949c5 fix: reduced risk of collision by using uuid.v4() instead of Date.now()
b493506 feat: added sasl scram 256 and 512 functionality
fcc8aef (origin/master, origin/HEAD, master) Fix ready event emitting for producers (#1349)
6d58279 V5.0.0 (#1347)
127d437 V4.1.3 (#1346)
26fac1c Improve recovering from broker deploys (#1345)
51b6292 Fix describe configs for multiple brokers (#1772) (#1280)

bota@LAPTOP-0GU1T19V:~/IAP/apps/kafka-node-scram$ git diff --name-only fcc8aef af8c7c6
README.md
docker/docker-compose.0.11.yml
docker/docker-compose.1.0.yml
docker/docker-compose.1.1.yml
docker/docker-compose.2.0.yml
docker/sasl/sasl.conf
docker/start-kafka.sh
lib/baseClient.js
lib/kafkaClient.js
lib/protocol/protocol.js
lib/protocol/scram.js
package-lock.json
test/test.kafkaClient.js
test/test.producer.js
bota@LAPTOP-0GU1T19V:~/IAP/apps/kafka-node-scram$

TESTS:

TEST1: Adapter startup config:

{
  "name": "kafka-2",
  "model": "@itentialopensource/adapter-kafka",
  "type": "Adapter",
  "properties": {
    "id": "kafka-2",
    "type": "Kafka",
    "brokers": [],
    "groups": [],
    "properties": {
      "registry_url": "",
      "writeTime": 30000,
      "stub": false,
      "host": "kafka_sasl",
      "port": 9092,
      "client": {
        "connectTimeout": 10000,
        "requestTimeout": 3000,
        "autoConnect": true,
        "connectRetryOptions": {},
        "idleConnection": 300000,
        "reconnectOnIdle": true,
        "maxAsyncRequests": 10,
        "sslOptions": {
          "rejectUnauthorized": false,
          "minVersion": "TLSv1.2"
        },
        "sasl": {
          "mechanism": "SCRAM-SHA-512",
          "username": "foo",
          "password": "bar"
        }
      },
      "producer": {
        "requireAcks": 1,
        "ackTimeoutMs": 100,
        "partitionerType": 0
      },
      "consumer": {
        "groupId": "kafka-node-group",
        "autoCommit": false,
        "autoCommitIntervalMs": 5000,
        "fetchMaxWaitMs": 100,
        "fetchMinBytes": 1,
        "fetchMaxBytes": 1048576,
        "fromOffset": true,
        "encoding": "utf8",
        "keyEncoding": "utf8"
      }
    }
  },
  "isEncrypted": true,
  "loggerProps": {
    "description": "Logging",
    "log_directory": "/opt/itential/logs",
    "log_filename": "kafka-2.log",
    "log_max_file_size": 10485760,
    "log_max_files": 10,
    "log_timezone_offset": 0,
    "syslog": {
      "level": "warning",
      "host": "127.0.0.1",
      "port": 514,
      "protocol": "udp4",
      "facility": "local0",
      "type": "BSD",
      "path": "",
      "pid": "process.pid",
      "localhost": "",
      "app_name": "",
      "eol": ""
    },
    "log_level": "info",
    "console_level": "info"
  },
  "virtual": false
}

logs:

automation-platform    | kafka-2 ready to work.
automation-platform    | 2022-09-21T11:48:18.870Z - info: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itential/pronghorn-core/core/Sheriff.bin:934, message=[ 'kafka-2 is OFFLINE!' ]
automation-platform    | 2022-09-21T11:48:18.871Z - info: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itential/pronghorn-core/core/Alarm.bin:255, message=[
automation-platform    |   'Alarms module sending AdapterUnreachable adapter = kafka-2 host = null port = null'
automation-platform    | ]
automation-platform    | 2022-09-21T11:48:18.873Z - debug: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itentialopensource/adapter-kafka/node_modules/kafka-node/lib/kafkaClient.js:511, message=[ 'kafka-node-client sending versions request to kafka_sasl:9092' ]
automation-platform    | 2022-09-21T11:48:18.877Z - debug: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itentialopensource/adapter-kafka/node_modules/kafka-node/lib/kafkaClient.js:242, message=[ 'broker socket connected %j', { host: 'kafka_sasl', port: 9092 } ]
automation-platform    | 2022-09-21T11:48:18.878Z - debug: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itentialopensource/adapter-kafka/node_modules/kafka-node/lib/kafkaClient.js:146, message=[ 'connected to socket, trying to load initial metadata' ]
automation-platform    | 2022-09-21T11:48:18.879Z - debug: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itentialopensource/adapter-kafka/node_modules/kafka-node/lib/kafkaClient.js:930, message=[
automation-platform    |   'missing apiSupport waiting until broker is ready...(loadMetadataForTopics)'
automation-platform    | ]
automation-platform    | 2022-09-21T11:48:18.879Z - debug: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itentialopensource/adapter-kafka/node_modules/kafka-node/lib/kafkaClient.js:1025, message=[
automation-platform    |   'waitUntilReady [BrokerWrapper kafka_sasl:9092 (connected: true) (ready: false) (idle: false) (needAuthentication: true) (authenticated: false)]'
automation-platform    | ]
automation-platform    | 2022-09-21T11:48:18.891Z - debug: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itentialopensource/adapter-kafka/node_modules/kafka-node/lib/kafkaClient.js:644, message=[ 'Received versions response from kafka_sasl:9092' ]
automation-platform    | 2022-09-21T11:48:18.891Z - debug: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itentialopensource/adapter-kafka/node_modules/kafka-node/lib/kafkaClient.js:651, message=[
automation-platform    |   'setting api support to %j',
automation-platform    |   [Object: null prototype] {
automation-platform    |     '21': { min: 0, max: 2, usable: false },
automation-platform    |     '22': { min: 0, max: 4, usable: false },
automation-platform    |     '23': { min: 0, max: 4, usable: false },
automation-platform    |     '24': { min: 0, max: 3, usable: false },
automation-platform    |     '25': { min: 0, max: 3, usable: false },
automation-platform    |     '26': { min: 0, max: 3, usable: false },
automation-platform    |     '27': { min: 0, max: 1, usable: false },
automation-platform    |     '28': { min: 0, max: 3, usable: false },
automation-platform    |     '29': { min: 0, max: 2, usable: false },
automation-platform    |     '30': { min: 0, max: 2, usable: false },
automation-platform    |     '31': { min: 0, max: 2, usable: false },
automation-platform    |     '33': { min: 0, max: 2, usable: false },
automation-platform    |     '34': { min: 0, max: 2, usable: false },
automation-platform    |     '35': { min: 0, max: 2, usable: false },
automation-platform    |     '37': { min: 0, max: 3, usable: false },
automation-platform    |     '38': { min: 0, max: 2, usable: false },
automation-platform    |     '39': { min: 0, max: 2, usable: false },
automation-platform    |     '40': { min: 0, max: 2, usable: false },
automation-platform    |     '41': { min: 0, max: 2, usable: false },
automation-platform    |     '42': { min: 0, max: 2, usable: false },
automation-platform    |     '43': { min: 0, max: 2, usable: false },
automation-platform    |     '44': { min: 0, max: 1, usable: false },
automation-platform    |     '45': { min: 0, max: 0, usable: false },
automation-platform    |     '46': { min: 0, max: 0, usable: false },
automation-platform    |     '47': { min: 0, max: 0, usable: false },
automation-platform    |     '48': { min: 0, max: 1, usable: false },
automation-platform    |     '49': { min: 0, max: 1, usable: false },
automation-platform    |     '50': { min: 0, max: 0, usable: false },
automation-platform    |     '51': { min: 0, max: 0, usable: false },
automation-platform    |     '56': { min: 0, max: 0, usable: false },
automation-platform    |     '57': { min: 0, max: 0, usable: false },
automation-platform    |     '60': { min: 0, max: 0, usable: false },
automation-platform    |     '61': { min: 0, max: 0, usable: false },
automation-platform    |     '65': { min: 0, max: 0, usable: false },
automation-platform    |     '66': { min: 0, max: 0, usable: false },
automation-platform    |     '67': { min: 0, max: 0, usable: false },
automation-platform    |     produce: { min: 0, max: 9, usable: 2 },
automation-platform    |     fetch: { min: 0, max: 13, usable: 2 },
automation-platform    |     offset: { min: 0, max: 7, usable: 0 },
automation-platform    |     metadata: { min: 0, max: 12, usable: 1 },
automation-platform    |     leader: { min: 0, max: 5, usable: false },
automation-platform    |     stopReplica: { min: 0, max: 3, usable: false },
automation-platform    |     updateMetadata: { min: 0, max: 7, usable: false },
automation-platform    |     controlledShutdown: { min: 0, max: 3, usable: false },
automation-platform    |     offsetCommit: { min: 0, max: 8, usable: 2 },
automation-platform    |     offsetFetch: { min: 0, max: 8, usable: 1 },
automation-platform    |     groupCoordinator: { min: 0, max: 4, usable: 0 },
automation-platform    |     joinGroup: { min: 0, max: 7, usable: 0 },
automation-platform    |     heartbeat: { min: 0, max: 4, usable: 0 },
automation-platform    |     leaveGroup: { min: 0, max: 4, usable: 0 },
automation-platform    |     syncGroup: { min: 0, max: 5, usable: 0 },
automation-platform    |     describeGroups: { min: 0, max: 5, usable: 0 },
automation-platform    |     listGroups: { min: 0, max: 4, usable: 0 },
automation-platform    |     saslHandshake: { min: 0, max: 1, usable: 1 },
automation-platform    |     apiVersions: { min: 0, max: 3, usable: 0 },
automation-platform    |     createTopics: { min: 0, max: 7, usable: 1 },
automation-platform    |     deleteTopics: { min: 0, max: 6, usable: false },
automation-platform    |     describeConfigs: { min: 0, max: 4, usable: 2 },
automation-platform    |     saslAuthenticate: { min: 0, max: 2, usable: 0 }
automation-platform    |   }
automation-platform    | ]
automation-platform    | 2022-09-21T11:48:18.892Z - debug: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itentialopensource/adapter-kafka/node_modules/kafka-node/lib/kafkaClient.js:685, message=[
automation-platform    |   'Sending SASL/SCRAM-SHA-512 handshake request to [BrokerWrapper kafka_sasl:9092 (connected: true) (ready: false) (idle: false) (needAuthentication: true) (authenticated: false)]'
automation-platform    | ]
automation-platform    | 2022-09-21T11:48:18.899Z - debug: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itentialopensource/adapter-kafka/node_modules/kafka-node/lib/kafkaClient.js:693, message=[
automation-platform    |   'Sending SASL/SCRAM-SHA-512 authentication request to kafka_sasl:9092'
automation-platform    | ]
automation-platform    | 2022-09-21T11:48:18.909Z - debug: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itentialopensource/adapter-kafka/node_modules/kafka-node/lib/kafkaClient.js:1029, message=[ 'broker is now ready' ]
automation-platform    | 2022-09-21T11:48:18.917Z - debug: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itentialopensource/adapter-kafka/node_modules/kafka-node/lib/kafkaClient.js:435, message=[ 'kafka-node-client updated internal metadata' ]
automation-platform    | 2022-09-21T11:48:18.919Z - info: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itentialopensource/adapter-kafka/adapter.js:419, message=[ 'EMITTED ONLINE' ]
automation-platform    | 2022-09-21T11:48:18.920Z - debug: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itentialopensource/adapter-kafka/adapter.js:437, message=[
automation-platform    |   'kafka-2-adapter-connect',
automation-platform    |   'consumerPayloads',
automation-platform    |   [
automation-platform    |     { topic: 't1', partition: 0, offset: 887 },
automation-platform    |     { topic: 'travis', partition: 0, offset: 25 },
automation-platform    |     { topic: 'winter', partition: 0, offset: 7 },
automation-platform    |     { topic: 'kafka', partition: 0, offset: 11 },
automation-platform    |     { topic: 'traivs', partition: 0, offset: 36 }
automation-platform    |   ],
automation-platform    |   {
automation-platform    |     groupId: 'kafka-node-group',
automation-platform    |     autoCommit: false,
automation-platform    |     autoCommitIntervalMs: 5000,
automation-platform    |     fetchMaxWaitMs: 100,
automation-platform    |     fetchMinBytes: 1,
automation-platform    |     fetchMaxBytes: 1048576,
automation-platform    |     fromOffset: true,
automation-platform    |     encoding: 'utf8',
automation-platform    |     keyEncoding: 'utf8'
automation-platform    |   }
automation-platform    | ]
automation-platform    | 2022-09-21T11:48:18.925Z - debug: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itentialopensource/adapter-kafka/node_modules/kafka-node/lib/kafkaClient.js:435, message=[ 'kafka-node-client updated internal metadata' ]
automation-platform    | 2022-09-21T11:48:18.929Z - error: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itentialopensource/adapter-kafka/adapter.js:442, message=[
automation-platform    |   'Consumer Error: LeaderNotAvailable,LeaderNotAvailable,LeaderNotAvailable,LeaderNotAvailable,LeaderNotAvailable'
automation-platform    | ]
automation-platform    | 2022-09-21T11:48:19.052Z - warn: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itential/pronghorn-core/core/WebServer.bin:609, message=[
automation-platform    |   'Failed to load schema for NSOManager getQueueItemDetails queueItemId: Error: schema is invalid: data/properties/queueItemId/examples should be array'
automation-platform    | ]
automation-platform    | 2022-09-21T11:48:19.360Z - info: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itential/pronghorn-core/core/Alarm.bin:289, message=[
automation-platform    |   'Alarms module sending AdapterConnected adapter = kafka-2 host = null port = null'
automation-platform    | ]
automation-platform    | 2022-09-21T11:48:48.844Z - debug: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itentialopensource/adapter-kafka/adapter.js:387, message=[ 'Update .topics.json file' ]
automation-platform    | 2022-09-21T11:49:18.842Z - debug: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itentialopensource/adapter-kafka/adapter.js:387, message=[ 'Update .topics.json file' ]
automation-platform    | 2022-09-21T11:49:48.844Z - debug: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itentialopensource/adapter-kafka/adapter.js:387, message=[ 'Update .topics.json file' ]
automation-platform    | 2022-09-21T11:50:18.843Z - debug: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itentialopensource/adapter-kafka/adapter.js:387, message=[ 'Update .topics.json file' ]
automation-platform    | 2022-09-21T11:50:48.845Z - debug: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itentialopensource/adapter-kafka/adapter.js:387, message=[ 'Update .topics.json file' ]
automation-platform    | 2022-09-21T11:51:18.844Z - debug: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itentialopensource/adapter-kafka/adapter.js:387, message=[ 'Update .topics.json file' ]
automation-platform    | 2022-09-21T11:51:48.845Z - debug: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itentialopensource/adapter-kafka/adapter.js:387, message=[ 'Update .topics.json file' ]
automation-platform    | 2022-09-21T11:52:18.845Z - debug: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itentialopensource/adapter-kafka/adapter.js:387, message=[ 'Update .topics.json file' ]
automation-platform    | 2022-09-21T11:52:48.845Z - debug: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itentialopensource/adapter-kafka/adapter.js:387, message=[ 'Update .topics.json file' ]
automation-platform    | 2022-09-21T11:53:18.845Z - debug: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itentialopensource/adapter-kafka/adapter.js:387, message=[ 'Update .topics.json file' ]
automation-platform    | 2022-09-21T11:53:48.846Z - debug: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itentialopensource/adapter-kafka/adapter.js:387, message=[ 'Update .topics.json file' ]
automation-platform    | 2022-09-21T11:54:18.846Z - debug: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itentialopensource/adapter-kafka/adapter.js:387, message=[ 'Update .topics.json file' ]
automation-platform    | 2022-09-21T11:54:48.846Z - debug: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itentialopensource/adapter-kafka/adapter.js:387, message=[ 'Update .topics.json file' ]
automation-platform    | 2022-09-21T11:55:18.846Z - debug: audit_id=undefined, origin=/opt/itential/automation-platform/node_modules/@itentialopensource/adapter-kafka/adapter.js:387, message=[ 'Update .topics.json file' ]

Edited Sep 21, 2022 by Bogdan Tabor

Added patch for SCRAM-SHA-256/512 support.

Merge request reports