From 2d63821cc65e6c112bf97c7d3166759069499b34 Mon Sep 17 00:00:00 2001 From: Ragul Shanmugam Date: Thu, 23 Jan 2025 11:41:27 -0800 Subject: [PATCH 01/27] Testing ODSE and SRTE in same container --- charts/debezium/templates/configmap.yaml | 2 ++ charts/debezium/templates/deployment.yaml | 3 ++ charts/debezium/values-dts1.yaml | 40 +++++++++++++++++++++++ 3 files changed, 45 insertions(+) diff --git a/charts/debezium/templates/configmap.yaml b/charts/debezium/templates/configmap.yaml index d7fccc729..6e583ff13 100644 --- a/charts/debezium/templates/configmap.yaml +++ b/charts/debezium/templates/configmap.yaml @@ -7,6 +7,8 @@ metadata: data: connector.json: |- {{- .Values.connect.sqlserverconnector| toPrettyJson | nindent 4 }} + connector1.json: |- + {{- .Values.connect.sqlserverconnector1| toPrettyJson | nindent 4 }} connect-distributed.properties: | group.id={{ .Values.connect.properties.group_id }} diff --git a/charts/debezium/templates/deployment.yaml b/charts/debezium/templates/deployment.yaml index 54470fadb..a830e1b8a 100644 --- a/charts/debezium/templates/deployment.yaml +++ b/charts/debezium/templates/deployment.yaml @@ -43,6 +43,9 @@ spec: - name: config mountPath: /etc/debezium/connector.json subPath: connector.json + - name: config + mountPath: /etc/debezium/connector1.json + subPath: connector1.json - name: config mountPath: /kafka/config/log4j.properties subPath: log4j.properties diff --git a/charts/debezium/values-dts1.yaml b/charts/debezium/values-dts1.yaml index 6d9fa6a38..7a2cbd147 100644 --- a/charts/debezium/values-dts1.yaml +++ b/charts/debezium/values-dts1.yaml @@ -81,6 +81,46 @@ connect: } } + sqlserverconnector1: { + "name": "debezium-connector-v010925", + "config": { + "connector.class": "io.debezium.connector.sqlserver.SqlServerConnector", + "database.hostname": "nbs-db.private-dts1.nbspreview.com", + "database.port": "1433", + "database.user": "", + "database.password": "", + "database.dbname": "nbs_srte", + "database.names": "nbs_srte", + "database.server.name": "srte", + "database.history.kafka.bootstrap.servers": "b-1.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092", + "database.history.kafka.topic": "dbhistory.database_server_name.database_name", + # Uncomment following to manually bypass the sqlserver agent status query results + #"database.sqlserver.agent.status.query": "select dbo.IsSqlAgentRunning()", + "database.trustServerCertificate": "true", + "include.schema.changes": "true", + "key.converter": "org.apache.kafka.connect.json.JsonConverter", + "key.converter.schemas.enable": "true", + "producer.max.request.size": "10000000", #10MB + "producer.message.max.bytes": "10000000", #10MB + "snapshot.mode": "initial", + "schema.history.internal.kafka.topic": "srte-schema-history", + "schema.history.internal.kafka.bootstrap.servers": "b-1.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092", + "tasks.max": "1", + "topic.prefix": "cdc", + "topic.creation.default.replication.factor": 2, + "topic.creation.default.partitions": 10, + "topic.creation.default.cleanup.policy": "compact", + "transforms": "dropPrefix, convertTimezone", + "transforms.dropPrefix.regex": "cdc\\.NBS_SRTE\\.dbo\\.(.+)", + "transforms.dropPrefix.type": "org.apache.kafka.connect.transforms.RegexRouter", + "transforms.dropPrefix.replacement": "nrt_srte_$1", + "transforms.convertTimezone.type": "io.debezium.transforms.TimezoneConverter", + "transforms.convertTimezone.converted.timezone": "UTC", + "value.converter": "org.apache.kafka.connect.json.JsonConverter", + "value.converter.schemas.enable": "true", + } + } + env: - name: BOOTSTRAP_SERVERS value: "b-1.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092" From 81aadc9ed233a5be38885d51afbe446789a5c7ca Mon Sep 17 00:00:00 2001 From: Ragul Shanmugam Date: Tue, 28 Jan 2025 09:34:26 -0800 Subject: [PATCH 02/27] Changed connecter value --- charts/debezium/values-dts1.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/charts/debezium/values-dts1.yaml b/charts/debezium/values-dts1.yaml index 7a2cbd147..5d7b2e70b 100644 --- a/charts/debezium/values-dts1.yaml +++ b/charts/debezium/values-dts1.yaml @@ -31,7 +31,7 @@ connect: memory: 1Gi properties: - group_id: "debezium-connector-v010925" + group_id: "debezium-connector-v012825" topics_basename: "debezium-connector" default_replication_factor: 2 default_partitions: 10 @@ -41,7 +41,7 @@ connect: bootstrap_server: "b-1.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092" sqlserverconnector: { - "name": "debezium-connector-v010925", + "name": "debezium-connector-v012825", "config": { "connector.class": "io.debezium.connector.sqlserver.SqlServerConnector", "database.hostname": "nbs-db.private-dts1.nbspreview.com", @@ -82,7 +82,7 @@ connect: } sqlserverconnector1: { - "name": "debezium-connector-v010925", + "name": "debezium-connector-v012825", "config": { "connector.class": "io.debezium.connector.sqlserver.SqlServerConnector", "database.hostname": "nbs-db.private-dts1.nbspreview.com", @@ -129,7 +129,7 @@ connect: - name: KAFKA_LOG4J_OPTS value: "-Dlog4j.configuration=file:/kafka/config/log4j.properties" - name: NAME - value: "debezium-connector-v010925" + value: "debezium-connector-v012825" - name: TZ value: "UTC" From 7090c8524656c5d21e44e3e0801845ca31cf536b Mon Sep 17 00:00:00 2001 From: Ragul Shanmugam Date: Tue, 28 Jan 2025 09:45:46 -0800 Subject: [PATCH 03/27] Changed srte config name --- charts/debezium/values-dts1.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/debezium/values-dts1.yaml b/charts/debezium/values-dts1.yaml index 5d7b2e70b..87a1c4dfc 100644 --- a/charts/debezium/values-dts1.yaml +++ b/charts/debezium/values-dts1.yaml @@ -82,7 +82,7 @@ connect: } sqlserverconnector1: { - "name": "debezium-connector-v012825", + "name": "debezium-connector-srte-v012825", "config": { "connector.class": "io.debezium.connector.sqlserver.SqlServerConnector", "database.hostname": "nbs-db.private-dts1.nbspreview.com", From b4d012f976660373acb62cce83aee25501b0d416 Mon Sep 17 00:00:00 2001 From: Ragul Shanmugam Date: Tue, 28 Jan 2025 10:05:38 -0800 Subject: [PATCH 04/27] Changed poststart command --- charts/debezium/templates/deployment.yaml | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/charts/debezium/templates/deployment.yaml b/charts/debezium/templates/deployment.yaml index a830e1b8a..417670d18 100644 --- a/charts/debezium/templates/deployment.yaml +++ b/charts/debezium/templates/deployment.yaml @@ -60,7 +60,23 @@ spec: lifecycle: postStart: exec: - command: [ '/bin/sh', '-c', 'sleep 60; curl --request POST --header "Accept:application/json" --header "Content-Type:application/json" "http://127.0.0.1:8083/connectors/" --data "@/etc/debezium/connector.json"' ] + #command: [ '/bin/sh', '-c', 'sleep 60; curl --request POST --header "Accept:application/json" --header "Content-Type:application/json" "http://127.0.0.1:8083/connectors/" --data "@/etc/debezium/connector.json"' ] + command: + - "/bin/sh" + - "-c" + - | + sleep 60; + curl --request POST \ + --header "Accept:application/json" \ + --header "Content-Type:application/json" \ + --data '@/etc/debezium/connector1.json' \ + "http://127.0.0.1:8083/connectors/"; + curl --request POST \ + --header "Accept:application/json" \ + --header "Content-Type:application/json" \ + --data '@/etc/debezium/connector2.json' \ + "http://127.0.0.1:8083/connectors/"; + {{- with .Values.connect.nodeSelector }} nodeSelector: From 77dcc5afc59e56a317fd7808192e8042e7ef4720 Mon Sep 17 00:00:00 2001 From: Ragul Shanmugam Date: Tue, 28 Jan 2025 10:13:34 -0800 Subject: [PATCH 05/27] Changed poststart command connector --- charts/debezium/templates/deployment.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/debezium/templates/deployment.yaml b/charts/debezium/templates/deployment.yaml index 417670d18..e0bc4f489 100644 --- a/charts/debezium/templates/deployment.yaml +++ b/charts/debezium/templates/deployment.yaml @@ -69,12 +69,12 @@ spec: curl --request POST \ --header "Accept:application/json" \ --header "Content-Type:application/json" \ - --data '@/etc/debezium/connector1.json' \ + --data '@/etc/debezium/connector.json' \ "http://127.0.0.1:8083/connectors/"; curl --request POST \ --header "Accept:application/json" \ --header "Content-Type:application/json" \ - --data '@/etc/debezium/connector2.json' \ + --data '@/etc/debezium/connector1.json' \ "http://127.0.0.1:8083/connectors/"; From 42cdc3063f487281ae36082e17087b91901df8a7 Mon Sep 17 00:00:00 2001 From: Ragul Shanmugam Date: Tue, 28 Jan 2025 11:38:23 -0800 Subject: [PATCH 06/27] Changed poststart command connector name --- charts/debezium/values-dts1.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/debezium/values-dts1.yaml b/charts/debezium/values-dts1.yaml index 87a1c4dfc..5d7b2e70b 100644 --- a/charts/debezium/values-dts1.yaml +++ b/charts/debezium/values-dts1.yaml @@ -82,7 +82,7 @@ connect: } sqlserverconnector1: { - "name": "debezium-connector-srte-v012825", + "name": "debezium-connector-v012825", "config": { "connector.class": "io.debezium.connector.sqlserver.SqlServerConnector", "database.hostname": "nbs-db.private-dts1.nbspreview.com", From 9e76b9f00be0d7549ac4c84041e1b80ff8d4aa01 Mon Sep 17 00:00:00 2001 From: Ragul Shanmugam Date: Wed, 29 Jan 2025 07:46:09 -0800 Subject: [PATCH 07/27] Adding dlq config to dts1 --- charts/debezium/values-dts1.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/charts/debezium/values-dts1.yaml b/charts/debezium/values-dts1.yaml index 5d7b2e70b..40c315936 100644 --- a/charts/debezium/values-dts1.yaml +++ b/charts/debezium/values-dts1.yaml @@ -118,6 +118,10 @@ connect: "transforms.convertTimezone.converted.timezone": "UTC", "value.converter": "org.apache.kafka.connect.json.JsonConverter", "value.converter.schemas.enable": "true", + "errors.tolerance": "all", + "errors.log.enable": "true", + "errors.deadletterqueue.topic.name": "debezium-deadletter", + "errors.deadletterqueue.context.headers.enable": "true" } } From 946cc3d02a14f8eafd685e290af58698d7812fc8 Mon Sep 17 00:00:00 2001 From: Ragul Shanmugam Date: Wed, 29 Jan 2025 08:25:56 -0800 Subject: [PATCH 08/27] Removing snapshot config to dts1 --- charts/debezium/values-dts1.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/debezium/values-dts1.yaml b/charts/debezium/values-dts1.yaml index 40c315936..141deef37 100644 --- a/charts/debezium/values-dts1.yaml +++ b/charts/debezium/values-dts1.yaml @@ -102,7 +102,7 @@ connect: "key.converter.schemas.enable": "true", "producer.max.request.size": "10000000", #10MB "producer.message.max.bytes": "10000000", #10MB - "snapshot.mode": "initial", + "snapshot.mode": "no_data", "schema.history.internal.kafka.topic": "srte-schema-history", "schema.history.internal.kafka.bootstrap.servers": "b-1.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092", "tasks.max": "1", From da9e338a9b110c6c5642df6b8bb3070c07e8a2ac Mon Sep 17 00:00:00 2001 From: Ragul Shanmugam Date: Wed, 29 Jan 2025 08:39:37 -0800 Subject: [PATCH 09/27] Removing snapshot config to dts1 delete --- charts/debezium/values-dts1.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/debezium/values-dts1.yaml b/charts/debezium/values-dts1.yaml index 141deef37..8102aedca 100644 --- a/charts/debezium/values-dts1.yaml +++ b/charts/debezium/values-dts1.yaml @@ -109,7 +109,7 @@ connect: "topic.prefix": "cdc", "topic.creation.default.replication.factor": 2, "topic.creation.default.partitions": 10, - "topic.creation.default.cleanup.policy": "compact", + "topic.creation.default.cleanup.policy": "delete", "transforms": "dropPrefix, convertTimezone", "transforms.dropPrefix.regex": "cdc\\.NBS_SRTE\\.dbo\\.(.+)", "transforms.dropPrefix.type": "org.apache.kafka.connect.transforms.RegexRouter", From 44c215eacc54c3cccbb139e880cdfa5bcf26f1e3 Mon Sep 17 00:00:00 2001 From: Ragul Shanmugam Date: Wed, 29 Jan 2025 09:44:23 -0800 Subject: [PATCH 10/27] Test conn1 --- charts/debezium/templates/deployment.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/charts/debezium/templates/deployment.yaml b/charts/debezium/templates/deployment.yaml index e0bc4f489..6a00f7b62 100644 --- a/charts/debezium/templates/deployment.yaml +++ b/charts/debezium/templates/deployment.yaml @@ -66,11 +66,11 @@ spec: - "-c" - | sleep 60; - curl --request POST \ - --header "Accept:application/json" \ - --header "Content-Type:application/json" \ - --data '@/etc/debezium/connector.json' \ - "http://127.0.0.1:8083/connectors/"; +{{/* curl --request POST \*/}} +{{/* --header "Accept:application/json" \*/}} +{{/* --header "Content-Type:application/json" \*/}} +{{/* --data '@/etc/debezium/connector.json' \*/}} +{{/* "http://127.0.0.1:8083/connectors/";*/}} curl --request POST \ --header "Accept:application/json" \ --header "Content-Type:application/json" \ From 44e29268499366f855476f02dac1e715d98cbac9 Mon Sep 17 00:00:00 2001 From: Ragul Shanmugam Date: Wed, 29 Jan 2025 09:54:53 -0800 Subject: [PATCH 11/27] Test conn2 --- charts/debezium/templates/deployment.yaml | 10 +++++----- charts/debezium/values-dts1.yaml | 1 + 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/charts/debezium/templates/deployment.yaml b/charts/debezium/templates/deployment.yaml index 6a00f7b62..e0bc4f489 100644 --- a/charts/debezium/templates/deployment.yaml +++ b/charts/debezium/templates/deployment.yaml @@ -66,11 +66,11 @@ spec: - "-c" - | sleep 60; -{{/* curl --request POST \*/}} -{{/* --header "Accept:application/json" \*/}} -{{/* --header "Content-Type:application/json" \*/}} -{{/* --data '@/etc/debezium/connector.json' \*/}} -{{/* "http://127.0.0.1:8083/connectors/";*/}} + curl --request POST \ + --header "Accept:application/json" \ + --header "Content-Type:application/json" \ + --data '@/etc/debezium/connector.json' \ + "http://127.0.0.1:8083/connectors/"; curl --request POST \ --header "Accept:application/json" \ --header "Content-Type:application/json" \ diff --git a/charts/debezium/values-dts1.yaml b/charts/debezium/values-dts1.yaml index 8102aedca..2e628692c 100644 --- a/charts/debezium/values-dts1.yaml +++ b/charts/debezium/values-dts1.yaml @@ -106,6 +106,7 @@ connect: "schema.history.internal.kafka.topic": "srte-schema-history", "schema.history.internal.kafka.bootstrap.servers": "b-1.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092", "tasks.max": "1", + "offset.storage.cleanup.policy": "compact", "topic.prefix": "cdc", "topic.creation.default.replication.factor": 2, "topic.creation.default.partitions": 10, From 0c5ef7534c5759c6b1c492350d3ad1a7c16c6e7f Mon Sep 17 00:00:00 2001 From: Ragul Shanmugam Date: Wed, 29 Jan 2025 09:58:17 -0800 Subject: [PATCH 12/27] Test conn2 --- charts/debezium/templates/configmap.yaml | 4 ++-- charts/debezium/templates/deployment.yaml | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/charts/debezium/templates/configmap.yaml b/charts/debezium/templates/configmap.yaml index 6e583ff13..cbfc4e9ed 100644 --- a/charts/debezium/templates/configmap.yaml +++ b/charts/debezium/templates/configmap.yaml @@ -5,8 +5,8 @@ metadata: labels: {{- include "debezium.labels.connect" . | nindent 4 }} data: - connector.json: |- - {{- .Values.connect.sqlserverconnector| toPrettyJson | nindent 4 }} +{{/* connector.json: |-*/}} +{{/* {{- .Values.connect.sqlserverconnector| toPrettyJson | nindent 4 }}*/}} connector1.json: |- {{- .Values.connect.sqlserverconnector1| toPrettyJson | nindent 4 }} diff --git a/charts/debezium/templates/deployment.yaml b/charts/debezium/templates/deployment.yaml index e0bc4f489..9271dc72d 100644 --- a/charts/debezium/templates/deployment.yaml +++ b/charts/debezium/templates/deployment.yaml @@ -40,9 +40,9 @@ spec: resources: {{- toYaml .Values.connect.resources | nindent 12 }} volumeMounts: - - name: config - mountPath: /etc/debezium/connector.json - subPath: connector.json +{{/* - name: config*/}} +{{/* mountPath: /etc/debezium/connector.json*/}} +{{/* subPath: connector.json*/}} - name: config mountPath: /etc/debezium/connector1.json subPath: connector1.json @@ -66,11 +66,11 @@ spec: - "-c" - | sleep 60; - curl --request POST \ - --header "Accept:application/json" \ - --header "Content-Type:application/json" \ - --data '@/etc/debezium/connector.json' \ - "http://127.0.0.1:8083/connectors/"; +{{/* curl --request POST \*/}} +{{/* --header "Accept:application/json" \*/}} +{{/* --header "Content-Type:application/json" \*/}} +{{/* --data '@/etc/debezium/connector.json' \*/}} +{{/* "http://127.0.0.1:8083/connectors/";*/}} curl --request POST \ --header "Accept:application/json" \ --header "Content-Type:application/json" \ From c509a55a337543a735b34c925f4dcd64c5890f8e Mon Sep 17 00:00:00 2001 From: Ragul Shanmugam Date: Wed, 29 Jan 2025 10:13:51 -0800 Subject: [PATCH 13/27] Test table include list --- charts/debezium/values-dts1.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/debezium/values-dts1.yaml b/charts/debezium/values-dts1.yaml index 2e628692c..77d75da7c 100644 --- a/charts/debezium/values-dts1.yaml +++ b/charts/debezium/values-dts1.yaml @@ -105,12 +105,12 @@ connect: "snapshot.mode": "no_data", "schema.history.internal.kafka.topic": "srte-schema-history", "schema.history.internal.kafka.bootstrap.servers": "b-1.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092", + "table.include.list": "dbo.Codeset", "tasks.max": "1", - "offset.storage.cleanup.policy": "compact", "topic.prefix": "cdc", "topic.creation.default.replication.factor": 2, "topic.creation.default.partitions": 10, - "topic.creation.default.cleanup.policy": "delete", + "topic.creation.default.cleanup.policy": "compact", "transforms": "dropPrefix, convertTimezone", "transforms.dropPrefix.regex": "cdc\\.NBS_SRTE\\.dbo\\.(.+)", "transforms.dropPrefix.type": "org.apache.kafka.connect.transforms.RegexRouter", From 4dfda8a132f73c853d146ab39b72a20ce509efb8 Mon Sep 17 00:00:00 2001 From: Ragul Shanmugam Date: Wed, 29 Jan 2025 11:48:06 -0800 Subject: [PATCH 14/27] Changing bootstrap servers --- charts/debezium/values-dts1.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/debezium/values-dts1.yaml b/charts/debezium/values-dts1.yaml index 77d75da7c..311fd8dce 100644 --- a/charts/debezium/values-dts1.yaml +++ b/charts/debezium/values-dts1.yaml @@ -38,7 +38,7 @@ connect: default_cleanup: "compact" sql_server_agent_override: false sql_server_agent_status: "" - bootstrap_server: "b-1.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092" + bootstrap_server: "b-1.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092" sqlserverconnector: { "name": "debezium-connector-v012825", @@ -51,7 +51,7 @@ connect: "database.dbname": "nbs_odse", "database.names": "nbs_odse", "database.server.name": "odse", - "database.history.kafka.bootstrap.servers": "b-1.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092", + "database.history.kafka.bootstrap.servers": "b-1.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092", "database.history.kafka.topic": "dbhistory.database_server_name.database_name", # Uncomment following to manually bypass the sqlserver agent status query results #"database.sqlserver.agent.status.query": "select dbo.IsSqlAgentRunning()", @@ -63,7 +63,7 @@ connect: "producer.message.max.bytes": "10000000", #10MB "snapshot.mode": "no_data", "schema.history.internal.kafka.topic": "odse-schema-history", - "schema.history.internal.kafka.bootstrap.servers": "b-1.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092", + "schema.history.internal.kafka.bootstrap.servers": "b-1.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092", "table.include.list": "dbo.Person, dbo.Organization, dbo.Observation, dbo.Public_health_case, dbo.state_defined_field_data, dbo.Notification, dbo.Interview, dbo.CN_transportq_out", "tasks.max": "1", "topic.prefix": "cdc", @@ -92,7 +92,7 @@ connect: "database.dbname": "nbs_srte", "database.names": "nbs_srte", "database.server.name": "srte", - "database.history.kafka.bootstrap.servers": "b-1.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092", + "database.history.kafka.bootstrap.servers": "b-1.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092", "database.history.kafka.topic": "dbhistory.database_server_name.database_name", # Uncomment following to manually bypass the sqlserver agent status query results #"database.sqlserver.agent.status.query": "select dbo.IsSqlAgentRunning()", @@ -104,7 +104,7 @@ connect: "producer.message.max.bytes": "10000000", #10MB "snapshot.mode": "no_data", "schema.history.internal.kafka.topic": "srte-schema-history", - "schema.history.internal.kafka.bootstrap.servers": "b-1.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092", + "schema.history.internal.kafka.bootstrap.servers": "b-1.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092", "table.include.list": "dbo.Codeset", "tasks.max": "1", "topic.prefix": "cdc", @@ -128,7 +128,7 @@ connect: env: - name: BOOTSTRAP_SERVERS - value: "b-1.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092" + value: "b-1.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092" - name: LOG_LEVEL value: "INFO" - name: KAFKA_LOG4J_OPTS From a88711f6a883dcd468950790688a6800615c7252 Mon Sep 17 00:00:00 2001 From: Ragul Shanmugam Date: Wed, 29 Jan 2025 12:18:39 -0800 Subject: [PATCH 15/27] Adding offset deletion --- charts/debezium/values-dts1.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/charts/debezium/values-dts1.yaml b/charts/debezium/values-dts1.yaml index 311fd8dce..40b2d1005 100644 --- a/charts/debezium/values-dts1.yaml +++ b/charts/debezium/values-dts1.yaml @@ -105,12 +105,13 @@ connect: "snapshot.mode": "no_data", "schema.history.internal.kafka.topic": "srte-schema-history", "schema.history.internal.kafka.bootstrap.servers": "b-1.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092", - "table.include.list": "dbo.Codeset", + #"table.include.list": "dbo.Codeset", "tasks.max": "1", + "offset.storage.cleanup.policy": "compact", "topic.prefix": "cdc", "topic.creation.default.replication.factor": 2, "topic.creation.default.partitions": 10, - "topic.creation.default.cleanup.policy": "compact", + "topic.creation.default.cleanup.policy": "delete", "transforms": "dropPrefix, convertTimezone", "transforms.dropPrefix.regex": "cdc\\.NBS_SRTE\\.dbo\\.(.+)", "transforms.dropPrefix.type": "org.apache.kafka.connect.transforms.RegexRouter", From 969450d6542260948e4999c09eb6be4513eb0f4c Mon Sep 17 00:00:00 2001 From: Ragul Shanmugam Date: Wed, 29 Jan 2025 12:32:46 -0800 Subject: [PATCH 16/27] Adding snapshot --- charts/debezium/values-dts1.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/debezium/values-dts1.yaml b/charts/debezium/values-dts1.yaml index 40b2d1005..865d8587c 100644 --- a/charts/debezium/values-dts1.yaml +++ b/charts/debezium/values-dts1.yaml @@ -102,7 +102,7 @@ connect: "key.converter.schemas.enable": "true", "producer.max.request.size": "10000000", #10MB "producer.message.max.bytes": "10000000", #10MB - "snapshot.mode": "no_data", + "snapshot.mode": "initial", "schema.history.internal.kafka.topic": "srte-schema-history", "schema.history.internal.kafka.bootstrap.servers": "b-1.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092", #"table.include.list": "dbo.Codeset", From 6afb9cb0f5e95e783db0fe0836dd2c6d73c6b49a Mon Sep 17 00:00:00 2001 From: Ragul Shanmugam Date: Wed, 29 Jan 2025 12:57:21 -0800 Subject: [PATCH 17/27] Adding include list back --- charts/debezium/values-dts1.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/debezium/values-dts1.yaml b/charts/debezium/values-dts1.yaml index 865d8587c..cfe485e7f 100644 --- a/charts/debezium/values-dts1.yaml +++ b/charts/debezium/values-dts1.yaml @@ -105,7 +105,7 @@ connect: "snapshot.mode": "initial", "schema.history.internal.kafka.topic": "srte-schema-history", "schema.history.internal.kafka.bootstrap.servers": "b-1.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092", - #"table.include.list": "dbo.Codeset", + "table.include.list": "dbo.Anatomic_site_code, dbo.City_code_value, dbo.Cntycity_code_value, dbo.Code_value_clinical, dbo.Code_value_general, dbo.Codeset, dbo.Codeset_Group_Metadata, dbo.Country_code, dbo.Country_Code_ISO, dbo.Country_XREF, dbo.ELR_XREF, dbo.IMRDBMapping, dbo.Investigation_code, dbo.Jurisdiction_code, dbo.Jurisdiction_participation, dbo.Lab_coding_system, dbo.Lab_result, dbo.Lab_result_Snomed, dbo.Lab_test, dbo.Labtest_loinc, dbo.Labtest_Progarea_Mapping, dbo.Language_code, dbo.LDF_page_set, dbo.LOINC_code, dbo.Loinc_condition, dbo.Loinc_snomed_condition, dbo.NAICS_Industry_code, dbo.Occupation_code, dbo.Participation_type, dbo.Program_area_code, dbo.Race_code, dbo.Snomed_code, dbo.Specimen_source_code, dbo.Standard_XREF, dbo.State_code, dbo.State_county_code_value, dbo.State_model, dbo.TotalIDM, dbo.Treatment_code, dbo.Unit_code, dbo.Zip_code_value, dbo.Zipcnty_code_value", "tasks.max": "1", "offset.storage.cleanup.policy": "compact", "topic.prefix": "cdc", From 3883ee696b33d40789551cd6d25cc76c5dcba9c7 Mon Sep 17 00:00:00 2001 From: Ragul Shanmugam Date: Wed, 29 Jan 2025 13:05:17 -0800 Subject: [PATCH 18/27] Adding include list back compact --- charts/debezium/values-dts1.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/charts/debezium/values-dts1.yaml b/charts/debezium/values-dts1.yaml index cfe485e7f..4f8ffd98e 100644 --- a/charts/debezium/values-dts1.yaml +++ b/charts/debezium/values-dts1.yaml @@ -107,11 +107,10 @@ connect: "schema.history.internal.kafka.bootstrap.servers": "b-1.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092", "table.include.list": "dbo.Anatomic_site_code, dbo.City_code_value, dbo.Cntycity_code_value, dbo.Code_value_clinical, dbo.Code_value_general, dbo.Codeset, dbo.Codeset_Group_Metadata, dbo.Country_code, dbo.Country_Code_ISO, dbo.Country_XREF, dbo.ELR_XREF, dbo.IMRDBMapping, dbo.Investigation_code, dbo.Jurisdiction_code, dbo.Jurisdiction_participation, dbo.Lab_coding_system, dbo.Lab_result, dbo.Lab_result_Snomed, dbo.Lab_test, dbo.Labtest_loinc, dbo.Labtest_Progarea_Mapping, dbo.Language_code, dbo.LDF_page_set, dbo.LOINC_code, dbo.Loinc_condition, dbo.Loinc_snomed_condition, dbo.NAICS_Industry_code, dbo.Occupation_code, dbo.Participation_type, dbo.Program_area_code, dbo.Race_code, dbo.Snomed_code, dbo.Specimen_source_code, dbo.Standard_XREF, dbo.State_code, dbo.State_county_code_value, dbo.State_model, dbo.TotalIDM, dbo.Treatment_code, dbo.Unit_code, dbo.Zip_code_value, dbo.Zipcnty_code_value", "tasks.max": "1", - "offset.storage.cleanup.policy": "compact", "topic.prefix": "cdc", "topic.creation.default.replication.factor": 2, "topic.creation.default.partitions": 10, - "topic.creation.default.cleanup.policy": "delete", + "topic.creation.default.cleanup.policy": "compact", "transforms": "dropPrefix, convertTimezone", "transforms.dropPrefix.regex": "cdc\\.NBS_SRTE\\.dbo\\.(.+)", "transforms.dropPrefix.type": "org.apache.kafka.connect.transforms.RegexRouter", From b96d2d2f2861e8ae71087d62247ac7f5e2466de4 Mon Sep 17 00:00:00 2001 From: Ragul Shanmugam Date: Wed, 29 Jan 2025 13:13:04 -0800 Subject: [PATCH 19/27] Removing include list --- charts/debezium/values-dts1.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/charts/debezium/values-dts1.yaml b/charts/debezium/values-dts1.yaml index 4f8ffd98e..19dbc6e0a 100644 --- a/charts/debezium/values-dts1.yaml +++ b/charts/debezium/values-dts1.yaml @@ -105,12 +105,13 @@ connect: "snapshot.mode": "initial", "schema.history.internal.kafka.topic": "srte-schema-history", "schema.history.internal.kafka.bootstrap.servers": "b-1.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092", - "table.include.list": "dbo.Anatomic_site_code, dbo.City_code_value, dbo.Cntycity_code_value, dbo.Code_value_clinical, dbo.Code_value_general, dbo.Codeset, dbo.Codeset_Group_Metadata, dbo.Country_code, dbo.Country_Code_ISO, dbo.Country_XREF, dbo.ELR_XREF, dbo.IMRDBMapping, dbo.Investigation_code, dbo.Jurisdiction_code, dbo.Jurisdiction_participation, dbo.Lab_coding_system, dbo.Lab_result, dbo.Lab_result_Snomed, dbo.Lab_test, dbo.Labtest_loinc, dbo.Labtest_Progarea_Mapping, dbo.Language_code, dbo.LDF_page_set, dbo.LOINC_code, dbo.Loinc_condition, dbo.Loinc_snomed_condition, dbo.NAICS_Industry_code, dbo.Occupation_code, dbo.Participation_type, dbo.Program_area_code, dbo.Race_code, dbo.Snomed_code, dbo.Specimen_source_code, dbo.Standard_XREF, dbo.State_code, dbo.State_county_code_value, dbo.State_model, dbo.TotalIDM, dbo.Treatment_code, dbo.Unit_code, dbo.Zip_code_value, dbo.Zipcnty_code_value", +# "table.include.list": "dbo.Anatomic_site_code, dbo.City_code_value, dbo.Cntycity_code_value, dbo.Code_value_clinical, dbo.Code_value_general, dbo.Codeset, dbo.Codeset_Group_Metadata, dbo.Country_code, dbo.Country_Code_ISO, dbo.Country_XREF, dbo.ELR_XREF, dbo.IMRDBMapping, dbo.Investigation_code, dbo.Jurisdiction_code, dbo.Jurisdiction_participation, dbo.Lab_coding_system, dbo.Lab_result, dbo.Lab_result_Snomed, dbo.Lab_test, dbo.Labtest_loinc, dbo.Labtest_Progarea_Mapping, dbo.Language_code, dbo.LDF_page_set, dbo.LOINC_code, dbo.Loinc_condition, dbo.Loinc_snomed_condition, dbo.NAICS_Industry_code, dbo.Occupation_code, dbo.Participation_type, dbo.Program_area_code, dbo.Race_code, dbo.Snomed_code, dbo.Specimen_source_code, dbo.Standard_XREF, dbo.State_code, dbo.State_county_code_value, dbo.State_model, dbo.TotalIDM, dbo.Treatment_code, dbo.Unit_code, dbo.Zip_code_value, dbo.Zipcnty_code_value", "tasks.max": "1", + "offset.storage.cleanup.policy": "compact", "topic.prefix": "cdc", "topic.creation.default.replication.factor": 2, "topic.creation.default.partitions": 10, - "topic.creation.default.cleanup.policy": "compact", + "topic.creation.default.cleanup.policy": "delete", "transforms": "dropPrefix, convertTimezone", "transforms.dropPrefix.regex": "cdc\\.NBS_SRTE\\.dbo\\.(.+)", "transforms.dropPrefix.type": "org.apache.kafka.connect.transforms.RegexRouter", From c4921ebae97837c0ac0cc7511c3f56a9fc5b494c Mon Sep 17 00:00:00 2001 From: Ragul Shanmugam Date: Wed, 29 Jan 2025 13:40:46 -0800 Subject: [PATCH 20/27] Update bootstrap servers --- charts/debezium/values-dts1.yaml | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/charts/debezium/values-dts1.yaml b/charts/debezium/values-dts1.yaml index 19dbc6e0a..b32335b08 100644 --- a/charts/debezium/values-dts1.yaml +++ b/charts/debezium/values-dts1.yaml @@ -31,17 +31,17 @@ connect: memory: 1Gi properties: - group_id: "debezium-connector-v012825" + group_id: "debezium-connector-v013025" topics_basename: "debezium-connector" default_replication_factor: 2 default_partitions: 10 default_cleanup: "compact" sql_server_agent_override: false sql_server_agent_status: "" - bootstrap_server: "b-1.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092" + bootstrap_server: "b-1.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092" sqlserverconnector: { - "name": "debezium-connector-v012825", + "name": "debezium-connector-v013025", "config": { "connector.class": "io.debezium.connector.sqlserver.SqlServerConnector", "database.hostname": "nbs-db.private-dts1.nbspreview.com", @@ -51,7 +51,7 @@ connect: "database.dbname": "nbs_odse", "database.names": "nbs_odse", "database.server.name": "odse", - "database.history.kafka.bootstrap.servers": "b-1.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092", + "database.history.kafka.bootstrap.servers": "b-1.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092", "database.history.kafka.topic": "dbhistory.database_server_name.database_name", # Uncomment following to manually bypass the sqlserver agent status query results #"database.sqlserver.agent.status.query": "select dbo.IsSqlAgentRunning()", @@ -63,7 +63,7 @@ connect: "producer.message.max.bytes": "10000000", #10MB "snapshot.mode": "no_data", "schema.history.internal.kafka.topic": "odse-schema-history", - "schema.history.internal.kafka.bootstrap.servers": "b-1.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092", + "schema.history.internal.kafka.bootstrap.servers": "b-1.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092", "table.include.list": "dbo.Person, dbo.Organization, dbo.Observation, dbo.Public_health_case, dbo.state_defined_field_data, dbo.Notification, dbo.Interview, dbo.CN_transportq_out", "tasks.max": "1", "topic.prefix": "cdc", @@ -82,7 +82,7 @@ connect: } sqlserverconnector1: { - "name": "debezium-connector-v012825", + "name": "debezium-connector-v013025", "config": { "connector.class": "io.debezium.connector.sqlserver.SqlServerConnector", "database.hostname": "nbs-db.private-dts1.nbspreview.com", @@ -92,7 +92,7 @@ connect: "database.dbname": "nbs_srte", "database.names": "nbs_srte", "database.server.name": "srte", - "database.history.kafka.bootstrap.servers": "b-1.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092", + "database.history.kafka.bootstrap.servers": "b-1.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092", "database.history.kafka.topic": "dbhistory.database_server_name.database_name", # Uncomment following to manually bypass the sqlserver agent status query results #"database.sqlserver.agent.status.query": "select dbo.IsSqlAgentRunning()", @@ -104,8 +104,7 @@ connect: "producer.message.max.bytes": "10000000", #10MB "snapshot.mode": "initial", "schema.history.internal.kafka.topic": "srte-schema-history", - "schema.history.internal.kafka.bootstrap.servers": "b-1.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092", -# "table.include.list": "dbo.Anatomic_site_code, dbo.City_code_value, dbo.Cntycity_code_value, dbo.Code_value_clinical, dbo.Code_value_general, dbo.Codeset, dbo.Codeset_Group_Metadata, dbo.Country_code, dbo.Country_Code_ISO, dbo.Country_XREF, dbo.ELR_XREF, dbo.IMRDBMapping, dbo.Investigation_code, dbo.Jurisdiction_code, dbo.Jurisdiction_participation, dbo.Lab_coding_system, dbo.Lab_result, dbo.Lab_result_Snomed, dbo.Lab_test, dbo.Labtest_loinc, dbo.Labtest_Progarea_Mapping, dbo.Language_code, dbo.LDF_page_set, dbo.LOINC_code, dbo.Loinc_condition, dbo.Loinc_snomed_condition, dbo.NAICS_Industry_code, dbo.Occupation_code, dbo.Participation_type, dbo.Program_area_code, dbo.Race_code, dbo.Snomed_code, dbo.Specimen_source_code, dbo.Standard_XREF, dbo.State_code, dbo.State_county_code_value, dbo.State_model, dbo.TotalIDM, dbo.Treatment_code, dbo.Unit_code, dbo.Zip_code_value, dbo.Zipcnty_code_value", + "schema.history.internal.kafka.bootstrap.servers": "b-1.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092", "tasks.max": "1", "offset.storage.cleanup.policy": "compact", "topic.prefix": "cdc", @@ -120,22 +119,18 @@ connect: "transforms.convertTimezone.converted.timezone": "UTC", "value.converter": "org.apache.kafka.connect.json.JsonConverter", "value.converter.schemas.enable": "true", - "errors.tolerance": "all", - "errors.log.enable": "true", - "errors.deadletterqueue.topic.name": "debezium-deadletter", - "errors.deadletterqueue.context.headers.enable": "true" } } env: - name: BOOTSTRAP_SERVERS - value: "b-1.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092" + value: "b-1.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092" - name: LOG_LEVEL value: "INFO" - name: KAFKA_LOG4J_OPTS value: "-Dlog4j.configuration=file:/kafka/config/log4j.properties" - name: NAME - value: "debezium-connector-v012825" + value: "debezium-connector-v013025" - name: TZ value: "UTC" From aa4ec01227a06b21987d79133e9efa2acc7e9eeb Mon Sep 17 00:00:00 2001 From: Ragul Shanmugam Date: Wed, 29 Jan 2025 13:57:20 -0800 Subject: [PATCH 21/27] Adding odse back --- charts/debezium/templates/configmap.yaml | 4 ++-- charts/debezium/templates/deployment.yaml | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/charts/debezium/templates/configmap.yaml b/charts/debezium/templates/configmap.yaml index cbfc4e9ed..6e583ff13 100644 --- a/charts/debezium/templates/configmap.yaml +++ b/charts/debezium/templates/configmap.yaml @@ -5,8 +5,8 @@ metadata: labels: {{- include "debezium.labels.connect" . | nindent 4 }} data: -{{/* connector.json: |-*/}} -{{/* {{- .Values.connect.sqlserverconnector| toPrettyJson | nindent 4 }}*/}} + connector.json: |- + {{- .Values.connect.sqlserverconnector| toPrettyJson | nindent 4 }} connector1.json: |- {{- .Values.connect.sqlserverconnector1| toPrettyJson | nindent 4 }} diff --git a/charts/debezium/templates/deployment.yaml b/charts/debezium/templates/deployment.yaml index 9271dc72d..e0bc4f489 100644 --- a/charts/debezium/templates/deployment.yaml +++ b/charts/debezium/templates/deployment.yaml @@ -40,9 +40,9 @@ spec: resources: {{- toYaml .Values.connect.resources | nindent 12 }} volumeMounts: -{{/* - name: config*/}} -{{/* mountPath: /etc/debezium/connector.json*/}} -{{/* subPath: connector.json*/}} + - name: config + mountPath: /etc/debezium/connector.json + subPath: connector.json - name: config mountPath: /etc/debezium/connector1.json subPath: connector1.json @@ -66,11 +66,11 @@ spec: - "-c" - | sleep 60; -{{/* curl --request POST \*/}} -{{/* --header "Accept:application/json" \*/}} -{{/* --header "Content-Type:application/json" \*/}} -{{/* --data '@/etc/debezium/connector.json' \*/}} -{{/* "http://127.0.0.1:8083/connectors/";*/}} + curl --request POST \ + --header "Accept:application/json" \ + --header "Content-Type:application/json" \ + --data '@/etc/debezium/connector.json' \ + "http://127.0.0.1:8083/connectors/"; curl --request POST \ --header "Accept:application/json" \ --header "Content-Type:application/json" \ From 359f5885b38c26bb5afba58897aba79c4b07a6c6 Mon Sep 17 00:00:00 2001 From: Ragul Shanmugam Date: Wed, 29 Jan 2025 14:01:17 -0800 Subject: [PATCH 22/27] Adding odse back diff name --- charts/debezium/values-dts1.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/debezium/values-dts1.yaml b/charts/debezium/values-dts1.yaml index b32335b08..b14121c24 100644 --- a/charts/debezium/values-dts1.yaml +++ b/charts/debezium/values-dts1.yaml @@ -41,7 +41,7 @@ connect: bootstrap_server: "b-1.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092" sqlserverconnector: { - "name": "debezium-connector-v013025", + "name": "debezium-odse-connector-v013025", "config": { "connector.class": "io.debezium.connector.sqlserver.SqlServerConnector", "database.hostname": "nbs-db.private-dts1.nbspreview.com", @@ -82,7 +82,7 @@ connect: } sqlserverconnector1: { - "name": "debezium-connector-v013025", + "name": "debezium-srte-connector-v013025", "config": { "connector.class": "io.debezium.connector.sqlserver.SqlServerConnector", "database.hostname": "nbs-db.private-dts1.nbspreview.com", From 2da90cdb08c22b955c109ff9712db40cb86908e2 Mon Sep 17 00:00:00 2001 From: Ragul Shanmugam Date: Wed, 29 Jan 2025 14:17:01 -0800 Subject: [PATCH 23/27] changing to new cluster --- charts/debezium/values-dts1.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/charts/debezium/values-dts1.yaml b/charts/debezium/values-dts1.yaml index b14121c24..3f398db63 100644 --- a/charts/debezium/values-dts1.yaml +++ b/charts/debezium/values-dts1.yaml @@ -31,14 +31,14 @@ connect: memory: 1Gi properties: - group_id: "debezium-connector-v013025" - topics_basename: "debezium-connector" + group_id: "debezium-odse-srte-connector-v013025" + topics_basename: "debezium-odse-srte-connector" default_replication_factor: 2 default_partitions: 10 default_cleanup: "compact" sql_server_agent_override: false sql_server_agent_status: "" - bootstrap_server: "b-1.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092" + bootstrap_server: "b-1.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092" sqlserverconnector: { "name": "debezium-odse-connector-v013025", @@ -51,7 +51,7 @@ connect: "database.dbname": "nbs_odse", "database.names": "nbs_odse", "database.server.name": "odse", - "database.history.kafka.bootstrap.servers": "b-1.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092", + "database.history.kafka.bootstrap.servers": "b-1.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092", "database.history.kafka.topic": "dbhistory.database_server_name.database_name", # Uncomment following to manually bypass the sqlserver agent status query results #"database.sqlserver.agent.status.query": "select dbo.IsSqlAgentRunning()", @@ -63,7 +63,7 @@ connect: "producer.message.max.bytes": "10000000", #10MB "snapshot.mode": "no_data", "schema.history.internal.kafka.topic": "odse-schema-history", - "schema.history.internal.kafka.bootstrap.servers": "b-1.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092", + "schema.history.internal.kafka.bootstrap.servers": "b-1.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092", "table.include.list": "dbo.Person, dbo.Organization, dbo.Observation, dbo.Public_health_case, dbo.state_defined_field_data, dbo.Notification, dbo.Interview, dbo.CN_transportq_out", "tasks.max": "1", "topic.prefix": "cdc", @@ -92,7 +92,7 @@ connect: "database.dbname": "nbs_srte", "database.names": "nbs_srte", "database.server.name": "srte", - "database.history.kafka.bootstrap.servers": "b-1.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092", + "database.history.kafka.bootstrap.servers": "b-1.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092", "database.history.kafka.topic": "dbhistory.database_server_name.database_name", # Uncomment following to manually bypass the sqlserver agent status query results #"database.sqlserver.agent.status.query": "select dbo.IsSqlAgentRunning()", @@ -104,7 +104,7 @@ connect: "producer.message.max.bytes": "10000000", #10MB "snapshot.mode": "initial", "schema.history.internal.kafka.topic": "srte-schema-history", - "schema.history.internal.kafka.bootstrap.servers": "b-1.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092", + "schema.history.internal.kafka.bootstrap.servers": "b-1.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092", "tasks.max": "1", "offset.storage.cleanup.policy": "compact", "topic.prefix": "cdc", @@ -124,13 +124,13 @@ connect: env: - name: BOOTSTRAP_SERVERS - value: "b-1.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092" + value: "b-1.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092" - name: LOG_LEVEL value: "INFO" - name: KAFKA_LOG4J_OPTS value: "-Dlog4j.configuration=file:/kafka/config/log4j.properties" - name: NAME - value: "debezium-connector-v013025" + value: "debezium-odse-srte-connector-v013025" - name: TZ value: "UTC" From 315b6f78960dd50e0df88491cb6628d9e0891e72 Mon Sep 17 00:00:00 2001 From: Ragul Shanmugam Date: Wed, 29 Jan 2025 14:26:45 -0800 Subject: [PATCH 24/27] changing to old cluster --- charts/debezium/values-dts1.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/debezium/values-dts1.yaml b/charts/debezium/values-dts1.yaml index 3f398db63..4c59c4606 100644 --- a/charts/debezium/values-dts1.yaml +++ b/charts/debezium/values-dts1.yaml @@ -38,7 +38,7 @@ connect: default_cleanup: "compact" sql_server_agent_override: false sql_server_agent_status: "" - bootstrap_server: "b-1.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092" + bootstrap_server: "b-1.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092" sqlserverconnector: { "name": "debezium-odse-connector-v013025", @@ -51,7 +51,7 @@ connect: "database.dbname": "nbs_odse", "database.names": "nbs_odse", "database.server.name": "odse", - "database.history.kafka.bootstrap.servers": "b-1.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092", + "database.history.kafka.bootstrap.servers": "b-1.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092", "database.history.kafka.topic": "dbhistory.database_server_name.database_name", # Uncomment following to manually bypass the sqlserver agent status query results #"database.sqlserver.agent.status.query": "select dbo.IsSqlAgentRunning()", @@ -63,7 +63,7 @@ connect: "producer.message.max.bytes": "10000000", #10MB "snapshot.mode": "no_data", "schema.history.internal.kafka.topic": "odse-schema-history", - "schema.history.internal.kafka.bootstrap.servers": "b-1.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092", + "schema.history.internal.kafka.bootstrap.servers": "b-1.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092", "table.include.list": "dbo.Person, dbo.Organization, dbo.Observation, dbo.Public_health_case, dbo.state_defined_field_data, dbo.Notification, dbo.Interview, dbo.CN_transportq_out", "tasks.max": "1", "topic.prefix": "cdc", @@ -92,7 +92,7 @@ connect: "database.dbname": "nbs_srte", "database.names": "nbs_srte", "database.server.name": "srte", - "database.history.kafka.bootstrap.servers": "b-1.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092", + "database.history.kafka.bootstrap.servers": "b-1.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092", "database.history.kafka.topic": "dbhistory.database_server_name.database_name", # Uncomment following to manually bypass the sqlserver agent status query results #"database.sqlserver.agent.status.query": "select dbo.IsSqlAgentRunning()", @@ -104,7 +104,7 @@ connect: "producer.message.max.bytes": "10000000", #10MB "snapshot.mode": "initial", "schema.history.internal.kafka.topic": "srte-schema-history", - "schema.history.internal.kafka.bootstrap.servers": "b-1.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092", + "schema.history.internal.kafka.bootstrap.servers": "b-1.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092", "tasks.max": "1", "offset.storage.cleanup.policy": "compact", "topic.prefix": "cdc", @@ -124,7 +124,7 @@ connect: env: - name: BOOTSTRAP_SERVERS - value: "b-1.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092" + value: "b-1.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092" - name: LOG_LEVEL value: "INFO" - name: KAFKA_LOG4J_OPTS From bf0aa2f6041f15072a2de049279a09d68687440d Mon Sep 17 00:00:00 2001 From: Ragul Shanmugam Date: Wed, 29 Jan 2025 14:43:58 -0800 Subject: [PATCH 25/27] Updating name and new cluster --- charts/debezium/templates/configmap.yaml | 8 ++++---- charts/debezium/templates/deployment.yaml | 12 ++++++------ charts/debezium/values-dts1.yaml | 16 ++++++++-------- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/charts/debezium/templates/configmap.yaml b/charts/debezium/templates/configmap.yaml index 6e583ff13..82cd72c06 100644 --- a/charts/debezium/templates/configmap.yaml +++ b/charts/debezium/templates/configmap.yaml @@ -5,10 +5,10 @@ metadata: labels: {{- include "debezium.labels.connect" . | nindent 4 }} data: - connector.json: |- - {{- .Values.connect.sqlserverconnector| toPrettyJson | nindent 4 }} - connector1.json: |- - {{- .Values.connect.sqlserverconnector1| toPrettyJson | nindent 4 }} + connector-odse.json: |- + {{- .Values.connect.sqlserverconnector_odse| toPrettyJson | nindent 4 }} + connector-srte.json: |- + {{- .Values.connect.sqlserverconnector_srte| toPrettyJson | nindent 4 }} connect-distributed.properties: | group.id={{ .Values.connect.properties.group_id }} diff --git a/charts/debezium/templates/deployment.yaml b/charts/debezium/templates/deployment.yaml index e0bc4f489..d9615bb11 100644 --- a/charts/debezium/templates/deployment.yaml +++ b/charts/debezium/templates/deployment.yaml @@ -41,11 +41,11 @@ spec: {{- toYaml .Values.connect.resources | nindent 12 }} volumeMounts: - name: config - mountPath: /etc/debezium/connector.json - subPath: connector.json + mountPath: /etc/debezium/connector-odse.json + subPath: connector-odse.json - name: config - mountPath: /etc/debezium/connector1.json - subPath: connector1.json + mountPath: /etc/debezium/connector-srte.json + subPath: connector-srte.json - name: config mountPath: /kafka/config/log4j.properties subPath: log4j.properties @@ -69,12 +69,12 @@ spec: curl --request POST \ --header "Accept:application/json" \ --header "Content-Type:application/json" \ - --data '@/etc/debezium/connector.json' \ + --data '@/etc/debezium/connector-odse.json' \ "http://127.0.0.1:8083/connectors/"; curl --request POST \ --header "Accept:application/json" \ --header "Content-Type:application/json" \ - --data '@/etc/debezium/connector1.json' \ + --data '@/etc/debezium/connector-srte.json' \ "http://127.0.0.1:8083/connectors/"; diff --git a/charts/debezium/values-dts1.yaml b/charts/debezium/values-dts1.yaml index 4c59c4606..ede866b08 100644 --- a/charts/debezium/values-dts1.yaml +++ b/charts/debezium/values-dts1.yaml @@ -38,9 +38,9 @@ connect: default_cleanup: "compact" sql_server_agent_override: false sql_server_agent_status: "" - bootstrap_server: "b-1.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092" + bootstrap_server: "b-1.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092" - sqlserverconnector: { + sqlserverconnector_odse: { "name": "debezium-odse-connector-v013025", "config": { "connector.class": "io.debezium.connector.sqlserver.SqlServerConnector", @@ -51,7 +51,7 @@ connect: "database.dbname": "nbs_odse", "database.names": "nbs_odse", "database.server.name": "odse", - "database.history.kafka.bootstrap.servers": "b-1.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092", + "database.history.kafka.bootstrap.servers": "b-1.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092", "database.history.kafka.topic": "dbhistory.database_server_name.database_name", # Uncomment following to manually bypass the sqlserver agent status query results #"database.sqlserver.agent.status.query": "select dbo.IsSqlAgentRunning()", @@ -63,7 +63,7 @@ connect: "producer.message.max.bytes": "10000000", #10MB "snapshot.mode": "no_data", "schema.history.internal.kafka.topic": "odse-schema-history", - "schema.history.internal.kafka.bootstrap.servers": "b-1.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092", + "schema.history.internal.kafka.bootstrap.servers": "b-1.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092", "table.include.list": "dbo.Person, dbo.Organization, dbo.Observation, dbo.Public_health_case, dbo.state_defined_field_data, dbo.Notification, dbo.Interview, dbo.CN_transportq_out", "tasks.max": "1", "topic.prefix": "cdc", @@ -81,7 +81,7 @@ connect: } } - sqlserverconnector1: { + sqlserverconnector_srte: { "name": "debezium-srte-connector-v013025", "config": { "connector.class": "io.debezium.connector.sqlserver.SqlServerConnector", @@ -92,7 +92,7 @@ connect: "database.dbname": "nbs_srte", "database.names": "nbs_srte", "database.server.name": "srte", - "database.history.kafka.bootstrap.servers": "b-1.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092", + "database.history.kafka.bootstrap.servers": "b-1.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092", "database.history.kafka.topic": "dbhistory.database_server_name.database_name", # Uncomment following to manually bypass the sqlserver agent status query results #"database.sqlserver.agent.status.query": "select dbo.IsSqlAgentRunning()", @@ -104,7 +104,7 @@ connect: "producer.message.max.bytes": "10000000", #10MB "snapshot.mode": "initial", "schema.history.internal.kafka.topic": "srte-schema-history", - "schema.history.internal.kafka.bootstrap.servers": "b-1.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092", + "schema.history.internal.kafka.bootstrap.servers": "b-1.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092", "tasks.max": "1", "offset.storage.cleanup.policy": "compact", "topic.prefix": "cdc", @@ -124,7 +124,7 @@ connect: env: - name: BOOTSTRAP_SERVERS - value: "b-1.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092" + value: "b-1.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092" - name: LOG_LEVEL value: "INFO" - name: KAFKA_LOG4J_OPTS From 43143768dbccadbc08aab8c537c79429351fdf4f Mon Sep 17 00:00:00 2001 From: Ragul Shanmugam Date: Thu, 30 Jan 2025 15:58:54 -0800 Subject: [PATCH 26/27] Adding int1 config for srte --- charts/debezium/values-dts1.yaml | 12 ++++---- charts/debezium/values-int1.yaml | 51 ++++++++++++++++++++++++++++---- charts/debezium/values.yaml | 51 ++++++++++++++++++++++++++++---- 3 files changed, 98 insertions(+), 16 deletions(-) diff --git a/charts/debezium/values-dts1.yaml b/charts/debezium/values-dts1.yaml index ede866b08..93d2e1507 100644 --- a/charts/debezium/values-dts1.yaml +++ b/charts/debezium/values-dts1.yaml @@ -31,7 +31,7 @@ connect: memory: 1Gi properties: - group_id: "debezium-odse-srte-connector-v013025" + group_id: "debezium-odse-srte-connector-v013125" topics_basename: "debezium-odse-srte-connector" default_replication_factor: 2 default_partitions: 10 @@ -41,7 +41,7 @@ connect: bootstrap_server: "b-1.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092" sqlserverconnector_odse: { - "name": "debezium-odse-connector-v013025", + "name": "debezium-odse-connector-v013125", "config": { "connector.class": "io.debezium.connector.sqlserver.SqlServerConnector", "database.hostname": "nbs-db.private-dts1.nbspreview.com", @@ -82,7 +82,7 @@ connect: } sqlserverconnector_srte: { - "name": "debezium-srte-connector-v013025", + "name": "debezium-srte-connector-v013125", "config": { "connector.class": "io.debezium.connector.sqlserver.SqlServerConnector", "database.hostname": "nbs-db.private-dts1.nbspreview.com", @@ -105,12 +105,12 @@ connect: "snapshot.mode": "initial", "schema.history.internal.kafka.topic": "srte-schema-history", "schema.history.internal.kafka.bootstrap.servers": "b-1.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreportingdebezium.89ln12.c11.kafka.us-east-1.amazonaws.com:9092", + "table.include.list": "dbo.Anatomic_site_code, dbo.City_code_value, dbo.Cntycity_code_value, dbo.Code_value_clinical, dbo.Code_value_general, dbo.Codeset, dbo.Codeset_Group_Metadata, dbo.Country_code, dbo.Country_Code_ISO, dbo.Country_XREF, dbo.ELR_XREF, dbo.IMRDBMapping, dbo.Investigation_code, dbo.Jurisdiction_code, dbo.Jurisdiction_participation, dbo.Lab_coding_system, dbo.Lab_result, dbo.Lab_result_Snomed, dbo.Lab_test, dbo.Labtest_loinc, dbo.Labtest_Progarea_Mapping, dbo.Language_code, dbo.LDF_page_set, dbo.LOINC_code, dbo.Loinc_condition, dbo.Loinc_snomed_condition, dbo.NAICS_Industry_code, dbo.Occupation_code, dbo.Participation_type, dbo.Program_area_code, dbo.Race_code, dbo.Snomed_code, dbo.Specimen_source_code, dbo.Standard_XREF, dbo.State_code, dbo.State_county_code_value, dbo.State_model, dbo.TotalIDM, dbo.Treatment_code, dbo.Unit_code, dbo.Zip_code_value, dbo.Zipcnty_code_value", "tasks.max": "1", - "offset.storage.cleanup.policy": "compact", "topic.prefix": "cdc", "topic.creation.default.replication.factor": 2, "topic.creation.default.partitions": 10, - "topic.creation.default.cleanup.policy": "delete", + "topic.creation.default.cleanup.policy": "compact", "transforms": "dropPrefix, convertTimezone", "transforms.dropPrefix.regex": "cdc\\.NBS_SRTE\\.dbo\\.(.+)", "transforms.dropPrefix.type": "org.apache.kafka.connect.transforms.RegexRouter", @@ -130,7 +130,7 @@ connect: - name: KAFKA_LOG4J_OPTS value: "-Dlog4j.configuration=file:/kafka/config/log4j.properties" - name: NAME - value: "debezium-odse-srte-connector-v013025" + value: "debezium-odse-srte-connector-v013125" - name: TZ value: "UTC" diff --git a/charts/debezium/values-int1.yaml b/charts/debezium/values-int1.yaml index 8e45c9746..9c46005a7 100644 --- a/charts/debezium/values-int1.yaml +++ b/charts/debezium/values-int1.yaml @@ -31,8 +31,8 @@ connect: memory: 1Gi properties: - group_id: "debezium-connector-v010925" - topics_basename: "debezium-connector" + group_id: "debezium-odse-srte-connector-v013125" + topics_basename: "debezium-odse-srte-connector" default_replication_factor: 2 default_partitions: 10 default_cleanup: "compact" @@ -40,8 +40,8 @@ connect: sql_server_agent_status: "" bootstrap_server: "b-2.cdcnbsint1developmentm.8u6ipl.c2.kafka.us-east-1.amazonaws.com:9092,b-1.cdcnbsint1developmentm.8u6ipl.c2.kafka.us-east-1.amazonaws.com:9092" - sqlserverconnector: { - "name": "debezium-connector-v010925", + sqlserverconnector_odse: { + "name": "debezium-odse-connector-v013125", "config": { "connector.class": "io.debezium.connector.sqlserver.SqlServerConnector", "database.hostname": "nbs-db.private-int1.nbspreview.com", @@ -81,6 +81,47 @@ connect: } } + sqlserverconnector_srte: { + "name": "debezium-srte-connector-v013125", + "config": { + "connector.class": "io.debezium.connector.sqlserver.SqlServerConnector", + "database.hostname": "nbs-db.private-int1.nbspreview.com", + "database.port": "1433", + "database.user": "", + "database.password": "", + "database.dbname": "nbs_srte", + "database.names": "nbs_srte", + "database.server.name": "srte", + "database.history.kafka.bootstrap.servers": "b-2.cdcnbsint1developmentm.8u6ipl.c2.kafka.us-east-1.amazonaws.com:9092,b-1.cdcnbsint1developmentm.8u6ipl.c2.kafka.us-east-1.amazonaws.com:9092", + "database.history.kafka.topic": "dbhistory.database_server_name.database_name", + # Uncomment following to manually bypass the sqlserver agent status query results + #"database.sqlserver.agent.status.query": "select dbo.IsSqlAgentRunning()", + "database.trustServerCertificate": "true", + "include.schema.changes": "true", + "key.converter": "org.apache.kafka.connect.json.JsonConverter", + "key.converter.schemas.enable": "true", + "producer.max.request.size": "10000000", #10MB + "producer.message.max.bytes": "10000000", #10MB + "snapshot.mode": "initial", + "schema.history.internal.kafka.topic": "srte-schema-history", + "schema.history.internal.kafka.bootstrap.servers": "b-2.cdcnbsint1developmentm.8u6ipl.c2.kafka.us-east-1.amazonaws.com:9092,b-1.cdcnbsint1developmentm.8u6ipl.c2.kafka.us-east-1.amazonaws.com:9092", + "table.include.list": "dbo.Anatomic_site_code, dbo.City_code_value, dbo.Cntycity_code_value, dbo.Code_value_clinical, dbo.Code_value_general, dbo.Codeset, dbo.Codeset_Group_Metadata, dbo.Country_code, dbo.Country_Code_ISO, dbo.Country_XREF, dbo.ELR_XREF, dbo.IMRDBMapping, dbo.Investigation_code, dbo.Jurisdiction_code, dbo.Jurisdiction_participation, dbo.Lab_coding_system, dbo.Lab_result, dbo.Lab_result_Snomed, dbo.Lab_test, dbo.Labtest_loinc, dbo.Labtest_Progarea_Mapping, dbo.Language_code, dbo.LDF_page_set, dbo.LOINC_code, dbo.Loinc_condition, dbo.Loinc_snomed_condition, dbo.NAICS_Industry_code, dbo.Occupation_code, dbo.Participation_type, dbo.Program_area_code, dbo.Race_code, dbo.Snomed_code, dbo.Specimen_source_code, dbo.Standard_XREF, dbo.State_code, dbo.State_county_code_value, dbo.State_model, dbo.TotalIDM, dbo.Treatment_code, dbo.Unit_code, dbo.Zip_code_value, dbo.Zipcnty_code_value", + "tasks.max": "1", + "topic.prefix": "cdc", + "topic.creation.default.replication.factor": 2, + "topic.creation.default.partitions": 10, + "topic.creation.default.cleanup.policy": "compact", + "transforms": "dropPrefix, convertTimezone", + "transforms.dropPrefix.regex": "cdc\\.NBS_SRTE\\.dbo\\.(.+)", + "transforms.dropPrefix.type": "org.apache.kafka.connect.transforms.RegexRouter", + "transforms.dropPrefix.replacement": "nrt_srte_$1", + "transforms.convertTimezone.type": "io.debezium.transforms.TimezoneConverter", + "transforms.convertTimezone.converted.timezone": "UTC", + "value.converter": "org.apache.kafka.connect.json.JsonConverter", + "value.converter.schemas.enable": "true", + } + } + env: - name: BOOTSTRAP_SERVERS value: "b-2.cdcnbsint1developmentm.8u6ipl.c2.kafka.us-east-1.amazonaws.com:9092,b-1.cdcnbsint1developmentm.8u6ipl.c2.kafka.us-east-1.amazonaws.com:9092" @@ -89,7 +130,7 @@ connect: - name: KAFKA_LOG4J_OPTS value: "-Dlog4j.configuration=file:/kafka/config/log4j.properties" - name: NAME - value: "debezium-connector-v010925" + value: "debezium-odse-srte-connector-v013125" - name: TZ value: "UTC" diff --git a/charts/debezium/values.yaml b/charts/debezium/values.yaml index 7cba3cf1f..1a986c4bc 100644 --- a/charts/debezium/values.yaml +++ b/charts/debezium/values.yaml @@ -31,8 +31,8 @@ connect: memory: 1Gi properties: - group_id: "debezium-connector-v010925" - topics_basename: "debezium-connector" + group_id: "debezium-odse-srte-connector-v013125" + topics_basename: "debezium-odse-srte-connector" default_replication_factor: 2 default_partitions: 10 default_cleanup: "compact" @@ -42,8 +42,8 @@ connect: # dbserver: "EXAMPLE_DB_ENDPOINT" - sqlserverconnector: { - "name": "debezium-connector-v010925", + sqlserverconnector_odse: { + "name": "debezium-odse-connector-v013125", "config": { "connector.class": "io.debezium.connector.sqlserver.SqlServerConnector", "database.hostname": "nbs-db.private-EXAMPLE_DOMAIN", @@ -83,6 +83,47 @@ connect: } } + sqlserverconnector_srte: { + "name": "debezium-srte-connector-v013125", + "config": { + "connector.class": "io.debezium.connector.sqlserver.SqlServerConnector", + "database.hostname": "nbs-db.private-EXAMPLE_DOMAIN", + "database.port": "1433", + "database.user": "EXAMPLE_DB_USER", + "database.password": "EXAMPLE_DB_USER_PASSWORD", + "database.dbname": "nbs_srte", + "database.names": "nbs_srte", + "database.server.name": "srte", + "database.history.kafka.bootstrap.servers": "EXAMPLE_MSK_KAFKA_ENDPOINT", + "database.history.kafka.topic": "dbhistory.database_server_name.database_name", + # Uncomment following to manually bypass the sqlserver agent status query results + #"database.sqlserver.agent.status.query": "select dbo.IsSqlAgentRunning()", + "database.trustServerCertificate": "true", + "include.schema.changes": "true", + "key.converter": "org.apache.kafka.connect.json.JsonConverter", + "key.converter.schemas.enable": "true", + "producer.max.request.size": "10000000", #10MB + "producer.message.max.bytes": "10000000", #10MB + "snapshot.mode": "initial", + "schema.history.internal.kafka.topic": "srte-schema-history", + "schema.history.internal.kafka.bootstrap.servers": "EXAMPLE_MSK_KAFKA_ENDPOINT", + "table.include.list": "dbo.Anatomic_site_code, dbo.City_code_value, dbo.Cntycity_code_value, dbo.Code_value_clinical, dbo.Code_value_general, dbo.Codeset, dbo.Codeset_Group_Metadata, dbo.Country_code, dbo.Country_Code_ISO, dbo.Country_XREF, dbo.ELR_XREF, dbo.IMRDBMapping, dbo.Investigation_code, dbo.Jurisdiction_code, dbo.Jurisdiction_participation, dbo.Lab_coding_system, dbo.Lab_result, dbo.Lab_result_Snomed, dbo.Lab_test, dbo.Labtest_loinc, dbo.Labtest_Progarea_Mapping, dbo.Language_code, dbo.LDF_page_set, dbo.LOINC_code, dbo.Loinc_condition, dbo.Loinc_snomed_condition, dbo.NAICS_Industry_code, dbo.Occupation_code, dbo.Participation_type, dbo.Program_area_code, dbo.Race_code, dbo.Snomed_code, dbo.Specimen_source_code, dbo.Standard_XREF, dbo.State_code, dbo.State_county_code_value, dbo.State_model, dbo.TotalIDM, dbo.Treatment_code, dbo.Unit_code, dbo.Zip_code_value, dbo.Zipcnty_code_value", + "tasks.max": "1", + "topic.prefix": "cdc", + "topic.creation.default.replication.factor": 2, + "topic.creation.default.partitions": 10, + "topic.creation.default.cleanup.policy": "compact", + "transforms": "dropPrefix, convertTimezone", + "transforms.dropPrefix.regex": "cdc\\.NBS_SRTE\\.dbo\\.(.+)", + "transforms.dropPrefix.type": "org.apache.kafka.connect.transforms.RegexRouter", + "transforms.dropPrefix.replacement": "nrt_srte_$1", + "transforms.convertTimezone.type": "io.debezium.transforms.TimezoneConverter", + "transforms.convertTimezone.converted.timezone": "UTC", + "value.converter": "org.apache.kafka.connect.json.JsonConverter", + "value.converter.schemas.enable": "true", + } + } + env: - name: BOOTSTRAP_SERVERS value: "EXAMPLE_MSK_KAFKA_ENDPOINT" @@ -91,7 +132,7 @@ connect: - name: KAFKA_LOG4J_OPTS value: "-Dlog4j.configuration=file:/kafka/config/log4j.properties" - name: NAME - value: "debezium-connector-v010925" + value: "debezium-odse-srte-connector-v013125" - name: TZ value: "UTC" From 1d086ed8479510033d563c1c4a93de654baf6613 Mon Sep 17 00:00:00 2001 From: Ragul Shanmugam Date: Mon, 3 Feb 2025 08:03:56 -0800 Subject: [PATCH 27/27] Updated debezium envs for odse_srte and removed srte chart --- charts/debezium-srte/.helmignore | 23 --- charts/debezium-srte/Chart.yaml | 10 -- charts/debezium-srte/readme.md | 82 ----------- charts/debezium-srte/templates/NOTES.txt | 18 --- charts/debezium-srte/templates/_helpers.tpl | 78 ---------- charts/debezium-srte/templates/configmap.yaml | 48 ------- .../debezium-srte/templates/deployment.yaml | 128 ----------------- charts/debezium-srte/templates/ingress.yaml | 22 --- charts/debezium-srte/templates/service.yaml | 33 ----- charts/debezium-srte/values-azure.yaml | 133 ----------------- charts/debezium-srte/values-dts1.yaml | 132 ----------------- charts/debezium-srte/values-feature.yaml | 132 ----------------- charts/debezium-srte/values-int1.yaml | 132 ----------------- charts/debezium-srte/values-test1.yaml | 132 ----------------- charts/debezium-srte/values.yaml | 135 ------------------ charts/debezium/values-azure.yaml | 50 ++++++- charts/debezium/values-feature.yaml | 51 ++++++- charts/debezium/values-test1.yaml | 51 ++++++- charts/debezium/values.yaml | 2 +- 19 files changed, 138 insertions(+), 1254 deletions(-) delete mode 100644 charts/debezium-srte/.helmignore delete mode 100644 charts/debezium-srte/Chart.yaml delete mode 100644 charts/debezium-srte/readme.md delete mode 100644 charts/debezium-srte/templates/NOTES.txt delete mode 100644 charts/debezium-srte/templates/_helpers.tpl delete mode 100644 charts/debezium-srte/templates/configmap.yaml delete mode 100644 charts/debezium-srte/templates/deployment.yaml delete mode 100644 charts/debezium-srte/templates/ingress.yaml delete mode 100644 charts/debezium-srte/templates/service.yaml delete mode 100644 charts/debezium-srte/values-azure.yaml delete mode 100644 charts/debezium-srte/values-dts1.yaml delete mode 100644 charts/debezium-srte/values-feature.yaml delete mode 100644 charts/debezium-srte/values-int1.yaml delete mode 100644 charts/debezium-srte/values-test1.yaml delete mode 100644 charts/debezium-srte/values.yaml diff --git a/charts/debezium-srte/.helmignore b/charts/debezium-srte/.helmignore deleted file mode 100644 index 0e8a0eb36..000000000 --- a/charts/debezium-srte/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/charts/debezium-srte/Chart.yaml b/charts/debezium-srte/Chart.yaml deleted file mode 100644 index e9a1c5494..000000000 --- a/charts/debezium-srte/Chart.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: v2 -name: debezium -description: Debezium & Debezium UI Chart -icon: https://debezium.io/assets/images/color_white_debezium_type_600px.svg - -type: application - -version: 0.0.2 - -appVersion: v0-0-2 diff --git a/charts/debezium-srte/readme.md b/charts/debezium-srte/readme.md deleted file mode 100644 index d54b930ba..000000000 --- a/charts/debezium-srte/readme.md +++ /dev/null @@ -1,82 +0,0 @@ -## Prerequisite - -Database scripts to be run before deploying Debezium Change Data Capture helm chart - -### Grant `master` database View Server State to user `nbs_ods` -```tsql -USE MASTER -GO -GRANT VIEW SERVER STATE TO nbs_ods; -``` - -### Enable Change data capture on Database level. -[Change Data Capture](https://learn.microsoft.com/en-us/sql/relational-databases/track-changes/enable-and-disable-change-data-capture-sql-server?view=sql-server-ver16) - -```tsql -USE nbs_srte -GO -EXEC sys.sp_cdc_enable_db -GO -``` - -### Enable Change data capture on table level. -Run the below SQL script, copy the contents of the result to query window and execute them all. - -```tsql -use nbs_srte; - -select concat('exec sys.sp_cdc_enable_table @source_schema = N', '''', TABLE_SCHEMA, ''',@source_name = N', '''', -TABLE_NAME, ''',@role_name = NULL;') -from INFORMATION_SCHEMA.TABLES t -where t.TABLE_catalog = 'nbs_srte' -and t.TABLE_SCHEMA = 'dbo' -and TABLE_TYPE = 'Base Table' -and t.TABLE_NAME in (select distinct TABLE_NAME -from INFORMATION_SCHEMA.TABLE_CONSTRAINTS tc -where tc.CONSTRAINT_catalog = 'nbs_srte' -and tc.table_catalog = 'nbs_srte' -and tc.CONSTRAINT_TYPE = 'PRIMARY KEY' -and tc.CONSTRAINT_SCHEMA = 'dbo' -and tc.TABLE_NAME not like '%TEMP%' -and tc.TABLE_SCHEMA = 'dbo') -order by t.table_name; -``` - -### Verify if Change data capture is enabled - -```tsql -select - name, - is_cdc_enabled -from sys.databases; -``` - -### Verify the tables tracked by Sql Server for Change Data Capture -```tsql -select - name, - is_tracked_by_cdc -from sys.tables; -``` - -### If SqlServer edition is lesser than 2016 then Upgrade compatibility level to allow inbuilt functions such as StringSplit - -```tsql -ALTER DATABASE nbs_srte SET COMPATIBILITY_LEVEL = 130; -``` - -### For Azure managed SqlServer Instance only -Sql Server agent run status is not exposed hence it needs to be mimicked - -```tsql -use nbs_srte; -CREATE or alter FUNCTION dbo.IsSqlAgentRunning() -RETURNS BIT -AS -BEGIN -DECLARE @IsRunning BIT = 1; - - RETURN @IsRunning; - -END; -``` \ No newline at end of file diff --git a/charts/debezium-srte/templates/NOTES.txt b/charts/debezium-srte/templates/NOTES.txt deleted file mode 100644 index f8183232b..000000000 --- a/charts/debezium-srte/templates/NOTES.txt +++ /dev/null @@ -1,18 +0,0 @@ -1. Get the application URL by running these commands: -{{- if .Values.ui.ingress.enabled }} - http://{{ .Values.ui.ingress.host }} -{{- else if contains "NodePort" .Values.connect.service.type }} - export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "debezium.fullname" . }}) - export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") - echo http://$NODE_IP:$NODE_PORT -{{- else if contains "LoadBalancer" .Values.connect.service.type }} - NOTE: It may take a few minutes for the LoadBalancer IP to be available. - You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "debezium.fullname" . }}' - export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "debezium.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") - echo http://$SERVICE_IP:{{ .Values.connect.service.port }} -{{- else if contains "ClusterIP" .Values.connect.service.type }} - export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "debezium.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") - export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") - echo "Visit http://127.0.0.1:8080 to use your application" - kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT -{{- end }} diff --git a/charts/debezium-srte/templates/_helpers.tpl b/charts/debezium-srte/templates/_helpers.tpl deleted file mode 100644 index f32b39c0e..000000000 --- a/charts/debezium-srte/templates/_helpers.tpl +++ /dev/null @@ -1,78 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "debezium.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "debezium.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "debezium.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "debezium.labels.connect" -}} -helm.sh/chart: {{ include "debezium.chart" . }} -{{ include "debezium.selectorLabels.connect" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{- define "debezium.labels.ui" -}} -helm.sh/chart: {{ include "debezium.chart" . }} -{{ include "debezium.selectorLabels.ui" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "debezium.selectorLabels.connect" -}} -app.kubernetes.io/name: {{ include "debezium.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -app.kubernetes.io/app: connect -{{- end }} - -{{- define "debezium.selectorLabels.ui" -}} -app.kubernetes.io/name: {{ include "debezium.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -app.kubernetes.io/app: ui -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "debezium.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "debezium.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} diff --git a/charts/debezium-srte/templates/configmap.yaml b/charts/debezium-srte/templates/configmap.yaml deleted file mode 100644 index d7fccc729..000000000 --- a/charts/debezium-srte/templates/configmap.yaml +++ /dev/null @@ -1,48 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "debezium.name" . }}-connect - labels: - {{- include "debezium.labels.connect" . | nindent 4 }} -data: - connector.json: |- - {{- .Values.connect.sqlserverconnector| toPrettyJson | nindent 4 }} - - connect-distributed.properties: | - group.id={{ .Values.connect.properties.group_id }} - plugin.path=/kafka/connect - producer.acks=1 - bootstrap.servers={{ .Values.connect.properties.bootstrap_server }} - #client.config.override.policy=all - key.converter=org.apache.kafka.connect.json.JsonConverter - key.converter.schemas.enable=false - min.in.sync.replicas=2 - max.request.size=10000000 - message.max.bytes=10000000 - offset.flush.timeout.ms=30000 - snapshot.mode=no_data - value.converter=org.apache.kafka.connect.json.JsonConverter - value.converter.schemas.enable=false - include.schema.changes=true - - config.storage.topic={{ .Values.connect.properties.topics_basename }}_config - config.storage.replication.factor={{ .Values.connect.properties.default_replication_factor }} - - status.storage.topic={{ .Values.connect.properties.topics_basename }}_status - status.storage.replication.factor={{ .Values.connect.properties.default_replication_factor }} - - offset.storage.topic={{ .Values.connect.properties.topics_basename }}_offset - offset.storage.replication.factor={{ .Values.connect.properties.default_replication_factor }} - - topic.creation.default.replication.factor={{ .Values.connect.properties.default_replication_factor }} - topic.creation.default.partitions={{ .Values.connect.properties.default_partitions }} - topic.creation.default.cleanup.policy={{ .Values.connect.properties.default_cleanup }} - {{- if .Values.connect.properties.sql_server_agent_override }} - database.sqlserver.agent.status.query={{ .Values.connect.properties.sql_server_agent_status }} - {{end}} - - log4j.properties: | - log4j.rootLogger={{ .Values.log_level }}, stdout - log4j.appender.stdout=org.apache.log4j.ConsoleAppender - log4j.appender.stdout.layout=org.apache.log4j.PatternLayout - log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c:%L)%n diff --git a/charts/debezium-srte/templates/deployment.yaml b/charts/debezium-srte/templates/deployment.yaml deleted file mode 100644 index 54470fadb..000000000 --- a/charts/debezium-srte/templates/deployment.yaml +++ /dev/null @@ -1,128 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "debezium.fullname" . }}-connect - labels: - {{- include "debezium.labels.connect" . | nindent 4 }} -spec: - {{- if not .Values.connect.autoscaling.enabled }} - replicas: {{ .Values.connect.replicaCount }} - {{- end }} - selector: - matchLabels: - {{- include "debezium.selectorLabels.connect" . | nindent 6 }} - template: - metadata: - {{- with .Values.connect.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - {{- include "debezium.selectorLabels.connect" . | nindent 8 }} - spec: - {{- with .Values.connect.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - volumes: - - name: config - configMap: - name: {{ include "debezium.name" . }}-connect - defaultMode: 0744 - containers: - - name: {{ .Chart.Name }}-connect - image: "{{ .Values.connect.image.repository }}:{{ .Values.connect.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.connect.image.pullPolicy }} - ports: - - name: {{ .Values.connect.service.name }} - containerPort: {{ .Values.connect.service.port }} - protocol: {{ .Values.connect.service.protocol }} - resources: - {{- toYaml .Values.connect.resources | nindent 12 }} - volumeMounts: - - name: config - mountPath: /etc/debezium/connector.json - subPath: connector.json - - name: config - mountPath: /kafka/config/log4j.properties - subPath: log4j.properties - - name: config - mountPath: /kafka/config/connect-distributed.properties - subPath: connect-distributed.properties - {{- with .Values.connect.env }} - env: - {{- toYaml . | nindent 12}} - {{- end}} - command: ['/kafka/bin/connect-distributed.sh', '/kafka/config/connect-distributed.properties'] - lifecycle: - postStart: - exec: - command: [ '/bin/sh', '-c', 'sleep 60; curl --request POST --header "Accept:application/json" --header "Content-Type:application/json" "http://127.0.0.1:8083/connectors/" --data "@/etc/debezium/connector.json"' ] - - {{- with .Values.connect.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.connect.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.connect.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} ---- -{{- if .Values.ui.enabled -}} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "debezium.fullname" . }}-ui - labels: - {{- include "debezium.labels.ui" . | nindent 4 }} -spec: - {{- if not .Values.ui.autoscaling.enabled }} - replicas: {{ .Values.ui.replicaCount }} - {{- end }} - selector: - matchLabels: - {{- include "debezium.selectorLabels.ui" . | nindent 6 }} - template: - metadata: - {{- with .Values.ui.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - {{- include "debezium.selectorLabels.ui" . | nindent 8 }} - spec: - {{- with .Values.ui.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - containers: - - name: {{ .Chart.Name }}-ui - image: "{{ .Values.ui.image.repository }}:{{ .Values.ui.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.ui.image.pullPolicy }} - ports: - - name: {{ .Values.ui.service.name}} - containerPort: 80 - protocol: TCP - resources: - {{- toYaml .Values.ui.resources | nindent 12 }} - {{- with .Values.ui.env }} - env: - {{- toYaml . | nindent 12}} - {{- end}} - {{- with .Values.ui.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.ui.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.ui.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} -{{- end}} \ No newline at end of file diff --git a/charts/debezium-srte/templates/ingress.yaml b/charts/debezium-srte/templates/ingress.yaml deleted file mode 100644 index 837e1a61c..000000000 --- a/charts/debezium-srte/templates/ingress.yaml +++ /dev/null @@ -1,22 +0,0 @@ -{{- if and .Values.ui.ingress.enabled .Values.ui.enabled -}} -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: {{ include "debezium.fullname" $ }}-ui - labels: - router: {{ .Values.ui.ingress.router }} - {{- include "debezium.labels.ui" $ | nindent 4 }} -spec: - rules: - host: {{ .Values.ui.ingress.host | quote }} - http: - paths: - - path: "/nbs/redirect/" - pathType: Prefix - backend: - service: - name: {{ include "debezium.fullname" $ }}-ui - port: - number: {{ $.Values.ui.service.port }} -{{- end }} - diff --git a/charts/debezium-srte/templates/service.yaml b/charts/debezium-srte/templates/service.yaml deleted file mode 100644 index 86d65375a..000000000 --- a/charts/debezium-srte/templates/service.yaml +++ /dev/null @@ -1,33 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ include "debezium.fullname" . }}-connect - labels: - {{- include "debezium.labels.connect" . | nindent 4 }} -spec: - type: {{ .Values.connect.service.type }} - ports: - - port: {{ .Values.connect.service.port }} - targetPort: {{ .Values.connect.service.port}} - protocol: {{ .Values.connect.service.protocol}} - name: {{ .Values.connect.service.name}} - selector: - {{- include "debezium.selectorLabels.connect" . | nindent 4 }} ---- -{{- if .Values.ui.enabled -}} -apiVersion: v1 -kind: Service -metadata: - name: {{ include "debezium.fullname" . }}-ui - labels: - {{- include "debezium.labels.ui" . | nindent 4 }} -spec: - type: {{ .Values.ui.service.type }} - ports: - - port: {{ .Values.ui.service.port }} - targetPort: {{ .Values.ui.service.port}} - protocol: {{ .Values.ui.service.protocol}} - name: {{ .Values.ui.service.name}} - selector: - {{- include "debezium.selectorLabels.ui" . | nindent 4 }} -{{- end}} \ No newline at end of file diff --git a/charts/debezium-srte/values-azure.yaml b/charts/debezium-srte/values-azure.yaml deleted file mode 100644 index 6b7a0ea0a..000000000 --- a/charts/debezium-srte/values-azure.yaml +++ /dev/null @@ -1,133 +0,0 @@ -nameOverride: "" -fullnameOverride: "" -log_level: INFO # DEBUG, INFO, ERROR - -connect: - replicaCount: 1 - imagePullSecrets: [ ] - - image: - repository: quay.io/debezium/connect - pullPolicy: IfNotPresent - tag: "2.7" - - service: - type: ClusterIP - port: 8083 - protocol: TCP - name: http - - ingress: - enabled: false - - autoscaling: - enabled: false - - resources: - limits: - cpu: 1000m - memory: 2Gi - requests: - cpu: 500m - memory: 1Gi - - properties: - group_id: "debezium-connector-v010925" - topics_basename: "debezium-connector" - default_replication_factor: 2 - default_partitions: 10 - default_cleanup: "compact" - bootstrap_server: "wn0-dev-hd.dsnw3omymczu3cea2rtxoridtc.bx.internal.cloudapp.net:9092,wn1-dev-hd.dsnw3omymczu3cea2rtxoridtc.bx.internal.cloudapp.net:9092,wn2-dev-hd.dsnw3omymczu3cea2rtxoridtc.bx.internal.cloudapp.net:9092" - sql_server_agent_override: true - sql_server_agent_status: "select dbo.IsSqlAgentRunning()" - - sqlserverconnector: { - "name": "debezium-connector-v010925", - "config": { - "connector.class": "io.debezium.connector.sqlserver.SqlServerConnector", - "database.hostname": "tf-cdc-nbs6-sql-managed-instance.cdf5734b998e.database.windows.net", - "database.port": "1433", - "database.user": "nbs_ods", - "database.password": "ods", - "database.dbname": "nbs_srte", - "database.names": "nbs_srte", - "database.server.name": "srte", - "database.history.kafka.bootstrap.servers": "wn0-dev-hd.dsnw3omymczu3cea2rtxoridtc.bx.internal.cloudapp.net:9092,wn1-dev-hd.dsnw3omymczu3cea2rtxoridtc.bx.internal.cloudapp.net:9092,wn2-dev-hd.dsnw3omymczu3cea2rtxoridtc.bx.internal.cloudapp.net:9092", - "database.history.kafka.topic": "dbhistory.database_server_name.database_name", -# manually bypass the sqlserver agent status query - "database.sqlserver.agent.status.query": "select dbo.IsSqlAgentRunning()", - "database.trustServerCertificate": "true", - "include.schema.changes": "true", - "key.converter": "org.apache.kafka.connect.json.JsonConverter", - "key.converter.schemas.enable": "true", - "producer.max.request.size": "10000000", #10MB - "producer.message.max.bytes": "10000000", #10MB - "snapshot.mode": "no_data", - "schema.history.internal.kafka.topic": "srte-schema-history", - "schema.history.internal.kafka.bootstrap.servers": "wn0-dev-hd.dsnw3omymczu3cea2rtxoridtc.bx.internal.cloudapp.net:9092,wn1-dev-hd.dsnw3omymczu3cea2rtxoridtc.bx.internal.cloudapp.net:9092,wn2-dev-hd.dsnw3omymczu3cea2rtxoridtc.bx.internal.cloudapp.net:9092", - "tasks.max": "1", - "topic.prefix": "cdc", - "topic.creation.default.replication.factor": 2, - "topic.creation.default.partitions": 10, - "topic.creation.default.cleanup.policy": "compact", - "transforms": "dropPrefix, convertTimezone", - "transforms.dropPrefix.regex": "cdc\\.NBS_SRTE\\.dbo\\.(.+)", - "transforms.dropPrefix.type": "org.apache.kafka.connect.transforms.RegexRouter", - "transforms.dropPrefix.replacement": "nrt_srte_$1", - "transforms.convertTimezone.type": "io.debezium.transforms.TimezoneConverter", - "transforms.convertTimezone.converted.timezone": "UTC", - "value.converter": "org.apache.kafka.connect.json.JsonConverter", - "value.converter.schemas.enable": "true", - } - } - - env: - - name: BOOTSTRAP_SERVERS - value: "wn0-dev-hd.dsnw3omymczu3cea2rtxoridtc.bx.internal.cloudapp.net:9092,wn1-dev-hd.dsnw3omymczu3cea2rtxoridtc.bx.internal.cloudapp.net:9092,wn2-dev-hd.dsnw3omymczu3cea2rtxoridtc.bx.internal.cloudapp.net:9092" - - name: LOG_LEVEL - value: "INFO" - - name: KAFKA_LOG4J_OPTS - value: "-Dlog4j.configuration=file:/kafka/config/log4j.properties" - - name: NAME - value: "debezium-connector-v010925" - - name: TZ - value: "UTC" - -ui: - enabled: false - replicaCount: 1 - imagePullSecrets: [ ] - - image: - repository: debezium/debezium-ui - pullPolicy: Always - tag: "2.7" - - service: - type: ClusterIP - port: "8080" - protocol: TCP - name: http - - ingress: - enabled: true - router: private - host: DEBEZIUM_UI_HOST_ADDRESS - - autoscaling: - enabled: false - - resources: - limits: - cpu: 500m - memory: 1Gi - requests: - cpu: 500m - memory: 1Gi - - env: - - name: KAFKA_CONNECT_URIS - value: "http://debezium-connect:8083" - - name: TZ - value: "UTC" - diff --git a/charts/debezium-srte/values-dts1.yaml b/charts/debezium-srte/values-dts1.yaml deleted file mode 100644 index fdb2334d5..000000000 --- a/charts/debezium-srte/values-dts1.yaml +++ /dev/null @@ -1,132 +0,0 @@ -nameOverride: "" -fullnameOverride: "" -log_level: INFO -connect: - replicaCount: 1 - imagePullSecrets: [ ] - - image: - repository: quay.io/debezium/connect - pullPolicy: IfNotPresent - tag: "2.7" - - service: - type: ClusterIP - port: 8083 - protocol: TCP - name: http - - ingress: - enabled: false - - autoscaling: - enabled: false - - resources: - limits: - cpu: 1000m - memory: 2Gi - requests: - cpu: 500m - memory: 1Gi - - properties: - group_id: "debezium-connector-v010925" - topics_basename: "debezium-connector" - default_replication_factor: 2 - default_partitions: 10 - default_cleanup: "compact" - sql_server_agent_override: false - sql_server_agent_status: "" - bootstrap_server: "b-1.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092" - - sqlserverconnector: { - "name": "debezium-connector-v010925", - "config": { - "connector.class": "io.debezium.connector.sqlserver.SqlServerConnector", - "database.hostname": "nbs-db.private-dts1.nbspreview.com", - "database.port": "1433", - "database.user": "", - "database.password": "", - "database.dbname": "nbs_srte", - "database.names": "nbs_srte", - "database.server.name": "srte", - "database.history.kafka.bootstrap.servers": "b-1.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092", - "database.history.kafka.topic": "dbhistory.database_server_name.database_name", - # Uncomment following to manually bypass the sqlserver agent status query results - #"database.sqlserver.agent.status.query": "select dbo.IsSqlAgentRunning()", - "database.trustServerCertificate": "true", - "include.schema.changes": "true", - "key.converter": "org.apache.kafka.connect.json.JsonConverter", - "key.converter.schemas.enable": "true", - "producer.max.request.size": "10000000", #10MB - "producer.message.max.bytes": "10000000", #10MB - "snapshot.mode": "initial", - "schema.history.internal.kafka.topic": "srte-schema-history", - "schema.history.internal.kafka.bootstrap.servers": "b-1.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092", - "tasks.max": "1", - "topic.prefix": "cdc", - "topic.creation.default.replication.factor": 2, - "topic.creation.default.partitions": 10, - "topic.creation.default.cleanup.policy": "compact", - "transforms": "dropPrefix, convertTimezone", - "transforms.dropPrefix.regex": "cdc\\.NBS_SRTE\\.dbo\\.(.+)", - "transforms.dropPrefix.type": "org.apache.kafka.connect.transforms.RegexRouter", - "transforms.dropPrefix.replacement": "nrt_srte_$1", - "transforms.convertTimezone.type": "io.debezium.transforms.TimezoneConverter", - "transforms.convertTimezone.converted.timezone": "UTC", - "value.converter": "org.apache.kafka.connect.json.JsonConverter", - "value.converter.schemas.enable": "true", - } - } - - env: - - name: BOOTSTRAP_SERVERS - value: "b-1.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092" - - name: LOG_LEVEL - value: "INFO" - - name: KAFKA_LOG4J_OPTS - value: "-Dlog4j.configuration=file:/kafka/config/log4j.properties" - - name: NAME - value: "debezium-connector-v010925" - - name: TZ - value: "UTC" - -ui: - enabled: false - replicaCount: 1 - imagePullSecrets: [ ] - - image: - repository: debezium/debezium-ui - pullPolicy: Always - tag: "2.7" - - service: - type: ClusterIP - port: "8080" - protocol: TCP - name: http - - ingress: - enabled: true - router: private - host: DEBEZIUM_UI_HOST_ADDRESS - - autoscaling: - enabled: false - - resources: - limits: - cpu: 500m - memory: 1Gi - requests: - cpu: 500m - memory: 1Gi - - env: - - name: KAFKA_CONNECT_URIS - value: "http://debezium-connect:8083" - - name: TZ - value: "UTC" - diff --git a/charts/debezium-srte/values-feature.yaml b/charts/debezium-srte/values-feature.yaml deleted file mode 100644 index 766b21307..000000000 --- a/charts/debezium-srte/values-feature.yaml +++ /dev/null @@ -1,132 +0,0 @@ -nameOverride: "" -fullnameOverride: "" -log_level: INFO -connect: - replicaCount: 1 - imagePullSecrets: [ ] - - image: - repository: quay.io/debezium/connect - pullPolicy: IfNotPresent - tag: "2.7" - - service: - type: ClusterIP - port: 8083 - protocol: TCP - name: http - - ingress: - enabled: false - - autoscaling: - enabled: false - - resources: - limits: - cpu: 1000m - memory: 2Gi - requests: - cpu: 500m - memory: 1Gi - - properties: - group_id: "debezium-connector-v010925" - topics_basename: "debezium-connector" - default_replication_factor: 2 - default_partitions: 10 - default_cleanup: "compact" - sql_server_agent_override: false - sql_server_agent_status: "" - bootstrap_server: "b-2.cdcnbsfeaturedevelopme.f4c8vq.c9.kafka.us-east-1.amazonaws.com:9092,b-1.cdcnbsfeaturedevelopme.f4c8vq.c9.kafka.us-east-1.amazonaws.com:9092" - - sqlserverconnector: { - "name": "debezium-connector-v010925", - "config": { - "connector.class": "io.debezium.connector.sqlserver.SqlServerConnector", - "database.hostname": "nbs-db.private-feature.nbspreview.com", - "database.port": "1433", - "database.user": "", - "database.password": "", - "database.dbname": "nbs_srte", - "database.names": "nbs_srte", - "database.server.name": "srte", - "database.history.kafka.bootstrap.servers": "b-2.cdcnbsfeaturedevelopme.f4c8vq.c9.kafka.us-east-1.amazonaws.com:9092,b-1.cdcnbsfeaturedevelopme.f4c8vq.c9.kafka.us-east-1.amazonaws.com:9092", - "database.history.kafka.topic": "dbhistory.database_server_name.database_name", - # Uncomment following to manually bypass the sqlserver agent status query results - #"database.sqlserver.agent.status.query": "select dbo.IsSqlAgentRunning()", - "database.trustServerCertificate": "true", - "include.schema.changes": "true", - "key.converter": "org.apache.kafka.connect.json.JsonConverter", - "key.converter.schemas.enable": "true", - "producer.max.request.size": "10000000", #10MB - "producer.message.max.bytes": "10000000", #10MB - "snapshot.mode": "no_data", - "schema.history.internal.kafka.topic": "srte-schema-history", - "schema.history.internal.kafka.bootstrap.servers": "b-2.cdcnbsfeaturedevelopme.f4c8vq.c9.kafka.us-east-1.amazonaws.com:9092,b-1.cdcnbsfeaturedevelopme.f4c8vq.c9.kafka.us-east-1.amazonaws.com:9092", - "tasks.max": "1", - "topic.prefix": "cdc", - "topic.creation.default.replication.factor": 2, - "topic.creation.default.partitions": 10, - "topic.creation.default.cleanup.policy": "compact", - "transforms": "dropPrefix, convertTimezone", - "transforms.dropPrefix.regex": "cdc\\.NBS_SRTE\\.dbo\\.(.+)", - "transforms.dropPrefix.type": "org.apache.kafka.connect.transforms.RegexRouter", - "transforms.dropPrefix.replacement": "nrt_srte_$1", - "transforms.convertTimezone.type": "io.debezium.transforms.TimezoneConverter", - "transforms.convertTimezone.converted.timezone": "UTC", - "value.converter": "org.apache.kafka.connect.json.JsonConverter", - "value.converter.schemas.enable": "true", - } - } - - env: - - name: BOOTSTRAP_SERVERS - value: "b-2.cdcnbsfeaturedevelopme.f4c8vq.c9.kafka.us-east-1.amazonaws.com:9092,b-1.cdcnbsfeaturedevelopme.f4c8vq.c9.kafka.us-east-1.amazonaws.com:9092" - - name: LOG_LEVEL - value: "INFO" - - name: KAFKA_LOG4J_OPTS - value: "-Dlog4j.configuration=file:/kafka/config/log4j.properties" - - name: NAME - value: "debezium-connector-v010925" - - name: TZ - value: "UTC" - -ui: - enabled: false - replicaCount: 1 - imagePullSecrets: [ ] - - image: - repository: debezium/debezium-ui - pullPolicy: Always - tag: "2.7" - - service: - type: ClusterIP - port: "8080" - protocol: TCP - name: http - - ingress: - enabled: true - router: private - host: DEBEZIUM_UI_HOST_ADDRESS - - autoscaling: - enabled: false - - resources: - limits: - cpu: 500m - memory: 1Gi - requests: - cpu: 500m - memory: 1Gi - - env: - - name: KAFKA_CONNECT_URIS - value: "http://debezium-connect:8083" - - name: TZ - value: "UTC" - diff --git a/charts/debezium-srte/values-int1.yaml b/charts/debezium-srte/values-int1.yaml deleted file mode 100644 index 6bb3aa809..000000000 --- a/charts/debezium-srte/values-int1.yaml +++ /dev/null @@ -1,132 +0,0 @@ -nameOverride: "" -fullnameOverride: "" -log_level: INFO -connect: - replicaCount: 1 - imagePullSecrets: [ ] - - image: - repository: quay.io/debezium/connect - pullPolicy: IfNotPresent - tag: "2.7" - - service: - type: ClusterIP - port: 8083 - protocol: TCP - name: http - - ingress: - enabled: false - - autoscaling: - enabled: false - - resources: - limits: - cpu: 1000m - memory: 2Gi - requests: - cpu: 500m - memory: 1Gi - - properties: - group_id: "debezium-connector-v010925" - topics_basename: "debezium-connector" - default_replication_factor: 2 - default_partitions: 10 - default_cleanup: "compact" - sql_server_agent_override: false - sql_server_agent_status: "" - bootstrap_server: "b-2.cdcnbsint1developmentm.8u6ipl.c2.kafka.us-east-1.amazonaws.com:9092,b-1.cdcnbsint1developmentm.8u6ipl.c2.kafka.us-east-1.amazonaws.com:9092" - - sqlserverconnector: { - "name": "debezium-connector-v010925", - "config": { - "connector.class": "io.debezium.connector.sqlserver.SqlServerConnector", - "database.hostname": "nbs-db.private-int1.nbspreview.com", - "database.port": "1433", - "database.user": "", - "database.password": "", - "database.dbname": "nbs_srte", - "database.names": "nbs_srte", - "database.server.name": "srte", - "database.history.kafka.bootstrap.servers": "b-2.cdcnbsint1developmentm.8u6ipl.c2.kafka.us-east-1.amazonaws.com:9092,b-1.cdcnbsint1developmentm.8u6ipl.c2.kafka.us-east-1.amazonaws.com:9092", - "database.history.kafka.topic": "dbhistory.database_server_name.database_name", - # Uncomment following to manually bypass the sqlserver agent status query results - #"database.sqlserver.agent.status.query": "select dbo.IsSqlAgentRunning()", - "database.trustServerCertificate": "true", - "include.schema.changes": "true", - "key.converter": "org.apache.kafka.connect.json.JsonConverter", - "key.converter.schemas.enable": "true", - "producer.max.request.size": "10000000", #10MB - "producer.message.max.bytes": "10000000", #10MB - "snapshot.mode": "no_data", - "schema.history.internal.kafka.topic": "srte-schema-history", - "schema.history.internal.kafka.bootstrap.servers": "b-2.cdcnbsint1developmentm.8u6ipl.c2.kafka.us-east-1.amazonaws.com:9092,b-1.cdcnbsint1developmentm.8u6ipl.c2.kafka.us-east-1.amazonaws.com:9092", - "tasks.max": "1", - "topic.prefix": "cdc", - "topic.creation.default.replication.factor": 2, - "topic.creation.default.partitions": 10, - "topic.creation.default.cleanup.policy": "compact", - "transforms": "dropPrefix, convertTimezone", - "transforms.dropPrefix.regex": "cdc\\.NBS_SRTE\\.dbo\\.(.+)", - "transforms.dropPrefix.type": "org.apache.kafka.connect.transforms.RegexRouter", - "transforms.dropPrefix.replacement": "nrt_srte_$1", - "transforms.convertTimezone.type": "io.debezium.transforms.TimezoneConverter", - "transforms.convertTimezone.converted.timezone": "UTC", - "value.converter": "org.apache.kafka.connect.json.JsonConverter", - "value.converter.schemas.enable": "true", - } - } - - env: - - name: BOOTSTRAP_SERVERS - value: "b-2.cdcnbsint1developmentm.8u6ipl.c2.kafka.us-east-1.amazonaws.com:9092,b-1.cdcnbsint1developmentm.8u6ipl.c2.kafka.us-east-1.amazonaws.com:9092" - - name: LOG_LEVEL - value: "INFO" - - name: KAFKA_LOG4J_OPTS - value: "-Dlog4j.configuration=file:/kafka/config/log4j.properties" - - name: NAME - value: "debezium-connector-v010925" - - name: TZ - value: "UTC" - -ui: - enabled: false - replicaCount: 1 - imagePullSecrets: [ ] - - image: - repository: debezium/debezium-ui - pullPolicy: Always - tag: "2.7" - - service: - type: ClusterIP - port: "8080" - protocol: TCP - name: http - - ingress: - enabled: true - router: private - host: DEBEZIUM_UI_HOST_ADDRESS - - autoscaling: - enabled: false - - resources: - limits: - cpu: 500m - memory: 1Gi - requests: - cpu: 500m - memory: 1Gi - - env: - - name: KAFKA_CONNECT_URIS - value: "http://debezium-connect:8083" - - name: TZ - value: "UTC" - diff --git a/charts/debezium-srte/values-test1.yaml b/charts/debezium-srte/values-test1.yaml deleted file mode 100644 index 087c2635a..000000000 --- a/charts/debezium-srte/values-test1.yaml +++ /dev/null @@ -1,132 +0,0 @@ -nameOverride: "" -fullnameOverride: "" -log_level: INFO -connect: - replicaCount: 1 - imagePullSecrets: [ ] - - image: - repository: quay.io/debezium/connect - pullPolicy: IfNotPresent - tag: "2.7" - - service: - type: ClusterIP - port: 8083 - protocol: TCP - name: http - - ingress: - enabled: false - - autoscaling: - enabled: false - - resources: - limits: - cpu: 1000m - memory: 2Gi - requests: - cpu: 500m - memory: 1Gi - - properties: - group_id: "debezium-connector-v010925" - topics_basename: "debezium-connector" - default_replication_factor: 2 - default_partitions: 10 - default_cleanup: "compact" - sql_server_agent_override: false - sql_server_agent_status: "" - bootstrap_server: "b-1.cdcnbstestdevelopment.toyoo7.c10.kafka.us-east-1.amazonaws.com:9092,b-2.cdcnbstestdevelopment.toyoo7.c10.kafka.us-east-1.amazonaws.com:9092" - - sqlserverconnector: { - "name": "debezium-connector-v010925", - "config": { - "connector.class": "io.debezium.connector.sqlserver.SqlServerConnector", - "database.hostname": "nbs-db.private-test.nbspreview.com", - "database.port": "1433", - "database.user": "", - "database.password": "", - "database.dbname": "nbs_srte", - "database.names": "nbs_srte", - "database.server.name": "srte", - "database.history.kafka.bootstrap.servers": "b-1.cdcnbstestdevelopment.toyoo7.c10.kafka.us-east-1.amazonaws.com:9092,b-2.cdcnbstestdevelopment.toyoo7.c10.kafka.us-east-1.amazonaws.com:9092", - "database.history.kafka.topic": "dbhistory.database_server_name.database_name", - # Uncomment following to manually bypass the sqlserver agent status query results - #"database.sqlserver.agent.status.query": "select dbo.IsSqlAgentRunning()", - "database.trustServerCertificate": "true", - "include.schema.changes": "true", - "key.converter": "org.apache.kafka.connect.json.JsonConverter", - "key.converter.schemas.enable": "true", - "producer.max.request.size": "10000000", #10MB - "producer.message.max.bytes": "10000000", #10MB - "snapshot.mode": "no_data", - "schema.history.internal.kafka.topic": "srte-schema-history", - "schema.history.internal.kafka.bootstrap.servers": "b-1.cdcnbstestdevelopment.toyoo7.c10.kafka.us-east-1.amazonaws.com:9092,b-2.cdcnbstestdevelopment.toyoo7.c10.kafka.us-east-1.amazonaws.com:9092", - "tasks.max": "1", - "topic.prefix": "cdc", - "topic.creation.default.replication.factor": 2, - "topic.creation.default.partitions": 10, - "topic.creation.default.cleanup.policy": "compact", - "transforms": "dropPrefix, convertTimezone", - "transforms.dropPrefix.regex": "cdc\\.NBS_SRTE\\.dbo\\.(.+)", - "transforms.dropPrefix.type": "org.apache.kafka.connect.transforms.RegexRouter", - "transforms.dropPrefix.replacement": "nrt_srte_$1", - "transforms.convertTimezone.type": "io.debezium.transforms.TimezoneConverter", - "transforms.convertTimezone.converted.timezone": "UTC", - "value.converter": "org.apache.kafka.connect.json.JsonConverter", - "value.converter.schemas.enable": "true", - } - } - - env: - - name: BOOTSTRAP_SERVERS - value: "b-1.cdcnbstestdevelopment.toyoo7.c10.kafka.us-east-1.amazonaws.com:9092,b-2.cdcnbstestdevelopment.toyoo7.c10.kafka.us-east-1.amazonaws.com:9092" - - name: LOG_LEVEL - value: "INFO" - - name: KAFKA_LOG4J_OPTS - value: "-Dlog4j.configuration=file:/kafka/config/log4j.properties" - - name: NAME - value: "debezium-connector-v010925" - - name: TZ - value: "UTC" - -ui: - enabled: false - replicaCount: 1 - imagePullSecrets: [ ] - - image: - repository: debezium/debezium-ui - pullPolicy: Always - tag: "2.7" - - service: - type: ClusterIP - port: "8080" - protocol: TCP - name: http - - ingress: - enabled: true - router: private - host: DEBEZIUM_UI_HOST_ADDRESS - - autoscaling: - enabled: false - - resources: - limits: - cpu: 500m - memory: 1Gi - requests: - cpu: 500m - memory: 1Gi - - env: - - name: KAFKA_CONNECT_URIS - value: "http://debezium-connect:8083" - - name: TZ - value: "UTC" - diff --git a/charts/debezium-srte/values.yaml b/charts/debezium-srte/values.yaml deleted file mode 100644 index 7c8176438..000000000 --- a/charts/debezium-srte/values.yaml +++ /dev/null @@ -1,135 +0,0 @@ -nameOverride: "" -fullnameOverride: "" -log_level: INFO -connect: - replicaCount: 1 - imagePullSecrets: [ ] - - image: - repository: quay.io/debezium/connect - pullPolicy: IfNotPresent - tag: "2.7" - - service: - type: ClusterIP - port: 8083 - protocol: TCP - name: http - - ingress: - enabled: false - - autoscaling: - enabled: false - - resources: - limits: - cpu: 1000m - memory: 2Gi - requests: - cpu: 500m - memory: 1Gi - - properties: - group_id: "debezium-connector-v010925" - topics_basename: "debezium-connector" - default_replication_factor: 2 - default_partitions: 10 - default_cleanup: "compact" - sql_server_agent_override: false - sql_server_agent_status: "" - bootstrap_server: "EXAMPLE_MSK_KAFKA_ENDPOINT" - - # dbserver: "EXAMPLE_DB_ENDPOINT" - - sqlserverconnector: { - "name": "debezium-connector-v010925", - "config": { - "connector.class": "io.debezium.connector.sqlserver.SqlServerConnector", - "database.hostname": "nbs-db.private-EXAMPLE_DOMAIN", - "database.port": "1433", - "database.user": "EXAMPLE_DB_USER", - "database.password": "EXAMPLE_DB_USER_PASSWORD", - "database.dbname": "nbs_srte", - "database.names": "nbs_srte", - "database.server.name": "srte", - "database.history.kafka.bootstrap.servers": "EXAMPLE_MSK_KAFKA_ENDPOINT", - "database.history.kafka.topic": "dbhistory.database_server_name.database_name", - # Uncomment following to manually bypass the sqlserver agent status query results - #"database.sqlserver.agent.status.query": "select dbo.IsSqlAgentRunning()", - "database.trustServerCertificate": "true", - "include.schema.changes": "true", - "key.converter": "org.apache.kafka.connect.json.JsonConverter", - "key.converter.schemas.enable": "true", - "producer.max.request.size": "10000000", #10MB - "producer.message.max.bytes": "10000000", #10MB - "snapshot.mode": "no_data", - "schema.history.internal.kafka.topic": "srte-schema-history", - "schema.history.internal.kafka.bootstrap.servers": "EXAMPLE_MSK_KAFKA_ENDPOINT", - "tasks.max": "1", - "topic.prefix": "cdc", - "topic.creation.default.replication.factor": 2, - "topic.creation.default.partitions": 10, - "topic.creation.default.cleanup.policy": "compact", - "transforms": "dropPrefix, convertTimezone", - "transforms.dropPrefix.regex": "cdc\\.NBS_SRTE\\.dbo\\.(.+)", - "transforms.dropPrefix.type": "org.apache.kafka.connect.transforms.RegexRouter", - "transforms.dropPrefix.replacement": "nrt_srte_$1", - "transforms.convertTimezone.type": "io.debezium.transforms.TimezoneConverter", - "transforms.convertTimezone.converted.timezone": "UTC", - "value.converter": "org.apache.kafka.connect.json.JsonConverter", - "value.converter.schemas.enable": "true", - } - } - - env: - - name: BOOTSTRAP_SERVERS - value: "EXAMPLE_MSK_KAFKA_ENDPOINT" - - name: LOG_LEVEL - value: "INFO" - - name: KAFKA_LOG4J_OPTS - value: "-Dlog4j.configuration=file:/kafka/config/log4j.properties" - - name: NAME - value: "debezium-connector-v010925" - - name: TZ - value: "UTC" - -ui: - enabled: false - replicaCount: 1 - imagePullSecrets: [ ] - - image: - repository: debezium/debezium-ui - pullPolicy: Always - tag: "2.7" - - service: - type: ClusterIP - port: "8080" - protocol: TCP - name: http - - ingress: - enabled: true - router: private - # host: DEBEZIUM_UI_HOST_ADDRESS - host: DEBEZIUM_UI_HOST_ADDRESS - - autoscaling: - enabled: false - - resources: - limits: - cpu: 500m - memory: 1Gi - requests: - cpu: 500m - memory: 1Gi - - env: - - name: KAFKA_CONNECT_URIS - value: "http://debezium-connect:8083" - - name: TZ - value: "UTC" - diff --git a/charts/debezium/values-azure.yaml b/charts/debezium/values-azure.yaml index 721b1ad12..4cb36f218 100644 --- a/charts/debezium/values-azure.yaml +++ b/charts/debezium/values-azure.yaml @@ -32,8 +32,8 @@ connect: memory: 1Gi properties: - group_id: "debezium-connector-v010925" - topics_basename: "debezium-connector" + group_id: "debezium-odse-srte-connector-v013125" + topics_basename: "debezium-odse-srte-connector" default_replication_factor: 2 default_partitions: 10 default_cleanup: "compact" @@ -41,8 +41,8 @@ connect: sql_server_agent_override: true sql_server_agent_status: "select dbo.IsSqlAgentRunning()" - sqlserverconnector: { - "name": "debezium-connector-v010925", + sqlserverconnector_odse: { + "name": "debezium-odse-srte-connector-v013125", "config": { "connector.class": "io.debezium.connector.sqlserver.SqlServerConnector", "database.hostname": "tf-cdc-nbs6-sql-managed-instance.cdf5734b998e.database.windows.net", @@ -80,6 +80,46 @@ connect: "value.converter.schemas.enable": "true", } } + sqlserverconnector_srte: { + "name": "debezium-odse-srte-connector-v013125", + "config": { + "connector.class": "io.debezium.connector.sqlserver.SqlServerConnector", + "database.hostname": "tf-cdc-nbs6-sql-managed-instance.cdf5734b998e.database.windows.net", + "database.port": "1433", + "database.user": "nbs_ods", + "database.password": "ods", + "database.dbname": "nbs_srte", + "database.names": "nbs_srte", + "database.server.name": "srte", + "database.history.kafka.bootstrap.servers": "wn0-dev-hd.dsnw3omymczu3cea2rtxoridtc.bx.internal.cloudapp.net:9092,wn1-dev-hd.dsnw3omymczu3cea2rtxoridtc.bx.internal.cloudapp.net:9092,wn2-dev-hd.dsnw3omymczu3cea2rtxoridtc.bx.internal.cloudapp.net:9092", + "database.history.kafka.topic": "dbhistory.database_server_name.database_name", + # manually bypass the sqlserver agent status query + "database.sqlserver.agent.status.query": "select dbo.IsSqlAgentRunning()", + "database.trustServerCertificate": "true", + "include.schema.changes": "true", + "key.converter": "org.apache.kafka.connect.json.JsonConverter", + "key.converter.schemas.enable": "true", + "producer.max.request.size": "10000000", #10MB + "producer.message.max.bytes": "10000000", #10MB + "snapshot.mode": "initial", + "schema.history.internal.kafka.topic": "srte-schema-history", + "schema.history.internal.kafka.bootstrap.servers": "wn0-dev-hd.dsnw3omymczu3cea2rtxoridtc.bx.internal.cloudapp.net:9092,wn1-dev-hd.dsnw3omymczu3cea2rtxoridtc.bx.internal.cloudapp.net:9092,wn2-dev-hd.dsnw3omymczu3cea2rtxoridtc.bx.internal.cloudapp.net:9092", + "table.include.list": "dbo.Anatomic_site_code, dbo.City_code_value, dbo.Cntycity_code_value, dbo.Code_value_clinical, dbo.Code_value_general, dbo.Codeset, dbo.Codeset_Group_Metadata, dbo.Country_code, dbo.Country_Code_ISO, dbo.Country_XREF, dbo.ELR_XREF, dbo.IMRDBMapping, dbo.Investigation_code, dbo.Jurisdiction_code, dbo.Jurisdiction_participation, dbo.Lab_coding_system, dbo.Lab_result, dbo.Lab_result_Snomed, dbo.Lab_test, dbo.Labtest_loinc, dbo.Labtest_Progarea_Mapping, dbo.Language_code, dbo.LDF_page_set, dbo.LOINC_code, dbo.Loinc_condition, dbo.Loinc_snomed_condition, dbo.NAICS_Industry_code, dbo.Occupation_code, dbo.Participation_type, dbo.Program_area_code, dbo.Race_code, dbo.Snomed_code, dbo.Specimen_source_code, dbo.Standard_XREF, dbo.State_code, dbo.State_county_code_value, dbo.State_model, dbo.TotalIDM, dbo.Treatment_code, dbo.Unit_code, dbo.Zip_code_value, dbo.Zipcnty_code_value", + "tasks.max": "1", + "topic.prefix": "cdc", + "topic.creation.default.replication.factor": 2, + "topic.creation.default.partitions": 10, + "topic.creation.default.cleanup.policy": "compact", + "transforms": "dropPrefix, convertTimezone", + "transforms.dropPrefix.regex": "cdc\\.NBS_SRTE\\.dbo\\.(.+)", + "transforms.dropPrefix.type": "org.apache.kafka.connect.transforms.RegexRouter", + "transforms.dropPrefix.replacement": "nrt_srte_$1", + "transforms.convertTimezone.type": "io.debezium.transforms.TimezoneConverter", + "transforms.convertTimezone.converted.timezone": "UTC", + "value.converter": "org.apache.kafka.connect.json.JsonConverter", + "value.converter.schemas.enable": "true", + } + } env: - name: BOOTSTRAP_SERVERS @@ -89,7 +129,7 @@ connect: - name: KAFKA_LOG4J_OPTS value: "-Dlog4j.configuration=file:/kafka/config/log4j.properties" - name: NAME - value: "debezium-connector-v010925" + value: "debezium-odse-srte-connector-v013125" - name: TZ value: "UTC" diff --git a/charts/debezium/values-feature.yaml b/charts/debezium/values-feature.yaml index 5cd77c1f5..dd4cde70d 100644 --- a/charts/debezium/values-feature.yaml +++ b/charts/debezium/values-feature.yaml @@ -31,8 +31,8 @@ connect: memory: 1Gi properties: - group_id: "debezium-connector-v010925" - topics_basename: "debezium-connector" + group_id: "debezium-odse-srte-connector-v013125" + topics_basename: "debezium-odse-srte-connector" default_replication_factor: 2 default_partitions: 10 default_cleanup: "compact" @@ -40,8 +40,8 @@ connect: sql_server_agent_status: "" bootstrap_server: "b-2.cdcnbsfeaturedevelopme.f4c8vq.c9.kafka.us-east-1.amazonaws.com:9092,b-1.cdcnbsfeaturedevelopme.f4c8vq.c9.kafka.us-east-1.amazonaws.com:9092" - sqlserverconnector: { - "name": "debezium-connector-v010925", + sqlserverconnector_odse: { + "name": "debezium-odse-srte-connector-v013125", "config": { "connector.class": "io.debezium.connector.sqlserver.SqlServerConnector", "database.hostname": "nbs-db.private-feature.nbspreview.com", @@ -81,6 +81,47 @@ connect: } } + sqlserverconnector_srte: { + "name": "debezium-odse-srte-connector-v013125", + "config": { + "connector.class": "io.debezium.connector.sqlserver.SqlServerConnector", + "database.hostname": "nbs-db.private-feature.nbspreview.com", + "database.port": "1433", + "database.user": "", + "database.password": "", + "database.dbname": "nbs_srte", + "database.names": "nbs_srte", + "database.server.name": "srte", + "database.history.kafka.bootstrap.servers": "b-2.cdcnbsfeaturedevelopme.f4c8vq.c9.kafka.us-east-1.amazonaws.com:9092,b-1.cdcnbsfeaturedevelopme.f4c8vq.c9.kafka.us-east-1.amazonaws.com:9092", + "database.history.kafka.topic": "dbhistory.database_server_name.database_name", + # Uncomment following to manually bypass the sqlserver agent status query results + #"database.sqlserver.agent.status.query": "select dbo.IsSqlAgentRunning()", + "database.trustServerCertificate": "true", + "include.schema.changes": "true", + "key.converter": "org.apache.kafka.connect.json.JsonConverter", + "key.converter.schemas.enable": "true", + "producer.max.request.size": "10000000", #10MB + "producer.message.max.bytes": "10000000", #10MB + "snapshot.mode": "initial", + "schema.history.internal.kafka.topic": "srte-schema-history", + "schema.history.internal.kafka.bootstrap.servers": "b-2.cdcnbsfeaturedevelopme.f4c8vq.c9.kafka.us-east-1.amazonaws.com:9092,b-1.cdcnbsfeaturedevelopme.f4c8vq.c9.kafka.us-east-1.amazonaws.com:9092", + "table.include.list": "dbo.Anatomic_site_code, dbo.City_code_value, dbo.Cntycity_code_value, dbo.Code_value_clinical, dbo.Code_value_general, dbo.Codeset, dbo.Codeset_Group_Metadata, dbo.Country_code, dbo.Country_Code_ISO, dbo.Country_XREF, dbo.ELR_XREF, dbo.IMRDBMapping, dbo.Investigation_code, dbo.Jurisdiction_code, dbo.Jurisdiction_participation, dbo.Lab_coding_system, dbo.Lab_result, dbo.Lab_result_Snomed, dbo.Lab_test, dbo.Labtest_loinc, dbo.Labtest_Progarea_Mapping, dbo.Language_code, dbo.LDF_page_set, dbo.LOINC_code, dbo.Loinc_condition, dbo.Loinc_snomed_condition, dbo.NAICS_Industry_code, dbo.Occupation_code, dbo.Participation_type, dbo.Program_area_code, dbo.Race_code, dbo.Snomed_code, dbo.Specimen_source_code, dbo.Standard_XREF, dbo.State_code, dbo.State_county_code_value, dbo.State_model, dbo.TotalIDM, dbo.Treatment_code, dbo.Unit_code, dbo.Zip_code_value, dbo.Zipcnty_code_value", + "tasks.max": "1", + "topic.prefix": "cdc", + "topic.creation.default.replication.factor": 2, + "topic.creation.default.partitions": 10, + "topic.creation.default.cleanup.policy": "compact", + "transforms": "dropPrefix, convertTimezone", + "transforms.dropPrefix.regex": "cdc\\.NBS_SRTE\\.dbo\\.(.+)", + "transforms.dropPrefix.type": "org.apache.kafka.connect.transforms.RegexRouter", + "transforms.dropPrefix.replacement": "nrt_srte_$1", + "transforms.convertTimezone.type": "io.debezium.transforms.TimezoneConverter", + "transforms.convertTimezone.converted.timezone": "UTC", + "value.converter": "org.apache.kafka.connect.json.JsonConverter", + "value.converter.schemas.enable": "true", + } + } + env: - name: BOOTSTRAP_SERVERS value: "b-2.cdcnbsfeaturedevelopme.f4c8vq.c9.kafka.us-east-1.amazonaws.com:9092,b-1.cdcnbsfeaturedevelopme.f4c8vq.c9.kafka.us-east-1.amazonaws.com:9092" @@ -89,7 +130,7 @@ connect: - name: KAFKA_LOG4J_OPTS value: "-Dlog4j.configuration=file:/kafka/config/log4j.properties" - name: NAME - value: "debezium-connector-v010925" + value: "debezium-odse-srte-connector-v013125" - name: TZ value: "UTC" diff --git a/charts/debezium/values-test1.yaml b/charts/debezium/values-test1.yaml index 74d7dde68..f4b764406 100644 --- a/charts/debezium/values-test1.yaml +++ b/charts/debezium/values-test1.yaml @@ -31,8 +31,8 @@ connect: memory: 1Gi properties: - group_id: "debezium-connector-v010925" - topics_basename: "debezium-connector" + group_id: "debezium-odse-srte-connector-v013125" + topics_basename: "debezium-odse-srte-connector" default_replication_factor: 2 default_partitions: 10 default_cleanup: "compact" @@ -40,8 +40,8 @@ connect: sql_server_agent_status: "" bootstrap_server: "b-1.cdcnbstestdevelopment.toyoo7.c10.kafka.us-east-1.amazonaws.com:9092,b-2.cdcnbstestdevelopment.toyoo7.c10.kafka.us-east-1.amazonaws.com:9092" - sqlserverconnector: { - "name": "debezium-connector-v010925", + sqlserverconnector_odse: { + "name": "debezium-odse-srte-connector-v013125", "config": { "connector.class": "io.debezium.connector.sqlserver.SqlServerConnector", "database.hostname": "nbs-db.private-test.nbspreview.com", @@ -81,6 +81,47 @@ connect: } } + sqlserverconnector_srte: { + "name": "debezium-odse-srte-connector-v013125", + "config": { + "connector.class": "io.debezium.connector.sqlserver.SqlServerConnector", + "database.hostname": "nbs-db.private-test.nbspreview.com", + "database.port": "1433", + "database.user": "", + "database.password": "", + "database.dbname": "nbs_srte", + "database.names": "nbs_srte", + "database.server.name": "srte", + "database.history.kafka.bootstrap.servers": "b-1.cdcnbstestdevelopment.toyoo7.c10.kafka.us-east-1.amazonaws.com:9092,b-2.cdcnbstestdevelopment.toyoo7.c10.kafka.us-east-1.amazonaws.com:9092", + "database.history.kafka.topic": "dbhistory.database_server_name.database_name", + # Uncomment following to manually bypass the sqlserver agent status query results + #"database.sqlserver.agent.status.query": "select dbo.IsSqlAgentRunning()", + "database.trustServerCertificate": "true", + "include.schema.changes": "true", + "key.converter": "org.apache.kafka.connect.json.JsonConverter", + "key.converter.schemas.enable": "true", + "producer.max.request.size": "10000000", #10MB + "producer.message.max.bytes": "10000000", #10MB + "snapshot.mode": "initial", + "schema.history.internal.kafka.topic": "srte-schema-history", + "schema.history.internal.kafka.bootstrap.servers": "b-1.cdcnbstestdevelopment.toyoo7.c10.kafka.us-east-1.amazonaws.com:9092,b-2.cdcnbstestdevelopment.toyoo7.c10.kafka.us-east-1.amazonaws.com:9092", + "table.include.list": "dbo.Anatomic_site_code, dbo.City_code_value, dbo.Cntycity_code_value, dbo.Code_value_clinical, dbo.Code_value_general, dbo.Codeset, dbo.Codeset_Group_Metadata, dbo.Country_code, dbo.Country_Code_ISO, dbo.Country_XREF, dbo.ELR_XREF, dbo.IMRDBMapping, dbo.Investigation_code, dbo.Jurisdiction_code, dbo.Jurisdiction_participation, dbo.Lab_coding_system, dbo.Lab_result, dbo.Lab_result_Snomed, dbo.Lab_test, dbo.Labtest_loinc, dbo.Labtest_Progarea_Mapping, dbo.Language_code, dbo.LDF_page_set, dbo.LOINC_code, dbo.Loinc_condition, dbo.Loinc_snomed_condition, dbo.NAICS_Industry_code, dbo.Occupation_code, dbo.Participation_type, dbo.Program_area_code, dbo.Race_code, dbo.Snomed_code, dbo.Specimen_source_code, dbo.Standard_XREF, dbo.State_code, dbo.State_county_code_value, dbo.State_model, dbo.TotalIDM, dbo.Treatment_code, dbo.Unit_code, dbo.Zip_code_value, dbo.Zipcnty_code_value", + "tasks.max": "1", + "topic.prefix": "cdc", + "topic.creation.default.replication.factor": 2, + "topic.creation.default.partitions": 10, + "topic.creation.default.cleanup.policy": "compact", + "transforms": "dropPrefix, convertTimezone", + "transforms.dropPrefix.regex": "cdc\\.NBS_SRTE\\.dbo\\.(.+)", + "transforms.dropPrefix.type": "org.apache.kafka.connect.transforms.RegexRouter", + "transforms.dropPrefix.replacement": "nrt_srte_$1", + "transforms.convertTimezone.type": "io.debezium.transforms.TimezoneConverter", + "transforms.convertTimezone.converted.timezone": "UTC", + "value.converter": "org.apache.kafka.connect.json.JsonConverter", + "value.converter.schemas.enable": "true", + } + } + env: - name: BOOTSTRAP_SERVERS value: "b-1.cdcnbstestdevelopment.toyoo7.c10.kafka.us-east-1.amazonaws.com:9092,b-2.cdcnbstestdevelopment.toyoo7.c10.kafka.us-east-1.amazonaws.com:9092" @@ -89,7 +130,7 @@ connect: - name: KAFKA_LOG4J_OPTS value: "-Dlog4j.configuration=file:/kafka/config/log4j.properties" - name: NAME - value: "debezium-connector-v010925" + value: "debezium-odse-srte-connector-v013125" - name: TZ value: "UTC" diff --git a/charts/debezium/values.yaml b/charts/debezium/values.yaml index 1a986c4bc..e4bcd4bd1 100644 --- a/charts/debezium/values.yaml +++ b/charts/debezium/values.yaml @@ -107,7 +107,7 @@ connect: "snapshot.mode": "initial", "schema.history.internal.kafka.topic": "srte-schema-history", "schema.history.internal.kafka.bootstrap.servers": "EXAMPLE_MSK_KAFKA_ENDPOINT", - "table.include.list": "dbo.Anatomic_site_code, dbo.City_code_value, dbo.Cntycity_code_value, dbo.Code_value_clinical, dbo.Code_value_general, dbo.Codeset, dbo.Codeset_Group_Metadata, dbo.Country_code, dbo.Country_Code_ISO, dbo.Country_XREF, dbo.ELR_XREF, dbo.IMRDBMapping, dbo.Investigation_code, dbo.Jurisdiction_code, dbo.Jurisdiction_participation, dbo.Lab_coding_system, dbo.Lab_result, dbo.Lab_result_Snomed, dbo.Lab_test, dbo.Labtest_loinc, dbo.Labtest_Progarea_Mapping, dbo.Language_code, dbo.LDF_page_set, dbo.LOINC_code, dbo.Loinc_condition, dbo.Loinc_snomed_condition, dbo.NAICS_Industry_code, dbo.Occupation_code, dbo.Participation_type, dbo.Program_area_code, dbo.Race_code, dbo.Snomed_code, dbo.Specimen_source_code, dbo.Standard_XREF, dbo.State_code, dbo.State_county_code_value, dbo.State_model, dbo.TotalIDM, dbo.Treatment_code, dbo.Unit_code, dbo.Zip_code_value, dbo.Zipcnty_code_value", + "table.include.list": "dbo.EXAMPLE_TABLES", "tasks.max": "1", "topic.prefix": "cdc", "topic.creation.default.replication.factor": 2,