debezium步骤--postgres到oracle

debezium步骤--postgres到oracle

centos7环境下

------docker-compose启动容器

------外网执行如下，下载镜像文件
 

[root@dbz dbz]# docker-compose --compatibility -f docker-compose-dbz-origin-pg.yaml up -d
WARNING: The following deploy sub-keys are not supported in compatibility mode and have been ignored: resources.reservations.cpus
WARNING: The following deploy sub-keys are not supported in compatibility mode and have been ignored: resources.reservations.cpus
WARNING: The following deploy sub-keys are not supported in compatibility mode and have been ignored: resources.reservations.cpus
Pulling dbz-zookeeper-pg (registry.cn-hangzhou.aliyuncs.com/bytesynch/sync_platform_zookeeper:1.7)...
1.7: Pulling from bytesynch/sync_platform_zookeeper
fd26973701b3: Pull complete
b2f8ccfb9486: Pull complete
794aaa403bf7: Pull complete
e7ce3abc4ee6: Pull complete
a921fccd379f: Pull complete
fa3da9645270: Pull complete
48f3a5fa7670: Pull complete
7811b5a57195: Pull complete
2e927026f1c2: Pull complete
0592c6f2399b: Pull complete
bc49236761dd: Pull complete
69e5c696384c: Pull complete
ad6e89e259a9: Pull complete
630bc63f5365: Pull complete
Digest: sha256:ef239835301c385bfa6037665d83600a7be5e6afff3524307191164302ddea2c
Pulling dbz-kafka-pg (registry.cn-hangzhou.aliyuncs.com/bytesynch/sync_platform_kafka:1.7)...
1.7: Pulling from bytesynch/sync_platform_kafka
1456cdf7dfbf: Pull complete
b2bcec70c95e: Pull complete
481c08bce215: Pull complete
489d7fe862c0: Pull complete
448cacd2877b: Pull complete
6e82bf8f9418: Pull complete
ff1256dd4fa3: Pull complete
dbce187a8d48: Pull complete
9e21f8c5e3d5: Pull complete
05cbb871c0e7: Pull complete
db423c813e29: Pull complete
dc5024e1959e: Pull complete
81089ab682bd: Pull complete
Digest: sha256:4aaa5f001684de0b9b85210de69b5fb52e0ef423fc6ed0666911fa77ae0a9b57
Pulling dbz-connect-pg (registry.cn-hangzhou.aliyuncs.com/bytesynch/sync_platform_connect:1.7)...
1.7: Pulling from bytesynch/sync_platform_connect
fd26973701b3: Already exists
bea6e81ab0ea: Pull complete
4b1a08b4ac47: Pull complete
96bf206b626f: Pull complete
381214ea4a0e: Pull complete
4519d9373bf1: Pull complete
e65ee62e1915: Pull complete
5b2f20a158ba: Pull complete
ad021eba2efa: Pull complete
1da944d6cd31: Pull complete
4f4da1f7d4bc: Pull complete
dae97f8da078: Pull complete
bf9dd8fa74c2: Pull complete
1838277ee8cf: Pull complete
0216ee168670: Pull complete
5f83cb27cca4: Pull complete
7d867b848a0d: Pull complete
f155d9376c77: Pull complete
af4c57b3ffb9: Pull complete
babaf2fe5169: Pull complete
2bd4dcab1a6a: Pull complete
aef3b318b180: Pull complete
0076e86736e0: Pull complete
Digest: sha256:ff22f4188fc96f77b36f074aaabfbc7806be508aebe7d68f73f0ca2c69ce5290
Creating dbz-zookeeper-pg ... done
Creating dbz-kafka-pg     ... done
Creating dbz-connect-pg   ... done

------创建source文件

[root@dbz dbz]# vi source-postgresql.json
{
  "name": "fulfillment-postgresql-connector",
  "config": {
    "connector.class": "io.debezium.connector.postgresql.PostgresConnector",
    "database.hostname": "192.168.194.155",
    "database.port": "5432",
    "database.user": "newsoft",
    "database.password": "123456",
    "database.dbname" : "data",
    "database.server.name": "fulfillment",
    "database.history.kafka.bootstrap.servers" : "dbz-kafka-pg:9092",
    "database.history.kafka.topic": "pg-changes.inventory",
    "tasks.max" : "1",
    "plugin.name":"pgoutput",
    "table.include.list": "user.table,user1.table2"
  }
}

------创建sink文件

[root@dbz dbz]# vi sink-oracle.json
{
"name":"sink-connector-pg-oracle",
"config":{
    "topics.regex":"fulfillment.newsoft.(.*)",
    "connector.class":"io.confluent.connect.jdbc.JdbcSinkConnector",
    "connection.url": "jdbc:oracle:thin:@192.168.194.121:1521/ORCL",
    "connection.user": "test",
    "connection.password":"test",
    "tasks.max":"1",
    "auto.create": true,
    "auto.evolve": true,
    "insert.mode":"upsert",
    "batch.size":0,
    "delete.enabled": true,
    "pk.mode":"record_key",
    "transforms":"dropPrefix,AddPrefix,ExtractField",
    "transforms.ExtractField.type":"org.apache.kafka.connect.transforms.ExtractField$Value",
    "transforms.ExtractField.field":"after",
    "transforms.dropPrefix.type":"org.apache.kafka.connect.transforms.RegexRouter",
    "transforms.dropPrefix.regex":"fulfillment.newsoft.(.*)",
    "transforms.dropPrefix.replacement":"",
    "transforms.AddPrefix.type":"org.apache.kafka.connect.transforms.RegexRouter",
    "transforms.AddPrefix.regex":".*",
    "transforms.unwrap.drop.tombstones": "false",
    "transforms.unwrap.delete.handling.mode": "rewrite",
    "transforms.AddPrefix.replacement":"
[root@dbz dbz]# curl -i -X POST -H "Accept:application/json" -H "Content-Type:application/json" http://localhost:8069/connectors/ -d @source-postgresql.json 
HTTP/1.1 201 Created
Date: Fri, 24 Dec 2021 13:59:53 GMT
Location: http://localhost:8069/connectors/fulfillment-postgresql-connector
Content-Type: application/json
Content-Length: 536
Server: Jetty(9.4.43.v20210629)

{"name":"fulfillment-postgresql-connector","config":{"connector.class":"io.debezium.connector.postgresql.PostgresConnector","database.hostname":"192.168.56.128","database.port":"5866","database.user":"highgo","database.password":"Hello@123","database.dbname":"dzalldate","database.server.name":"fulfillment","database.history.kafka.bootstrap.servers":"dbz-kafka-pg:9092","database.history.kafka.topic":"pg-changes.inventory","tasks.max":"1","plugin.name":"pgoutput","name":"fulfillment-postgresql-connector"},"tasks":[],"type":"source"}[root@dbz dbz]# 

[root@dbz dbz]# curl -i -X POST -H "Accept:application/json" -H "Content-Type:application/json" http://localhost:8069/connectors/ -d @sink-oracle.json 
HTTP/1.1 100 Continue

HTTP/1.1 201 Created
Date: Fri, 24 Dec 2021 14:02:39 GMT
Location: http://localhost:8069/connectors/sink-connector-pg-oracle
Content-Type: application/json
Content-Length: 1072
Server: Jetty(9.4.43.v20210629)

{"name":"sink-connector-pg-oracle","config":{"topics.regex":"fulfillment.public.(.*)","connector.class":"io.confluent.connect.jdbc.JdbcSinkConnector","connection.url":"jdbc:oracle:thin:@192.168.56.124:1521/ORCL","connection.user":"hvr","connection.password":"hvr","tasks.max":"1","auto.create":"true","auto.evolve":"true","insert.mode":"upsert","batch.size":"3000","delete.enabled":"true","pk.mode":"record_key","transforms":"dropPrefix,AddPrefix,ExtractField","transforms.ExtractField.type":"org.apache.kafka.connect.transforms.ExtractField$Value","transforms.ExtractField.field":"after","transforms.dropPrefix.type":"org.apache.kafka.connect.transforms.RegexRouter","transforms.dropPrefix.regex":"fulfillment.public.(.*)","transforms.dropPrefix.replacement":"","transforms.AddPrefix.type":"org.apache.kafka.connect.transforms.RegexRouter","transforms.AddPrefix.regex":".*","transforms.unwrap.drop.tombstones":"false","transforms.unwrap.delete.handling.mode":"rewrite","transforms.AddPrefix.replacement":"
docker-compose  -f docker-compose-dbz-origin-pg.yaml logs -t -f dbz-connect-pg
","name":"sink-connector-pg-oracle"},"tasks":[],"type":"sink"}[root@dbz dbz]# 
"
 }
}

------数据同步部署

[root@dbz dbz]# curl -i -X POST -H "Accept:application/json" -H "Content-Type:application/json" http://localhost:8069/connectors/ -d @sink-oracle.json 

------日志查看：

------查看同步表的信息