阿里云elasticsearch如何同步阿里云（Aliyun）RDS

阿里云elasticsearch如何同步阿里云（Aliyun）RDS

aliyun RDS同步这里主要使用阿里云logstash工具来同步，如下图所示

logstash创建后需要创建数据管道方式来实现DB与ES的数据增量同步

 管道配置如下：

input {
  jdbc {
    jdbc_driver_class => "com.mysql.jdbc.Driver"
    jdbc_driver_library => ""
    jdbc_connection_string => "jdbc:mysql://数据库地址:端口/数据库名?useUnicode=true&characterEncoding=utf-8&useSSL=false&allowLoadLocalInfile=false&autoDeserialize=false"
    jdbc_user => "数据库账号"
    jdbc_password => "数据库密码"
    jdbc_paging_enabled => "true"
    jdbc_page_size => "50000"
    statement => "SELECt * from 表名 where updated_at>=:sql_last_value"
    schedule => "* * * * * *"
    record_last_run => true
    last_run_metadata_path => ""
    clean_run => false
    tracking_column_type => "timestamp"
    use_column_value => true
    tracking_column => "updated_at"
  }
}
filter {
  json {
   source => "message"
  }
}
output {
  stdout {
    codec => json_lines
  }
  elasticsearch {
    hosts => "ES地址:9200"
    user => "ES账号"
    password => "ES密码"
    index => "ES的索引"
    document_id => "%{DB or ES唯一主键}"
  }
  file_extend {
    path => "debug日志文件path"
  }
}