DataX之MySQL-overwrite-HIVE

DataX之MySQL-overwrite-HIVE

1、json脚本

{
  "job": {
    "setting": {
      "speed": {
        "channel": 3
      },
      "cipher": {
        "sfjm": "Y",
        "instance": "AES128"
      },
      "errorLimit": {
        "record": "-1"
      }
    },
    "content": [{
      "reader": {
        "name": "mysqlreader",
        "parameter": {
          "password": "123456",
          "username": "root",
          "connection": [{
            "jdbcUrl": ["jdbc:mysql://192.168.30.20:3306/test?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai"],
            "table": ["test.jkmzmxx_plqd"]
          }],
          "column": ["`xm`", "`zjhm`", "`lxdh`", "`city`", "`region`", "`street`", "`community`", "`dz`", "`sjly`", "`lysj`", "`zmyy`", "`zmqzt`", "`nzzt`", "`sffk`", "`jcfkrq`"]
        }
      },
      "writer": {
        "name": "hdfswriter",
        "parameter": {
          "password": "123456",
          "username": "hadoop",
          "jdbcUrl": "jdbc:hive2://bigcity1:2181,bigcity2:2181,bigcity3:2181/stg;serviceDiscoveryMode=zooKeeper;zooKeeperNamespace=hiveserver2",
          "analyzeSql": "ANALYZE TABLE stg.stg_szsj_jkmzmxx_plqd partition(day_id='20211204') COMPUTE STATISTICS",
          "defaultFS": "hdfs://bigdata:8020",
          "hadoopConfig": {
            "dfs.client.use.datanode.hostname": "true"
          },
          "fileType": "orc",
          "fieldDelimiter": ",",
          "fileName": "stg.stg_szsj_jkmzmxx_plqd",
          "path": "/warehouse/tablespace/managed/hive/stg.db/stg_szsj_jkmzmxx_plqd/day_id=20211204",
          "hdfsUser": "hdfs",
          "writeMode": "overwrite",
          "column": [{
            "name": "xm",
            "type": "VARCHAR"
          }, {
            "name": "zjhm",
            "type": "VARCHAR"
          }, {
            "name": "lxdh",
            "type": "VARCHAR"
          }, {
            "name": "city",
            "type": "VARCHAR"
          }, {
            "name": "region",
            "type": "VARCHAR"
          }, {
            "name": "street",
            "type": "VARCHAR"
          }, {
            "name": "community",
            "type": "VARCHAR"
          }, {
            "name": "dz",
            "type": "VARCHAR"
          }, {
            "name": "sjly",
            "type": "VARCHAR"
          }, {
            "name": "lysj",
            "type": "TIMESTAMP"
          }, {
            "name": "zmyy",
            "type": "VARCHAR"
          }, {
            "name": "zmqzt",
            "type": "VARCHAR"
          }, {
            "name": "nzzt",
            "type": "VARCHAR"
          }, {
            "name": "sffk",
            "type": "VARCHAR"
          }, {
            "name": "jcfkrq",
            "type": "TIMESTAMP"
          }],
          "originColumn": [{
            "name": "xm",
            "type": "VARCHAR"
          }, {
            "name": "zjhm",
            "type": "VARCHAR"
          }, {
            "name": "lxdh",
            "type": "VARCHAR"
          }, {
            "name": "city",
            "type": "VARCHAR"
          }, {
            "name": "region",
            "type": "VARCHAR"
          }, {
            "name": "street",
            "type": "VARCHAR"
          }, {
            "name": "community",
            "type": "VARCHAR"
          }, {
            "name": "dz",
            "type": "VARCHAR"
          }, {
            "name": "sjly",
            "type": "VARCHAR"
          }, {
            "name": "lysj",
            "type": "TIMESTAMP"
          }, {
            "name": "zmyy",
            "type": "VARCHAR"
          }, {
            "name": "zmqzt",
            "type": "VARCHAR"
          }, {
            "name": "nzzt",
            "type": "VARCHAR"
          }, {
            "name": "sffk",
            "type": "VARCHAR"
          }, {
            "name": "jcfkrq",
            "type": "TIMESTAMP"
          }]
        }
      }
    }]
  }
}