DataX之MySQL-append-Hive

DataX之MySQL-append-Hive

1、mysql-to-hive.json

{
	"setting": {},
	"job": {
		"setting": {
			"speed": {
				"channel": 2
			}
		},
		"content": [
			{
				"reader": {
					"name": "mysqlreader",
					"parameter": {
						"username": "root",
						"password": "123456",
						"connection": [
							{
								"querySql": [
									"select comany_id as mcn_comany_id,corporation_code as mcn_corporation_code,company as mcn_company,corporation_person as mcn_corporation_person,province_code as mcn_province_code,province as mcn_province,city_code as mcn_city_code,city as mcn_city from enterprise_basic_info where city = '南京'"
								],
								"jdbcUrl": [
									"jdbc:mysql://192.168.30.130:30005/test?useUnicode=true&characterEncoding=utf-8"
								]
							}
						]
					}
				},
				"writer": {
					"name": "hdfswriter",
					"parameter": {
						"defaultFS": "hdfs://192.168.30.160:8020",
						"fileType": "text",
						"path": "/user/hive/warehouse/ods.db/enterprise_basic_info_ccm/batch_no=20211203",
						"fileName": "202104",
						"column": [
							{
								"type": "string",
								"name": "mcn_corporation_code"
							},
							{
								"type": "string",
								"name": "mcn_comany_id"
							},
							{
								"type": "string",
								"name": "mcn_company"
							},
							{
								"type": "string",
								"name": "mcn_corporation_person"
							},
							{
								"type": "string",
								"name": "mcn_province_code"
							},
							{
								"type": "string",
								"name": "mcn_province"
							},
							{
								"type": "string",
								"name": "mcn_city_code"
							},
							{
								"type": "string",
								"name": "mcn_city"
							}
						],
						"writeMode": "append",
						"fieldDelimiter": "|"
					}
				}
			}
		]
	}
}

2、同步 Hive 元数据的语句

ANALYZE TABLE ods.enterprise_basic_info_ccm partition(batch_no='20211203') COMPUTE STATISTICS;