【使用python脚本筛分重复日志】

使用python脚本筛分重复日志

需求：业务涉及企微回调，近期发现重复回调攀升，所以从日志分析一下重复比例
思路：
1、根据关键词筛选命中日志到新文件中
2、使用python脚本筛选出重复key，结果集打印到新文件

环境准备

1. 安装Anaconda下载地址
2. 配置环境变量
   
3. 验证
   

实战

1. 拆分日志

#筛选日志（三台ims，4.13号一天）
zgrep "**客户回调信息" server-2022-04-13-* | grep "add_external_contact" -> add_callback_4.13.log
#将日志拉取到本地环境
scp /home/xxx/ims/logs/all/add_callback_4.13.log root@xxx.xxx.xxx.xxx:/home/logdev/add_callback_1.log
#日志合并
cat add_callback_1.log >> add_callback_result.log
cat add_callback_2.log >> add_callback_result.log
cat add_callback_3.log >> add_callback_result.log

2. 编写py文件add_callback_1.0.py

import json
import sys

filepath=sys.argv[1]
list = []
total = 0
count = 0
count2 = 0
temp={}
temp2={}
result={}
result2={}
resultStr=''


def dataClear(s):
	global count,total,count2
	total = total + 1
	#获取回调对象
	a='**客户回调信息：'
	b='logSeq:0'
	objson= s[s.find(a, 0, len(s))+len(a):s.find(b, 0, len(s))-1]
	o=json.loads(objson)
	#获取回调uri
	c='uri:'
	d='method:'
	uriStr = s[s.find(c, 0, len(s)):s.find(d, 0, len(s))-3]
	uriArr = uriStr.split('/')
	uriStr = uriArr[len(uriArr)-1]
	key = o['toUserName']
	val = o['userID']+'_'+o['externalUserID']+'_'+str(o['createTime'])
	val2 = o['userID']+'_'+o['externalUserID']+'_'+str(o['createTime'])+'_'+uriStr
	if key in temp:
		if temp[key].count(val) > 0:
			count = count+1
			putResult(key,val)			
		else:
			temp[key].append(val)	
	else:
		list = []
		list.append(val)
		temp[key] = list

	if key in temp2:
		if temp2[key].count(val2) > 0:
			count2 = count2 + 1
			putResult2(key,val2)
		else:
			temp2[key].append(val2)			
	else:
		list2 = []
		list2.append(val2)
		temp2[key] = list2
		
	
		
			
def putResult(key,val):
	if key in result:
		result[key].append(val)			
	else:
		list = []
		list.append(val)
		result[key] = list

def putResult2(key,val):
	if key in result2:
		result2[key].append(val)			
	else:
		list = []
		list.append(val)
		result2[key] = list
		
with open(filepath,'r',encoding='utf-8') as f:

	for line in f.readlines():
		line = line.strip('\n')
		dataClear(line)
		
	for keys in result.keys():
		resultStr = resultStr + '公司:'+keys+',重复:'+str(len(result[keys]))+'条,同uri重复:'+str(len(result2.get(keys,[])))+'条'+'\n'
		for vals in result[keys]:
			print('公司:'+keys+',重复回调:'+vals)
		if keys in result2:
			for vals in result2[keys]:
				print('公司:'+keys+',同uri重复回调:'+vals)
	print('总回调数:'+str(total)+',重复条数:'+str(count)+'条,同uri重复条数:'+str(count2)+'条')
	print(resultStr)

3. 执行py文件查询数据

python3 add_callback_1.0.py /home/logdev/add_callback_1.log