kubernetes 日志收集

K8s系统的组件日志

# 除了kubelet外，其他组件都容器化了，可以使用journalctl 来查看systemd管理的组件kubelet
# journalctl  -u kubelet | grep error
11月 01 20:26:00 k8s-master kubelet[985]: E1101 20:26:00.402299     985 controller.go:144] failed to ensure lease exists, will retry in 800ms, error: Get "https://10.0.0.100:6443/apis/coordination.k8s.io/v1/namespaces/kube-node-lease/leases/k8s-master?timeout=10s": dial tcp 10.0.0.100:6443: connect: connection refused
​

K8s Cluster里面部署的应用程序日志

• 标准输出

  # 容器中的组件用kubectl logs podName查看，输出到控制台(被docker接管)
  # kubectl logs my-dep-5b7868d854-z9k5r  -f
  192.168.36.64 - - [01/Nov/2021:12:50:31 +0000] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36" "-"
  # 日志获取流程
  kubectl logs -->apiserver -->kubelet-->container
  ​
  # 查看proxy组件的日志
  # kubectl logs kube-proxy-7rxb6 -n kube-system
  ​
  # 容器中的日志在本地宿主机的位置：
  /var/lib/docker/containers//-json.log
  ​
  1. 查找过程：
  先确定pod在那个node上，通过 -o wide 可以确定
  #  kubectl get pods kube-proxy-9dch7  -n kube-system -o wide
  NAME               READY   STATUS    RESTARTS      AGE    IP           NODE        NOMINATED NODE   READINESS GATES
  kube-proxy-9dch7   1/1     Running   4 (21h ago)   2d7h   10.0.0.101   k8s-node1              
  2. 进入对应节点：查找container-id
  [root@k8s-node1 ~]# docker ps | grep kube-proxy-9dch7
  aefd56e468d3   6120bd723dce                                        "/usr/local/bin/kube…"   33 minutes ago   Up 33 minutes             k8s_kube-proxy_kube-proxy-9dch7_kube-system_5552bec6-86e9-4d46-ad7d-63da81a27058_4
  f0218b79a478   registry.aliyuncs.com/google_containers/pause:3.5   "/pause"                 33 minutes ago   Up 33 minutes             k8s_POD_kube-proxy-9dch7_kube-system_5552bec6-86e9-4d46-ad7d-63da81a27058_4
  container-id = aefd56e468d3
  3. 根据container-id查找宿主机上对应容器的日志，宿主机中日志为json格式
   cat /var/lib/docker/containers/aefd56e468d3/aefd56e468d3-json.log 
    ....
   {"log":"I1101 12:26:58.206292       1 shared_informer.go:247] Caches are synced for endpoint slice config n","stream":"stderr","time":"2021-11-01T12:26:58.206436402Z"}
   4. 查看容器日志是否一致（master 节点执行）
   # kubectl logs kube-proxy-9dch7 -n kube-system
   ...
   I1101 12:26:58.206292       1 shared_informer.go:247] Caches are synced for endpoint slice config 

  • 日志文件

    # cat two-files.yaml 
    apiVersion: v1
    kind: Pod
    metadata:
      name: counter
    spec:
      containers:
      - name: count
        image: busybox
        args:
        - /bin/sh
        - -c
        - >
          i=0;
          while true;
          do
            echo "$i: $(date)" >> /var/log/1.log;
            echo "$(date) INFO $i" >> /var/log/2.log;
            i=$((i+1));
            sleep 1;
          done      
        volumeMounts:
        - name: varlog
          mountPath: /var/log
      volumes:
      - name: varlog
        emptyDir: {}
    ​
    # kubectl apply -f two-files.yaml
    pod/counter created
    ​
    # kubectl get pod -o wide
    NAME                      READY   STATUS    RESTARTS      AGE   IP                NODE        NOMINATED NODE   READINESS GATES
    counter                   1/1     Running   0             81s   192.168.36.75     k8s-node1              
    ​
    ​
    ​
    # 文件日志在宿主机中的位置：/var/lib/kubelet/pods//volumes/kubernetes.io~empty-dir/
    获取Pod-id（到对应node上）
    ]# docker ps | grep counter
    ed231e225433   busybox                                             "/bin/sh -c 'i=0; wh…"   about a minute ago   Up about a minute             k8s_count_counter_default_0d70c728-eb23-4c15-a2ea-68743d856023_0
    9123abb4fa34   registry.aliyuncs.com/google_containers/pause:3.5   "/pause"                 2 minutes ago        Up 2 minutes                  k8s_POD_counter_default_0d70c728-eb23-4c15-a2ea-68743d856023_0
    ​
     根据pod-id在宿主机上查找日志
    # # cat /var/lib/kubelet/pods/0d70c728-eb23-4c15-a2ea-68743d856023/volumes/kubernetes.io~empty-dir/varlog/1.log 
    ​
    ​
    ​

    边车模式

    apiVersion: v1
    kind: Pod
    metadata:
      name: counter1
    spec:
      containers:
      - name: count1
        image: busybox
        args:
        - /bin/sh
        - -c
        - >
          i=0;
          while true;
          do
            echo "$i: $(date)" >> /var/log/1.log;
            echo "$(date) INFO $i" >> /var/log/2.log;
            i=$((i+1));
            sleep 1;
          done      
        volumeMounts:
        - name: varlog
          mountPath: /var/log
          
      - name: count1-log
        image: busybox
        args: [/bin/sh, -c, 'tail -n+1 -f /var/log/1.log']
        volumeMounts:
        - name: varlog
          mountPath: /var/log
      - name: count2-log
        image: busybox
        args: [/bin/sh, -c, 'tail -n+1 -f /var/log/2.log']
        volumeMounts:
        - name: varlog
          mountPath: /var/log
    
      volumes:
      - name: varlog
        emptyDir: {}
    
    ​
    ​

    1. 查看日志
    # kubectl logs counter1 count1-log
    2. 宿主机上查看
    ​
    [root@k8s-node2 web1-log]# docker ps | grep counter1
    7a92417f9aa3   busybox                                             "/bin/sh -c 'tail -n…"   2 minutes ago   Up 2 minutes             k8s_count1-log_counter1_default_c95a2ad0-3dd6-43fe-962a-6eb82284a418_0
    5e204be05e75   busybox                                             "/bin/sh -c 'i=0; wh…"   2 minutes ago   Up 2 minutes             k8s_count1_counter1_default_c95a2ad0-3dd6-43fe-962a-6eb82284a418_0
    6577994c54ac   registry.aliyuncs.com/google_containers/pause:3.5   "/pause"                 3 minutes ago   Up 3 minutes             k8s_POD_counter1_default_c95a2ad0-3dd6-43fe-962a-6eb82284a418_0
    ​
    # cat /var/lib/kubelet/pods/c95a2ad0-3dd6-43fe-962a-6eb82284a418/volumes/kubernetes.io~empty-dir/varlog/2.log 
    ​