k8s+ceph+jenkins持续集成

k8s+ceph+jenkins持续集成

背景：
这两天在家里没事，想着搭建一个持续集成系统，采用8台虚以机，其中两台做F5软负载均衡（haproxy+keepalive），两台做k8s集群控制节点，四台即做ceph存储，也做集群工作节点。ceph存储集群可以单独拿出来，但没有多余的机器，我记录下本次的 *** 作过程，文章重点记录持续集成。
1、机器列表如下：
ceph1: 10.168.1.165 cpu2 8G (控制节点)
ceph2: 10.168.1.192 cpu2 8G （控制节点）
ceph3: 10.168.1.204 cpu2 8G （计算节点/存储节点）
ceph4: 10.168.1.160 cpu2 8G （计算节点/存储节点）
ceph5: 10.168.1.170 cpu2 8G （计算节点/存储节点）
ceph6: 10.168.1.181 cpu2 8G （计算节点/存储节点）
haproxy1: 10.168.1.139 cpu2 4G （F5）vip 10.168.1.199
haproxy2: 10.168.1.111 cpu2 4G （F5）vip 10.168.1.199

2、k8s集群的搭建这里不记录，我采用的是kubeadm方式搭建的集群，两个控制节点通过两台F5跟计算节点通信。

3、ceph存储搭建这里也不记录，我采用的是rook方式搭建的存储集群。

4、jenkins跟harbor、nexus的安装都采用helm方式，这种安装简单方便。

5、本次实验使用的是分布式系统，有订单模块，账户模块、仓库模块、逻辑模块、公共模块组成。实现的效果是jenkins能并发构建各个模块，并自动部署到k8s集群上。

6、分布式系统代码只是简单的实验代码，可能无法运行，比如需要配置apollo，seata分布式事务等，以上提到的这些都会有单独的章节来记录 代码访问：https://github.com/jackbauer123/mytest。

7、k8s里所有http访问方式都是采用ingress方式。
1)、jenkins的ingress.yaml文件：

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
    name: jenkins
    namespace: default
spec:
    ingressClassName: nginx
    rules:
      - host: jenkins.yuanzhibin.com
        http:
          paths:
            - backend:
                service: 
                  name: my-jenkins
                  port:
                    number: 80
              path: /
              pathType: Prefix
    # This section is only required if TLS is to be enabled for the Ingress
    #tls:
    #    - hosts:
    #        - www.example.com
    #      secretName: example-tls

2）、kubeapps（helm）的ingress.yaml文件：

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
    name: kubeapps
    namespace: kubeapps
spec:
    ingressClassName: nginx
    rules:
      - host: kubeapps.yuanzhibin.com
        http:
          paths:
            - backend:
                service: 
                  name: kubeapps
                  port:
                    number: 80
              path: /
              pathType: Prefix
    # This section is only required if TLS is to be enabled for the Ingress
    #tls:
    #    - hosts:
    #        - www.example.com
    #      secretName: example-tls

3)、harbor的ingress.yaml文件：

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  annotations:
    ingress.kubernetes.io/proxy-body-size: "0"
    ingress.kubernetes.io/ssl-redirect: "true"
    meta.helm.sh/release-name: my-harbor
    meta.helm.sh/release-namespace: default
    nginx.ingress.kubernetes.io/proxy-body-size: "0"
    nginx.ingress.kubernetes.io/ssl-redirect: "true"
  labels:
    app: harbor
    app.kubernetes.io/managed-by: Helm
    chart: harbor
    heritage: Helm
    release: my-harbor
  name: my-harbor-ingress
  namespace: default
spec:
  ingressClassName: nginx
  rules:
  - host: harbor.yuanzhibin.com
    http:
      paths:
      - backend:
          service:
            name: my-harbor-portal
            port:
              number: 80
        path: /
        pathType: Prefix
      - backend:
          service:
            name: my-harbor-core
            port:
              number: 80
        path: /api/
        pathType: Prefix
      - backend:
          service:
            name: my-harbor-core
            port:
              number: 80
        path: /service/
        pathType: Prefix
      - backend:
          service:
            name: my-harbor-core
            port:
              number: 80
        path: /v2
        pathType: Prefix
      - backend:
          service:
            name: my-harbor-core
            port:
              number: 80
        path: /chartrepo/
        pathType: Prefix
      - backend:
          service:
            name: my-harbor-core
            port:
              number: 80
        path: /c/
        pathType: Prefix
  tls:
  - hosts:
    - harbor.yuanzhibin.com
    secretName: my-harbor-ingress
status:
  loadBalancer: {}

8、访问界面如下：

9、并发部署Jenkinsfile文件：

def label_scm = "mypod-${UUID.randomUUID().toString()}"


parameters {
	string(name: 'buid_id', defaultValue: "${env.BUILD_NUMBER}", description: 'What should I say?')
  }


podTemplate(label: label_scm,
		cloud: "kubernetes",
		containers: [containerTemplate(name: 'maven', image: 'maven:3.8.4-jdk-8',command: 'sleep', args: '99d'),
				containerTemplate(name: 'docker', image: 'docker', command: 'sleep', args: '99d'),
    				containerTemplate(name: 'kubectl', image: 'cnych/kubectl', command: 'cat', ttyEnabled: true)],
		volumes: [configMapVolume(configMapName: 'settings.xml', mountPath: '/config'),
			//configMapVolume(configMapName: 'ca.crt', mountPath: '/etc/docker/certs.d/harbor.yuanzhibin.com'),
			hostPathVolume(hostPath: '/var/run/docker.sock', mountPath: '/var/run/docker.sock')]
    ){
    
   parallel ( 
        
        "storage":  { 
            	node(label_scm) {
                    	stage('build storage jar'){
                      		git credentialsId: 'github', url: 'git@github.com:jackbauer123/mytest.git'
				script {
				    build_tag = sh(returnStdout: true, script: 'git rev-parse --short HEAD').trim()
				    if (env.BRANCH_NAME != 'master') {
					build_tag = "${build_tag}-${env.BUILD_NUMBER}"
				    }
				}
			      	container('maven') {

					sh 'mvn -B -ntp clean package -DskipTests -f samples-common/pom.xml -s  /config/settings.xml'
					sh 'mvn install -pl samples-common  -s /config/settings.xml'
					sh 'mvn -B -ntp clean package -DskipTests -f storage/pom.xml  -s /config/settings.xml'
				}
                 	}
			stage('build storage image') {
			  
				  container('docker'){
					  storage = docker.build("harbor.yuanzhibin.com/library/storage:${build_tag}","storage")

				  }
			 
			}
			stage('Push storage image') {
				  container('docker'){
					docker.withRegistry('https://harbor.yuanzhibin.com', 'harbor-admin') {

						storage.push("${build_tag}")

					} 

				  }	  
	  
	 	 	}
			stage('deploy storage'){
			
			
				container('kubectl') {

					withKubeConfig([credentialsId: 'kube2', serverUrl: 'https://10.168.1.199:6443']) {

						sh "sed -i 's//${build_tag}/' storage/storage.yaml"
						sh 'kubectl apply -f storage/storage.yaml'


				    }
				}	
			
			
	  
	  		}
	  


		}

        },
         "account": {
              node(label_scm) {
                    	stage('build account jar'){
				git credentialsId: 'github', url: 'git@github.com:jackbauer123/mytest.git'
				script {
				    build_tag = sh(returnStdout: true, script: 'git rev-parse --short HEAD').trim()
				    if (env.BRANCH_NAME != 'master') {
					build_tag = "${build_tag}-${env.BUILD_NUMBER}"
				    }
				}
				container('maven') {
					sh 'mvn -B -ntp clean package -DskipTests -f samples-common/pom.xml  -s /config/settings.xml'
					sh 'mvn install -pl samples-common  -s /config/settings.xml'
					sh 'mvn -B -ntp clean package -DskipTests -f account/pom.xml -s /config/settings.xml'
				}
                     	}
		     	stage('build account image') {
			  
				  container('docker'){
					  account = docker.build("harbor.yuanzhibin.com/library/account:${build_tag}","account")

				  }
			 
			}
			stage('Push account image') {
				  container('docker'){
					docker.withRegistry('https://harbor.yuanzhibin.com', 'harbor-admin') {

						account.push("${build_tag}")

					} 

				  }	  
	  
	 	 	}
			stage('deploy account'){
			
			
				container('kubectl') {

					withKubeConfig([credentialsId: 'kube2', serverUrl: 'https://10.168.1.199:6443']) {

						sh "sed -i 's//${build_tag}/' account/account.yaml"
						sh 'kubectl apply -f account/account.yaml'


				    }
				}	
			
			
	  
	  		}


              }
	},
         "order": {
              node(label_scm) {
                    	stage('build order jar'){
                        	git credentialsId: 'github', url: 'git@github.com:jackbauer123/mytest.git'
				script {
				    build_tag = sh(returnStdout: true, script: 'git rev-parse --short HEAD').trim()
				    if (env.BRANCH_NAME != 'master') {
					build_tag = "${build_tag}-${env.BUILD_NUMBER}"
				    }
				}
				container('maven') {
					sh 'mvn -B -ntp clean package -DskipTests  -f samples-common/pom.xml -s /config/settings.xml'
					sh 'mvn install -pl samples-common -s /config/settings.xml'
					sh 'mvn -B -ntp clean package -DskipTests -f order/pom.xml -s /config/settings.xml'
				}
                     	}
		     	stage('build order image') {
			  
				  container('docker'){
					  order = docker.build("harbor.yuanzhibin.com/library/order:${build_tag}","order")

				  }
			 
			}
			
			stage('Push order image') {
				  container('docker'){
					docker.withRegistry('https://harbor.yuanzhibin.com', 'harbor-admin') {

						order.push("${build_tag}")

					} 

				  }	  
	  
	 		 }
			 stage('deploy order'){
			
			
				container('kubectl') {

					withKubeConfig([credentialsId: 'kube2', serverUrl: 'https://10.168.1.199:6443']) {

						sh "sed -i 's//${build_tag}/' order/order.yaml"
						sh 'kubectl apply -f order/order.yaml'


				    }
				}	
			
			
	  
	  		}


              }
	},
         "logic": {
              node(label_scm) {
                    	stage('build logic jar'){
				git credentialsId: 'github', url: 'git@github.com:jackbauer123/mytest.git'
				script {
				    build_tag = sh(returnStdout: true, script: 'git rev-parse --short HEAD').trim()
				    if (env.BRANCH_NAME != 'master') {
					build_tag = "${build_tag}-${env.BUILD_NUMBER}"
				    }
				}
				container('maven') {
					sh 'mvn -B -ntp clean package -DskipTests  -f samples-common/pom.xml -s /config/settings.xml'
					sh 'mvn install -pl samples-common -s /config/settings.xml'
					sh 'mvn -B -ntp clean package -DskipTests -f logic/pom.xml -s /config/settings.xml'
				}
                     	}
		     
		     	stage('build logic image') {
			  
				  container('docker'){
					  logic = docker.build("harbor.yuanzhibin.com/library/logic:${build_tag}","logic")

				  }
			 
			}
			
			stage('Push  logic image') {
				  container('docker'){
					docker.withRegistry('https://harbor.yuanzhibin.com', 'harbor-admin') {

						logic.push("${build_tag}")

					} 

				  }	  
	  
	 	 	}
			stage('deploy storage'){
			
			
				container('kubectl') {

					withKubeConfig([credentialsId: 'kube2', serverUrl: 'https://10.168.1.199:6443']) {

						sh "sed -i 's//${build_tag}/' logic/logic.yaml"
						sh 'kubectl apply -f logic/logic.yaml'


				    }
				}	
			
			
	  
	  		}


              }
	}	
	)	
	
	


}

10、执行的结果：


11、pods运行截图：

欢迎交流！！！