官方文档: https://kubernetes.io/

社区: https://ask.kubesphere.io/forum/

KubeSphere官网: https://kubesphere.io/zh/docs/v3.4/

KubeSphere DevOps系统

安装文档: https://kubesphere.io/zh/docs/v3.4/pluggable-components/devops/

基于 Jenkins 的 KubeSphere DevOps 系统是专为 Kubernetes 中的 CI/CD 工作流设计的,它提供了一站式的解决方案,帮助开发和运维团队用非常简单的方式构建、测试和发布应用到 Kubernetes。它还具有插件管理、Binary-to-Image (B2I)Source-to-Image (S2I)、代码依赖缓存、代码质量分析、流水线日志等功能。

DevOps 系统为用户提供了一个自动化的环境,应用可以自动发布到同一个平台。它还兼容第三方私有镜像仓库(如 Harbor)和代码库(如 GitLab/GitHub/SVN/BitBucket)。它为用户提供了全面的、可视化的 CI/CD 流水线,打造了极佳的用户体验,而且这种兼容性强的流水线能力在离线环境中非常有用

分支选型

由于公司代码仓库有上百个,jenkinsFile DockerFile维护在代码仓库实在不是个可以简单推进下去的方案。

于是新开了一个代码仓库用于发布, 在创建流水线时,流水线类别选择多分支流水线

注意事项

kubeSphere里的jenkins不支持升级,下载的插件一定要对应jenkins的版本,否则jenkins会无法启动!

创建流水线凭证

image-20240816181114066

创建保密字典

image-20240816182226757

后端代码部署示例
  • JenkinsFile

    仓库代码布置路径: ${PROJECT}/${APP_NAME}/${WORK_SPACE}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
pipeline {
agent {
node {
label 'maven'
}
}

parameters {
string(name:'COMMIT_ID', defaultValue: '', description:'')
}

environment {
// ID之类的务必需要创建好凭证
DOCKERHUB_CREDENTIAL_ID = 'dockerhub-id' // 连接镜像仓库使用的账号密码
GITLAB_CREDENTIAL_ID = 'gitlab-id' // gitlab的ssh 私钥
KUBECONFIG_CREDENTIAL_ID = 'kubeconfig'
REGISTRY = 'registry.in.dtmiller.com'
DOCKERHUB_NAMESPACE = 'qingmu-image'
APP_NAME = 'jellycat-service'
NAME_SPACE = 'staging-jellycat'
WORK_SPACE = 'staging'
PROJECT = 'jellycat'
WORK_DIR = "/home/jenkins/agent/_work/${WORK_SPACE}_${APP_NAME}"

}

stages {
stage('拉取项目代码') {
agent none
steps {
script {
checkout([
$class: 'GitSCM',
branches: [[name: "${params.COMMIT_ID}"]],
doGenerateSubmoduleConfigurations: false,
extensions: [],
submoduleCfg: [],
userRemoteConfigs: [[credentialsId: "${env.GITLAB_CREDENTIAL_ID}",
url: 'git@gitlab.dtmiller.com:micro-service/wechat-mini.git']]
])
}
wrap([$class: 'BuildUser']) {
script {
BUILD_USER = "${env.BUILD_USER}"
env.TAG_NAME = sh(returnStdout: true, script: 'git rev-parse --short HEAD').trim()
}
}
println "BUILD_USER = ${BUILD_USER} TAG_NAME = ${env.TAG_NAME} COMMIT_ID = ${params.COMMIT_ID}"
sh("mkdir -p $WORK_DIR && mv * .git $WORK_DIR && ls $WORK_DIR")
}
}


stage('拉取部署代码') {
agent none
steps {
git(url: 'git@gitlab.dtmiller.com:ops/k8sdeploy.git', credentialsId: "$GITLAB_CREDENTIAL_ID", branch: 'master', changelog: true, poll: false)
sh("mv ${PROJECT} $WORK_DIR && ls -al $WORK_DIR")

}
}

stage('保存制品') {
agent none
steps {
container('maven') {
sh 'cd $WORK_DIR && ls -al && mvn clean package -B -U -Dmaven.test.skip=true -f ./pom.xml'
// archiveArtifacts(artifacts: 'target/*.jar', followSymlinks: true)
}
}
}

stage('编译和推送镜像') {
agent none
steps {
container('maven') {
// input(message: '@admin', submitter: '')
withCredentials([usernamePassword(credentialsId: "$DOCKERHUB_CREDENTIAL_ID", passwordVariable: 'DOCKER_PASSWORD', usernameVariable: 'DOCKER_USERNAME')]) {
sh 'echo "$DOCKER_PASSWORD" | podman login $REGISTRY -u "$DOCKER_USERNAME" --password-stdin'
sh 'cd $WORK_DIR && podman build -f ./${PROJECT}/${APP_NAME}/${WORK_SPACE}/Dockerfile -t $REGISTRY/$DOCKERHUB_NAMESPACE/$APP_NAME:$TAG_NAME .'
sh 'podman push $REGISTRY/$DOCKERHUB_NAMESPACE/$APP_NAME:$TAG_NAME'
}

}

}
}

stage('部署') {
steps {
container('maven') {
withCredentials([kubeconfigContent(credentialsId: "$KUBECONFIG_CREDENTIAL_ID", variable: 'KUBECONFIG_CONTENT')]) {
sh '''mkdir ~/.kube
echo "$KUBECONFIG_CONTENT" > ~/.kube/config
cd $WORK_DIR
envsubst < ./${PROJECT}/${APP_NAME}/${WORK_SPACE}/deployment.yaml | kubectl apply -f -'''
}

}

}
}

}


post {
success {
dingtalk (
robot: "d59d224d-f53f-446d-9641-4ed24757df65",
type: "ACTION_CARD",
atAll: false,
title: "构建成功:${env.JOB_NAME}",
text: [
"### [${env.JOB_NAME}](${env.JOB_URL}) ",
'---',
"- 任务:${APP_NAME}:${TAG_NAME} [${currentBuild.displayName}](${env.BUILD_URL})",
"- 镜像:<font color=#00CD00 >$REGISTRY/$DOCKERHUB_NAMESPACE/$APP_NAME:$TAG_NAME</font>",
'- 企业空间:<font color=#00CD00 >${WORK_SPACE}</font>',
"- 分支:<font color=#00CD00 >${params.COMMIT_ID}</font>",
'- 状态:<font color=#00CD00 >成功</font>',
"- 持续时间:${currentBuild.durationString}".split("and counting")[0],
"- 执行人:${BUILD_USER}",
]
)
}
failure {
dingtalk (
robot: "d59d224d-f53f-446d-9641-4ed24757df65",
type: "ACTION_CARD",
atAll: false,
title: "构建失败:${env.JOB_NAME}",
text: [
"### [${env.JOB_NAME}](${env.JOB_URL}) ",
'---',
"- 任务:${APP_NAME}:${TAG_NAME} [${currentBuild.displayName}](${env.BUILD_URL})",
"- 镜像:<font color=#EE0000 >$REGISTRY/$DOCKERHUB_NAMESPACE/$APP_NAME:$TAG_NAME</font>",
'- 状态:<font color=#EE0000 >失败</font>',
'- 企业空间:<font color=#EE0000 >${WORK_SPACE}</font>',
"- 分支:<font color=#EE0000 >${params.COMMIT_ID}</font>",
"- 持续时间:${currentBuild.durationString}".split("and counting")[0],
"- 执行人:${BUILD_USER}",
]
)
}
}

}

  • DockerFile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
FROM registry.in.dtmiller.com/qingmu-image/minijdk8-pinpoint-1.8.0:base

ENV APPNAME="sit-wechatmini-service" \
RUNTIME_ENV="sit" \
LOG_PATH="/data/apps/logs" \
APOLLO_META="http://10.6.16.93:8085" \
JAVA_OPTS="-Xmx4096m -Xms4096m -Xmn2048m -Xss228k" \
POD_NAME=""

COPY qm-admin/target/*.jar /${APPNAME}.jar


RUN echo '/opt/jdk/bin/java -server ${JAVA_OPTS} \
-Djava.security.egd=file:/dev/urandom \
-XX:-OmitStackTraceInFastThrow \
-XX:+UseConcMarkSweepGC \
-XX:CMSInitiatingOccupancyFraction=75 \
-XX:+PrintGCDetails \
-XX:+PrintGCDateStamps -XX:+PrintGCTimeStamps \
-XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=100M \
-Xloggc:${LOG_PATH}/${APP_NAME}.gc -XX:ErrorFile=${LOG_PATH}/hs_err_${APP_NAME}_%t_%p.log \
-XX:HeapDumpPath=${LOG_PATH}/java_pid_${APP_NAME}_%t_%p.hprof \
-XX:+HeapDumpOnOutOfMemoryError \
-jar /${APPNAME}.jar \
-Dapollo.meta=${APOLLO_META} \
--spring.profiles.active=${RUNTIME_ENV}' > /start.sh \
&& chmod +x /start.sh \
&& /bin/ln -s ${LOG_PATH}

ENTRYPOINT ["sh", "-c", "/start.sh && sleep 3650d"]
  • deployment.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
---
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app.kubernetes.io/name: ${APP_NAME}
app.kubernetes.io/component: wechatmini-service
app.kubernetes.io/part-of: jellycat
app.kubernetes.io/logging: filebeat
name: ${APP_NAME}
namespace: ${NAME_SPACE}
annotations:
kubesphere.io/imagepullsecrets: '{"${APP_NAME}": "aliyun-registry"}'
spec:
progressDeadlineSeconds: 600
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: ${APP_NAME}
app.kubernetes.io/component: wechatmini-service
app.kubernetes.io/part-of: jellycat
strategy:
rollingUpdate:
maxSurge: 100%
maxUnavailable: 100%
type: RollingUpdate
template:
metadata:
labels:
app.kubernetes.io/name: ${APP_NAME}
app.kubernetes.io/component: wechatmini-service
app.kubernetes.io/part-of: jellycat
app.kubernetes.io/logging: filebeat
spec:
volumes:
- name: log-volume
hostPath:
path: /data/logs
type: Directory
initContainers:
- command:
- sh
- -c
- sysopt
image: ${REGISTRY}/${DOCKERHUB_NAMESPACE}/sysopt:base
imagePullPolicy: Always
name: init-0-sysopt
resources: { }
securityContext:
privileged: true
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
containers:
- image: ${REGISTRY}/${DOCKERHUB_NAMESPACE}/${APP_NAME}:${TAG_NAME}
env:
- name: APP_NAME
value: ${APP_NAME}
- name: POD_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.name
- name: NAME_SPACE
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.namespace
- name: HOST_IP
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: status.hostIP
- name: POD_IP
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: status.podIP
- name: JAVA_OPTS
value: "-Xmx4096M -Xms4096M -Xmn1536M -XX:MaxMetaspaceSize=512M -XX:MetaspaceSize=512M"
imagePullPolicy: Always
name: ${APP_NAME}
volumeMounts:
- name: log-volume
mountPath: /data/apps/logs
subPathExpr: $(NAME_SPACE)/$(APP_NAME)/$(POD_NAME)
resources:
limits:
memory: 4Gi
requests:
memory: 4Gi
securityContext:
privileged: true
runAsUser: 0
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
dnsPolicy: ClusterFirst
restartPolicy: Always
terminationGracePeriodSeconds: 30
imagePullSecrets:
- name: aliyun-registry

---
apiVersion: v1
kind: Service
metadata:
labels:
app.kubernetes.io/name: ${APP_NAME}
app.kubernetes.io/component: wechatmini-service
app.kubernetes.io/part-of: jellycat
name: ${APP_NAME}
namespace: ${NAME_SPACE}
spec:
ports:
- name: http-8080
port: 8080
protocol: TCP
targetPort: 8080
selector:
app.kubernetes.io/name: ${APP_NAME}
app.kubernetes.io/component: wechatmini-service
app.kubernetes.io/part-of: jellycat
sessionAffinity: None
type: ClusterIP
internalTrafficPolicy: Cluster
前端项目部署
  • JenkinsFile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
pipeline {
agent {
node {
label 'base'
}
}

parameters {
string(name:'COMMIT_ID', defaultValue: '', description:'')
}

environment {
DOCKERHUB_CREDENTIAL_ID = 'dockerhub-id'
GITLAB_CREDENTIAL_ID = 'gitlab-id'
KUBECONFIG_CREDENTIAL_ID = 'kubeconfig'
REGISTRY = 'registry.in.dtmiller.com'
DOCKERHUB_NAMESPACE = 'qingmu-image'
APP_NAME = 'jellycat-manage'
NAME_SPACE = 'staging-jellycat'
WORK_SPACE = 'staging'
PROJECT = 'jellycat'
WORK_DIR = "/home/jenkins/agent/_work/${WORK_SPACE}_${APP_NAME}"

}

stages {
stage('拉取项目代码') {
agent none
steps {
script {
checkout([
$class: 'GitSCM',
branches: [[name: "${params.COMMIT_ID}"]],
doGenerateSubmoduleConfigurations: false,
extensions: [],
submoduleCfg: [],
userRemoteConfigs: [[credentialsId: "${env.GITLAB_CREDENTIAL_ID}",
url: 'git@gitlab.dtmiller.com:feprogram/jellycat-manage.git']]
])
}
wrap([$class: 'BuildUser']) {
script {
BUILD_USER = "${env.BUILD_USER}"
env.TAG_NAME = sh(returnStdout: true, script: 'git rev-parse --short HEAD').trim()
}
}
println "BUILD_USER = ${BUILD_USER} TAG_NAME = ${env.TAG_NAME} COMMIT_ID = ${params.COMMIT_ID}"
sh("mkdir -p $WORK_DIR && mv * .git $WORK_DIR")
}
}


stage('拉取部署代码') {
agent none
steps {
git(url: 'git@gitlab.dtmiller.com:ops/k8sdeploy.git', credentialsId: "$GITLAB_CREDENTIAL_ID", branch: 'master', changelog: true, poll: false)
sh("mv ${PROJECT} $WORK_DIR/../")

}
}


stage('编译和推送镜像') {
agent none
steps {
container('base') {
// input(message: '@admin', submitter: '')
withCredentials([usernamePassword(credentialsId: "$DOCKERHUB_CREDENTIAL_ID", passwordVariable: 'DOCKER_PASSWORD', usernameVariable: 'DOCKER_USERNAME')]) {
sh 'echo "$DOCKER_PASSWORD" | docker login $REGISTRY -u "$DOCKER_USERNAME" --password-stdin'
sh 'cd $WORK_DIR && docker build -f ../${PROJECT}/${APP_NAME}/${WORK_SPACE}/Dockerfile -t $REGISTRY/$DOCKERHUB_NAMESPACE/$APP_NAME:$TAG_NAME .'
sh 'docker push $REGISTRY/$DOCKERHUB_NAMESPACE/$APP_NAME:$TAG_NAME'
}

}

}
}

stage('部署') {
steps {
container('base') {
withCredentials([kubeconfigContent(credentialsId: "$KUBECONFIG_CREDENTIAL_ID", variable: 'KUBECONFIG_CONTENT')]) {
sh '''mkdir ~/.kube
echo "$KUBECONFIG_CONTENT" > ~/.kube/config
cd $WORK_DIR
envsubst < ../${PROJECT}/${APP_NAME}/${WORK_SPACE}/deployment.yaml | kubectl apply -f -'''
}

}

}
}

}


post {
success {
dingtalk (
robot: "d59d224d-f53f-446d-9641-4ed24757df65",
type: "ACTION_CARD",
atAll: false,
title: "构建成功:${env.JOB_NAME}",
text: [
"### [${env.JOB_NAME}](${env.JOB_URL}) ",
'---',
"- 任务:${APP_NAME}:${TAG_NAME} [${currentBuild.displayName}](${env.BUILD_URL})",
"- 镜像:<font color=#00CD00 >$REGISTRY/$DOCKERHUB_NAMESPACE/$APP_NAME:$TAG_NAME</font>",
'- 企业空间:<font color=#00CD00 >${WORK_SPACE}</font>',
"- 分支:<font color=#00CD00 >${params.COMMIT_ID}</font>",
'- 状态:<font color=#00CD00 >成功</font>',
"- 持续时间:${currentBuild.durationString}".split("and counting")[0],
"- 执行人:${BUILD_USER}",
]
)
}
failure {
dingtalk (
robot: "d59d224d-f53f-446d-9641-4ed24757df65",
type: "ACTION_CARD",
atAll: false,
title: "构建失败:${env.JOB_NAME}",
text: [
"### [${env.JOB_NAME}](${env.JOB_URL}) ",
'---',
"- 任务:${APP_NAME}:${TAG_NAME} [${currentBuild.displayName}](${env.BUILD_URL})",
"- 镜像:<font color=#EE0000 >$REGISTRY/$DOCKERHUB_NAMESPACE/$APP_NAME:$TAG_NAME</font>",
'- 状态:<font color=#EE0000 >失败</font>',
'- 企业空间:<font color=#EE0000 >${WORK_SPACE}</font>',
"- 分支:<font color=#EE0000 >${params.COMMIT_ID}</font>",
"- 持续时间:${currentBuild.durationString}".split("and counting")[0],
"- 执行人:${BUILD_USER}",
]
)
}
}

}
  • DockerFile
1
2
3
4
5
6
7
8
9
FROM registry.in.dtmiller.com/qingmu-image/node1.1.7-nginx:base

ENV RUNTIME_ENV=sit PORT=80

ADD ./dist/ /app/dist/
WORKDIR /app
EXPOSE ${PORT}

CMD [ "/usr/sbin/nginx" ]
  • deployment.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
---
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app.kubernetes.io/name: ${APP_NAME}
app.kubernetes.io/component: jellycat-manage
app.kubernetes.io/part-of: jellycat
app.kubernetes.io/env: ${WORK_SPACE}
name: ${APP_NAME}
namespace: ${NAME_SPACE}
annotations:
kubesphere.io/imagepullsecrets: '{"${APP_NAME}": "aliyun-registry"}'
spec:
progressDeadlineSeconds: 600
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: ${APP_NAME}
app.kubernetes.io/component: jellycat-manage
app.kubernetes.io/part-of: jellycat
app.kubernetes.io/env: ${WORK_SPACE}
strategy:
rollingUpdate:
maxSurge: 100%
maxUnavailable: 100%
type: RollingUpdate
template:
metadata:
labels:
app.kubernetes.io/name: ${APP_NAME}
app.kubernetes.io/component: jellycat-manage
app.kubernetes.io/part-of: jellycat
app.kubernetes.io/env: ${WORK_SPACE}
spec:
volumes:
- name: log-volume
hostPath:
path: /data/logs
type: Directory
containers:
- image: ${REGISTRY}/${DOCKERHUB_NAMESPACE}/${APP_NAME}:${TAG_NAME}
command:
- /usr/sbin/nginx
args:
- '-g'
- daemon off;
env:
- name: APP_NAME
value: ${APP_NAME}
- name: POD_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.name
- name: NAME_SPACE
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.namespace
- name: HOST_IP
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: status.hostIP
- name: POD_IP
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: status.podIP
imagePullPolicy: Always
name: ${APP_NAME}
volumeMounts:
- name: log-volume
mountPath: /data/apps/logs
subPathExpr: $(APP_NAME)/$(POD_NAME)
livenessProbe:
failureThreshold: 5
tcpSocket:
port: 80
initialDelaySeconds: 15
periodSeconds: 6
successThreshold: 1
timeoutSeconds: 5
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
dnsPolicy: ClusterFirst
restartPolicy: Always
terminationGracePeriodSeconds: 30
imagePullSecrets:
- name: aliyun-registry

---
apiVersion: v1
kind: Service
metadata:
labels:
app.kubernetes.io/name: ${APP_NAME}
app.kubernetes.io/component: jellycat-manage
app.kubernetes.io/part-of: jellycat
app.kubernetes.io/env: ${WORK_SPACE}
name: ${APP_NAME}
namespace: ${NAME_SPACE}
spec:
ports:
- name: http-80
port: 80
protocol: TCP
targetPort: 80
selector:
app.kubernetes.io/name: ${APP_NAME}
app.kubernetes.io/component: jellycat-manage
app.kubernetes.io/part-of: jellycat
app.kubernetes.io/env: ${WORK_SPACE}
sessionAffinity: None
type: ClusterIP
internalTrafficPolicy: Cluster