helm部署OpenVPN 4.2.3
k8s部署openvpn打通k8s网络将Kubernetes集群网络暴露给本地开发网络
简单介绍：
VPN直译就是虚拟专用通道，是提供给企业之间或者个人与公司之间安全数据传输的隧道，OpenVPN无疑是Linux下开源VPN的先锋，提供了良好的性能和友好的用户GUI。
它大量使用了OpenSSL加密库中的SSLv3/TLSv1协议函数库。
目前OpenVPN能在Solaris、Linux、OpenBSD、FreeBSD、NetBSD、Mac OS X与Microsoft Windows以及Android和iOS上运行，并包含了许多安全性的功能。它并不是一个基于Web的VPN软件，也不与IPsec及其他VPN软件包兼容
官方仓库地址：Kubeapps Hub
客户端下载地址：下载地址
若该网址无法访问，可以到下面的链接进行下载
百度网盘：点击直达 提取码：l0qv
确认集群helm安装正确
1
2
|
[root@master01 ~]# helm version
version.BuildInfo{Version:"v3.8.2", GitCommit:"6e3701edea09e5d55a8ca2aae03a68917630e91b", GitTreeState:"clean", GoVersion:"go1.17.5"}
|
下载Chart包，修改values.yaml文件
1
2
3
4
5
6
7
|
helm repo add stable https://kubernetes-charts.storage.googleapis.com
helm pull stable/openvpn
或者
wget http://mirror.azure.cn/kubernetes/charts/openvpn-4.2.5.tgz
tar -zxvf openvpn-4.2.5.tgz
cd openvpn
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
|
cp values.yaml openvpn-values.yaml
replicaCount: 1
updateStrategy: {}
imagePullSecretName:
image:
repository: jfelten/openvpn-docker
tag: 1.1.0
pullPolicy: IfNotPresent
service:
type: ClusterIP
#externalPort: 443
#internalPort: 443
# hostPort: 443
externalIPs: []
nodePort: 32085
# clusterIP: None
# LoadBalancerSourceRanges: 0.0.0.0/0
# loadBalancerIP: 10.0.0.1
## Here annotations can be added to the openvpn service
# annotations:
# external-dns.alpha.kubernetes.io/hostname: vpn.example.com
annotations: {}
## Here annotations can be added to the openvpn pod
# podAnnotations:
# backup.ark.heptio.com/backup-volumes: certs
podAnnotations: {}
# Add privileged init container to enable IPv4 forwarding
ipForwardInitContainer: true
resources:
limits:
cpu: 300m
memory: 128Mi
requests:
cpu: 300m
memory: 128Mi
readinessProbe:
initialDelaySeconds: 5
periodSeconds: 5
successThreshold: 2
persistence:
enabled: true
storageClass: "nfs-client"
accessMode: ReadWriteOnce
size: 2M
openvpn:
# Network allocated for openvpn clients (default: 10.240.0.0).
OVPN_NETWORK: 10.240.0.0
# Network subnet allocated for openvpn client (default: 255.255.0.0).
OVPN_SUBNET: 255.255.0.0
# Protocol used by openvpn tcp or udp (default: udp).
OVPN_PROTO: udp
OVPN_K8S_POD_NETWORK: "172.20.0.0" #k8s pod地址
OVPN_K8S_POD_SUBNET: "255.255.0.0"
OVPN_K8S_SVC_NETWORK: "172.21.0.0" #k8s svc地址
OVPN_K8S_SVC_SUBNET: "255.255.0.0"
DEFAULT_ROUTE_ENABLED: true
dhcpOptionDomain: true
# Redirect all client traffic through VPN
redirectGateway: true
useCrl: false
taKey: true
cipher: AES-256-CBC
istio:
enabled: false
proxy:
port: 15001
iptablesExtra: []
ccd:
enabled: false
config: {}
nodeSelector: {}
tolerations: []
|
相关文档参考：
www.1nth.com/post/k8s-op…
zhuanlan.zhihu.com/p/491…
pythontaotao.github.io/2…
部署并进行验证
可以直接helm部署，也可以生成yaml文件部署
方法一：
1
2
3
|
kubectl create ns openvpn
helm install openvpn -n openvpn ./ -f openvpn-values.yaml
helm status openvpn -n openvpn
|
方法二：我才用此方法
1
2
3
|
helm install openvpn -n openvpn ./ -f openvpn-values.yaml --dry-run --debug
生成yaml ，在进行部署
|
生成的文件
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
|
---
# Source: openvpn/templates/config-openvpn.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: openvpn
labels:
app: openvpn
chart: openvpn-4.2.5
release: openvpn
heritage: Helm
data:
setup-certs.sh: |-
#!/bin/bash
EASY_RSA_LOC="/etc/openvpn/certs"
cd $EASY_RSA_LOC
SERVER_CERT="${EASY_RSA_LOC}/pki/issued/server.crt"
if [ -e "$SERVER_CERT" ]
then
echo "found existing certs - reusing"
if [ ! -e ${EASY_RSA_LOC}/pki/ta.key ]
then
echo "generating missed ta.key"
openvpn --genkey --secret ${EASY_RSA_LOC}/pki/ta.key
fi
else
cp -R /usr/share/easy-rsa/* $EASY_RSA_LOC
./easyrsa init-pki
echo "ca\n" | ./easyrsa build-ca nopass
./easyrsa build-server-full server nopass
./easyrsa gen-dh
openvpn --genkey --secret ${EASY_RSA_LOC}/pki/ta.key
fi
newClientCert.sh: |-
#!/bin/bash
EASY_RSA_LOC="/etc/openvpn/certs"
cd $EASY_RSA_LOC
MY_IP_ADDR="$2"
./easyrsa build-client-full $1 nopass
cat >${EASY_RSA_LOC}/pki/$1.ovpn <<EOF
client
nobind
dev tun
remote ${MY_IP_ADDR} 443 udp
cipher AES-256-CBC
redirect-gateway def1
<key>
`cat ${EASY_RSA_LOC}/pki/private/$1.key`
</key>
<cert>
`cat ${EASY_RSA_LOC}/pki/issued/$1.crt`
</cert>
<ca>
`cat ${EASY_RSA_LOC}/pki/ca.crt`
</ca>
<tls-auth>
`cat ${EASY_RSA_LOC}/pki/ta.key`
</tls-auth>
key-direction 1
EOF
cat pki/$1.ovpn
revokeClientCert.sh: |-
#!/bin/bash
EASY_RSA_LOC="/etc/openvpn/certs"
cd $EASY_RSA_LOC
./easyrsa revoke $1
./easyrsa gen-crl
cp ${EASY_RSA_LOC}/pki/crl.pem ${EASY_RSA_LOC}
chmod 644 ${EASY_RSA_LOC}/crl.pem
configure.sh: |-
#!/bin/sh
cidr2mask() {
# Number of args to shift, 255..255, first non-255 byte, zeroes
set -- $(( 5 - ($1 / 8) )) 255 255 255 255 $(( (255 << (8 - ($1 % 8))) & 255 )) 0 0 0
[ $1 -gt 1 ] && shift "$1" || shift
echo ${1-0}.${2-0}.${3-0}.${4-0}
}
cidr2net() {
local i ip mask netOctets octets
ip="${1%/*}"
mask="${1#*/}"
octets=$(echo "$ip" | tr '.' '\n')
for octet in $octets; do
i=$((i+1))
if [ $i -le $(( mask / 8)) ]; then
netOctets="$netOctets.$octet"
elif [ $i -eq $(( mask / 8 +1 )) ]; then
netOctets="$netOctets.$((((octet / ((256 / ((2**((mask % 8)))))))) * ((256 / ((2**((mask % 8))))))))"
else
netOctets="$netOctets.0"
fi
done
echo ${netOctets#.}
}
/etc/openvpn/setup/setup-certs.sh
iptables -t nat -A POSTROUTING -s 10.240.0.0/255.255.0.0 -o eth0 -j MASQUERADE
mkdir -p /dev/net
if [ ! -c /dev/net/tun ]; then
mknod /dev/net/tun c 10 200
fi
if [ "$DEBUG" == "1" ]; then
echo ========== ${OVPN_CONFIG} ==========
cat "${OVPN_CONFIG}"
echo ====================================
fi
intAndIP="$(ip route get 8.8.8.8 | awk '/8.8.8.8/ {print $5 "-" $7}')"
int="${intAndIP%-*}"
ip="${intAndIP#*-}"
cidr="$(ip addr show dev "$int" | awk -vip="$ip" '($2 ~ ip) {print $2}')"
NETWORK="$(cidr2net $cidr)"
NETMASK="$(cidr2mask ${cidr#*/})"
DNS=$(cat /etc/resolv.conf | grep -v '^#' | grep nameserver | awk '{print $2}')
SEARCH=$(cat /etc/resolv.conf | grep -v '^#' | grep search | awk '{$1=""; print $0}')
FORMATTED_SEARCH=""
for DOMAIN in $SEARCH; do
FORMATTED_SEARCH="${FORMATTED_SEARCH}push \"dhcp-option DOMAIN-SEARCH ${DOMAIN}\"\n"
done
cp -f /etc/openvpn/setup/openvpn.conf /etc/openvpn/
sed 's|OVPN_K8S_SEARCH|'"${FORMATTED_SEARCH}"'|' -i /etc/openvpn/openvpn.conf
sed 's|OVPN_K8S_DNS|'"${DNS}"'|' -i /etc/openvpn/openvpn.conf
sed 's|NETWORK|'"${NETWORK}"'|' -i /etc/openvpn/openvpn.conf
sed 's|NETMASK|'"${NETMASK}"'|' -i /etc/openvpn/openvpn.conf
# exec openvpn process so it receives lifecycle signals
exec openvpn --config /etc/openvpn/openvpn.conf
openvpn.conf: |-
server 10.240.0.0 255.255.0.0
verb 3
key /etc/openvpn/certs/pki/private/server.key
ca /etc/openvpn/certs/pki/ca.crt
cert /etc/openvpn/certs/pki/issued/server.crt
dh /etc/openvpn/certs/pki/dh.pem
tls-auth /etc/openvpn/certs/pki/ta.key 0
cipher AES-256-CBC
key-direction 0
keepalive 10 60
persist-key
persist-tun
proto udp
port 443
dev tun0
status /tmp/openvpn-status.log
user nobody
group nogroup
push "route NETWORK NETMASK"
push "route 172.20.0.0 255.255.0.0"
push "route 172.21.0.0 255.255.0.0"
OVPN_K8S_SEARCH
push "dhcp-option DNS 172.21.16.10"
---
# Source: openvpn/templates/certs-pvc.yaml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: openvpn
labels:
app: openvpn
chart: openvpn-4.2.5
release: openvpn
heritage: Helm
spec:
accessModes:
- "ReadWriteOnce"
resources:
requests:
storage: "2M"
storageClassName: "nfs-client"
---
# Source: openvpn/templates/openvpn-service.yaml
apiVersion: v1
kind: Service
metadata:
name: openvpn
labels:
app: openvpn
chart: openvpn-4.2.5
release: openvpn
heritage: Helm
spec:
ports:
- name: openvpn
port: 443
targetPort: 443
protocol: UDP
selector:
app: openvpn
release: openvpn
type: ClusterIP
---
# Source: openvpn/templates/openvpn-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: openvpn
labels:
app: openvpn
chart: openvpn-4.2.5
release: openvpn
heritage: Helm
spec:
replicas: 1
selector:
matchLabels:
app: openvpn
release: openvpn
template:
metadata:
labels:
app: openvpn
release: openvpn
annotations:
checksum/config: c6d7908b4a71d5de2ee5f823eac2522e8de7c15ab026fcc0f4149a270886df1f
spec:
initContainers:
- args:
- -c
- sysctl -w net.ipv4.ip_forward=1
command:
- /bin/sh
image: busybox:1.29
imagePullPolicy: IfNotPresent
name: sysctl
resources:
requests:
cpu: 1m
memory: 1Mi
securityContext:
privileged: true
containers:
- name: openvpn
image: "jfelten/openvpn-docker:1.1.0"
imagePullPolicy: IfNotPresent
command: ["/etc/openvpn/setup/configure.sh"]
ports:
- containerPort: 443
name: openvpn
securityContext:
capabilities:
add:
- NET_ADMIN
readinessProbe:
initialDelaySeconds: 5
periodSeconds: 5
successThreshold: 2
exec:
command:
- nc
- -u
- -z
- 127.0.0.1
- "443"
resources:
requests:
cpu: "300m"
memory: "128Mi"
limits:
cpu: "300m"
memory: "128Mi"
volumeMounts:
- mountPath: /etc/openvpn/setup
name: openvpn
readOnly: false
- mountPath: /etc/openvpn/certs
name: certs
readOnly: false
volumes:
- name: openvpn
configMap:
name: openvpn
defaultMode: 0775
- name: certs
persistentVolumeClaim:
claimName: openvpn
|
账号管理脚本
下面整理一下生成clientkey的脚本,这里面的信息就是上面部署完成后输出的信息，需要修改参数KEY_NAME和SERVICE_IP,执行完成后会生成wangwu.ovpn的文件，客户端使用该文件作为连接凭证
create-client-key.sh 创建用户
1
2
3
4
5
6
7
8
9
10
11
12
|
$ cat create-client-key.sh
#!/bin/bash
POD_NAME=$(kubectl get pods --namespace "default" -l "app=openvpn,release=my-openvpn" -o jsonpath='{ .items[0].metadata.name }')
SERVICE_NAME=$(kubectl get svc --namespace "default" -l "app=openvpn,release=my-openvpn" -o jsonpath='{ .items[0].metadata.name }')
SERVICE_IP=10.169.68.142
KEY_NAME=wangwu
kubectl --namespace "default" exec -it "$POD_NAME" /etc/openvpn/setup/newClientCert.sh "$KEY_NAME" "$SERVICE_IP"
kubectl --namespace "default" exec -it "$POD_NAME" cat "/etc/openvpn/certs/pki/$KEY_NAME.ovpn" > "$KEY_NAME.ovpn"
$ chmod +x get_clientkey.sh && ./get_clientkey.sh
|
rm-client-key.sh 注销用户
1
2
3
4
5
|
#!/usr/bin/env bash
KEY_NAME=$1
POD_NAME=$(kubectl get pods -n "openvpn" -l "app=openvpn,release=openvpn" -o jsonpath='{.items[0].metadata.name}')
kubectl -n "openvpn" exec -it "$POD_NAME" /etc/openvpn/setup/revokeClientCert.sh $KEY_NAME
|
创建客户端密钥
1
2
3
4
5
6
7
|
[root@iZwz98n1yl2yxjc2xpwt8dZ openvpn]# kubectl --namespace "openvpn" get svc
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
openvpn LoadBalancer 172.19.77.8 119.23.xxx.xx 443:30924/TCP 8d
[root@iZwz98n1yl2yxjc2xpwt8dZ openvpn]# ./create-client-key.sh sunnyhaha
[root@iZwz98n1yl2yxjc2xpwt8dZ openvpn]# ll user-openvpn-20220402/sunnyhaha.ovpn
-rw-r--r-- 1 root root 7445 Apr 2 09:50 user-openvpn-20220402/sunnyhaha.ovpn
|
上面的命令会生成client.ovpn配置文件，因为前面用Helm部署openvpn时，创建的Servie是ClusterIP类型的，生成的配置文件 中的VPN服务地址会有错误，需要修改这个配置文件，将remote配置修改如下地址为k8s边缘节点的ip，端口为前面暴露的1194端口，同时删除redirect-gateway def1这行内容
解析k8s内部svc
部署完成之后有个坑,不能解析k8s内部svc
1
2
3
4
5
6
7
|
nslookup kubernetes.default.svc.cluster.local
服务器: UnKnown
Address: 192.168.1.1
DNS request timed out.
timeout was 2 seconds.
DNS request timed out.
|
查看openvpn服务端配置
1
2
3
|
kubectl get cm -n openvpn openvpn -oyaml
......
push "dhcp-option DNS OVPN_K8S_DNS"
|
因为我用的是node-local-dns,pod的DNS地址会变成__PILLAR__LOCAL__DNS__的默认地址169.254.20.10所以导致不能解析
修改服务端配置
1
2
3
4
|
kubectl get svc -n kube-system -l k8s-app=kube-dns -o jsonpath='{$.items[*].spec.clusterIP}'
10.68.0.2
kubectl get cm -n openvpn openvpn -oyaml
push "dhcp-option DNS 10.68.0.2"
|
