@zhangsiming65965
2019-04-22T06:43:16.000000Z
字数 14296
阅读 254
VPN

已知,公司的openvpnclient可以成功连接aws的openvpnserver的内网服务器,这是一组openvpn client-server;
需求一: 在自建机房架构openvpnclient,使得IDC机房内网gpu服务器可以双向连通aws内网服务器;
需求二: 通过aws的openvpnserver的client to client配置使得IDC机房client可以和公司client实现数据互通。
root@ip-172-31-35-87:/openvpn-server/openvpn-master# ll总用量 36-rw-r--r-- 1 root root 1205 4月 16 19:21 docker-compose.yml#docker-compose.yaml启动文件-rw-r--r-- 1 root root 444 4月 16 17:32 Dockerfile#构建openvpn镜像文件-rwxr-xr-x 1 root root 172 4月 16 19:36 entrypoint.sh*#openvpn-docker启动入口-rwxr-xr-x 1 root root 1500 4月 16 17:34 openvpn_init.sh*#初始化脚本drwxrwxr-x 2 root root 4096 4月 16 19:36 openvpn_need/#生成文件目录root@ip-172-31-35-87:/openvpn-server/openvpn-master# ll openvpn_need/总用量 24drwxrwxr-x 2 root root 4096 4月 16 19:36 ./drwxrwxr-x 4 root root 4096 4月 19 20:10 ../-rw-r--r-- 1 root root 855 4月 16 17:26 base.conf#客户端配置文件基准文件-rwxr-xr-x 1 root root 1714 4月 16 17:26 make_cert.sh*#制作证书文件-rw-r--r-- 1 root root 1533 4月 16 17:31 server.conf#服务端配置文件-rw-r--r-- 1 root root 2130 4月 4 17:07 vars#生成证书文件
root@ip-172-31-35-87:/openvpn-server/openvpn-master# cat openvpn_need/base.confclientdev tunproto tcp#客户端和服务端的传输协议需要一致remote 119.18.195.149 8888#openvpn-server的远程地址和端口resolv-retry infinitenobindpersist-keypersist-tunca /etc/openvpn/ca.crtcert /etc/openvpn/client.crtkey /etc/openvpn/client.key#连接用的证书和keyns-cert-type servercomp-lzoverb 3
root@ip-172-31-35-87:/openvpn-server/openvpn-master# cat openvpn_need/make_cert.sh#!/bin/bashecho -e "-------make_cert.sh start--------\n\n"source vars./clean-all#先生效vars文件,再生成密钥#生成ca证书echo -e "\n\n----------build-ca-----------"/usr/bin/expect << EOFspawn ./build-caexpect "Country Name" {send "\r"}expect "State or Province Name" {send "\r"}expect "Locality Name" {send "\r"}expect "Organization Name" {send "\r"}expect "Organizational Unit Name" {send "\r"}expect "Common Name" {send "\r"}expect "Name" {send "\r"}expect "Email Address" {send "\r"}expect eofEOF#基于ca生成server证书echo -e "\n\n-------build-key-server-------"/usr/bin/expect << EOFspawn ./build-key-server serverexpect "Country Name" {send "\r"}expect "State or Province Name" {send "\r"}expect "Locality Name" {send "\r"}expect "Organization Name" {send "\r"}expect "Organizational Unit Name" {send "\r"}expect "Common Name" {send "\r"}expect "Name" {send "\r"}expect "Email Address" {send "\r"}expect "A challenge password" {send "\r"}expect "An optional company name" {send "\r"}expect "Sign the certificate?" {send "y\r"}expect "commit?" {send "y\r"}expect eofEOF#基于ca生成client证书echo -e "\n\n-------build-key-client-------"/usr/bin/expect << EOFspawn ./build-key client#后面传入的client(可以任意指定),为证书的COMMON NAMEexpect "Country Name" {send "\r"}expect "State or Province Name" {send "\r"}expect "Locality Name" {send "\r"}expect "Organization Name" {send "\r"}expect "Organizational Unit Name" {send "\r"}expect "Common Name" {send "\r"}expect "Name" {send "\r"}expect "Email Address" {send "\r"}expect "A challenge password" {send "\r"}expect "An optional company name" {send "\r"}expect "Sign the certificate?" {send "y\r"}expect "commit?" {send "y\r"}expect eofEOFecho -e "\n\n--------build-dh-----------"#生成证书交换文件./build-dhecho -e "\n\n-------make_cert.sh done--------"
root@ip-172-31-35-87:/openvpn-server/openvpn-master# cat openvpn_need/server.confport 52115#openvpn-server监听端口proto tcpdev tunserver {{ OPENVPN_SERVER }}#给openvpn服务tun0网卡分配的地址,需要唯一,不要和双方任意的私网或者公网一样,在docker-compose用ENV导入push "route 1.10.0.0 255.255.0.0"#给客户端添加到server本地私网的路由,在docker-compose的ENV声明,在openvpn_init.sh中对应更改route 11.11.0.0 255.255.0.0#给本地添加一条到openvpn-client的回程路由client-config-dir /etc/openvpn/ccd#ccd目录必须写绝对路径,且要存在与docker中;openvpn-server动态读取之中的配置,直接修改ccd中的内容之后无需重启openvpn-server服务,客户端直接连接即可生效;ccd中的文件名需要和客户端证书的COMMON NAME一致ifconfig-pool-persist /etc/openvpn/logs/ipp.txtstatus /etc/openvpn/logs/status.loglog /etc/openvpn/logs/openvpn.logca /etc/openvpn/server/ca.crtcert /etc/openvpn/server/server.crtkey /etc/openvpn/server/server.keydh /etc/openvpn/server/dh2048.pemclient-to-client#client-to-client允许连接到同一个openvpn-server的不同客户端之间通信keepalive 10 120duplicate-cn#duplicate-cn允许不同用户使用同一证书连接openvpn-servercomp-lzopersist-keypersist-tunverb 3
root@ip-172-31-35-87:/openvpn-server/openvpn-master# tail -17 openvpn_need/vars | head -13export KEY_COUNTRY="$KEY_COUNTRY"export KEY_PROVINCE="$KEY_PROVINCE"export KEY_CITY="$KEY_CITY"export KEY_ORG="$KEY_ORG"export KEY_EMAIL="$KEY_EMAIL"export KEY_OU="$KEY_OU"export KEY_CN="$KEY_CN"# X509 Subject Fieldexport KEY_NAME="$KEY_NAME"export KEY_ALTNAMES="something"# PKCS11 Smart Card# export PKCS11_MODULE_PATH="/usr/lib/changeme.so"# export PKCS11_PIN=1234#里面声明了证书的信息,城市邮箱等...
root@ip-172-31-35-87:/openvpn-server/openvpn-master# cat openvpn_init.sh#!/bin/bashOPENVPN_DIR=/etc/openvpnOPENVPN_NEED=/usr/local/bin/if [ ! -f ${OPENVPN_DIR}/server/server.crt ]thenecho 'openvpn initializing ...'# make server ca and server crtcd ${OPENVPN_DIR}mkdir -p server client logscd /root/mkdir -p openvpn-cacp /usr/share/easy-rsa/\* openvpn-cacp ${OPENVPN_NEED}/make_cert.sh ${OPENVPN_NEED}/vars openvpn-cacp openvpn-ca/openssl-1.0.0.cnf openvpn-ca/openssl.cnfcd openvpn-ca./make_cert.shcd ./keys#创建证书并把server和client的证书放到对应位置cp ca.crt server.crt server.key dh* ${OPENVPN_DIR}/server/cp ca.crt client.crt client.key ${OPENVPN_DIR}/client/if [ ! -f ${OPENVPN_DIR}/server/server.conf ]thencp ${OPENVPN_NEED}/server.conf ${OPENVPN_DIR}/server/server.conffi# fill environment variable in#修改.dh文件sed -i "s|^dh.*|dh ${OPENVPN_DIR}/server/dh${KEY_SIZE}.pem|" ${OPENVPN_DIR}/server/server.conf#修改server.conf的tun网卡网段,即openvpn提供隧道服务的网段sed -i "s|{{ OPENVPN_SERVER }}|${OPENVPN_SERVER}|g" ${OPENVPN_DIR}/server/server.conf#修改给客户端添加的路由(指向openvpn-server的私网网卡)sed -i "s|^push \"route.*|push \"route ${PUSH}\"|" ${OPENVPN_DIR}/server/server.conf# make client confcp ${OPENVPN_NEED}/base.conf ${OPENVPN_DIR}/client/client.conf#修改client.conf指向远程openvpn-server的IP+端口sed -i "s|^remote.*|remote ${CLIENTREMOTE} 52115|" ${OPENVPN_DIR}/client/client.confrm -rf /usr/local/bin/*rm -rf /root/openvpn-caelseecho 'openvpn has been initialized, skip.'fi#添加tun网卡,隧道网卡提供vpn服务# add tunif [ ! -c /dev/net/tun ]thenecho "creating /dev/net/tun"mkdir /dev/netmknod /dev/net/tun c 10 200fi
root@ip-172-31-35-87:/openvpn-server/openvpn-master# cat entrypoint.sh#!/bin/bashcase "$1" inopenvpn)#先执行初始化进行配置/root/openvpn_init.sh#之后指向server.conf启动openpvn-server服务exec openvpn --dev tun --config /etc/openvpn/server/server.conf ${@:2};;*)exec "$@";;esac
root@ip-172-31-35-87:/openvpn-server/openvpn-master# cat DockerfileFROM registry.cn-beijing.aliyuncs.com/shannonai/openvpn:v1.1.1#安装证书生成工具,openvpn和expect等无交互辅助工具# runtime dependenciesRUN set -ex \&& apt-get -yqq update && apt-get install -yqq --no-install-recommends \openvpn \expect \easy-rsa \&& rm -rf /var/lib/apt/lists/*VOLUME /etc/openvpnEXPOSE 52115COPY openvpn_init.sh /root/COPY entrypoint.sh /root/COPY openvpn_need/ /usr/local/bin/#把文件放到指定路径,方便openvpn_init.sh的时候用ENTRYPOINT ["/root/entrypoint.sh"]CMD ["openvpn"]
root@ip-172-31-35-87:/openvpn-server/openvpn-master# cat docker-compose.ymlversion: '3.2'services:openvpn:image: openvpncontainer_name: openvpn-servernetwork_mode: host#使用宿主机网络restart: alwaysvolumes:- ./data:/etc/openvpn#挂载当前目录的data目录到docker中的/etc/openvpnports:- "52115"#暴露端口cap_add:- NET_ADMIN#cap_add不加会报错environment:OPENVPN_SERVER: 10.66.0.0 255.255.0.0#tun网卡网段,openvpn网卡服务网段PUSH: 172.31.0.0 255.255.0.0#给客户端添加到openvpn-server内网的路由CLIENTREMOTE: 52.82.24.178#给client.conf添加的远程openvpn-server公网地址KEY_COUNTRY: CN # The state abbreviations(for example: CN)KEY_PROVINCE: BJ # Province shorthand(for example: BJ)KEY_CITY: Beijing # City(for example: BeiJing)KEY_ORG: shannonai # organisation(for example: shannon.ai)KEY_OU: shannonai # affiliated unit(for example: shannon.ai)KEY_CN: shannonaiKEY_EMAIL: siming_zhang@shannonai.com # Email(for example: contact@shannonai.com)KEY_SIZE: 2048 # The private key size(for example: 2048)
root@ip-172-31-35-87:/openvpn-server/openvpn-master# lsdocker-compose.yml Dockerfile entrypoint.sh openvpn_init.sh openvpn_needroot@ip-172-31-35-87:/openvpn-server/openvpn-master# docker build -t openvpn .root@ip-172-31-35-87:/openvpn-server/openvpn-master# docker-compose up -droot@ip-172-31-35-87:/openvpn-server/openvpn-master# docker-compose psName Command State Ports------------------------------------------------------------openvpn-server /root/entrypoint.sh openvpn Up#成功启动root@ip-172-31-35-87:/openvpn-server/openvpn-master# ls data/ccd client logs serverroot@ip-172-31-35-87:/openvpn-server/openvpn-master# cat data/ccd/clientiroute 11.11.0.0 255.255.0.0#ccd目录手动创建,添加iroute,地址写openpvn-client的内网网段,我理解的是回城tun point-to-point的指向客户端路由root@ip-172-31-35-87:/openvpn-server/openvpn-master# ls data/client/ca.crt client.conf client.crt client.key make.sh#将客户端的证书、密钥和配置文件都拷贝得到openvpn-client服务器的指定目录连接备用root@ip-172-31-35-87:/openvpn-server/openvpn-master# ls data/server/ca.crt client.conf dh2048.pem server.conf server.crt server.key#查看监听端口root@ip-172-31-35-87:/openvpn-server/openvpn-master# netstat -antup | grep 52115tcp 0 0 0.0.0.0:52115 0.0.0.0:* LISTEN 1289/openvpn
root@jumpserver:~/openvpn-client# lltotal 32drwxr-xr-x 2 root root 4096 Apr 18 00:03 client/#client目录中放我们刚刚server生成的ca证书、client证书密钥和client.conf-rw-r--r-- 1 root root 282 Apr 15 08:07 docker-compose.yml-rw-r--r-- 1 root root 417 Apr 8 09:39 Dockerfile-rwxr-xr-x 1 root root 169 Apr 15 08:06 entrypoint.sh*-rwxr-xr-x 1 root root 139 Apr 8 08:14 openvpn_init.sh*
root@jumpserver:~/openvpn-client# cat client/client.confclientdev tunproto tcpremote 52.82.24.178 52115#连接openvpn-server的地址和端口resolv-retry infinitenobindpersist-keypersist-tunca /etc/openvpn/ca.crtcert /etc/openvpn/client.crtkey /etc/openvpn/client.keyns-cert-type servercomp-lzoverb 3
root@jumpserver:~/openvpn-client# cat entrypoint.sh#!/bin/bashcase "$1" inopenvpn)sh /root/openvpn_init.shexec openvpn --dev tun --config /etc/openvpn/client.conf ${@:2};;*)exec "$@";;esac#先初始化之后指向client.conf启动openvpn-client
root@jumpserver:~/openvpn-client# cat DockerfileFROM registry.cn-beijing.aliyuncs.com/shannonai/openvpn:v1.1.1USER root# runtime dependenciesRUN set -ex \&& apt-get -yqq update && apt-get install -yqq --no-install-recommends \openvpn \expect \easy-rsa \&& rm -rf /var/lib/apt/lists/*VOLUME /etc/openvpnEXPOSE 52115COPY openvpn_init.sh /root/COPY entrypoint.sh /root/#与openvpn-server的Dockerfile几乎一样,注意细节ENTRYPOINT ["/root/entrypoint.sh"]CMD ["openvpn"]
root@jumpserver:~/openvpn-client# cat docker-compose.ymlversion: '3.2'services:openvpn:image: openvpncontainer_name: openvpn-clientnetwork_mode: hostrestart: alwaysvolumes:- ./client:/etc/openvpn#挂载当前目录的client目录到/etc/openvpn中ports:- "52115"cap_add:- NET_ADMIN
root@jumpserver:~/openvpn-client# lsclient Dockerfile openvpn_init.shdocker-compose.yml entrypoint.sh README.mdroot@jumpserver:~/openvpn-client# docker build -t openvpn .root@jumpserver:~/openvpn-client# docker-compose up -droot@jumpserver:~/openvpn-client# docker-compose psName Command State Ports------------------------------------------------------------openvpn-client /root/entrypoint.sh openvpn Uproot@jumpserver:~/openvpn-client# netstat -antup | grep 52115tcp 0 0 11.11.22.100:53996 52.82.24.178:52115 ESTABLISHED 30650/openvpn#成功启动
#openvpn-clientroot@jumpserver:~/openvpn-client# route -n | grep tun010.66.0.0 10.66.0.5 255.255.0.0 UG 0 0 0 tun010.66.0.5 0.0.0.0 255.255.255.255 UH 0 0 0 tun1172.31.0.0 10.66.0.5 255.255.0.0 UG 0 0 0 tun0root@jumpserver:~/openvpn-client# ifconfig | grep -A 1 tun0tun0 Link encap:UNSPEC HWaddr 00-00-00-00-00-00-00-00-00-00-00-00-00-00-00-00inet addr:10.66.0.6 P-t-P:10.66.0.5 Mask:255.255.255.255#openvpn-serverroot@ip-172-31-35-87:/openvpn-server/openvpn-master# route -n | grep tun10.66.0.0 10.66.0.2 255.255.0.0 UG 0 0 0 tun010.66.0.2 0.0.0.0 255.255.255.255 UH 0 0 0 tun011.11.0.0 10.66.0.2 255.255.0.0 UG 0 0 0 tun0root@ip-172-31-35-87:/openvpn-server/openvpn-master# ifconfig | grep -A 1 tuntun0 Link encap:未指定 硬件地址 00-00-00-00-00-00-00-00-00-00-00-00-00-00-00-00inet 地址:10.66.0.1 点对点:10.66.0.2 掩码:255.255.255.255
#openvpn-client ping openvpn-serverroot@jumpserver:~/openvpn-client# ping 172.31.35.87PING 172.31.35.87 (172.31.35.87) 56(84) bytes of data.64 bytes from 172.31.35.87: icmp_seq=1 ttl=64 time=30.0 ms64 bytes from 172.31.35.87: icmp_seq=2 ttl=64 time=29.9 ms64 bytes from 172.31.35.87: icmp_seq=3 ttl=64 time=29.8 ms^C--- 172.31.35.87 ping statistics ---3 packets transmitted, 3 received, 0% packet loss, time 2000msrtt min/avg/max/mdev = 29.892/29.936/30.002/0.149 ms#openvpn-server ping openvpn-clientroot@ip-172-31-35-87:/openvpn-server/openvpn-master# ping 11.11.22.100PING 11.11.22.100 (11.11.22.100) 56(84) bytes of data.64 bytes from 11.11.22.100: icmp_seq=1 ttl=64 time=30.0 ms64 bytes from 11.11.22.100: icmp_seq=2 ttl=64 time=29.9 ms64 bytes from 11.11.22.100: icmp_seq=3 ttl=64 time=29.9 ms^C--- 11.11.22.100 ping statistics ---3 packets transmitted, 3 received, 0% packet loss, time 2000msrtt min/avg/max/mdev = 29.936/29.967/30.028/0.204 ms#成功互通
正常情况下开启了openvpn-server和openvpn-client就可以实现openvpn-server和openvpn-client私网间的互通。但是如果openvpn-server想要找到openvpn-client通以局域网下的其他主机,还需要在openvpn-client子网服务器添加一条路由规则。因为即使发送的数据包可以到达openvpn-client通一局域网内的其他服务器,这些服务器并不知道怎么发回来。
#tcpdump抓一下包root@gpuserver004:/home/zhangsiming# tcpdump -v icmptcpdump: listening on enp14s0, link-type EN10MB (Ethernet), capture size 262144 bytes10:59:45.181306 IP (tos 0x0, ttl 62, id 42159, offset 0, flags [DF], proto ICMP (1), length 84)172.31.40.247 > 11.11.22.4: ICMP echo request, id 7117, seq 48, length 6410:59:45.181343 IP (tos 0x0, ttl 64, id 13077, offset 0, flags [none], proto ICMP (1), length 84)11.11.22.4 > 172.31.40.247: ICMP echo reply, id 7117, seq 48, length 64#可见,需要把回程的目标地址172.31.40.247发送给openvpn-client处理才行root@gpuserver004:/home/zhangsiming# route add -net 172.31.0.0/16 gw 11.11.22.100#添加上面这条路由即可。
同理,在aws上面,也需要设置回程路由。





#IDC内网服务器pingaws内网服务器zhangsiming@gpuserver004:~$ ping 172.31.40.247PING 172.31.40.247 (172.31.40.247) 56(84) bytes of data.64 bytes from 172.31.40.247: icmp_seq=1 ttl=62 time=59.9 ms64 bytes from 172.31.40.247: icmp_seq=2 ttl=62 time=30.2 ms64 bytes from 172.31.40.247: icmp_seq=3 ttl=62 time=30.2 ms^C--- 172.31.40.247 ping statistics ---3 packets transmitted, 3 received, 0% packet loss, time 2002msrtt min/avg/max/mdev = 30.254/40.160/59.955/13.997 ms#aws内网服务器pingIDC内网服务器root@172:/# ping 11.11.22.4PING 11.11.22.4 (11.11.22.4) 56(84) bytes of data.64 bytes from 11.11.22.4: icmp_seq=1 ttl=62 time=30.2 ms64 bytes from 11.11.22.4: icmp_seq=2 ttl=62 time=30.2 ms64 bytes from 11.11.22.4: icmp_seq=3 ttl=62 time=30.2 ms^C--- 11.11.22.4 ping statistics ---3 packets transmitted, 3 received, 0% packet loss, time 2002msrtt min/avg/max/mdev = 30.201/30.241/30.279/0.031 ms
成功实现互通,实验完成。