@zhangsiming65965
2019-04-22T06:43:16.000000Z
字数 14296
阅读 223
VPN
已知,公司的openvpnclient可以成功连接aws的openvpnserver的内网服务器,这是一组openvpn client-server;
需求一: 在自建机房架构openvpnclient,使得IDC机房内网gpu服务器可以双向连通aws内网服务器;
需求二: 通过aws的openvpnserver的client to client配置使得IDC机房client可以和公司client实现数据互通。
root@ip-172-31-35-87:/openvpn-server/openvpn-master# ll
总用量 36
-rw-r--r-- 1 root root 1205 4月 16 19:21 docker-compose.yml
#docker-compose.yaml启动文件
-rw-r--r-- 1 root root 444 4月 16 17:32 Dockerfile
#构建openvpn镜像文件
-rwxr-xr-x 1 root root 172 4月 16 19:36 entrypoint.sh*
#openvpn-docker启动入口
-rwxr-xr-x 1 root root 1500 4月 16 17:34 openvpn_init.sh*
#初始化脚本
drwxrwxr-x 2 root root 4096 4月 16 19:36 openvpn_need/
#生成文件目录
root@ip-172-31-35-87:/openvpn-server/openvpn-master# ll openvpn_need/
总用量 24
drwxrwxr-x 2 root root 4096 4月 16 19:36 ./
drwxrwxr-x 4 root root 4096 4月 19 20:10 ../
-rw-r--r-- 1 root root 855 4月 16 17:26 base.conf
#客户端配置文件基准文件
-rwxr-xr-x 1 root root 1714 4月 16 17:26 make_cert.sh*
#制作证书文件
-rw-r--r-- 1 root root 1533 4月 16 17:31 server.conf
#服务端配置文件
-rw-r--r-- 1 root root 2130 4月 4 17:07 vars
#生成证书文件
root@ip-172-31-35-87:/openvpn-server/openvpn-master# cat openvpn_need/base.conf
client
dev tun
proto tcp
#客户端和服务端的传输协议需要一致
remote 119.18.195.149 8888
#openvpn-server的远程地址和端口
resolv-retry infinite
nobind
persist-key
persist-tun
ca /etc/openvpn/ca.crt
cert /etc/openvpn/client.crt
key /etc/openvpn/client.key
#连接用的证书和key
ns-cert-type server
comp-lzo
verb 3
root@ip-172-31-35-87:/openvpn-server/openvpn-master# cat openvpn_need/make_cert.sh
#!/bin/bash
echo -e "-------make_cert.sh start--------\n\n"
source vars
./clean-all
#先生效vars文件,再生成密钥
#生成ca证书
echo -e "\n\n----------build-ca-----------"
/usr/bin/expect << EOF
spawn ./build-ca
expect "Country Name" {send "\r"}
expect "State or Province Name" {send "\r"}
expect "Locality Name" {send "\r"}
expect "Organization Name" {send "\r"}
expect "Organizational Unit Name" {send "\r"}
expect "Common Name" {send "\r"}
expect "Name" {send "\r"}
expect "Email Address" {send "\r"}
expect eof
EOF
#基于ca生成server证书
echo -e "\n\n-------build-key-server-------"
/usr/bin/expect << EOF
spawn ./build-key-server server
expect "Country Name" {send "\r"}
expect "State or Province Name" {send "\r"}
expect "Locality Name" {send "\r"}
expect "Organization Name" {send "\r"}
expect "Organizational Unit Name" {send "\r"}
expect "Common Name" {send "\r"}
expect "Name" {send "\r"}
expect "Email Address" {send "\r"}
expect "A challenge password" {send "\r"}
expect "An optional company name" {send "\r"}
expect "Sign the certificate?" {send "y\r"}
expect "commit?" {send "y\r"}
expect eof
EOF
#基于ca生成client证书
echo -e "\n\n-------build-key-client-------"
/usr/bin/expect << EOF
spawn ./build-key client
#后面传入的client(可以任意指定),为证书的COMMON NAME
expect "Country Name" {send "\r"}
expect "State or Province Name" {send "\r"}
expect "Locality Name" {send "\r"}
expect "Organization Name" {send "\r"}
expect "Organizational Unit Name" {send "\r"}
expect "Common Name" {send "\r"}
expect "Name" {send "\r"}
expect "Email Address" {send "\r"}
expect "A challenge password" {send "\r"}
expect "An optional company name" {send "\r"}
expect "Sign the certificate?" {send "y\r"}
expect "commit?" {send "y\r"}
expect eof
EOF
echo -e "\n\n--------build-dh-----------"
#生成证书交换文件
./build-dh
echo -e "\n\n-------make_cert.sh done--------"
root@ip-172-31-35-87:/openvpn-server/openvpn-master# cat openvpn_need/server.conf
port 52115
#openvpn-server监听端口
proto tcp
dev tun
server {{ OPENVPN_SERVER }}
#给openvpn服务tun0网卡分配的地址,需要唯一,不要和双方任意的私网或者公网一样,在docker-compose用ENV导入
push "route 1.10.0.0 255.255.0.0"
#给客户端添加到server本地私网的路由,在docker-compose的ENV声明,在openvpn_init.sh中对应更改
route 11.11.0.0 255.255.0.0
#给本地添加一条到openvpn-client的回程路由
client-config-dir /etc/openvpn/ccd
#ccd目录必须写绝对路径,且要存在与docker中;openvpn-server动态读取之中的配置,直接修改ccd中的内容之后无需重启openvpn-server服务,客户端直接连接即可生效;ccd中的文件名需要和客户端证书的COMMON NAME一致
ifconfig-pool-persist /etc/openvpn/logs/ipp.txt
status /etc/openvpn/logs/status.log
log /etc/openvpn/logs/openvpn.log
ca /etc/openvpn/server/ca.crt
cert /etc/openvpn/server/server.crt
key /etc/openvpn/server/server.key
dh /etc/openvpn/server/dh2048.pem
client-to-client
#client-to-client允许连接到同一个openvpn-server的不同客户端之间通信
keepalive 10 120
duplicate-cn
#duplicate-cn允许不同用户使用同一证书连接openvpn-server
comp-lzo
persist-key
persist-tun
verb 3
root@ip-172-31-35-87:/openvpn-server/openvpn-master# tail -17 openvpn_need/vars | head -13
export KEY_COUNTRY="$KEY_COUNTRY"
export KEY_PROVINCE="$KEY_PROVINCE"
export KEY_CITY="$KEY_CITY"
export KEY_ORG="$KEY_ORG"
export KEY_EMAIL="$KEY_EMAIL"
export KEY_OU="$KEY_OU"
export KEY_CN="$KEY_CN"
# X509 Subject Field
export KEY_NAME="$KEY_NAME"
export KEY_ALTNAMES="something"
# PKCS11 Smart Card
# export PKCS11_MODULE_PATH="/usr/lib/changeme.so"
# export PKCS11_PIN=1234
#里面声明了证书的信息,城市邮箱等...
root@ip-172-31-35-87:/openvpn-server/openvpn-master# cat openvpn_init.sh
#!/bin/bash
OPENVPN_DIR=/etc/openvpn
OPENVPN_NEED=/usr/local/bin/
if [ ! -f ${OPENVPN_DIR}/server/server.crt ]
then
echo 'openvpn initializing ...'
# make server ca and server crt
cd ${OPENVPN_DIR}
mkdir -p server client logs
cd /root/
mkdir -p openvpn-ca
cp /usr/share/easy-rsa/\* openvpn-ca
cp ${OPENVPN_NEED}/make_cert.sh ${OPENVPN_NEED}/vars openvpn-ca
cp openvpn-ca/openssl-1.0.0.cnf openvpn-ca/openssl.cnf
cd openvpn-ca
./make_cert.sh
cd ./keys
#创建证书并把server和client的证书放到对应位置
cp ca.crt server.crt server.key dh* ${OPENVPN_DIR}/server/
cp ca.crt client.crt client.key ${OPENVPN_DIR}/client/
if [ ! -f ${OPENVPN_DIR}/server/server.conf ]
then
cp ${OPENVPN_NEED}/server.conf ${OPENVPN_DIR}/server/server.conf
fi
# fill environment variable in
#修改.dh文件
sed -i "s|^dh.*|dh ${OPENVPN_DIR}/server/dh${KEY_SIZE}.pem|" ${OPENVPN_DIR}/server/server.conf
#修改server.conf的tun网卡网段,即openvpn提供隧道服务的网段
sed -i "s|{{ OPENVPN_SERVER }}|${OPENVPN_SERVER}|g" ${OPENVPN_DIR}/server/server.conf
#修改给客户端添加的路由(指向openvpn-server的私网网卡)
sed -i "s|^push \"route.*|push \"route ${PUSH}\"|" ${OPENVPN_DIR}/server/server.conf
# make client conf
cp ${OPENVPN_NEED}/base.conf ${OPENVPN_DIR}/client/client.conf
#修改client.conf指向远程openvpn-server的IP+端口
sed -i "s|^remote.*|remote ${CLIENTREMOTE} 52115|" ${OPENVPN_DIR}/client/client.conf
rm -rf /usr/local/bin/*
rm -rf /root/openvpn-ca
else
echo 'openvpn has been initialized, skip.'
fi
#添加tun网卡,隧道网卡提供vpn服务
# add tun
if [ ! -c /dev/net/tun ]
then
echo "creating /dev/net/tun"
mkdir /dev/net
mknod /dev/net/tun c 10 200
fi
root@ip-172-31-35-87:/openvpn-server/openvpn-master# cat entrypoint.sh
#!/bin/bash
case "$1" in
openvpn)
#先执行初始化进行配置
/root/openvpn_init.sh
#之后指向server.conf启动openpvn-server服务
exec openvpn --dev tun --config /etc/openvpn/server/server.conf ${@:2}
;;
*)
exec "$@"
;;
esac
root@ip-172-31-35-87:/openvpn-server/openvpn-master# cat Dockerfile
FROM registry.cn-beijing.aliyuncs.com/shannonai/openvpn:v1.1.1
#安装证书生成工具,openvpn和expect等无交互辅助工具
# runtime dependencies
RUN set -ex \
&& apt-get -yqq update && apt-get install -yqq --no-install-recommends \
openvpn \
expect \
easy-rsa \
&& rm -rf /var/lib/apt/lists/*
VOLUME /etc/openvpn
EXPOSE 52115
COPY openvpn_init.sh /root/
COPY entrypoint.sh /root/
COPY openvpn_need/ /usr/local/bin/
#把文件放到指定路径,方便openvpn_init.sh的时候用
ENTRYPOINT ["/root/entrypoint.sh"]
CMD ["openvpn"]
root@ip-172-31-35-87:/openvpn-server/openvpn-master# cat docker-compose.yml
version: '3.2'
services:
openvpn:
image: openvpn
container_name: openvpn-server
network_mode: host
#使用宿主机网络
restart: always
volumes:
- ./data:/etc/openvpn
#挂载当前目录的data目录到docker中的/etc/openvpn
ports:
- "52115"
#暴露端口
cap_add:
- NET_ADMIN
#cap_add不加会报错
environment:
OPENVPN_SERVER: 10.66.0.0 255.255.0.0
#tun网卡网段,openvpn网卡服务网段
PUSH: 172.31.0.0 255.255.0.0
#给客户端添加到openvpn-server内网的路由
CLIENTREMOTE: 52.82.24.178
#给client.conf添加的远程openvpn-server公网地址
KEY_COUNTRY: CN # The state abbreviations(for example: CN)
KEY_PROVINCE: BJ # Province shorthand(for example: BJ)
KEY_CITY: Beijing # City(for example: BeiJing)
KEY_ORG: shannonai # organisation(for example: shannon.ai)
KEY_OU: shannonai # affiliated unit(for example: shannon.ai)
KEY_CN: shannonai
KEY_EMAIL: siming_zhang@shannonai.com # Email(for example: contact@shannonai.com)
KEY_SIZE: 2048 # The private key size(for example: 2048)
root@ip-172-31-35-87:/openvpn-server/openvpn-master# ls
docker-compose.yml Dockerfile entrypoint.sh openvpn_init.sh openvpn_need
root@ip-172-31-35-87:/openvpn-server/openvpn-master# docker build -t openvpn .
root@ip-172-31-35-87:/openvpn-server/openvpn-master# docker-compose up -d
root@ip-172-31-35-87:/openvpn-server/openvpn-master# docker-compose ps
Name Command State Ports
------------------------------------------------------------
openvpn-server /root/entrypoint.sh openvpn Up
#成功启动
root@ip-172-31-35-87:/openvpn-server/openvpn-master# ls data/
ccd client logs server
root@ip-172-31-35-87:/openvpn-server/openvpn-master# cat data/ccd/client
iroute 11.11.0.0 255.255.0.0
#ccd目录手动创建,添加iroute,地址写openpvn-client的内网网段,我理解的是回城tun point-to-point的指向客户端路由
root@ip-172-31-35-87:/openvpn-server/openvpn-master# ls data/client/
ca.crt client.conf client.crt client.key make.sh
#将客户端的证书、密钥和配置文件都拷贝得到openvpn-client服务器的指定目录连接备用
root@ip-172-31-35-87:/openvpn-server/openvpn-master# ls data/server/
ca.crt client.conf dh2048.pem server.conf server.crt server.key
#查看监听端口
root@ip-172-31-35-87:/openvpn-server/openvpn-master# netstat -antup | grep 52115
tcp 0 0 0.0.0.0:52115 0.0.0.0:* LISTEN 1289/openvpn
root@jumpserver:~/openvpn-client# ll
total 32
drwxr-xr-x 2 root root 4096 Apr 18 00:03 client/
#client目录中放我们刚刚server生成的ca证书、client证书密钥和client.conf
-rw-r--r-- 1 root root 282 Apr 15 08:07 docker-compose.yml
-rw-r--r-- 1 root root 417 Apr 8 09:39 Dockerfile
-rwxr-xr-x 1 root root 169 Apr 15 08:06 entrypoint.sh*
-rwxr-xr-x 1 root root 139 Apr 8 08:14 openvpn_init.sh*
root@jumpserver:~/openvpn-client# cat client/client.conf
client
dev tun
proto tcp
remote 52.82.24.178 52115
#连接openvpn-server的地址和端口
resolv-retry infinite
nobind
persist-key
persist-tun
ca /etc/openvpn/ca.crt
cert /etc/openvpn/client.crt
key /etc/openvpn/client.key
ns-cert-type server
comp-lzo
verb 3
root@jumpserver:~/openvpn-client# cat entrypoint.sh
#!/bin/bash
case "$1" in
openvpn)
sh /root/openvpn_init.sh
exec openvpn --dev tun --config /etc/openvpn/client.conf ${@:2}
;;
*)
exec "$@"
;;
esac
#先初始化之后指向client.conf启动openvpn-client
root@jumpserver:~/openvpn-client# cat Dockerfile
FROM registry.cn-beijing.aliyuncs.com/shannonai/openvpn:v1.1.1
USER root
# runtime dependencies
RUN set -ex \
&& apt-get -yqq update && apt-get install -yqq --no-install-recommends \
openvpn \
expect \
easy-rsa \
&& rm -rf /var/lib/apt/lists/*
VOLUME /etc/openvpn
EXPOSE 52115
COPY openvpn_init.sh /root/
COPY entrypoint.sh /root/
#与openvpn-server的Dockerfile几乎一样,注意细节
ENTRYPOINT ["/root/entrypoint.sh"]
CMD ["openvpn"]
root@jumpserver:~/openvpn-client# cat docker-compose.yml
version: '3.2'
services:
openvpn:
image: openvpn
container_name: openvpn-client
network_mode: host
restart: always
volumes:
- ./client:/etc/openvpn
#挂载当前目录的client目录到/etc/openvpn中
ports:
- "52115"
cap_add:
- NET_ADMIN
root@jumpserver:~/openvpn-client# ls
client Dockerfile openvpn_init.sh
docker-compose.yml entrypoint.sh README.md
root@jumpserver:~/openvpn-client# docker build -t openvpn .
root@jumpserver:~/openvpn-client# docker-compose up -d
root@jumpserver:~/openvpn-client# docker-compose ps
Name Command State Ports
------------------------------------------------------------
openvpn-client /root/entrypoint.sh openvpn Up
root@jumpserver:~/openvpn-client# netstat -antup | grep 52115
tcp 0 0 11.11.22.100:53996 52.82.24.178:52115 ESTABLISHED 30650/openvpn
#成功启动
#openvpn-client
root@jumpserver:~/openvpn-client# route -n | grep tun0
10.66.0.0 10.66.0.5 255.255.0.0 UG 0 0 0 tun0
10.66.0.5 0.0.0.0 255.255.255.255 UH 0 0 0 tun1
172.31.0.0 10.66.0.5 255.255.0.0 UG 0 0 0 tun0
root@jumpserver:~/openvpn-client# ifconfig | grep -A 1 tun0
tun0 Link encap:UNSPEC HWaddr 00-00-00-00-00-00-00-00-00-00-00-00-00-00-00-00
inet addr:10.66.0.6 P-t-P:10.66.0.5 Mask:255.255.255.255
#openvpn-server
root@ip-172-31-35-87:/openvpn-server/openvpn-master# route -n | grep tun
10.66.0.0 10.66.0.2 255.255.0.0 UG 0 0 0 tun0
10.66.0.2 0.0.0.0 255.255.255.255 UH 0 0 0 tun0
11.11.0.0 10.66.0.2 255.255.0.0 UG 0 0 0 tun0
root@ip-172-31-35-87:/openvpn-server/openvpn-master# ifconfig | grep -A 1 tun
tun0 Link encap:未指定 硬件地址 00-00-00-00-00-00-00-00-00-00-00-00-00-00-00-00
inet 地址:10.66.0.1 点对点:10.66.0.2 掩码:255.255.255.255
#openvpn-client ping openvpn-server
root@jumpserver:~/openvpn-client# ping 172.31.35.87
PING 172.31.35.87 (172.31.35.87) 56(84) bytes of data.
64 bytes from 172.31.35.87: icmp_seq=1 ttl=64 time=30.0 ms
64 bytes from 172.31.35.87: icmp_seq=2 ttl=64 time=29.9 ms
64 bytes from 172.31.35.87: icmp_seq=3 ttl=64 time=29.8 ms
^C
--- 172.31.35.87 ping statistics ---
3 packets transmitted, 3 received, 0% packet loss, time 2000ms
rtt min/avg/max/mdev = 29.892/29.936/30.002/0.149 ms
#openvpn-server ping openvpn-client
root@ip-172-31-35-87:/openvpn-server/openvpn-master# ping 11.11.22.100
PING 11.11.22.100 (11.11.22.100) 56(84) bytes of data.
64 bytes from 11.11.22.100: icmp_seq=1 ttl=64 time=30.0 ms
64 bytes from 11.11.22.100: icmp_seq=2 ttl=64 time=29.9 ms
64 bytes from 11.11.22.100: icmp_seq=3 ttl=64 time=29.9 ms
^C
--- 11.11.22.100 ping statistics ---
3 packets transmitted, 3 received, 0% packet loss, time 2000ms
rtt min/avg/max/mdev = 29.936/29.967/30.028/0.204 ms
#成功互通
正常情况下开启了openvpn-server和openvpn-client就可以实现openvpn-server和openvpn-client私网间的互通。但是如果openvpn-server想要找到openvpn-client通以局域网下的其他主机,还需要在openvpn-client子网服务器添加一条路由规则。因为即使发送的数据包可以到达openvpn-client通一局域网内的其他服务器,这些服务器并不知道怎么发回来。
#tcpdump抓一下包
root@gpuserver004:/home/zhangsiming# tcpdump -v icmp
tcpdump: listening on enp14s0, link-type EN10MB (Ethernet), capture size 262144 bytes
10:59:45.181306 IP (tos 0x0, ttl 62, id 42159, offset 0, flags [DF], proto ICMP (1), length 84)
172.31.40.247 > 11.11.22.4: ICMP echo request, id 7117, seq 48, length 64
10:59:45.181343 IP (tos 0x0, ttl 64, id 13077, offset 0, flags [none], proto ICMP (1), length 84)
11.11.22.4 > 172.31.40.247: ICMP echo reply, id 7117, seq 48, length 64
#可见,需要把回程的目标地址172.31.40.247发送给openvpn-client处理才行
root@gpuserver004:/home/zhangsiming# route add -net 172.31.0.0/16 gw 11.11.22.100
#添加上面这条路由即可。
同理,在aws上面,也需要设置回程路由。
#IDC内网服务器pingaws内网服务器
zhangsiming@gpuserver004:~$ ping 172.31.40.247
PING 172.31.40.247 (172.31.40.247) 56(84) bytes of data.
64 bytes from 172.31.40.247: icmp_seq=1 ttl=62 time=59.9 ms
64 bytes from 172.31.40.247: icmp_seq=2 ttl=62 time=30.2 ms
64 bytes from 172.31.40.247: icmp_seq=3 ttl=62 time=30.2 ms
^C
--- 172.31.40.247 ping statistics ---
3 packets transmitted, 3 received, 0% packet loss, time 2002ms
rtt min/avg/max/mdev = 30.254/40.160/59.955/13.997 ms
#aws内网服务器pingIDC内网服务器
root@172:/# ping 11.11.22.4
PING 11.11.22.4 (11.11.22.4) 56(84) bytes of data.
64 bytes from 11.11.22.4: icmp_seq=1 ttl=62 time=30.2 ms
64 bytes from 11.11.22.4: icmp_seq=2 ttl=62 time=30.2 ms
64 bytes from 11.11.22.4: icmp_seq=3 ttl=62 time=30.2 ms
^C
--- 11.11.22.4 ping statistics ---
3 packets transmitted, 3 received, 0% packet loss, time 2002ms
rtt min/avg/max/mdev = 30.201/30.241/30.279/0.031 ms
成功实现互通,实验完成。