[关闭]
@zhangsiming65965 2019-10-29T01:48:18.000000Z 字数 17364 阅读 184

kubernetes后续配置

Kubernetes系列

---Author：张思明 ZhangSiming

---Mail：siming_zhang@shannonai.com

---QQ：1030728296

1.进行rbac生成admin和nsadmin的config

这里创建一个serviceaccount，直接绑定clusterrole：cluster-admin的权限，之后读取生成的token即可。

  1. #!/bin/bash
  2. #Usage: bash k8stest-admin-generateconfig.sh 
  3. IP="nlb-k8s-test-apiserver-1f07f2e6c2c2b42c.elb.cn-northwest-1.amazonaws.com.cn"
  4. KUBE_APISERVER="https://$IP:6443"
  6. kubectl create serviceaccount shannon-admin
  7. kubectl create clusterrolebinding shannon-admin --clusterrole=cluster-admin --serviceaccount=default:shannon-admin
  9. TOKEN=`kubectl get secret $(kubectl get secrets | grep shannon-admin | awk '{print $1}') -o jsonpath={.data.token}|base64 -d`
  11. echo "${TOKEN}"
  1. apiVersion: rbac.authorization.k8s.io/v1
  2. kind: ClusterRole
  3. metadata:
  4.   name: k8stest-admin
  5. rules:
  6.   - apiGroups: 
  7.     - ""
  8.     resources: 
  9.     - namespaces
  10.     verbs: 
  11.     - get
  12.     - list
  13.     - watch
  14.   - apiGroups: 
  15.     - "kubeapps.com"
  16.     resources: 
  17.     - apprepositories
  18.     verbs: 
  19.     - get
  20.     - list
  21.     - watch
  22.   - apiGroups: 
  23.     - "apiextensions.k8s.io"
  24.     resources: 
  25.     - customresourcedefinitions
  26.     verbs: 
  27.     - create
  28.     - update
  29.     - get
  30.     - list
  31.     - watch
  32.     - delete
  33.   - apiGroups: 
  34.     - "rbac.authorization.k8s.io"
  35.     resources: 
  36.     - clusterroles
  37.     verbs: 
  38.     - create
  39.     - update
  40.     - get
  41.     - list
  42.     - watch
  43.   - apiGroups: 
  44.     - "rbac.authorization.k8s.io"
  45.     resources: 
  46.     - clusterrolebindings
  47.     verbs: 
  48.     - create
  49.     - update
  50.     - get
  51.     - list
  52.     - watch
  1. #!/bin/bash
  2. #Usage: /bin/bash k8stest-namespaceadmin-generateconfig.sh test prod dev
  4. IP="nlb-k8s-test-apiserver-1f07f2e6c2c2b42c.elb.cn-northwest-1.amazonaws.com.cn"
  5. KUBE_APISERVER="https://$IP:6443"
  6. USER=shannon
  7. CLUSTER=shannontest
  9. for i in $@
  10. do
  11.     kubectl create namespace $i
  12. done
  14. kubectl create serviceaccount ${USER} --namespace kube-system
  16. for i in $@
  17. do
  18.     kubectl create rolebinding k8stest-ns-$i --clusterrole=cluster-admin --serviceaccount=kube-system:${USER} --namespace $i
  19. done
  21. kubectl create clusterrolebinding k8stest-admin --clusterrole=k8stest-admin --serviceaccount=kube-system:${USER}
  23. kubectl config set-cluster ${CLUSTER} --insecure-skip-tls-verify=true --server=${KUBE_APISERVER} --kubeconfig=k8stest-admin.kubeconfig
  25. for i in $@
  26. do
  27. 	kubectl config set-context $i-admin@${CLUSTER} --cluster=${CLUSTER} --user=${USER} --namespace=$i --kubeconfig=k8stest-admin.kubeconfig
  28. done
  30. TOKEN=`kubectl get secret -n kube-system $(kubectl get secrets -n kube-system | grep ${USER} | awk '{print $1}') -o jsonpath={.data.token}|base64 -d`
  31. kubectl config set-credentials ${USER} --token=$TOKEN --kubeconfig=k8stest-admin.kubeconfig
  32. kubectl config use-context $1-admin@${CLUSTER} --kubeconfig=k8stest-admin.kubeconfig

2.使用kubernetes-dashboard-v2.0

  1. ---
  3. apiVersion: v1
  4. kind: ServiceAccount
  5. metadata:
  6.   labels:
  7.     k8s-app: kubernetes-dashboard
  8.   name: kubernetes-dashboard
  9.   namespace: kube-system
  11. ---
  13. kind: Service
  14. apiVersion: v1
  15. metadata:
  16.   labels:
  17.     k8s-app: kubernetes-dashboard
  18.   name: kubernetes-dashboard
  19.   namespace: kube-system
  20. spec:
  21.   ports:
  22.     - port: 443
  23.       targetPort: 8443
  24.   selector:
  25.     k8s-app: kubernetes-dashboard
  27. ---
  29. apiVersion: v1
  30. kind: Secret
  31. metadata:
  32.   labels:
  33.     k8s-app: kubernetes-dashboard
  34.   name: kubernetes-dashboard-certs
  35.   namespace: kube-system
  36. type: Opaque
  38. ---
  40. apiVersion: v1
  41. kind: Secret
  42. metadata:
  43.   labels:
  44.     k8s-app: kubernetes-dashboard
  45.   name: kubernetes-dashboard-csrf
  46.   namespace: kube-system
  47. type: Opaque
  48. data:
  49.   csrf: ""
  51. ---
  53. apiVersion: v1
  54. kind: Secret
  55. metadata:
  56.   labels:
  57.     k8s-app: kubernetes-dashboard
  58.   name: kubernetes-dashboard-key-holder
  59.   namespace: kube-system
  60. type: Opaque
  62. ---
  64. kind: ConfigMap
  65. apiVersion: v1
  66. metadata:
  67.   labels:
  68.     k8s-app: kubernetes-dashboard
  69.   name: kubernetes-dashboard-settings
  70.   namespace: kube-system
  72. ---
  74. kind: Role
  75. apiVersion: rbac.authorization.k8s.io/v1
  76. metadata:
  77.   labels:
  78.     k8s-app: kubernetes-dashboard
  79.   name: kubernetes-dashboard
  80.   namespace: kube-system
  81. rules:
  82.   # Allow Dashboard to get, update and delete Dashboard exclusive secrets.
  83.   - apiGroups: [""]
  84.     resources: ["secrets"]
  85.     resourceNames: ["kubernetes-dashboard-key-holder", "kubernetes-dashboard-certs", "kubernetes-dashboard-csrf"]
  86.     verbs: ["get", "update", "delete"]
  87.     # Allow Dashboard to get and update 'kubernetes-dashboard-settings' config map.
  88.   - apiGroups: [""]
  89.     resources: ["configmaps"]
  90.     resourceNames: ["kubernetes-dashboard-settings"]
  91.     verbs: ["get", "update"]
  92.     # Allow Dashboard to get metrics.
  93.   - apiGroups: [""]
  94.     resources: ["services"]
  95.     resourceNames: ["heapster", "dashboard-metrics-scraper"]
  96.     verbs: ["proxy"]
  97.   - apiGroups: [""]
  98.     resources: ["services/proxy"]
  99.     resourceNames: ["heapster", "http:heapster:", "https:heapster:", "dashboard-metrics-scraper", "http:dashboard-metrics-scraper"]
  100.     verbs: ["get"]
  102. ---
  104. kind: ClusterRole
  105. apiVersion: rbac.authorization.k8s.io/v1
  106. metadata:
  107.   labels:
  108.     k8s-app: kubernetes-dashboard
  109.   name: kubernetes-dashboard
  110. rules:
  111.   # Allow Metrics Scraper to get metrics from the Metrics server
  112.   - apiGroups: ["metrics.k8s.io"]
  113.     resources: ["pods", "nodes"]
  114.     verbs: ["get", "list", "watch"]
  116. ---
  118. apiVersion: rbac.authorization.k8s.io/v1
  119. kind: RoleBinding
  120. metadata:
  121.   labels:
  122.     k8s-app: kubernetes-dashboard
  123.   name: kubernetes-dashboard
  124.   namespace: kube-system
  125. roleRef:
  126.   apiGroup: rbac.authorization.k8s.io
  127.   kind: Role
  128.   name: kubernetes-dashboard
  129. subjects:
  130.   - kind: ServiceAccount
  131.     name: kubernetes-dashboard
  132.     namespace: kube-system
  134. ---
  136. apiVersion: rbac.authorization.k8s.io/v1
  137. kind: ClusterRoleBinding
  138. metadata:
  139.   name: kubernetes-dashboard
  140.   namespace: kube-system
  141. roleRef:
  142.   apiGroup: rbac.authorization.k8s.io
  143.   kind: ClusterRole
  144.   name: kubernetes-dashboard
  145. subjects:
  146.   - kind: ServiceAccount
  147.     name: kubernetes-dashboard
  148.     namespace: kube-system
  150. ---
  152. kind: Deployment
  153. apiVersion: apps/v1
  154. metadata:
  155.   labels:
  156.     k8s-app: kubernetes-dashboard
  157.   name: kubernetes-dashboard
  158.   namespace: kube-system
  159. spec:
  160.   replicas: 1
  161.   revisionHistoryLimit: 10
  162.   selector:
  163.     matchLabels:
  164.       k8s-app: kubernetes-dashboard
  165.   template:
  166.     metadata:
  167.       labels:
  168.         k8s-app: kubernetes-dashboard
  169.     spec:
  170.       containers:
  171.         - name: kubernetes-dashboard
  172.           image: kubernetesui/dashboard:v2.0.0-beta1
  173.           imagePullPolicy: Always
  174.           ports:
  175.             - containerPort: 8443
  176.               protocol: TCP
  177.           args:
  178.             - --auto-generate-certificates
  179.             - --namespace=kube-system
  180.             # Uncomment the following line to manually specify Kubernetes API server Host
  181.             # If not specified, Dashboard will attempt to auto discover the API server and connect
  182.             # to it. Uncomment only if the default does not work.
  183.             # - --apiserver-host=http://my-address:port
  184.           volumeMounts:
  185.             - name: kubernetes-dashboard-certs
  186.               mountPath: /certs
  187.               # Create on-disk volume to store exec logs
  188.             - mountPath: /tmp
  189.               name: tmp-volume
  190.           livenessProbe:
  191.             httpGet:
  192.               scheme: HTTPS
  193.               path: /
  194.               port: 8443
  195.             initialDelaySeconds: 30
  196.             timeoutSeconds: 30
  197.       volumes:
  198.         - name: kubernetes-dashboard-certs
  199.           secret:
  200.             secretName: kubernetes-dashboard-certs
  201.         - name: tmp-volume
  202.           emptyDir: {}
  203.       serviceAccountName: kubernetes-dashboard
  204.       # Comment the following tolerations if Dashboard must not be deployed on master
  205.       tolerations:
  206.         - key: node-role.kubernetes.io/master
  207.           effect: NoSchedule
  209. ---
  211. kind: Service
  212. apiVersion: v1
  213. metadata:
  214.   labels:
  215.     k8s-app: kubernetes-metrics-scraper
  216.   name: dashboard-metrics-scraper
  217.   namespace: kube-system
  218. spec:
  219.   ports:
  220.     - port: 8000
  221.       targetPort: 8000
  222.   selector:
  223.     k8s-app: kubernetes-metrics-scraper
  225. ---
  227. kind: Deployment
  228. apiVersion: apps/v1
  229. metadata:
  230.   labels:
  231.     k8s-app: kubernetes-metrics-scraper
  232.   name: kubernetes-metrics-scraper
  233.   namespace: kube-system
  234. spec:
  235.   replicas: 1
  236.   revisionHistoryLimit: 10
  237.   selector:
  238.     matchLabels:
  239.       k8s-app: kubernetes-metrics-scraper
  240.   template:
  241.     metadata:
  242.       labels:
  243.         k8s-app: kubernetes-metrics-scraper
  244.     spec:
  245.       containers:
  246.         - name: kubernetes-metrics-scraper
  247.           image: kubernetesui/metrics-scraper:v1.0.0
  248.           ports:
  249.             - containerPort: 8000
  250.               protocol: TCP
  251.           livenessProbe:
  252.             httpGet:
  253.               scheme: HTTP
  254.               path: /
  255.               port: 8000
  256.             initialDelaySeconds: 30
  257.             timeoutSeconds: 30
  258.       serviceAccountName: kubernetes-dashboard
  259.       # Comment the following tolerations if Dashboard must not be deployed on master
  260.       tolerations:
  261.         - key: node-role.kubernetes.io/master
  262.           effect: NoSchedule
  266. ---
  268. #允许匿名用户通过token的方式外部访问dashboard
  269. kind: ClusterRole
  270. apiVersion: rbac.authorization.k8s.io/v1
  271. metadata:
  272.   name: kubernetes-dashboard-anonymous
  273. rules:
  274. - apiGroups: [""]
  275.   resources: ["services/proxy"]
  276.   resourceNames: ["https:kubernetes-dashboard:"]
  277.   verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
  278. - nonResourceURLs: ["/ui", "/ui/*", "/api/v1/namespaces/kube-system/services/https:kubernetes-dashboard:/proxy/*"]
  279.   verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
  281. ---
  282. apiVersion: rbac.authorization.k8s.io/v1
  283. kind: ClusterRoleBinding
  284. metadata:
  285.   name: kubernetes-dashboard-anonymous
  286. roleRef:
  287.   apiGroup: rbac.authorization.k8s.io
  288.   kind: ClusterRole
  289.   name: kubernetes-dashboard-anonymous
  290. subjects:
  291. - kind: User
  292.   name: system:anonymous

kubernetes-dashboard的v2.0版本支持"metrics_server"

kubespray部署metrics_server: (部署、rbac一套做好)

  1. $ cat kubespray/inventory/aws-nx-k8s-test/group_vars/k8s-cluster/addons.yml
  3. metrics_server_enabled: true
  4. metrics_server_kubelet_insecure_tls: true
  5. metrics_server_metric_resolution: 60s
  6. metrics_server_kubelet_preferred_address_types: "InternalIP,Hostname,InternalDNS,ExternalDNS,ExternalIP"
  8. #需要两个镜像：
  9. 1.registry.cn-beijing.aliyuncs.com/shannonai-k8s/metrics-server-amd64:v0.3.3
  10. 2.registry.cn-beijing.aliyuncs.com/shannonai-k8s/addon-resizer:1.8.3

image_1dhqupgdloa69mtetpihf1ikv9.png-424.7kB

image_1dhqups0m18sk58qmhd4p4ai7m.png-589kB

2.挂载集群容器内数据到本地(nfs方式)

1.配置nfs

♥ 使用kubectl本地需要安装NFS-server

  1. #!/bin/bash
  2. DIR="挂载目录"
  3. apt-get update
  4. apt install -y nfs-kernel-server 
  5. mkdir -p $DIR
  6. chown -R nobody:nogroup $DIR
  7. chmod -R 777 $DIR
  8. echo "$DIR *(rw,sync,no_root_squash,no_subtree_check)" >> /etc/exports
  9. exportfs -a
  10. systemctl restart nfs-kernel-server
  11. systemctl enable nfs-kernel-server

♥ k8s集群内部所有节点安装NFS-client

  1. apt-get install -y nfs-common

2.创建pv

  1. kind: PersistentVolume
  2. apiVersion: v1
  3. metadata:
  4.   name: PVNAME
  5.   labels:
  6.     app.kubernetes.io/name: LABEL
  7. spec:
  8.   storageClassName: YOURPROJECT
  9.   capacity:
  10.     storage: 3Gi
  11.   accessModes:
  12.     - ReadWriteOnce
  13. #删除pvc之后pv资源自动释放，适合测试使用~
  14.   persistentVolumeReclaimPolicy: Recycle
  15.   nfs:
  16.     server: NFSSERVER-IP
  17.     path: DIR

3.创建pvc

  1. kind: PersistentVolumeClaim
  2. apiVersion: v1
  3. metadata:
  4.   name: PVCNAME
  5.   namespace: PVCNAMESPACE 
  6. spec:
  7.   accessModes:
  8.     - "ReadWriteOnce"
  9.   resources:
  10.     requests:
  11.       storage: "3Gi"
  12.   storageClassName: "YOURPROJECT"
  13.   selector:
  14.     matchLabels:
  15.       app.kubernetes.io/name: LABEL

4.Pod控制器挂载引用pvc

  1. #比如deployment等
  2. ...
  3. containers:
  4.   volumeMounts:
  5.   - name: MOUNTNAME
  6.   mountPath: "容器里的地址"
  7. ...
  9. volumes:
  10. - name: MOUNTNAME
  11.   persistentVolumeClaim: PVCNAME
  12. ...

3.coredns、dns-autoscaler、nodelocaldns详解

kubespray部署之后，会在集群中部署deployment：coredns、dns-autoscaler和daemonset：nodelocaldns，接下来我们针对这三个Pod进行研究与分析。

传统扩容方式"kubectl scale -n kube-system replicaset coredns-fd89744f4 --replicas=2"在这里不再生效，原因是coredns的副本数由dns-autoscaler进行管控，dns-autoscaler定义的coredns副本数由他的配置文件控制。

  1. $ kubectl get configmap dns-autoscaler -n kube-system
  2. NAME             DATA   AGE
  3. dns-autoscaler   1      8d
  4. $ kubectl get configmap dns-autoscaler -n kube-system -o yaml 
  5. apiVersion: v1
  6. data:
  7.   linear: '{"coresPerReplica":256,"min":1,"nodesPerReplica":16,"preventSinglePointFailure":false}'
  8. kind: ConfigMap
  9. metadata:
  10.   creationTimestamp: "2019-08-06T05:11:51Z"
  11.   name: dns-autoscaler
  12.   namespace: kube-system
  13.   resourceVersion: "1101773"
  14.   selfLink: /api/v1/namespaces/kube-system/configmaps/dns-autoscaler
  15.   uid: b34ad82f-b808-11e9-bba7-025e0c22fba8
  16. #修改linear中的min为2即可把最小副本数更新为2

dns-autoscaler控制的coredns副本数的计算公式为replicas = max( ceil( cores * 1/coresPerReplica ) , ceil( nodes * 1/nodesPerReplica ) )，可见linear中的"coresPerReplica"、"nodesPerReplica"和"min"决定了corends的副本数；当集群的节点比较多的时候，core数主要决定，当集群节点比较少的时候，node数主要决定，目前由于节点数比较少，副本数还是走的min值。

我们希望coredns可以实现三个需求：
1.去找我们gitlab、harbor这样的公司内部服务走DNS：172.31.15.168解析；
2.去www.baidu.com或者其他正常访问走默认的DNS；
3.去找Kubernetes内部服务的请求走内部解析直接访问。

看两张官方的介绍：

image_1di9jlgs52mfu85o8vlktg1e9.png-199.9kB

image_1di9jmp561qjiglgjke19qsrcim.png-320.8kB

也就是说，在启用了nodelocaldns的功能之后，每个node会有一个daemonset启动的nodelocaldns，节点上的Pod将不再去找集群中的其他coredns解析DNS，直接找相同node的nodelocaldns去解析，加快了解析速率，避免了一些集群中的网络问题；
所以要定制DNS，需要修改nodelocaldns的configmap才行。

  1. $ kubectl get configmap nodelocaldns -n kube-system -o yaml apiVersion: v1
  2. data:
  3.   Corefile: |
  4.     cluster.local:53 {
  5.         errors
  6.         cache {
  7.             success 9984 30
  8.             denial 9984 5
  9.         }
  10.         reload
  11.         loop
  12.         bind 169.254.25.10
  13.         forward . 172.60.0.3 {
  14.             force_tcp
  15.         }
  16.         prometheus :9253
  17. #默认这个health后面的8080将会作为监听端口占用在每个node，如果想过要更改监听端口，可以在这里更改，之后需要在daemonset中更改健康探测端口
  18.         health 169.254.25.10:8080
  19.     }
  20.     in-addr.arpa:53 {
  21.         errors
  22.         cache 30
  23.         reload
  24.         loop
  25.         bind 169.254.25.10
  26.         forward . 172.60.0.3 {
  27.             force_tcp
  28.         }
  29.         prometheus :9253
  30.     }
  31.     ip6.arpa:53 {
  32.         errors
  33.         cache 30
  34.         reload
  35.         loop
  36.         bind 169.254.25.10
  37.         forward . 172.60.0.3 {
  38.             force_tcp
  39.         }
  40.         prometheus :9253
  41.     }
  42.     .:53 {
  43.         errors
  44.         cache 30
  45.         reload
  46.         loop
  47.         bind 169.254.25.10
  48.         forward . /etc/resolv.conf
  49.         prometheus :9253
  50.     }
  51.     shannonai.com:53 {
  52.           errors
  53.           cache 30
  54.           reload
  55.           loop
  56.           bind 169.254.25.10
  57.           forward . 172.31.15.168
  58.           prometheus :9253
  59.       }
  60. #可以看到加入了shannonai.com的服务，意思是所有以"shannonai.com"为后缀的dns解析都去找forward定义的dns解析，本例为172.31.15.168地址

nodelocaldns相关参考地址：

Kubespray对于nodelocaldns的解释
官方对于nodelocaldns的解释
Corefile的插件使用手册

测试dns解析是否成功

一个test自带nslookup的测试yaml：

  1. apiVersion: apps/v1beta1 
  2. kind: Deployment
  3. metadata:
  4.   name: test
  5. spec:
  6.   replicas: 1
  7.   template:
  8.     metadata:
  9.       labels:
  10.         app: test
  11.     spec:
  12.       containers:
  13.       - name: test
  14.         image: harbor.shannonai.com/test/busybox
  15.         command:
  16.         - sleep
  17.         - "36000"
  18.         imagePullPolicy: IfNotPresent
  19.       restartPolicy: Always

翻墙下载镜像方法：curl -s https://zhangguanzhang.github.io/bash/pull.sh | bash -s -- 镜像地址:版本

开始测试：

  1. #切入一个集群中的Pod
  2. $ kubectl get pods
  3. NAME                                READY   STATUS    RESTARTS   AGE
  4. nginx-deployment-85f4877f96-pbnhw   1/1     Running   0          19h
  5. $ kubectl exec -it nginx-deployment-85f4877f96-pbnhw bash
  7. #测试集群内部DNS解析
  8. Pod$ nslookup kubernetes.default
  9. Server:     169.254.25.10
  10. Address:    169.254.25.10#53
  12. Name:   kubernetes.default.svc.cluster.local
  13. Address: 172.60.0.1
  15. #测试集群对外正常的DNS解析
  16. Pod$ nslookup www.baidu.com
  17. Server:     169.254.25.10
  18. Address:    169.254.25.10#53
  20. Non-authoritative answer:
  21. www.baidu.com   canonical name = www.a.shifen.com.
  22. Name:   www.a.shifen.com
  23. Address: 220.181.38.150
  24. Name:   www.a.shifen.com
  25. Address: 220.181.38.149
  27. #测试找北京内网服务的DNS解析
  28. nslookup git.shannonai.com
  29. Server:     169.254.25.10
  30. Address:    169.254.25.10#53
  32. Name:   git.shannonai.com
  33. Address: 172.31.51.21

附：Pod中的/etc/resolv.conf详解

  1. #kubespray代码中定义了nodelocaldns的IP地址
  2. kubespray/inventory/aws-nx-k8s-prod/group_vars/k8s-cluster/k8s-cluster.yml
  3. 136:nodelocaldns_ip: 169.254.25.10
  5. Pod$ cat /etc/resolv.conf 
  6. nameserver 169.254.25.10
  7. search default.svc.cluster.local svc.cluster.local cluster.local cn-northwest-1.compute.internal
  8. options ndots:5
  9. #上面的nameserver找的是nodelocaldns的地址
  10. #之后在访问的地址点数小于5点的时候(ndots)，回去search中一个一个进行试匹配解析，这种一般优先提供给集群内部服务做解析；之后有自定义比如我们的nodelocal配置，再继续进行匹配解析。

创建Pod的时候，默认的dnsPolicy为"ClusterFirst"，优先采用coredns的配置进行dns解析；如果想要修改Pod启动的时候"/etc/resolv.conf"的数值，可以在Pod字段选项的"dnsConfig"中配置，详情请见：Pod-dnsconfig

4.tiller(helm2)授权

安装helm2：

  1. #!/bin/bash
  2. mkdir -pv helm && cd helm
  3. wget https://storage.googleapis.com/kubernetes-helm/helm-v2.11.0-linux-amd64.tar.gz
  4. tar xf helm-v2.11.0-linux-amd64.tar.gz
  5. sudo mv linux-amd64/helm /usr/local/bin
  6. rm -rf linux-amd64

初始化helm2：

  1. helm init --upgrade -i registry.cn-beijing.aliyuncs.com/shannonai-k8s/tiller:v2.13.1 --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts --tiller-namespace kube-system

查看tiller的serviceaccount：

  1. $ kubectl get serviceaccount -n kube-system | grep tiller
  2. tiller                               1         3h21m

给tiller做rbac授权：

  1. ---
  2. apiVersion: rbac.authorization.k8s.io/v1beta1
  3. kind: ClusterRoleBinding
  4. metadata:
  5.   name: kubernetes-tiller
  6. roleRef:
  7.   apiGroup: rbac.authorization.k8s.io
  8.   kind: ClusterRole
  9.   name: cluster-admin
  10. subjects:
  11. - kind: ServiceAccount
  12.   name: tiller
  13.   namespace: kube-system

修复部署：

  1. kubectl patch deploy --namespace kube-system tiller-deploy -p '{"spec":{"template":{"spec":{"serviceAccount":"tiller"}}}}'

授权普通用户使用kube-system命名空间下的tiller：

  1. apiVersion: v1
  2. kind: ServiceAccount
  3. metadata:
  4.   name: USER
  5.   namespace: NAMESPACE
  6. ---
  7. apiVersion: rbac.authorization.k8s.io/v1
  8. kind: Role
  9. metadata:
  10.   name: use-tiller
  11.   namespace: kube-system
  12. rules:
  13. - apiGroups:
  14.   - ""
  15.   resources:
  16.   - pods/portforward
  17.   verbs:
  18.   - create
  19. - apiGroups:
  20.   - ""
  21.   resources:
  22.   - pods
  23.   verbs:
  24.   - list
  25. ---
  26. apiVersion: rbac.authorization.k8s.io/v1
  27. kind: RoleBinding
  28. metadata:
  29.   name: use-tiller-binding
  30.   namespace: kube-system
  31. roleRef:
  32.   apiGroup: rbac.authorization.k8s.io
  33.   kind: Role
  34.   name: use-tiller
  35. subjects:
  36. - kind: ServiceAccount
  37.   name: USER
  38.   namespace: NAMESPACE

之后生成普通用户的config即可

5.部署kubeapps

引用官方的chart部署kubeapps:v1.4.2

  1. $ helm repo add bitnami https://charts.bitnami.com/bitnami
  2. $ helm fetch bitnami/kubeapps
  3. $ tar xf kubeapps-2.0.4.tgz
  4. $ cd kubeapps/
  5. $ vim values.yaml
  6. ...修改内容...
  7.   annotations:
  8.     kubernetes.io/ingress.class: traefik-internal
  9. ...
  10.   hosts:
  11.   - name: kubeapps-nx.shannonai.com 
  12.     path: /
  13. ...
  14.   initialRepos:
  15.   - name: azure-mirror-stable
  16.     url: http://mirror.azure.cn/kubernetes/charts/
  17.   - name: azure-mirror-incubator
  18.     url: http://mirror.azure.cn/kubernetes/charts-incubator/
  19.   - name: bitnami
  20.     url: https://charts.bitnami.com/bitnami
  21.   - name: shannonai
  22.     url: https://chartmuseum.shannonai.com/
  23. ...修改内容...
  24. #开始部署
  25. $ helm install --name kubeapps-aws-nx --namespace ci-cd .

访问dashboard：

image_1din7qvpo20asob1eatug9nqa9.png-759.3kB

6.部署prometheus+grafana

  1. $ git clone ssh://git@git.shannonai.com:2222/shannon/charts.git
  2. $ cd prometheus-operator
  3. $ helm repo add kube-state-metrics https://chartmuseum.shannonai.com/
  4. $ helm repo add prometheus-node-exporter https://chartmuseum.shannonai.com/
  5. $ helm repo add grafana https://chartmuseum.shannonai.com/
  6. $ helm dep update
  7. $ vim values.yaml
  8. ...修改内容...
  9. 修改内容包括：ingress.class、三个ingress域名、ldap域名改为"ldap.shannonai.com"、两个"storageClassName"
  10. ...修改内容...
  12. $ rm -rf templates/grafana/dashboards/ceph.yaml
  13. $ vim vim templates/alertmanager/alert-dingtalk.yaml
  14. ...修改内容...
  15. 修改钉钉报警url和base64加密的报警信息模板
  16. ...修改内容...
  18. #开始部署安装
  19. $ helm install --name prometheus-aws-nx --namespace monitor .

注意：
1.注意servicemonitor的select需要匹配service的label；
2.如果promethues的target显示"0/0up"，更换"templates/exporters/"下的yaml为最新版是一个解决方向；
3.scheduler的servicemonitor问题仍未解决，但是其重要性不是很大，可以暂时忽略...

访问dashboard：

image_1dium5eequoijvu1t7vn014el9.png-803.7kB

7.部署fluentd-elasticsearch

  1. $ git clone ssh://git@git.shannonai.com:2222/shannon/charts.git
  2. $ cd fluentd-elasticsearch-nx
  3. #创建具有访问aws-elasticsearch的权限的用户的secret
  4. $ cat cretificate.yaml
  5. #由于secret是Opaque类型，"ACCESS_KEY_ID"、"ACCESS_KEY_ID"需要base64加密：echo -n "xxxx" | base64
  6. apiVersion: v1
  7. data:
  8.   ACCESS_KEY_ID: QUtJQTNQTkhaNUdURlFRWjc0QUg=
  9.   SECRET_ACCESS_KEY: eWVHYmRuNzdWSFJBY3I3K0tSM1pjTHp3ODlZWWphSXNNcnZwd3I2Lw==
  10. kind: Secret
  11. metadata:
  12.   labels:
  13.     app: fluentd
  14.   name: fluentd-aws-nx-es-logs-secret
  15.   namespace: kube-system
  16. type: Opaque
  18. $ vim values.yaml
  19. ...修改内容...
  20. elasticsearch的地址，前缀设置为fluentd
  21. #以secret的方式传入环境变量
  22. secret:
  23.  ACCESS_KEY_ID:
  24.    secret_name: fluentd-aws-nx-es-logs-secret
  25.    secret_key: ACCESS_KEY_ID
  26.  SECRET_ACCESS_KEY:
  27.    secret_name: fluentd-aws-nx-es-logs-secret
  28.    secret_key: SECRET_ACCESS_KEY 
  29. ...修改内容...
  31. #创建"system-node-critical"、"system-cluster-critical"的PriorityClass，注意这两个只能被kube-system命名空间的资源下被引用。
  32. $ cat PriorityClass.yaml 
  33. # NOTE: "system-node-critical" and "system-cluster-critical" can only be used in ns kube-system
  34. apiVersion: scheduling.k8s.io/v1beta1
  35. kind: PriorityClass
  36. metadata:
  37.   name: system-node-critical
  38. value: 2000001000
  40. ---
  41. apiVersion: scheduling.k8s.io/v1beta1
  42. kind: PriorityClass
  43. metadata:
  44.   name: system-cluster-critical
  45. value: 2000000000
  47. #开始部署
  48. $ helm install --namespace kube-system --name fluentd-elasticsearch-nx .

aaaaaa.gif-1527.2kB

8.部署kube-resource-report

添加新批注
在作者公开此批注前，只有你和作者可见。
回复批注