First hetzner changes

This commit is contained in:
Moritz Graf 2020-11-14 18:56:45 +01:00
parent 460accd206
commit 2dc555a9a3
10 changed files with 451 additions and 9 deletions

2
bootstrap_hetzner/.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
kubespray
venv

103
bootstrap_hetzner/README.md Normal file
View File

@ -0,0 +1,103 @@
# Bootstrap
The following lines document how to initalize a fresh cluster. On a real cluster, or using Vagrant. It therefore assumes to clone kubespray to this folder. It will be excluded in *.gitignore* and all files are in this folder.
Use [kubespray tag](https://github.com/kubernetes-sigs/kubespray/releases) as parameter
```sh
./init.sh "release-2.14"
```
See [kubespray.io](https://kubespray.io/) oj detailed information about kubespray. Though it seems to be a littlebit outdated.
## Vagrant
```sh
cd kubespray
vagrant up
# up and abkle to ssh
vagrant ssh k8s-1
```
## Prod
Prepare server:
* deactivate swap!
* `moritz username ALL=(ALL) NOPASSWD:ALL`
* `
```sh
ssh centos@<ip>
# auth via pw
sudo su - root
adduser moritz
visudo # add as sudo user
su - moritz
sudo yum -y install vim python3
ssh-keygen
vim .ssh/authorized_users # paste key
chmod 644 .ssh/authorized_keys
# check whether login works with ssh key
sudo vim /etc/ssh/sshd_config # remove pw auth & root login
sudo yum upgrade -y && sudo reboot
```
Install Kubernetes:
```sh
. ./init.sh
# follow instructions from output, sth like:
cd kubespray
ansible-playbook -i inventory/prod/inventory.ini cluster.yml
```
And get credentials:
```sh
ssh <ip>
sudo su - root
cd
cp -r .kube /home/moritz/
chown -R moritz. /home/moritz/.kube
#ctrl + d
kubectl get ns # test connection
#ctrl + d
scp haumdaucher.de:/home/moritz/.kube/config .kube/config
```
Foreward in k8s-directory.
## Upgrade cluster
Check the current default value of `kube_version` in cloned repository.
```sh
cd kubespray
ansible-playbook -i inventory/prod/inventory.ini -e kube_version=v1.18.8 -e upgrade_cluster_setup=true cluster.yml
# or just the newest version
ansible-playbook -i inventory/prod/inventory.ini -e upgrade_cluster_setup=true cluster.yml
# upgrade to specific calico version (did not trigger/ failed)
ansible-playbook -i inventory/prod/inventory.ini -e upgrade_cluster_setup=true -e calico_version=v3.15.2 cluster.yml --tags=network
```
History:
* 2020-04-18 kube_version=v1.16.8 kubespray_branch=release-2.12
## Add node
See [documentation](https://github.com/kubernetes-sigs/kubespray/blob/master/docs/nodes.md).
Note: This was more or less a trial and error approach. Running different playbooks over and over again got it right at some point.
```sh
ansible-playbook -i inventory/prod/inventory.ini --limit=ns3088070.ip-37-59-40.eu,ns3100058.ip-37-59-61.eu scale.yml
ansible-playbook -i inventory/prod/inventory.ini --limit=etcd,kube-master -e ignore_assert_errors=yes cluster.yml
```
This runs everything and is kind of idempotent:
```sh
ansible-playbook -i inventory/prod/inventory.ini cluster.yml
```

36
bootstrap_hetzner/dev.ini Normal file
View File

@ -0,0 +1,36 @@
# ## Configure 'ip' variable to bind kubernetes services on a
# ## different ip than the default iface
# ## We should set etcd_member_name for etcd cluster. The node that is not a etcd member do not need to set the value, or can set the empty string value.
[all]
# node1 ansible_host=95.54.0.12 # ip=10.3.0.1 etcd_member_name=etcd1
# node2 ansible_host=95.54.0.13 # ip=10.3.0.2 etcd_member_name=etcd2
# node3 ansible_host=95.54.0.14 # ip=10.3.0.3 etcd_member_name=etcd3
# node4 ansible_host=95.54.0.15 # ip=10.3.0.4 etcd_member_name=etcd4
# node5 ansible_host=95.54.0.16 # ip=10.3.0.5 etcd_member_name=etcd5
# node6 ansible_host=95.54.0.17 # ip=10.3.0.6 etcd_member_name=etcd6
# ## configure a bastion host if your nodes are not directly reachable
# bastion ansible_host=x.x.x.x ansible_user=some_user
[kube-master]
# node1
# node2
[etcd]
# node1
# node2
# node3
[kube-node]
# node2
# node3
# node4
# node5
# node6
[calico-rr]
[k8s-cluster:children]
kube-master
#kube-node
#calico-rr

52
bootstrap_hetzner/init.sh Executable file
View File

@ -0,0 +1,52 @@
#!/bin/sh
# check whether script is sourced
[[ "${BASH_SOURCE[0]}" == "${0}" ]] && echo "script is executed, but needs to be sourced" && exit 1
echo "######################################################################################"
echo "## Reinit repository"
rm -rf kubespray
VERSION=${1:-release-2.14}
git clone --branch $VERSION https://github.com/kubernetes-sigs/kubespray.git
echo "######################################################################################"
echo "## Activating pyenv venv"
eval "$(pyenv init -)"
eval "$(pyenv virtualenv-init -)"
pyenv virtualenv 3.8.3 infrapuzzle-bootstrap
pyenv activate infrapuzzle-bootstrap
python -m pip install -r kubespray/requirements.txt
echo "######################################################################################"
echo "## Customizing vagrant dev env"
mkdir -p kubespray/vagrant
cat << EOF > kubespray/vagrant/config.rb
\$instance_name_prefix = "k8s"
\$vm_cpus = 4
\$num_instances = 1
\$os = "centos"
\$subnet = "10.0.20"
\$network_plugin = "calico"
\$shared_folders = { 'temp/docker_rpms' => "/var/cache/yum/x86_64/7/docker-ce/packages" }
\$kube_node_instances_with_disks_number = 0
EOF
# make the rpm cache
mkdir -p kubespray/temp/docker_rpms
echo "###############"
echo "Now cd to kubespray and 'vagrant up'"
echo ""
echo "export KUBECONFIG=\"$( pwd )/kubespray/inventory/sample/artifacts/admin.conf\""
echo "######################################################################################"
echo "## Preparing real prod environment"
cp -r kubespray/inventory/sample kubespray/inventory/prod
rm kubespray/inventory/prod/inventory.ini
cp ./prod.ini kubespray/inventory/prod/inventory.ini
#gsed -i "s/kube_network_plugin: .*/kube_network_plugin: flannel/" ./kubespray/inventory/prod/group_vars/k8s-cluster/k8s-cluster.yml
gsed -i "s/# calico_iptables_backend: \"Legacy\"/calico_iptables_backend: \"NFT\"/" ./kubespray/inventory/prod/group_vars/k8s-cluster/k8s-net-calico.yml
#echo 'calico_iptables_backend: "NFT"' >> ./kubespray/inventory/prod/group_vars/k8s-cluster/k8s-net-cluster.yml
#gsed -i "s/metrics_server_enabled: .*/metrics_server_enabled: true/" ./kubespray/inventory/prod/group_vars/k8s-cluster/addons.yml

View File

@ -0,0 +1,33 @@
# ## Configure 'ip' variable to bind kubernetes services on a
# ## different ip than the default iface
# ## We should set etcd_member_name for etcd cluster. The node that is not a etcd member do not need to set the value, or can set the empty string value.
[all]
haumdaucher ansible_host=136.243.23.215 etcd_member_name=etcd1 ansible_become=yes ansible_become_method=sudo ansible_python_interpreter=/usr/bin/python3 metrics_server_enabled=false
#ns3088070.ip-37-59-40.eu ansible_host=37.59.40.95 ansible_become=yes ansible_become_method=sudo ansible_python_interpreter=/usr/bin/python3
#ns3100058.ip-37-59-61.eu ansible_host=37.59.61.198 ansible_become=yes ansible_become_method=sudo ansible_python_interpreter=/usr/bin/python3
# node1 ansible_host=95.54.0.12 # ip=10.3.0.1 etcd_member_name=etcd1
# node2 ansible_host=95.54.0.13 # ip=10.3.0.2 etcd_member_name=etcd2
# node3 ansible_host=95.54.0.14 # ip=10.3.0.3 etcd_member_name=etcd3
# node4 ansible_host=95.54.0.15 # ip=10.3.0.4 etcd_member_name=etcd4
# node5 ansible_host=95.54.0.16 # ip=10.3.0.5 etcd_member_name=etcd5
# node6 ansible_host=95.54.0.17 # ip=10.3.0.6 etcd_member_name=etcd6
# ## configure a bastion host if your nodes are not directly reachable
# bastion ansible_host=x.x.x.x ansible_user=some_user
[kube-master]
haumdaucher
#ns3088070.ip-37-59-40.eu
[etcd]
haumdaucher
[kube-node]
haumdaucher
[calico-rr]
[k8s-cluster:children]
kube-master
kube-node
calico-rr

View File

@ -83,10 +83,14 @@ $ kubectl delete -f kuard
Update with the follwoing command. Chart can be found [here](https://github.com/openebs/charts/tree/master/charts/openebs). Update with the follwoing command. Chart can be found [here](https://github.com/openebs/charts/tree/master/charts/openebs).
Pitfal:
* On fresh installation: activate *ndmOperator*, so that CRDs are correctly installed. It may be deactivated afterwards.
```sh ```sh
helm repo add openebs https://openebs.github.io/charts helm repo add openebs https://openebs.github.io/charts
helm repo update helm repo update
helm upgrade --install -f openebs/openebs.yml openebs --namespace openebs openebs/openebs helm upgrade --install --create-namespace -f openebs/openebs.yml openebs --namespace openebs openebs/openebs
k apply -f openebs/storageclass.yml
``` ```
## minio ## minio
@ -118,7 +122,8 @@ kubectl delete ns velero
A backup may be created using: A backup may be created using:
```sh ```sh
velero backup create full-backup --default-volumes-to-restic --include-namespaces datalab,development,nextcloud,tt-rss,zebrium --wait DATE=$( date +%Y%m%d )
velero backup create $DATE --default-volumes-to-restic --include-namespaces datalab,development,nextcloud,tt-rss,zebrium --wait
``` ```
## Add private docker registry ## Add private docker registry
@ -129,7 +134,7 @@ USER='moritz'
PASSWORD='xxx' PASSWORD='xxx'
docker run --entrypoint htpasswd --rm registry:2 -Bbn $USER $PASSWORD docker run --entrypoint htpasswd --rm registry:2 -Bbn $USER $PASSWORD
# # # #
helm upgrade --install docker-registry stable/docker-registry -n development -f development/registry.secret.yaml helm upgrade --install --create-namespace docker-registry stable/docker-registry -n development -f development/registry.secret.yaml
##kubectl apply -f development/registry.secret.yaml ##kubectl apply -f development/registry.secret.yaml
``` ```
@ -198,8 +203,19 @@ EOF
## metrics-server ## metrics-server
Getting resources (was already done):
```sh ```sh
helm upgrade --install -f kube-system/metrics-server.yaml metrics-server -n kube-system stable/metrics-server cd kube-system
curl -L -o metrics-server.yml https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml
# add parameters to deployment:
# - --kubelet-preferred-address-types=InternalIP
# - --v=2
# - --kubelet-insecure-tls
```
```sh
kubectl apply -n kube-system -f kube-system/metrics-server.yml
``` ```
## rstudio ## rstudio

View File

@ -1,4 +0,0 @@
args:
- "--v=2"
- "--kubelet-insecure-tls"
- "--kubelet-preferred-address-types=InternalIP"

View File

@ -0,0 +1,189 @@
apiVersion: v1
kind: ServiceAccount
metadata:
labels:
k8s-app: metrics-server
name: metrics-server
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
k8s-app: metrics-server
rbac.authorization.k8s.io/aggregate-to-admin: "true"
rbac.authorization.k8s.io/aggregate-to-edit: "true"
rbac.authorization.k8s.io/aggregate-to-view: "true"
name: system:aggregated-metrics-reader
rules:
- apiGroups:
- metrics.k8s.io
resources:
- pods
- nodes
verbs:
- get
- list
- watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
k8s-app: metrics-server
name: system:metrics-server
rules:
- apiGroups:
- ""
resources:
- pods
- nodes
- nodes/stats
- namespaces
- configmaps
verbs:
- get
- list
- watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
labels:
k8s-app: metrics-server
name: metrics-server-auth-reader
namespace: kube-system
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: extension-apiserver-authentication-reader
subjects:
- kind: ServiceAccount
name: metrics-server
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
labels:
k8s-app: metrics-server
name: metrics-server:system:auth-delegator
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:auth-delegator
subjects:
- kind: ServiceAccount
name: metrics-server
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
labels:
k8s-app: metrics-server
name: system:metrics-server
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:metrics-server
subjects:
- kind: ServiceAccount
name: metrics-server
namespace: kube-system
---
apiVersion: v1
kind: Service
metadata:
labels:
k8s-app: metrics-server
name: metrics-server
namespace: kube-system
spec:
ports:
- name: https
port: 443
protocol: TCP
targetPort: https
selector:
k8s-app: metrics-server
---
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
k8s-app: metrics-server
name: metrics-server
namespace: kube-system
spec:
selector:
matchLabels:
k8s-app: metrics-server
strategy:
rollingUpdate:
maxUnavailable: 0
template:
metadata:
labels:
k8s-app: metrics-server
spec:
containers:
- args:
- --cert-dir=/tmp
- --secure-port=4443
- --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
- --kubelet-use-node-status-port
- --kubelet-preferred-address-types=InternalIP
- --v=2
- --kubelet-insecure-tls
image: k8s.gcr.io/metrics-server/metrics-server:v0.4.0
imagePullPolicy: IfNotPresent
livenessProbe:
failureThreshold: 3
httpGet:
path: /livez
port: https
scheme: HTTPS
periodSeconds: 10
name: metrics-server
ports:
- containerPort: 4443
name: https
protocol: TCP
readinessProbe:
failureThreshold: 3
httpGet:
path: /readyz
port: https
scheme: HTTPS
periodSeconds: 10
securityContext:
readOnlyRootFilesystem: true
runAsNonRoot: true
runAsUser: 1000
volumeMounts:
- mountPath: /tmp
name: tmp-dir
nodeSelector:
kubernetes.io/os: linux
priorityClassName: system-cluster-critical
serviceAccountName: metrics-server
volumes:
- emptyDir: {}
name: tmp-dir
---
apiVersion: apiregistration.k8s.io/v1
kind: APIService
metadata:
labels:
k8s-app: metrics-server
name: v1beta1.metrics.k8s.io
spec:
group: metrics.k8s.io
groupPriorityMinimum: 100
insecureSkipTLSVerify: true
service:
name: metrics-server
namespace: kube-system
version: v1beta1
versionPriority: 100

View File

@ -1,4 +1,5 @@
apiserver:
enabled: true
analytics: analytics:
enabled: false enabled: false
@ -8,3 +9,5 @@ ndm:
enabled: false enabled: false
snapshotOperator: snapshotOperator:
enabled: true enabled: true
webhook:
enabled: true

View File

@ -0,0 +1,12 @@
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
annotations:
cas.openebs.io/config: "- name: StorageType\n value: \"hostpath\"\n- name: BasePath\n
\ value: /var/openebs/local \n"
openebs.io/cas-type: local
storageclass.kubernetes.io/is-default-class: "true"
name: openebs-hostpath
provisioner: openebs.io/local
reclaimPolicy: Delete
volumeBindingMode: WaitForFirstConsumer