Adding velero scheduled backup and dropbo sync

k8s/README.md

@@ -24,31 +24,7 @@ kubectl patch deployment $DEPLOYMENT -n $NAMESPACE -p "{\"spec\": {\"template\":
 
 # Deployment
 
-## namespaces
+## [ingress-nginx](https://github.com/kubernetes/ingress-nginx/tree/master/charts/ingress-nginx)
-
-DEPRECATED. Namespaces shall be created for the specific service.
-
-```sh
-namespaces="flux cert-manager nginx-ingress infrapuzzle kuard auth nextcloud datalab web development tt-rss backup monitoring nextcloud mailu"
-for i in $( echo $NAMESPACES ) ; do
-  k create ns $i
-done
-```
-
-## helm repositories
-
-
-DEPRECATED. Helm repo will be listed for the individual apps.
-
-```sh
-helm repo add stable https://kubernetes-charts.storage.googleapis.com
-helm repo add bitnami https://charts.bitnami.com/bitnami
-helm repo add k8s-land https://charts.k8s.land
-helm repo add mailu https://mailu.github.io/helm-charts/
-helm repo update
-```
-
-## [ingress-controller](https://github.com/helm/charts/tree/master/stable/nginx-ingress)
 
 Apply with helm:
 
@@ -66,7 +42,7 @@ Apply with helm. [See chart.](https://github.com/jetstack/cert-manager):
 helm repo add jetstack https://charts.jetstack.io
 helm repo update
 helm upgrade --install --create-namespace cert-manager jetstack/cert-manager -n cert-manager -f cert-manager/cert-manager.yaml
-# this is required:
+# apply the two issuer classes
 kubectl apply -f cert-manager/staging-issuer.yaml
 kubectl apply -f cert-manager/production-issuer.yaml
 ```
@@ -83,7 +59,7 @@ $ kubectl delete -f kuard
 
 Update with the follwoing command. Chart can be found [here](https://github.com/openebs/charts/tree/master/charts/openebs).
 
-Pitfal:
+Pitfall:
 * On fresh installation: activate *ndmOperator*, so that CRDs are correctly installed. It may be deactivated afterwards.
 
 ```sh
@@ -114,20 +90,26 @@ Backup tool. See chart [README](https://github.com/vmware-tanzu/helm-charts/blob
 helm repo add vmware-tanzu https://vmware-tanzu.github.io/helm-charts
 helm repo update
 helm upgrade --install --create-namespace --namespace velero -f ./velero/velero.secret.yaml velero vmware-tanzu/velero
+kubectl create secret generic rclone-config --from-file=./velero/rclone.secret
+kubectl apply -f velero/dropbox_sync.yml
 # #
 helm delete velero -n velero
 kubectl delete ns velero
 ```
 
-A backup may be created using:
+A manual backup may be created executing the following command. **Note: Keep backuped namespaces in sync with config from helm chart!!!**
 
 ```sh
 DATE=$( date +%Y%m%d )
-velero backup create $DATE --default-volumes-to-restic --include-namespaces datalab,development,nextcloud,tt-rss,zebrium --wait
+velero backup create $DATE --include-namespaces datalab,development,nextcloud,tt-rss,zebrium,mailu --wait
 ```
 
+
+
 ## Add private docker registry
 
+**TODO: chart no longer exists. Check how to replace this someday.**
+
 ```sh
 # create secret base64 encoded and put it in htpasswd helm chart
 USER='moritz'
@@ -143,7 +125,7 @@ helm upgrade --install --create-namespace docker-registry stable/docker-registry
 Create credentials secret [according to docu](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/#create-a-secret-by-providing-credentials-on-the-command-line):
 
 ```sh
-namespaces="datalab"
+namespaces="datalab web"
 for i in $( echo $namespaces ) ; do
   kubectl create secret docker-registry registry-haumdaucher-de \
     -n $i \
@@ -203,7 +185,7 @@ EOF
 
 ## metrics-server
 
-Getting resources (was already done):
+Getting resources (already done):
 
 ```sh
 cd kube-system
@@ -214,10 +196,16 @@ curl -L -o metrics-server.yml https://github.com/kubernetes-sigs/metrics-server/
 #           - --kubelet-insecure-tls
 ```
 
+Implement metrics-server:
+
 ```sh
 kubectl apply -n kube-system -f kube-system/metrics-server.yml
 ```
 
+# Persistent stuff
+
+From here everything should be covered by the backup. Implenting those objects should already be performed by the velero backup.
+
 ## rstudio
 
 Currently only for one user:
@@ -316,7 +304,7 @@ helm upgrade --install nextcloud-phpmyadmin bitnami/phpmyadmin -n nextcloud -f n
 
 ### backup & restore
 
-#TODO
+#TODO with Velero
 
 ## Jupyter
 
@@ -357,7 +345,7 @@ ssh moritzgraf.de "sudo su - docker -c 'cd /home/docker/mailu && docker-compose
 cd ../terraform && terraform apply
 # helm apply
 cd ../k8s
-helm upgrade --install mailu mailu/mailu -n mailu -f mailu/mailu.secret.yml
+helm upgrade --create-namespace --install mailu mailu/mailu -n mailu -f mailu/mailu.secret.yml
 # apply mailu and scale all to 0
 kc mailu
 k scale --replicas=0 --all=true deploy
@@ -411,5 +399,7 @@ k delete -f dbench
 ## Web
 
 ```sh
+kubectl create ns web
+kubectl apply -n web ./re
 kubectl apply -f web/
 ```

k8s/longhorn-system/longhorn-ingress.yaml

@@ -1,28 +0,0 @@
-# # ### status: works, but is not secure
-
-
-# apiVersion: extensions/v1beta1
-# kind: Ingress
-# metadata:
-#   name: longhorn-frontend
-#   namespace: longhorn-system
-#   annotations:
-#     kubernetes.io/ingress.class: "nginx"    
-#     nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
-#     cert-manager.io/cluster-issuer: "letsencrypt-prod"
-#     nginx.ingress.kubernetes.io/auth-type: basic
-#     nginx.ingress.kubernetes.io/auth-secret: htpasswd
-#     nginx.ingress.kubernetes.io/auth-realm: "Authentication Required - ok"
-# spec:
-#   tls:
-#    - hosts:
-#      - "longhorn.haumdaucher.de"
-#      secretName: longhorn-haumdaucher-de
-#   rules:
-#   - host: longhorn.haumdaucher.de
-#     http:
-#       paths:
-#       - path: /
-#         backend:
-#           serviceName: longhorn-frontend
-#           servicePort: 80

k8s/longhorn-system/longhorn.yaml

@@ -1,18 +0,0 @@
-# # status: implemented
-
-# apiVersion: helm.fluxcd.io/v1
-# kind: HelmRelease
-# metadata:
-#   name: helm-longhorn
-#   namespace: flux
-# spec:
-#   releaseName: longhorn
-#   targetNamespace: longhorn-system
-#   chart:
-#     git: https://github.com/longhorn/longhorn.git
-#     path: "chart/"
-#     ref: "v0.8.0"
-#   values:
-#     # defaultSettings:
-#     #   backupTarget:
-#     #   backupTargetCredentialSecret:

k8s/mailu/backup.secret.yml

@@ -1,45 +1,45 @@
-kind: ConfigMap
+# kind: ConfigMap
-metadata:
+# metadata:
-  creationTimestamp: null
+#   creationTimestamp: null
-  name: rclone-dropbox-config
+#   name: rclone-dropbox-config
-  namespace: mailu
+#   namespace: mailu
-apiVersion: v1
+# apiVersion: v1
-data:
+# data:
-  rclone.conf: |
+#   rclone.conf: |
-    [dropbox]
+#     [dropbox]
-    type = dropbox
+#     type = dropbox
-    token = {"access_token":"17hvEArIh3cAAAAAAAGTXUfwkGdz8EC_rDcEseLmSRltQoUz_AVceuUewx0sGMQ1","token_type":"bearer","expiry":"0001-01-01T00:00:00Z"}
+#     token = {"access_token":"17hvEArIh3cAAAAAAAGTXUfwkGdz8EC_rDcEseLmSRltQoUz_AVceuUewx0sGMQ1","token_type":"bearer","expiry":"0001-01-01T00:00:00Z"}
----
+# ---
-apiVersion: batch/v1beta1
+# apiVersion: batch/v1beta1
-kind: CronJob
+# kind: CronJob
-metadata:
+# metadata:
-  name: backup-mailu
+#   name: backup-mailu
-  namespace: mailu
+#   namespace: mailu
-spec:
+# spec:
-  schedule: "30 4 * * *"
+#   schedule: "30 4 * * *"
-  jobTemplate:
+#   jobTemplate:
-    spec:
+#     spec:
-      template:
+#       template:
-        spec:
+#         spec:
-          containers:
+#           containers:
-          - name: backup-mailu
+#           - name: backup-mailu
-            image: iptizer/swiss
+#             image: iptizer/swiss
-            volumeMounts:
+#             volumeMounts:
-            - name: rclone-dropbox-config
+#             - name: rclone-dropbox-config
-              mountPath: /rclone.conf
+#               mountPath: /rclone.conf
-              subPath: "rclone.conf"
+#               subPath: "rclone.conf"
-            - mountPath: /data/
+#             - mountPath: /data/
-              name: data
+#               name: data
-            command:
+#             command:
-              - "/bin/bash"
+#               - "/bin/bash"
-              - "-c"
+#               - "-c"
-            args: 
+#             args: 
-              - "cd / && tar zcvf - /data/ | /usr/bin/rclone --config /rclone.conf -v rcat dropbox:server_backup/$( /bin/expr $( date +%m ) % 3 )_$( /bin/expr $( date +%j ) % 3 )_mailu.tar.gz"
+#               - "cd / && tar zcvf - /data/ | /usr/bin/rclone --config /rclone.conf -v rcat dropbox:server_backup/$( /bin/expr $( date +%m ) % 3 )_$( /bin/expr $( date +%j ) % 3 )_mailu.tar.gz"
-          volumes:
+#           volumes:
-          - name: rclone-dropbox-config
+#           - name: rclone-dropbox-config
-            configMap:
+#             configMap:
-              name: rclone-dropbox-config
+#               name: rclone-dropbox-config
-          - name: data
+#           - name: data
-            persistentVolumeClaim:
+#             persistentVolumeClaim:
-              claimName: mailu-storage
+#               claimName: mailu-storage
-          restartPolicy: Never
+#           restartPolicy: Never

k8s/mailu/mailu.secret.yml

@@ -1,7 +1,6 @@
 mailuVersion: master
 hostnames: 
     - mail.moritzgraf.de
-    - moritzgraf.de
 domain: moritzgraf.de 
 secretKey: fa5faeD9aegietaesahbiequ5Pe9au
 subnet: 10.233.0.0/16

k8s/velero/dropbox_sync.yml

@@ -0,0 +1,29 @@
+apiVersion: batch/v1beta1
+kind: CronJob
+metadata:
+  name: dropbox-sync
+  namespace: velero
+spec:
+  schedule: "0 5 * * *"
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          serviceAccountName: default
+          containers:
+          - name: dopybox-sync
+            image: iptizer/swiss
+            volumeMounts:
+            - name: rclone-config
+              mountPath: /rclone.secret
+              subPath: "rclone.secret"
+            command:
+              - "/bin/bash"
+              - "-c"
+            args: 
+              - "/usr/bin/rclone --config /rclone.secret -v copy minio:backup/ dropbox:server_backup/velero/"
+          volumes:
+          - name: rclone-config
+            secret:
+              secretName: rclone-config
+          restartPolicy: Never

k8s/web/moritzgrafde.yaml

@@ -1,67 +1,65 @@
 #### Migrate at last
 
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: moritzgrafde
+  labels:
+    app: moritzgrafde
+  namespace: web
+spec:
+  selector:
+    matchLabels:
+      app: moritzgrafde
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: moritzgrafde
+    spec:
+      imagePullSecrets:
+      - name: "registry-haumdaucher-de"
+      containers:
+      - image: registry.haumdaucher.de/moritzgrafde:latest
+        imagePullPolicy: Always
+        name: moritzgrafde
+        ports:
+        - containerPort: 80
 
-
+---
-# ---
+apiVersion: v1
-# apiVersion: apps/v1
+kind: Service
-# kind: Deployment
+metadata:
-# metadata:
+  name: moritzgrafde
-#   name: moritzgrafde
+  namespace: web
-#   labels:
+spec:
-#     app: moritzgrafde
+  ports:
-#   namespace: web
+  - port: 80
-# spec:
+    targetPort: 80
-#   selector:
+    protocol: TCP
-#     matchLabels:
+  selector:
-#       app: moritzgrafde
+    app: moritzgrafde
-#   replicas: 1
+---
-#   template:
+apiVersion: extensions/v1beta1
-#     metadata:
+kind: Ingress
-#       labels:
+metadata:
-#         app: moritzgrafde
+  name: moritzgrafde
-#     spec:
+  namespace: web
-#       imagePullSecrets:
+  annotations:
-#       - name: "hub-moritzgraf-de"
+    kubernetes.io/ingress.class: "nginx"    
-#       containers:
+    nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
-#       - image: hub.moritzgraf.de:5000/moritzgrafde:latest
+    cert-manager.io/cluster-issuer: "letsencrypt-prod"
-#         imagePullPolicy: Always
+spec:
-#         name: moritzgrafde
+  tls:
-#         ports:
+   - hosts:
-#         - containerPort: 80
+     - "moritzgraf.de"
-
+     secretName: moritzgraf-de
-# ---
+  rules:
-# apiVersion: v1
+  - host: moritzgraf.de
-# kind: Service
+    http:
-# metadata:
+      paths:
-#   name: moritzgrafde
+      - path: /
-#   namespace: web
+        backend:
-# spec:
+          serviceName: moritzgrafde
-#   ports:
+          servicePort: 80
-#   - port: 80
-#     targetPort: 80
-#     protocol: TCP
-#   selector:
-#     app: moritzgrafde
-# ---
-# apiVersion: extensions/v1beta1
-# kind: Ingress
-# metadata:
-#   name: moritzgrafde
-#   namespace: web
-#   annotations:
-#     kubernetes.io/ingress.class: "nginx"    
-#     nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
-#     cert-manager.io/cluster-issuer: "letsencrypt-prod"
-# spec:
-#   tls:
-#    - hosts:
-#      - "moritzgraf.de"
-#      secretName: corona-moritzgraf-de
-#   rules:
-#   - host: moritzgraf.de
-#     http:
-#       paths:
-#       - path: /
-#         backend:
-#           serviceName: moritzgrafde
-#           servicePort: 80