Adding troubleshoot

k8s/troubleshoot/README.md

@@ -0,0 +1,15 @@
+# Troubleshoot
+
+
+## Trotubleshoot host-port problem
+
+See [GitHub issue](https://github.com/projectcalico/calico/issues/3412).
+
+### Steps to reproduce & analyze
+
+```sh
+k apply -f nginx.yml; sleep 15 ;k delete -f nginx.yml
+# on the host
+iptables -t nat --line-numbers -L CNI-HOSTPORT-DNAT
+# rules shown are not deleted anymore
+```

k8s/troubleshoot/busybox.yaml

@@ -1,30 +1,30 @@
-# status: ready
+# # status: ready
-# note: readym to debug your problem
+# # note: readym to debug your problem
-apiVersion: apps/v1
+# apiVersion: apps/v1
-kind: Deployment
+# kind: Deployment
-metadata:
+# metadata:
-  creationTimestamp: null
+#   creationTimestamp: null
-  labels:
+#   labels:
-    app: busybox
+#     app: busybox
-  name: busybox
+#   name: busybox
-spec:
+# spec:
-  replicas: 1
+#   replicas: 1
-  selector:
+#   selector:
-    matchLabels:
+#     matchLabels:
-      app: busybox
+#       app: busybox
-  strategy: {}
+#   strategy: {}
-  template:
+#   template:
-    metadata:
+#     metadata:
-      creationTimestamp: null
+#       creationTimestamp: null
-      labels:
+#       labels:
-        app: busybox
+#         app: busybox
-    spec:
+#     spec:
-      containers:
+#       containers:
-      - image: busybox
+#       - image: busybox
-        name: busybox
+#         name: busybox
-        resources: {}
+#         resources: {}
-        command:
+#         command:
-                - "sleep"
+#                 - "sleep"
-                - "3600"
+#                 - "3600"
-        stdin: true
+#         stdin: true
-        tty: true
+#         tty: true

k8s/troubleshoot/delete_duplicate_iptables.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+IPTABLES_SAVE="$( iptables-save 2> /dev/null )"
+KUBECTL_ALL_PODS=$( kubectl get pods --all-namespaces -o wide | sed "s/  */ /g" )
+
+# find all rules for "CNI-HOSTPORT-DNAT"
+CHAINS_CNI_HOSTPORT_DNAT=$( echo "$IPTABLES_SAVE" | grep -e "^-A CNI-HOSTPORT-DNAT" )
+echo "$CHAINS_CNI_HOSTPORT_DNAT" | while read chain || [[ -n $chain ]];
+do
+  # find all targets
+  TARGET=$( echo $chain | cut -d " " -f 18 )
+  echo "$TARGET" | while read target || [[ -n $target ]];
+  do
+    # search for all the targets and just use the one containing "--to-destination"
+    ALL_DESTINATION_RULES=$( echo "$IPTABLES_SAVE" | grep -e "^-A $target")
+    TO_DESTINATION_RULE=$( echo "$IPTABLES_SAVE" | grep -e "^-A $target.*--to-destination" )
+    echo "$TO_DESTINATION_RULE" | while read rule || [[ -n $rule ]];
+    do
+      echo "rule: $rule"
+      HOST_PORTS=$( echo "$rule" | cut -d " " -f 8 | tr ',' ' ' )
+      TO_DESTINATION_RULE_ADDRESS_PORT=$( echo "$rule" | cut -d " " -f 12)
+      CONTAINER_ADDRESS=$( echo "$TO_DESTINATION_RULE_ADDRESS_PORT" | cut -d ":" -f 1 )
+      CONTAINER_PORT=$( echo "$TO_DESTINATION_RULE_ADDRESS_PORT" | cut -d ":" -f 2 )
+      echo "hostport: $HOST_PORT container address: $CONTAINER_ADDRESS port: $CONTAINER_PORT"
+      # check whether there is a pod with that address
+      POD=$( echo "$KUBECTL_ALL_PODS" | grep $CONTAINER_ADDRESS | cut -d " " -f 2 )
+      NAMESPACE=$( echo "$KUBECTL_ALL_PODS" | grep $CONTAINER_ADDRESS | cut -d " " -f 1 )
+      POD_COUNT=$( echo "$POD" | wc -c )
+      if [[ "$POD_COUNT" == "1" ]]
+      then
+        echo "#No pod found for address $CONTAINER_ADDRESS deleting iptables rules"
+        echo "$ALL_DESTINATION_RULES" | while read deleteRule || [[ -n $deleteRule ]];
+        do
+          echo "iptables -t nat -D ${deleteRule#-A }"
+          #iptables -t nat -D "${to_delete_rule#-A }"
+        done
+        echo "iptables -t nat -D ${chain#-A }"
+        #iptables -t nat -D ${chain#-A }
+      else
+        echo "The pod $POD actually exists in namespace $NAMESPACE"
+      fi
+    done
+  done
+done
+
+#k get pod nginx -n troubleshoot -o jsonpath="{$.spec.containers[*].ports[*].containerPort}"

k8s/troubleshoot/nginx.yml

@@ -0,0 +1,20 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  creationTimestamp: null
+  labels:
+    run: nginx
+  name: nginx
+  namespace: troubleshoot
+spec:
+  containers:
+  - image: nginx
+    name: nginx
+    resources: {}
+    ports:
+    - containerPort: 80
+      hostPort: 10081
+      name: http
+    
+  dnsPolicy: ClusterFirst
+  restartPolicy: Never