diff --git a/k8s/troubleshoot/README.md b/k8s/troubleshoot/README.md new file mode 100644 index 0000000..e9c70a0 --- /dev/null +++ b/k8s/troubleshoot/README.md @@ -0,0 +1,15 @@ +# Troubleshoot + + +## Trotubleshoot host-port problem + +See [GitHub issue](https://github.com/projectcalico/calico/issues/3412). + +### Steps to reproduce & analyze + +```sh +k apply -f nginx.yml; sleep 15 ;k delete -f nginx.yml +# on the host +iptables -t nat --line-numbers -L CNI-HOSTPORT-DNAT +# rules shown are not deleted anymore +``` \ No newline at end of file diff --git a/k8s/troubleshoot/busybox.yaml b/k8s/troubleshoot/busybox.yaml index 45b26d4..f4da245 100644 --- a/k8s/troubleshoot/busybox.yaml +++ b/k8s/troubleshoot/busybox.yaml @@ -1,30 +1,30 @@ -# status: ready -# note: readym to debug your problem -apiVersion: apps/v1 -kind: Deployment -metadata: - creationTimestamp: null - labels: - app: busybox - name: busybox -spec: - replicas: 1 - selector: - matchLabels: - app: busybox - strategy: {} - template: - metadata: - creationTimestamp: null - labels: - app: busybox - spec: - containers: - - image: busybox - name: busybox - resources: {} - command: - - "sleep" - - "3600" - stdin: true - tty: true +# # status: ready +# # note: readym to debug your problem +# apiVersion: apps/v1 +# kind: Deployment +# metadata: +# creationTimestamp: null +# labels: +# app: busybox +# name: busybox +# spec: +# replicas: 1 +# selector: +# matchLabels: +# app: busybox +# strategy: {} +# template: +# metadata: +# creationTimestamp: null +# labels: +# app: busybox +# spec: +# containers: +# - image: busybox +# name: busybox +# resources: {} +# command: +# - "sleep" +# - "3600" +# stdin: true +# tty: true diff --git a/k8s/troubleshoot/delete_duplicate_iptables.sh b/k8s/troubleshoot/delete_duplicate_iptables.sh new file mode 100755 index 0000000..009ed3b --- /dev/null +++ b/k8s/troubleshoot/delete_duplicate_iptables.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +IPTABLES_SAVE="$( iptables-save 2> /dev/null )" +KUBECTL_ALL_PODS=$( kubectl get pods --all-namespaces -o wide | sed "s/ */ /g" ) + +# find all rules for "CNI-HOSTPORT-DNAT" +CHAINS_CNI_HOSTPORT_DNAT=$( echo "$IPTABLES_SAVE" | grep -e "^-A CNI-HOSTPORT-DNAT" ) +echo "$CHAINS_CNI_HOSTPORT_DNAT" | while read chain || [[ -n $chain ]]; +do + # find all targets + TARGET=$( echo $chain | cut -d " " -f 18 ) + echo "$TARGET" | while read target || [[ -n $target ]]; + do + # search for all the targets and just use the one containing "--to-destination" + ALL_DESTINATION_RULES=$( echo "$IPTABLES_SAVE" | grep -e "^-A $target") + TO_DESTINATION_RULE=$( echo "$IPTABLES_SAVE" | grep -e "^-A $target.*--to-destination" ) + echo "$TO_DESTINATION_RULE" | while read rule || [[ -n $rule ]]; + do + echo "rule: $rule" + HOST_PORTS=$( echo "$rule" | cut -d " " -f 8 | tr ',' ' ' ) + TO_DESTINATION_RULE_ADDRESS_PORT=$( echo "$rule" | cut -d " " -f 12) + CONTAINER_ADDRESS=$( echo "$TO_DESTINATION_RULE_ADDRESS_PORT" | cut -d ":" -f 1 ) + CONTAINER_PORT=$( echo "$TO_DESTINATION_RULE_ADDRESS_PORT" | cut -d ":" -f 2 ) + echo "hostport: $HOST_PORT container address: $CONTAINER_ADDRESS port: $CONTAINER_PORT" + # check whether there is a pod with that address + POD=$( echo "$KUBECTL_ALL_PODS" | grep $CONTAINER_ADDRESS | cut -d " " -f 2 ) + NAMESPACE=$( echo "$KUBECTL_ALL_PODS" | grep $CONTAINER_ADDRESS | cut -d " " -f 1 ) + POD_COUNT=$( echo "$POD" | wc -c ) + if [[ "$POD_COUNT" == "1" ]] + then + echo "#No pod found for address $CONTAINER_ADDRESS deleting iptables rules" + echo "$ALL_DESTINATION_RULES" | while read deleteRule || [[ -n $deleteRule ]]; + do + echo "iptables -t nat -D ${deleteRule#-A }" + #iptables -t nat -D "${to_delete_rule#-A }" + done + echo "iptables -t nat -D ${chain#-A }" + #iptables -t nat -D ${chain#-A } + else + echo "The pod $POD actually exists in namespace $NAMESPACE" + fi + done + done +done + +#k get pod nginx -n troubleshoot -o jsonpath="{$.spec.containers[*].ports[*].containerPort}" \ No newline at end of file diff --git a/k8s/troubleshoot/nginx.yml b/k8s/troubleshoot/nginx.yml new file mode 100644 index 0000000..f198293 --- /dev/null +++ b/k8s/troubleshoot/nginx.yml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Pod +metadata: + creationTimestamp: null + labels: + run: nginx + name: nginx + namespace: troubleshoot +spec: + containers: + - image: nginx + name: nginx + resources: {} + ports: + - containerPort: 80 + hostPort: 10081 + name: http + + dnsPolicy: ClusterFirst + restartPolicy: Never