test

README.md

@@ -1,3 +1,9 @@
+docker network create k3s --gateway 192.168.221.1 --subnet 192.168.221.0/24
+
 docker-compose up -d
 
 docker-compose exec vm1 bash -c "docker-compose up -d" && docker-compose exec vm2 bash -c "docker-compose up -d" && docker-compose exec vm3 bash -c "docker-compose up -d" && cp services1/output/kubeconfig.yaml ~/.kube/clusters/autodoor.yaml && source ~/.zshrc
+
+docker-compose exec vm1 bash -c "docker-compose down && docker volume prune -f" && docker-compose exec vm2 bash -c "docker-compose down && docker volume prune -f" && docker-compose exec vm3 bash -c "docker-compose down && docker volume prune -f"
+
+docker-compose down && docker volume prune -f 

docker-compose.yml

@@ -11,6 +11,8 @@ services:
     working_dir: /root/services
     ports:
     - 6443:6443
+    - 3001:3006
+    - 3002:3007
     networks:
       k3s:
         ipv4_address: 192.168.221.11
@@ -24,6 +26,9 @@ services:
     stdin_open: true # docker run -i
     tty: true        # docker run -t
     working_dir: /root/services
+    ports:
+    - 3003:3006
+    - 3004:3007
     networks:
       k3s:
         ipv4_address: 192.168.221.12
@@ -37,12 +42,24 @@ services:
     stdin_open: true # docker run -i
     tty: true        # docker run -t
     working_dir: /root/services
+    ports:
+    - 3005:3006
+    - 3006:3007
     networks:
       k3s:
         ipv4_address: 192.168.221.13
     volumes:
       - ./services3:/root/services
 
+  # nginx:
+  #   image: "nginx"
+  #   container_name: nginx
+  #   restart: always
+  #   ports:
+  #   - 4001:3001
+  #   volumes:
+  #     - ./nginx/default.conf:/etc/nginx/conf.d/default.conf
+          
 networks:
   k3s:
     external: true

nginx/default.conf

@@ -0,0 +1,16 @@
+upstream nginx1 {
+    server 192.168.1.107:3001;
+    server 192.168.1.107:3002;
+    server 192.168.1.107:3003;
+    server 192.168.1.107:3004;
+    server 192.168.1.107:3005;
+    server 192.168.1.107:3006;
+}
+
+server {
+    listen       3001;
+
+    location / {
+        proxy_pass http://nginx1;
+    }
+}

services1/docker-compose.yml

@@ -26,8 +26,9 @@ services:
   master1:
     image: "rancher/k3s:${K3S_VERSION:-latest}"
     container_name: master1
-    command: server --node-name=master1 --node-external-ip="${K3S_MASTER}"  --cluster-init --advertise-address ${K3S_MASTER} --tls-san ${K3S_MASTER} --tls-san ${K3S_MASTER2} --disable-cloud-controller --disable traefik --disable metrics-server --disable servicelb --node-label "node-type=master" --node-label "node-name=master1" --node-label "k8s=egate" 
+    command: server --node-name=master1 --kube-proxy-arg iptables-sync-period=2s --kube-proxy-arg masquerade-all=true --disable-network-policy --disable servicelb --disable traefik --disable local-storage --disable-helm-controller --disable metrics-server --disable coredns --node-external-ip="${K3S_MASTER}"  --cluster-init --advertise-address ${K3S_MASTER} --tls-san ${K3S_MASTER} --tls-san ${K3S_MASTER2} --node-label "node-type=master" --node-label "node-name=master1" --node-label "k8s=egate" 
-    # --disable coredns
+# --flannel-backend=wireguard-native --kube-proxy-arg proxy-mode=ipvs --kube-proxy-arg ipvs-scheduler=lc --kube-proxy-arg ipvs-strict-arp=true 
+# --kube-proxy-arg iptables-sync-period=2s --kube-proxy-arg masquerade-all=true
     tmpfs:
     - /run
     - /var/run
@@ -45,6 +46,7 @@ services:
     - K3S_DATASTORE_ENDPOINT=http://${K3S_MASTER}:2379
     volumes:
     - ./output:/output
+    - ./modules:/etc/modules-load.d/ipvs.conf
     # - ../registry/registries.yaml:/etc/rancher/k3s/registries.yaml
     ports:
     - 6443:6443
@@ -57,7 +59,8 @@ services:
   worker1:
     image: "rancher/k3s:${K3S_VERSION:-latest}"
     container_name: worker1
-    command: agent --node-name=worker1 --node-external-ip="${K3S_MASTER}" --node-label "node-type=worker" --node-label "node-name=worker1" --node-label "k8s=egate"
+    command: agent --node-name=worker1 --kube-proxy-arg iptables-sync-period=2s --kube-proxy-arg masquerade-all=true --node-external-ip="${K3S_MASTER}" --node-label "node-type=worker" --node-label "node-name=worker1" --node-label "k8s=egate"
+# --kube-proxy-arg proxy-mode=ipvs --kube-proxy-arg ipvs-scheduler=lc --kube-proxy-arg ipvs-strict-arp=true
     tmpfs:
     - /run
     - /var/run
@@ -71,7 +74,8 @@ services:
     environment:
     - K3S_URL=https://${K3S_MASTER}:6443
     - K3S_TOKEN=${K3S_TOKEN:?err}
-    # volumes:
+    volumes:
+    - ./modules:/etc/modules-load.d/ipvs.conf
     # - ../registry/registries.yaml:/etc/rancher/k3s/registries.yaml
     networks:
       vv:
@@ -87,6 +91,8 @@ services:
     - 3003:3003
     - 3004:3004
     - 3005:3005
+    - 3006:3006
+    - 3007:3007
     volumes:
       - ./nginx/default.conf:/etc/nginx/conf.d/default.conf
     networks:

services1/modules

@@ -0,0 +1,5 @@
+ip_vs
+ip_vs_rr
+ip_vs_wrr
+ip_vs_sh
+nf_conntrack_ipv4

services1/nginx/default.conf

@@ -23,6 +23,16 @@ upstream streamer {
     server 172.25.0.35:30005;
 }
 
+upstream nginx {
+    server 172.25.0.34:30013;
+    server 172.25.0.35:30013;
+}
+
+upstream nginx1 {
+    server 172.25.0.34:30014;
+    server 172.25.0.35:30014;
+}
+
 server {
     listen       3001;
 
@@ -62,3 +72,19 @@ server {
         proxy_pass http://streamer;
     }
 }
+
+server {
+    listen       3006;
+
+    location / {
+        proxy_pass http://nginx;
+    }
+}
+
+server {
+    listen       3007;
+
+    location / {
+        proxy_pass http://nginx1;
+    }
+}

services1/output/kubeconfig.yaml

@@ -1,7 +1,7 @@
 apiVersion: v1
 clusters:
 - cluster:
-    certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJkekNDQVIyZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWpNU0V3SHdZRFZRUUREQmhyTTNNdGMyVnkKZG1WeUxXTmhRREUyT0RrMk5USXhNRGd3SGhjTk1qTXdOekU0TURNME9ESTRXaGNOTXpNd056RTFNRE0wT0RJNApXakFqTVNFd0h3WURWUVFEREJock0zTXRjMlZ5ZG1WeUxXTmhRREUyT0RrMk5USXhNRGd3V1RBVEJnY3Foa2pPClBRSUJCZ2dxaGtqT1BRTUJCd05DQUFRMGFMakhoNXZqcUlsU2syYk5icWVNUkljazNnUFZiSW80Zk4yVS9MVzIKcjdOcENOMHNrQ3lvb3dxdVFNalZtU3F5NU1oSkl2bG16STJPcVNJcmFkTDVvMEl3UURBT0JnTlZIUThCQWY4RQpCQU1DQXFRd0R3WURWUjBUQVFIL0JBVXdBd0VCL3pBZEJnTlZIUTRFRmdRVWNQYk91ckl2emJkZkxWK25aWVQ2CnRVV001dTR3Q2dZSUtvWkl6ajBFQXdJRFNBQXdSUUlnZGcxQWhmT0x2UmpaMWhMQTJoRXh3Z2kvSkU4Tnh1VTgKcXJldldGc2lQcjRDSVFEM0Q2bjlmNE5BbEhRMzc5TzZtanRXWXlzcnluQkowMDNlSFBXaVZBUlhuZz09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
+    certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJkekNDQVIyZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWpNU0V3SHdZRFZRUUREQmhyTTNNdGMyVnkKZG1WeUxXTmhRREUyT0RrNE9Ea3pORFF3SGhjTk1qTXdOekl3TWpFME1qSTBXaGNOTXpNd056RTNNakUwTWpJMApXakFqTVNFd0h3WURWUVFEREJock0zTXRjMlZ5ZG1WeUxXTmhRREUyT0RrNE9Ea3pORFF3V1RBVEJnY3Foa2pPClBRSUJCZ2dxaGtqT1BRTUJCd05DQUFRVm01eENFWjI1NkNiWHl0TFBidVUzVmtYMnVjVXpwVG9FcTBrVWxjaFIKKzRENzVHV25vVTdoT1AyWjhJMVRrYXFhVlRFa3dNRkNsbjZWNWxKbEtOT1RvMEl3UURBT0JnTlZIUThCQWY4RQpCQU1DQXFRd0R3WURWUjBUQVFIL0JBVXdBd0VCL3pBZEJnTlZIUTRFRmdRVTZpZGhmdUExNFp3T3lvUVJRVm53Cjk1TkdUV2d3Q2dZSUtvWkl6ajBFQXdJRFNBQXdSUUlnY2IxTXRBVEZPZzJuRDk3TlNFWU04dHhHQ2hvNHZnNWgKbGp6RG5UNjJEYUVDSVFDVWNBQmI5YnQ4OHNsK2s1TC96NkdYbGpEamM0aXdydlA2UUNNQ2FFeXlydz09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
     server: https://127.0.0.1:6443
   name: default
 contexts:
@@ -15,5 +15,5 @@ preferences: {}
 users:
 - name: default
   user:
-    client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJrakNDQVRlZ0F3SUJBZ0lJWTVOOVpJWlhnV1l3Q2dZSUtvWkl6ajBFQXdJd0l6RWhNQjhHQTFVRUF3d1kKYXpOekxXTnNhV1Z1ZEMxallVQXhOamc1TmpVeU1UQTRNQjRYRFRJek1EY3hPREF6TkRneU9Gb1hEVEkwTURjeApOekF6TkRneU9Gb3dNREVYTUJVR0ExVUVDaE1PYzNsemRHVnRPbTFoYzNSbGNuTXhGVEFUQmdOVkJBTVRESE41CmMzUmxiVHBoWkcxcGJqQlpNQk1HQnlxR1NNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJLd1pDQ2RidWFkSHhhQnQKSUp2SUdwWGMrcG5YVUx6bjZxM0MzWWtycjhVbjBkcmNmVzVjZDhhS3VFb0R6d1A4ajgrSGNyd2p5aitvS0l5ZApBekNZSzcralNEQkdNQTRHQTFVZER3RUIvd1FFQXdJRm9EQVRCZ05WSFNVRUREQUtCZ2dyQmdFRkJRY0RBakFmCkJnTlZIU01FR0RBV2dCUUxZdEhLSnFRaXdvTXNub0ErVzZ1dElWUUZUVEFLQmdncWhrak9QUVFEQWdOSkFEQkcKQWlFQS9xSFNKRDJHY2QzTHZpak5VaGdub2xibmtPN3FOSGpVY1A4ZWZIalltTVlDSVFDTWx2S0RIRmRlYmducApMdVlsbDBFdzZnVzVmb1RvWGlPRjVxekJERDVTN1E9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCi0tLS0tQkVHSU4gQ0VSVElGSUNBVEUtLS0tLQpNSUlCZHpDQ0FSMmdBd0lCQWdJQkFEQUtCZ2dxaGtqT1BRUURBakFqTVNFd0h3WURWUVFEREJock0zTXRZMnhwClpXNTBMV05oUURFMk9EazJOVEl4TURnd0hoY05Nak13TnpFNE1ETTBPREk0V2hjTk16TXdOekUxTURNME9ESTQKV2pBak1TRXdId1lEVlFRRERCaHJNM010WTJ4cFpXNTBMV05oUURFMk9EazJOVEl4TURnd1dUQVRCZ2NxaGtqTwpQUUlCQmdncWhrak9QUU1CQndOQ0FBU3NUZldqLzVrWXI0bVhoRGdQSVNsOGQ3aGtQaHVlcDgwMEwrOUszVHBSCm4zeGlmTS90RFpER0xGS2JMRnJCR0Y3dndCYWEvS0VDSGR4cVVBZmNncm5ZbzBJd1FEQU9CZ05WSFE4QkFmOEUKQkFNQ0FxUXdEd1lEVlIwVEFRSC9CQVV3QXdFQi96QWRCZ05WSFE0RUZnUVVDMkxSeWlha0lzS0RMSjZBUGx1cgpyU0ZVQlUwd0NnWUlLb1pJemowRUF3SURTQUF3UlFJZ082VEY4UHV2US9GeUtTNFVTemNtVXppa0JjYXovMEJHCjhWRFcxTFdHU0pzQ0lRREl4Ti85bGFJMXhUVklxeVlSa1RTS3Y4Q3NNaG5vSmZhNWhHUDJWN2tibnc9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg==
+    client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJrRENDQVRlZ0F3SUJBZ0lJQTNSeGo0ekxoajR3Q2dZSUtvWkl6ajBFQXdJd0l6RWhNQjhHQTFVRUF3d1kKYXpOekxXTnNhV1Z1ZEMxallVQXhOamc1T0RnNU16UTBNQjRYRFRJek1EY3lNREl4TkRJeU5Gb1hEVEkwTURjeApPVEl4TkRJeU5Gb3dNREVYTUJVR0ExVUVDaE1PYzNsemRHVnRPbTFoYzNSbGNuTXhGVEFUQmdOVkJBTVRESE41CmMzUmxiVHBoWkcxcGJqQlpNQk1HQnlxR1NNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJHbWd4UEl4eWZ1YXBBN0MKNHlEbGFKdWJwQndZOTFaRlpGR1VYRjlJSkQ2MzFDa0tIVU9ESStYcVliS3BTYUs1SG9IQjJRRGxEL0NDSSt4YgpWcWNLb2d1alNEQkdNQTRHQTFVZER3RUIvd1FFQXdJRm9EQVRCZ05WSFNVRUREQUtCZ2dyQmdFRkJRY0RBakFmCkJnTlZIU01FR0RBV2dCUytIbEVTN09Ma295eGVZRmp5bFFRVlVwY25pekFLQmdncWhrak9QUVFEQWdOSEFEQkUKQWlCeFluL2ZhcG00MkxTdTJ3VmJ0SEF2QXB1THQ2eVY3VE9BUURnU1JmdVZPUUlnSmhPUHV5YXpwRW1JNkc3YwpUeTZIR3R0YVNiL1RBcFZER3A5UjJudzBmV1E9Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0KLS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJkekNDQVIyZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWpNU0V3SHdZRFZRUUREQmhyTTNNdFkyeHAKWlc1MExXTmhRREUyT0RrNE9Ea3pORFF3SGhjTk1qTXdOekl3TWpFME1qSTBXaGNOTXpNd056RTNNakUwTWpJMApXakFqTVNFd0h3WURWUVFEREJock0zTXRZMnhwWlc1MExXTmhRREUyT0RrNE9Ea3pORFF3V1RBVEJnY3Foa2pPClBRSUJCZ2dxaGtqT1BRTUJCd05DQUFUbkROdVV0QVRrdmJwMXVVL0RVN0FnSFRWRllIbEkvcDFZczVBWmdTeU8KbWtHMGxKZ3ZQdnN2OFhvZjlQRzhIMFVNOWpPTVpiNURhTUk2bXFWRFV6TWhvMEl3UURBT0JnTlZIUThCQWY4RQpCQU1DQXFRd0R3WURWUjBUQVFIL0JBVXdBd0VCL3pBZEJnTlZIUTRFRmdRVXZoNVJFdXppNUtNc1htQlk4cFVFCkZWS1hKNHN3Q2dZSUtvWkl6ajBFQXdJRFNBQXdSUUloQUtCRHlZRmMwMVJSM3ZCQ2tWeVdNNmx6U0g3MmJlSnMKWG90aUd0d0puZWx2QWlCY2JyT0cxZUN3QnZob094SGFEdDFWays4QlhWZ3hhZGNxMThUNkZuNXdjQT09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
-    client-key-data: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUJkTVo2cm5ZSmkzbU9wMHhNSTNwcjg1UTltcXRWNklVQVVQa2xqVjljZTJvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFckJrSUoxdTVwMGZGb0cwZ204Z2FsZHo2bWRkUXZPZnFyY0xkaVN1dnhTZlIydHg5Ymx4Mwp4b3E0U2dQUEEveVB6NGR5dkNQS1A2Z29qSjBETUpncnZ3PT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo=
+    client-key-data: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUJ3elVyOUp4SHlyZ2tmU1gxY2ZIQnA5YmtnaW1NYU1YcGNuREVDQjJGV0VvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFYWFERThqSEorNXFrRHNMaklPVm9tNXVrSEJqM1ZrVmtVWlJjWDBna1ByZlVLUW9kUTRNago1ZXBoc3FsSm9ya2VnY0haQU9VUDhJSWo3RnRXcHdxaUN3PT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo=

services2/docker-compose.yml

@@ -5,7 +5,7 @@ services:
   master2:
     image: "rancher/k3s:${K3S_VERSION:-latest}"
     container_name: master2
-    command: server --node-name=master2 --node-external-ip="${K3S_MASTER2}" --advertise-address ${K3S_MASTER} --tls-san ${K3S_MASTER} --tls-san ${K3S_MASTER2} --disable-cloud-controller --disable traefik --disable metrics-server --disable servicelb --node-label "node-type=master" --node-label "node-name=master2" --node-label "k8s=egate"
+    command: server --node-name=master2 --kube-proxy-arg iptables-sync-period=2s --kube-proxy-arg masquerade-all=true --disable-network-policy --disable servicelb --disable traefik --disable local-storage --disable-helm-controller --disable metrics-server --disable coredns --node-external-ip="${K3S_MASTER2}" --advertise-address ${K3S_MASTER} --tls-san ${K3S_MASTER} --tls-san ${K3S_MASTER2} --node-label "node-type=master" --node-label "node-name=master2" --node-label "k8s=egate"
     tmpfs:
     - /run
     - /var/run
@@ -31,7 +31,7 @@ services:
   worker2:
     image: "rancher/k3s:${K3S_VERSION:-latest}"
     container_name: worker2
-    command: agent --node-name=worker2 --node-external-ip="${K3S_MASTER2}" --node-label "node-type=worker" --node-label "node-name=worker2" --node-label "k8s=egate"
+    command: agent --node-name=worker2 --kube-proxy-arg iptables-sync-period=2s --kube-proxy-arg masquerade-all=true --node-external-ip="${K3S_MASTER2}" --node-label "node-type=worker" --node-label "node-name=worker2" --node-label "k8s=egate"
     tmpfs:
     - /run
     - /var/run
@@ -61,6 +61,8 @@ services:
     - 3003:3003
     - 3004:3004
     - 3005:3005
+    - 3006:3006
+    - 3007:3007
     volumes:
       - ./nginx/default.conf:/etc/nginx/conf.d/default.conf
     networks:

services2/nginx/default.conf

@@ -23,6 +23,16 @@ upstream streamer {
     server 172.25.0.37:30005;
 }
 
+upstream nginx {
+    server 172.25.0.36:30013;
+    server 172.25.0.37:30013;
+}
+
+upstream nginx1 {
+    server 172.25.0.36:30014;
+    server 172.25.0.37:30014;
+}
+
 server {
     listen       3001;
 
@@ -62,3 +72,19 @@ server {
         proxy_pass http://streamer;
     }
 }
+
+server {
+    listen       3006;
+
+    location / {
+        proxy_pass http://nginx;
+    }
+}
+
+server {
+    listen       3007;
+
+    location / {
+        proxy_pass http://nginx1;
+    }
+}

services3/docker-compose.yml

@@ -5,7 +5,7 @@ services:
   worker3:
     image: "rancher/k3s:${K3S_VERSION:-latest}"
     container_name: worker3
-    command: agent --node-name=worker3 --node-external-ip ${K3S_WORKER} --node-label "node-type=worker" --node-label "node-name=worker3" --node-label "k8s=egate"
+    command: agent --node-name=worker3 --kube-proxy-arg iptables-sync-period=2s --kube-proxy-arg masquerade-all=true --node-external-ip ${K3S_WORKER} --node-label "node-type=worker" --node-label "node-name=worker3" --node-label "k8s=egate"
     tmpfs:
     - /run
     - /var/run
@@ -35,6 +35,8 @@ services:
     - 3003:3003
     - 3004:3004
     - 3005:3005
+    - 3006:3006
+    - 3007:3007
     volumes:
       - ./nginx/default.conf:/etc/nginx/conf.d/default.conf
     networks:

services3/nginx/default.conf

@@ -18,6 +18,14 @@ upstream streamer {
     server 172.25.0.38:30005;
 }
 
+upstream nginx {
+    server 172.25.0.38:30013;
+}
+
+upstream nginx1 {
+    server 172.25.0.38:30014;
+}
+
 server {
     listen       3001;
 
@@ -57,3 +65,19 @@ server {
         proxy_pass http://streamer;
     }
 }
+
+server {
+    listen       3006;
+
+    location / {
+        proxy_pass http://nginx;
+    }
+}
+
+server {
+    listen       3007;
+
+    location / {
+        proxy_pass http://nginx1;
+    }
+}