diff --git a/README.md b/README.md index af882f7..5fe40c1 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,9 @@ +docker network create k3s --gateway 192.168.221.1 --subnet 192.168.221.0/24 + docker-compose up -d -docker-compose exec vm1 bash -c "docker-compose up -d" && docker-compose exec vm2 bash -c "docker-compose up -d" && docker-compose exec vm3 bash -c "docker-compose up -d" && cp services1/output/kubeconfig.yaml ~/.kube/clusters/autodoor.yaml && source ~/.zshrc \ No newline at end of file +docker-compose exec vm1 bash -c "docker-compose up -d" && docker-compose exec vm2 bash -c "docker-compose up -d" && docker-compose exec vm3 bash -c "docker-compose up -d" && cp services1/output/kubeconfig.yaml ~/.kube/clusters/autodoor.yaml && source ~/.zshrc + +docker-compose exec vm1 bash -c "docker-compose down && docker volume prune -f" && docker-compose exec vm2 bash -c "docker-compose down && docker volume prune -f" && docker-compose exec vm3 bash -c "docker-compose down && docker volume prune -f" + +docker-compose down && docker volume prune -f \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 31dd23b..95e0675 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,6 +11,8 @@ services: working_dir: /root/services ports: - 6443:6443 + - 3001:3006 + - 3002:3007 networks: k3s: ipv4_address: 192.168.221.11 @@ -24,6 +26,9 @@ services: stdin_open: true # docker run -i tty: true # docker run -t working_dir: /root/services + ports: + - 3003:3006 + - 3004:3007 networks: k3s: ipv4_address: 192.168.221.12 @@ -37,11 +42,23 @@ services: stdin_open: true # docker run -i tty: true # docker run -t working_dir: /root/services + ports: + - 3005:3006 + - 3006:3007 networks: k3s: ipv4_address: 192.168.221.13 volumes: - ./services3:/root/services + + # nginx: + # image: "nginx" + # container_name: nginx + # restart: always + # ports: + # - 4001:3001 + # volumes: + # - ./nginx/default.conf:/etc/nginx/conf.d/default.conf networks: k3s: diff --git a/nginx/default.conf b/nginx/default.conf new file mode 100644 index 0000000..7ea9705 --- /dev/null +++ b/nginx/default.conf @@ -0,0 +1,16 @@ +upstream nginx1 { + server 192.168.1.107:3001; + server 192.168.1.107:3002; + server 192.168.1.107:3003; + server 192.168.1.107:3004; + server 192.168.1.107:3005; + server 192.168.1.107:3006; +} + +server { + listen 3001; + + location / { + proxy_pass http://nginx1; + } +} \ No newline at end of file diff --git a/services1/docker-compose.yml b/services1/docker-compose.yml index 02d8027..445cdca 100644 --- a/services1/docker-compose.yml +++ b/services1/docker-compose.yml @@ -26,8 +26,9 @@ services: master1: image: "rancher/k3s:${K3S_VERSION:-latest}" container_name: master1 - command: server --node-name=master1 --node-external-ip="${K3S_MASTER}" --cluster-init --advertise-address ${K3S_MASTER} --tls-san ${K3S_MASTER} --tls-san ${K3S_MASTER2} --disable-cloud-controller --disable traefik --disable metrics-server --disable servicelb --node-label "node-type=master" --node-label "node-name=master1" --node-label "k8s=egate" - # --disable coredns + command: server --node-name=master1 --kube-proxy-arg iptables-sync-period=2s --kube-proxy-arg masquerade-all=true --disable-network-policy --disable servicelb --disable traefik --disable local-storage --disable-helm-controller --disable metrics-server --disable coredns --node-external-ip="${K3S_MASTER}" --cluster-init --advertise-address ${K3S_MASTER} --tls-san ${K3S_MASTER} --tls-san ${K3S_MASTER2} --node-label "node-type=master" --node-label "node-name=master1" --node-label "k8s=egate" +# --flannel-backend=wireguard-native --kube-proxy-arg proxy-mode=ipvs --kube-proxy-arg ipvs-scheduler=lc --kube-proxy-arg ipvs-strict-arp=true +# --kube-proxy-arg iptables-sync-period=2s --kube-proxy-arg masquerade-all=true tmpfs: - /run - /var/run @@ -45,6 +46,7 @@ services: - K3S_DATASTORE_ENDPOINT=http://${K3S_MASTER}:2379 volumes: - ./output:/output + - ./modules:/etc/modules-load.d/ipvs.conf # - ../registry/registries.yaml:/etc/rancher/k3s/registries.yaml ports: - 6443:6443 @@ -57,7 +59,8 @@ services: worker1: image: "rancher/k3s:${K3S_VERSION:-latest}" container_name: worker1 - command: agent --node-name=worker1 --node-external-ip="${K3S_MASTER}" --node-label "node-type=worker" --node-label "node-name=worker1" --node-label "k8s=egate" + command: agent --node-name=worker1 --kube-proxy-arg iptables-sync-period=2s --kube-proxy-arg masquerade-all=true --node-external-ip="${K3S_MASTER}" --node-label "node-type=worker" --node-label "node-name=worker1" --node-label "k8s=egate" +# --kube-proxy-arg proxy-mode=ipvs --kube-proxy-arg ipvs-scheduler=lc --kube-proxy-arg ipvs-strict-arp=true tmpfs: - /run - /var/run @@ -71,7 +74,8 @@ services: environment: - K3S_URL=https://${K3S_MASTER}:6443 - K3S_TOKEN=${K3S_TOKEN:?err} - # volumes: + volumes: + - ./modules:/etc/modules-load.d/ipvs.conf # - ../registry/registries.yaml:/etc/rancher/k3s/registries.yaml networks: vv: @@ -87,6 +91,8 @@ services: - 3003:3003 - 3004:3004 - 3005:3005 + - 3006:3006 + - 3007:3007 volumes: - ./nginx/default.conf:/etc/nginx/conf.d/default.conf networks: diff --git a/services1/modules b/services1/modules new file mode 100644 index 0000000..759bd4e --- /dev/null +++ b/services1/modules @@ -0,0 +1,5 @@ +ip_vs +ip_vs_rr +ip_vs_wrr +ip_vs_sh +nf_conntrack_ipv4 \ No newline at end of file diff --git a/services1/nginx/default.conf b/services1/nginx/default.conf index 22a9884..5e295e8 100644 --- a/services1/nginx/default.conf +++ b/services1/nginx/default.conf @@ -23,6 +23,16 @@ upstream streamer { server 172.25.0.35:30005; } +upstream nginx { + server 172.25.0.34:30013; + server 172.25.0.35:30013; +} + +upstream nginx1 { + server 172.25.0.34:30014; + server 172.25.0.35:30014; +} + server { listen 3001; @@ -61,4 +71,20 @@ server { location / { proxy_pass http://streamer; } +} + +server { + listen 3006; + + location / { + proxy_pass http://nginx; + } +} + +server { + listen 3007; + + location / { + proxy_pass http://nginx1; + } } \ No newline at end of file diff --git a/services1/output/kubeconfig.yaml b/services1/output/kubeconfig.yaml index 8c3e32a..fa285d3 100644 --- a/services1/output/kubeconfig.yaml +++ b/services1/output/kubeconfig.yaml @@ -1,7 +1,7 @@ apiVersion: v1 clusters: - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJkekNDQVIyZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWpNU0V3SHdZRFZRUUREQmhyTTNNdGMyVnkKZG1WeUxXTmhRREUyT0RrMk5USXhNRGd3SGhjTk1qTXdOekU0TURNME9ESTRXaGNOTXpNd056RTFNRE0wT0RJNApXakFqTVNFd0h3WURWUVFEREJock0zTXRjMlZ5ZG1WeUxXTmhRREUyT0RrMk5USXhNRGd3V1RBVEJnY3Foa2pPClBRSUJCZ2dxaGtqT1BRTUJCd05DQUFRMGFMakhoNXZqcUlsU2syYk5icWVNUkljazNnUFZiSW80Zk4yVS9MVzIKcjdOcENOMHNrQ3lvb3dxdVFNalZtU3F5NU1oSkl2bG16STJPcVNJcmFkTDVvMEl3UURBT0JnTlZIUThCQWY4RQpCQU1DQXFRd0R3WURWUjBUQVFIL0JBVXdBd0VCL3pBZEJnTlZIUTRFRmdRVWNQYk91ckl2emJkZkxWK25aWVQ2CnRVV001dTR3Q2dZSUtvWkl6ajBFQXdJRFNBQXdSUUlnZGcxQWhmT0x2UmpaMWhMQTJoRXh3Z2kvSkU4Tnh1VTgKcXJldldGc2lQcjRDSVFEM0Q2bjlmNE5BbEhRMzc5TzZtanRXWXlzcnluQkowMDNlSFBXaVZBUlhuZz09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJkekNDQVIyZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWpNU0V3SHdZRFZRUUREQmhyTTNNdGMyVnkKZG1WeUxXTmhRREUyT0RrNE9Ea3pORFF3SGhjTk1qTXdOekl3TWpFME1qSTBXaGNOTXpNd056RTNNakUwTWpJMApXakFqTVNFd0h3WURWUVFEREJock0zTXRjMlZ5ZG1WeUxXTmhRREUyT0RrNE9Ea3pORFF3V1RBVEJnY3Foa2pPClBRSUJCZ2dxaGtqT1BRTUJCd05DQUFRVm01eENFWjI1NkNiWHl0TFBidVUzVmtYMnVjVXpwVG9FcTBrVWxjaFIKKzRENzVHV25vVTdoT1AyWjhJMVRrYXFhVlRFa3dNRkNsbjZWNWxKbEtOT1RvMEl3UURBT0JnTlZIUThCQWY4RQpCQU1DQXFRd0R3WURWUjBUQVFIL0JBVXdBd0VCL3pBZEJnTlZIUTRFRmdRVTZpZGhmdUExNFp3T3lvUVJRVm53Cjk1TkdUV2d3Q2dZSUtvWkl6ajBFQXdJRFNBQXdSUUlnY2IxTXRBVEZPZzJuRDk3TlNFWU04dHhHQ2hvNHZnNWgKbGp6RG5UNjJEYUVDSVFDVWNBQmI5YnQ4OHNsK2s1TC96NkdYbGpEamM0aXdydlA2UUNNQ2FFeXlydz09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K server: https://127.0.0.1:6443 name: default contexts: @@ -15,5 +15,5 @@ preferences: {} users: - name: default user: - client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJrakNDQVRlZ0F3SUJBZ0lJWTVOOVpJWlhnV1l3Q2dZSUtvWkl6ajBFQXdJd0l6RWhNQjhHQTFVRUF3d1kKYXpOekxXTnNhV1Z1ZEMxallVQXhOamc1TmpVeU1UQTRNQjRYRFRJek1EY3hPREF6TkRneU9Gb1hEVEkwTURjeApOekF6TkRneU9Gb3dNREVYTUJVR0ExVUVDaE1PYzNsemRHVnRPbTFoYzNSbGNuTXhGVEFUQmdOVkJBTVRESE41CmMzUmxiVHBoWkcxcGJqQlpNQk1HQnlxR1NNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJLd1pDQ2RidWFkSHhhQnQKSUp2SUdwWGMrcG5YVUx6bjZxM0MzWWtycjhVbjBkcmNmVzVjZDhhS3VFb0R6d1A4ajgrSGNyd2p5aitvS0l5ZApBekNZSzcralNEQkdNQTRHQTFVZER3RUIvd1FFQXdJRm9EQVRCZ05WSFNVRUREQUtCZ2dyQmdFRkJRY0RBakFmCkJnTlZIU01FR0RBV2dCUUxZdEhLSnFRaXdvTXNub0ErVzZ1dElWUUZUVEFLQmdncWhrak9QUVFEQWdOSkFEQkcKQWlFQS9xSFNKRDJHY2QzTHZpak5VaGdub2xibmtPN3FOSGpVY1A4ZWZIalltTVlDSVFDTWx2S0RIRmRlYmducApMdVlsbDBFdzZnVzVmb1RvWGlPRjVxekJERDVTN1E9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCi0tLS0tQkVHSU4gQ0VSVElGSUNBVEUtLS0tLQpNSUlCZHpDQ0FSMmdBd0lCQWdJQkFEQUtCZ2dxaGtqT1BRUURBakFqTVNFd0h3WURWUVFEREJock0zTXRZMnhwClpXNTBMV05oUURFMk9EazJOVEl4TURnd0hoY05Nak13TnpFNE1ETTBPREk0V2hjTk16TXdOekUxTURNME9ESTQKV2pBak1TRXdId1lEVlFRRERCaHJNM010WTJ4cFpXNTBMV05oUURFMk9EazJOVEl4TURnd1dUQVRCZ2NxaGtqTwpQUUlCQmdncWhrak9QUU1CQndOQ0FBU3NUZldqLzVrWXI0bVhoRGdQSVNsOGQ3aGtQaHVlcDgwMEwrOUszVHBSCm4zeGlmTS90RFpER0xGS2JMRnJCR0Y3dndCYWEvS0VDSGR4cVVBZmNncm5ZbzBJd1FEQU9CZ05WSFE4QkFmOEUKQkFNQ0FxUXdEd1lEVlIwVEFRSC9CQVV3QXdFQi96QWRCZ05WSFE0RUZnUVVDMkxSeWlha0lzS0RMSjZBUGx1cgpyU0ZVQlUwd0NnWUlLb1pJemowRUF3SURTQUF3UlFJZ082VEY4UHV2US9GeUtTNFVTemNtVXppa0JjYXovMEJHCjhWRFcxTFdHU0pzQ0lRREl4Ti85bGFJMXhUVklxeVlSa1RTS3Y4Q3NNaG5vSmZhNWhHUDJWN2tibnc9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== - client-key-data: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUJkTVo2cm5ZSmkzbU9wMHhNSTNwcjg1UTltcXRWNklVQVVQa2xqVjljZTJvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFckJrSUoxdTVwMGZGb0cwZ204Z2FsZHo2bWRkUXZPZnFyY0xkaVN1dnhTZlIydHg5Ymx4Mwp4b3E0U2dQUEEveVB6NGR5dkNQS1A2Z29qSjBETUpncnZ3PT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= + client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJrRENDQVRlZ0F3SUJBZ0lJQTNSeGo0ekxoajR3Q2dZSUtvWkl6ajBFQXdJd0l6RWhNQjhHQTFVRUF3d1kKYXpOekxXTnNhV1Z1ZEMxallVQXhOamc1T0RnNU16UTBNQjRYRFRJek1EY3lNREl4TkRJeU5Gb1hEVEkwTURjeApPVEl4TkRJeU5Gb3dNREVYTUJVR0ExVUVDaE1PYzNsemRHVnRPbTFoYzNSbGNuTXhGVEFUQmdOVkJBTVRESE41CmMzUmxiVHBoWkcxcGJqQlpNQk1HQnlxR1NNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJHbWd4UEl4eWZ1YXBBN0MKNHlEbGFKdWJwQndZOTFaRlpGR1VYRjlJSkQ2MzFDa0tIVU9ESStYcVliS3BTYUs1SG9IQjJRRGxEL0NDSSt4YgpWcWNLb2d1alNEQkdNQTRHQTFVZER3RUIvd1FFQXdJRm9EQVRCZ05WSFNVRUREQUtCZ2dyQmdFRkJRY0RBakFmCkJnTlZIU01FR0RBV2dCUytIbEVTN09Ma295eGVZRmp5bFFRVlVwY25pekFLQmdncWhrak9QUVFEQWdOSEFEQkUKQWlCeFluL2ZhcG00MkxTdTJ3VmJ0SEF2QXB1THQ2eVY3VE9BUURnU1JmdVZPUUlnSmhPUHV5YXpwRW1JNkc3YwpUeTZIR3R0YVNiL1RBcFZER3A5UjJudzBmV1E9Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0KLS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJkekNDQVIyZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWpNU0V3SHdZRFZRUUREQmhyTTNNdFkyeHAKWlc1MExXTmhRREUyT0RrNE9Ea3pORFF3SGhjTk1qTXdOekl3TWpFME1qSTBXaGNOTXpNd056RTNNakUwTWpJMApXakFqTVNFd0h3WURWUVFEREJock0zTXRZMnhwWlc1MExXTmhRREUyT0RrNE9Ea3pORFF3V1RBVEJnY3Foa2pPClBRSUJCZ2dxaGtqT1BRTUJCd05DQUFUbkROdVV0QVRrdmJwMXVVL0RVN0FnSFRWRllIbEkvcDFZczVBWmdTeU8KbWtHMGxKZ3ZQdnN2OFhvZjlQRzhIMFVNOWpPTVpiNURhTUk2bXFWRFV6TWhvMEl3UURBT0JnTlZIUThCQWY4RQpCQU1DQXFRd0R3WURWUjBUQVFIL0JBVXdBd0VCL3pBZEJnTlZIUTRFRmdRVXZoNVJFdXppNUtNc1htQlk4cFVFCkZWS1hKNHN3Q2dZSUtvWkl6ajBFQXdJRFNBQXdSUUloQUtCRHlZRmMwMVJSM3ZCQ2tWeVdNNmx6U0g3MmJlSnMKWG90aUd0d0puZWx2QWlCY2JyT0cxZUN3QnZob094SGFEdDFWays4QlhWZ3hhZGNxMThUNkZuNXdjQT09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K + client-key-data: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUJ3elVyOUp4SHlyZ2tmU1gxY2ZIQnA5YmtnaW1NYU1YcGNuREVDQjJGV0VvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFYWFERThqSEorNXFrRHNMaklPVm9tNXVrSEJqM1ZrVmtVWlJjWDBna1ByZlVLUW9kUTRNago1ZXBoc3FsSm9ya2VnY0haQU9VUDhJSWo3RnRXcHdxaUN3PT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= diff --git a/services2/docker-compose.yml b/services2/docker-compose.yml index cebad18..8c1f63e 100644 --- a/services2/docker-compose.yml +++ b/services2/docker-compose.yml @@ -5,7 +5,7 @@ services: master2: image: "rancher/k3s:${K3S_VERSION:-latest}" container_name: master2 - command: server --node-name=master2 --node-external-ip="${K3S_MASTER2}" --advertise-address ${K3S_MASTER} --tls-san ${K3S_MASTER} --tls-san ${K3S_MASTER2} --disable-cloud-controller --disable traefik --disable metrics-server --disable servicelb --node-label "node-type=master" --node-label "node-name=master2" --node-label "k8s=egate" + command: server --node-name=master2 --kube-proxy-arg iptables-sync-period=2s --kube-proxy-arg masquerade-all=true --disable-network-policy --disable servicelb --disable traefik --disable local-storage --disable-helm-controller --disable metrics-server --disable coredns --node-external-ip="${K3S_MASTER2}" --advertise-address ${K3S_MASTER} --tls-san ${K3S_MASTER} --tls-san ${K3S_MASTER2} --node-label "node-type=master" --node-label "node-name=master2" --node-label "k8s=egate" tmpfs: - /run - /var/run @@ -31,7 +31,7 @@ services: worker2: image: "rancher/k3s:${K3S_VERSION:-latest}" container_name: worker2 - command: agent --node-name=worker2 --node-external-ip="${K3S_MASTER2}" --node-label "node-type=worker" --node-label "node-name=worker2" --node-label "k8s=egate" + command: agent --node-name=worker2 --kube-proxy-arg iptables-sync-period=2s --kube-proxy-arg masquerade-all=true --node-external-ip="${K3S_MASTER2}" --node-label "node-type=worker" --node-label "node-name=worker2" --node-label "k8s=egate" tmpfs: - /run - /var/run @@ -61,6 +61,8 @@ services: - 3003:3003 - 3004:3004 - 3005:3005 + - 3006:3006 + - 3007:3007 volumes: - ./nginx/default.conf:/etc/nginx/conf.d/default.conf networks: diff --git a/services2/nginx/default.conf b/services2/nginx/default.conf index cd723b5..7d71ab9 100644 --- a/services2/nginx/default.conf +++ b/services2/nginx/default.conf @@ -23,6 +23,16 @@ upstream streamer { server 172.25.0.37:30005; } +upstream nginx { + server 172.25.0.36:30013; + server 172.25.0.37:30013; +} + +upstream nginx1 { + server 172.25.0.36:30014; + server 172.25.0.37:30014; +} + server { listen 3001; @@ -61,4 +71,20 @@ server { location / { proxy_pass http://streamer; } +} + +server { + listen 3006; + + location / { + proxy_pass http://nginx; + } +} + +server { + listen 3007; + + location / { + proxy_pass http://nginx1; + } } \ No newline at end of file diff --git a/services3/docker-compose.yml b/services3/docker-compose.yml index da8cd8b..7aeab63 100644 --- a/services3/docker-compose.yml +++ b/services3/docker-compose.yml @@ -5,7 +5,7 @@ services: worker3: image: "rancher/k3s:${K3S_VERSION:-latest}" container_name: worker3 - command: agent --node-name=worker3 --node-external-ip ${K3S_WORKER} --node-label "node-type=worker" --node-label "node-name=worker3" --node-label "k8s=egate" + command: agent --node-name=worker3 --kube-proxy-arg iptables-sync-period=2s --kube-proxy-arg masquerade-all=true --node-external-ip ${K3S_WORKER} --node-label "node-type=worker" --node-label "node-name=worker3" --node-label "k8s=egate" tmpfs: - /run - /var/run @@ -35,6 +35,8 @@ services: - 3003:3003 - 3004:3004 - 3005:3005 + - 3006:3006 + - 3007:3007 volumes: - ./nginx/default.conf:/etc/nginx/conf.d/default.conf networks: diff --git a/services3/nginx/default.conf b/services3/nginx/default.conf index cc9cacd..e9ddc2b 100644 --- a/services3/nginx/default.conf +++ b/services3/nginx/default.conf @@ -18,6 +18,14 @@ upstream streamer { server 172.25.0.38:30005; } +upstream nginx { + server 172.25.0.38:30013; +} + +upstream nginx1 { + server 172.25.0.38:30014; +} + server { listen 3001; @@ -56,4 +64,20 @@ server { location / { proxy_pass http://streamer; } +} + +server { + listen 3006; + + location / { + proxy_pass http://nginx; + } +} + +server { + listen 3007; + + location / { + proxy_pass http://nginx1; + } } \ No newline at end of file