test

2023-07-25 01:17:22 +07:00 · 2023-07-25 01:17:22 +07:00 · b9afec7bf2
commit b9afec7bf2
parent 35f61ac3c0
10 changed files with 210 additions and 43 deletions
--- a/README.md
+++ b/README.md
@ -2,8 +2,8 @@ docker network create k3s --gateway 192.168.221.1 --subnet 192.168.221.0/24

 docker-compose up -d

-docker-compose exec vm1 bash -c "docker-compose up -d" && docker-compose exec vm2 bash -c "docker-compose up -d" && docker-compose exec vm3 bash -c "docker-compose up -d" && cp services1/output/kubeconfig.yaml ~/.kube/clusters/autodoor.yaml && source ~/.zshrc
+docker-compose exec vm1 bash -c "docker-compose up -d" && docker-compose exec vm2 bash -c "docker-compose up -d" && docker-compose exec vm3 bash -c "docker-compose up -d" && docker-compose exec vm4 bash -c "docker-compose up -d" && cp services1/output/kubeconfig.yaml ~/.kube/clusters/autodoor.yaml && source ~/.zshrc

-docker-compose exec vm1 bash -c "docker-compose down && docker volume prune -f" && docker-compose exec vm2 bash -c "docker-compose down && docker volume prune -f" && docker-compose exec vm3 bash -c "docker-compose down && docker volume prune -f"
+docker-compose exec vm1 bash -c "docker-compose down && docker volume prune -f" && docker-compose exec vm2 bash -c "docker-compose down && docker volume prune -f" && docker-compose exec vm3 bash -c "docker-compose down && docker volume prune -f" && docker-compose exec vm4 bash -c "docker-compose down && docker volume prune -f"

 docker-compose down && docker volume prune -f 
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -51,6 +51,22 @@ services:
    volumes:
      - ./services3:/root/services

+  vm4:
+    image: earth4242/dockervm:3.0
+    container_name: vm4
+    privileged: true
+    stdin_open: true # docker run -i
+    tty: true        # docker run -t
+    working_dir: /root/services
+    ports:
+    - 3007:3006
+    - 3008:3007
+    networks:
+      k3s:
+        ipv4_address: 192.168.221.14
+    volumes:
+      - ./services4:/root/services
+
  # nginx:
  #   image: "nginx"
  #   container_name: nginx
--- a/services1/docker-compose.yml
+++ b/services1/docker-compose.yml
@ -26,8 +26,7 @@ services:
  master1:
    image: "rancher/k3s:${K3S_VERSION:-latest}"
    container_name: master1
-    command: server --node-name=master1 --kube-proxy-arg iptables-sync-period=2s --kube-proxy-arg masquerade-all=true --disable-network-policy --disable traefik --disable local-storage --disable-helm-controller --disable metrics-server --disable coredns --node-external-ip="${K3S_MASTER}"  --cluster-init --advertise-address ${K3S_MASTER} --tls-san ${K3S_MASTER} --tls-san ${K3S_MASTER2} --node-label "node-type=master" --node-label "node-name=master1" --node-label "k8s=egate" 
-# --flannel-backend=wireguard-native --kube-proxy-arg proxy-mode=ipvs --kube-proxy-arg ipvs-scheduler=lc --kube-proxy-arg ipvs-strict-arp=true 
+    command: server --node-name=master1 --flannel-backend=wireguard-native --flannel-external-ip --disable-network-policy --disable traefik --disable local-storage --disable-helm-controller --disable metrics-server --disable coredns --node-external-ip="${K3S_MASTER}"  --cluster-init --advertise-address ${K3S_MASTER} --tls-san ${K3S_MASTER} --tls-san ${K3S_MASTER2} --node-label "node-type=master" --node-label "node-name=master1" --node-label "k8s=egate" 
 #  --kube-proxy-arg iptables-sync-period=2s --kube-proxy-arg masquerade-all=true
    tmpfs:
    - /run
@ -46,12 +45,9 @@ services:
    - K3S_DATASTORE_ENDPOINT=http://${K3S_MASTER}:2379
    volumes:
    - ./output:/output
-    - ./modules:/etc/modules-load.d/ipvs.conf
    # - ../registry/registries.yaml:/etc/rancher/k3s/registries.yaml
    ports:
    - 6443:6443
-    - 80:80      # Ingress controller port 80
-    - 443:443    # Ingress controller port 443
    networks:
      vv:
        ipv4_address: 172.25.0.34
@ -59,8 +55,7 @@ services:
  worker1:
    image: "rancher/k3s:${K3S_VERSION:-latest}"
    container_name: worker1
-    command: agent --node-name=worker1 --kube-proxy-arg iptables-sync-period=2s --kube-proxy-arg masquerade-all=true --node-external-ip="${K3S_MASTER}" --node-label "node-type=worker" --node-label "node-name=worker1" --node-label "k8s=egate"
-# --kube-proxy-arg proxy-mode=ipvs --kube-proxy-arg ipvs-scheduler=lc --kube-proxy-arg ipvs-strict-arp=true
+    command: agent --node-name=worker1 --node-external-ip="${K3S_MASTER}" --node-label "node-type=worker" --node-label "node-name=worker1" --node-label "k8s=egate"
    tmpfs:
    - /run
    - /var/run
@ -74,9 +69,12 @@ services:
    environment:
    - K3S_URL=https://${K3S_MASTER}:6443
    - K3S_TOKEN=${K3S_TOKEN:?err}
-    volumes:
-    - ./modules:/etc/modules-load.d/ipvs.conf
+    # volumes:
    # - ../registry/registries.yaml:/etc/rancher/k3s/registries.yaml
+    ports:
+    - 8472:8472/udp
+    - 51820:51820/udp
+    - 51821:51821/udp
    networks:
      vv:
        ipv4_address: 172.25.0.35
--- a/services1/output/kubeconfig.yaml
+++ b/services1/output/kubeconfig.yaml
@ -1,7 +1,7 @@
 apiVersion: v1
 clusters:
 - cluster:
-    certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJkakNDQVIyZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWpNU0V3SHdZRFZRUUREQmhyTTNNdGMyVnkKZG1WeUxXTmhRREUyT0RrNU16VTROamd3SGhjTk1qTXdOekl4TVRBek56UTRXaGNOTXpNd056RTRNVEF6TnpRNApXakFqTVNFd0h3WURWUVFEREJock0zTXRjMlZ5ZG1WeUxXTmhRREUyT0RrNU16VTROamd3V1RBVEJnY3Foa2pPClBRSUJCZ2dxaGtqT1BRTUJCd05DQUFTZTJKc3NLQkxpVGpaS0w4aXZ1dENjZnEzT3lBK2hhU0JVdllSLzk0Y0IKUlRlMjFFQk9pVTRWdkhRRDhKQzN2Q25laUtBQjJLWG52L2V5V2RRR0RWRlJvMEl3UURBT0JnTlZIUThCQWY4RQpCQU1DQXFRd0R3WURWUjBUQVFIL0JBVXdBd0VCL3pBZEJnTlZIUTRFRmdRVXJzeU5JMTg1eDc2clZ6NkxFbm1aCmVwSVd5YmN3Q2dZSUtvWkl6ajBFQXdJRFJ3QXdSQUlnRENaYytON3M1elJQMFVBcnY4RW5SSnZtaGhwVno2ZHUKY1J2VGlVeDIxaUVDSUdFcE1LY2lxZWFkZEYzM2JiWU1GWUg3ZHUySFA1ekFkRDZNRjhZNzlFdGwKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=
+    certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJkakNDQVIyZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWpNU0V3SHdZRFZRUUREQmhyTTNNdGMyVnkKZG1WeUxXTmhRREUyT1RBeU1qRTBNVFl3SGhjTk1qTXdOekkwTVRjMU5qVTJXaGNOTXpNd056SXhNVGMxTmpVMgpXakFqTVNFd0h3WURWUVFEREJock0zTXRjMlZ5ZG1WeUxXTmhRREUyT1RBeU1qRTBNVFl3V1RBVEJnY3Foa2pPClBRSUJCZ2dxaGtqT1BRTUJCd05DQUFSS1RER1d5ckNhMGE4Y2hxY0hDcVg4bjg3WXA0YTZuR05FaGZSYjBWYUQKV2hwcDVKSndaSVk1R2FVNy9CV1ptSmpvQUZ6YnRURWt2OHlVNHo0RUpkWDFvMEl3UURBT0JnTlZIUThCQWY4RQpCQU1DQXFRd0R3WURWUjBUQVFIL0JBVXdBd0VCL3pBZEJnTlZIUTRFRmdRVWh6MGJMdnFCSURlaFk4NmxaalcrCjRIb2dZVjh3Q2dZSUtvWkl6ajBFQXdJRFJ3QXdSQUlnUWxaRDFBUFhaUjE0SU0veE1YYUpUU0VOUTlHUS9YTEUKNDU5VGhCNXNkL0lDSUU3NUdGYUVuRWtleXEwOTR3THNjeFFHTmU0ZjhGVUpiM1pvdGNWZklZbVQKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=
    server: https://127.0.0.1:6443
  name: default
 contexts:
@ -15,5 +15,5 @@ preferences: {}
 users:
 - name: default
  user:
-    client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJrRENDQVRlZ0F3SUJBZ0lJQ25nTDNkejVhSmN3Q2dZSUtvWkl6ajBFQXdJd0l6RWhNQjhHQTFVRUF3d1kKYXpOekxXTnNhV1Z1ZEMxallVQXhOamc1T1RNMU9EWTRNQjRYRFRJek1EY3lNVEV3TXpjME9Gb1hEVEkwTURjeQpNREV3TXpjME9Gb3dNREVYTUJVR0ExVUVDaE1PYzNsemRHVnRPbTFoYzNSbGNuTXhGVEFUQmdOVkJBTVRESE41CmMzUmxiVHBoWkcxcGJqQlpNQk1HQnlxR1NNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJPUVJDN2Z0aDJOZ3RDNmoKUkFpNllxTTg3d0kwMmZjbEtoaXMzUDdwVDFkWW80bGIxUUx3S1RLWTZBYWNDcFpGRUpWMG9qUmFlRWJtSmw5cgpEQVpoWTJhalNEQkdNQTRHQTFVZER3RUIvd1FFQXdJRm9EQVRCZ05WSFNVRUREQUtCZ2dyQmdFRkJRY0RBakFmCkJnTlZIU01FR0RBV2dCUU1BUjJvQUgrTXU1RExOMUZISVZJVWRkSE94VEFLQmdncWhrak9QUVFEQWdOSEFEQkUKQWlBUUlRYnNDQ1I4K1ZIZmRUdUtwOFZJZlhzOEwwTVN1WGxwT3ExWFZZV1pHQUlnVnFPVTQ0d29aQ0N3aCtobgo1c1VYSkRhREcwMVZ3RUlWSXI3RnBpcGRCN1k9Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0KLS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJkakNDQVIyZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWpNU0V3SHdZRFZRUUREQmhyTTNNdFkyeHAKWlc1MExXTmhRREUyT0RrNU16VTROamd3SGhjTk1qTXdOekl4TVRBek56UTRXaGNOTXpNd056RTRNVEF6TnpRNApXakFqTVNFd0h3WURWUVFEREJock0zTXRZMnhwWlc1MExXTmhRREUyT0RrNU16VTROamd3V1RBVEJnY3Foa2pPClBRSUJCZ2dxaGtqT1BRTUJCd05DQUFUV2oweW4rYUtoSjZkQzFtNndRZTlQc0VCTHJjVWtoZXN5TnpaOXVqUzMKRTE1dDZRc29ZeUlKazF2Y2IrZDZQNnQxY2tDeVFHQU1LRFVoMnR0OGo4RlVvMEl3UURBT0JnTlZIUThCQWY4RQpCQU1DQXFRd0R3WURWUjBUQVFIL0JBVXdBd0VCL3pBZEJnTlZIUTRFRmdRVURBRWRxQUIvakx1UXl6ZFJSeUZTCkZIWFJ6c1V3Q2dZSUtvWkl6ajBFQXdJRFJ3QXdSQUlnR0dHSzJGb04vd2hkS2VEU2Y2ZFZXZ1JHL0pxQUh3RkoKSEtCc1g2OForZmtDSUI0U2FKWUxmRG9xb3llV3hpbzdvSkxnY2Y0U2RzTnlmVVozNEc5OURKcjAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=
-    client-key-data: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUN0R3hZcFhqdGMvNXJzd1lZNHpZcVNxZUp0emhTc2ROY1R4Z0NnVDNDYkNvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFNUJFTHQrMkhZMkMwTHFORUNMcGlvenp2QWpUWjl5VXFHS3pjL3VsUFYxaWppVnZWQXZBcApNcGpvQnB3S2xrVVFsWFNpTkZwNFJ1WW1YMnNNQm1GalpnPT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo=
+    client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJrVENDQVRlZ0F3SUJBZ0lJY0pTREhoQVVTMnd3Q2dZSUtvWkl6ajBFQXdJd0l6RWhNQjhHQTFVRUF3d1kKYXpOekxXTnNhV1Z1ZEMxallVQXhOamt3TWpJeE5ERTJNQjRYRFRJek1EY3lOREUzTlRZMU5sb1hEVEkwTURjeQpNekUzTlRZMU5sb3dNREVYTUJVR0ExVUVDaE1PYzNsemRHVnRPbTFoYzNSbGNuTXhGVEFUQmdOVkJBTVRESE41CmMzUmxiVHBoWkcxcGJqQlpNQk1HQnlxR1NNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJMdGs0eWd4RHlUSVZWcTUKcnN2dThoRGVUeEpLMUxnR2FvblJVcWNwUG1tK0pRRU4yNC9lVGJSNFU4VTRoK3I0YzZMcjdGV0FCSjhDQmlZaAozNXo4L082alNEQkdNQTRHQTFVZER3RUIvd1FFQXdJRm9EQVRCZ05WSFNVRUREQUtCZ2dyQmdFRkJRY0RBakFmCkJnTlZIU01FR0RBV2dCUmRJa2gyL0RPalkraWxNYnFuMEhheDJJRmE2REFLQmdncWhrak9QUVFEQWdOSUFEQkYKQWlFQXIyaEMrenFhMExUM3NlZ1d6bkVhazUvMis0Z1lta3I2SlhBNW5GemZCQk1DSUhFNjJGS2krbXRNSytUOApLV0lGUmtMSG5KU3hDN1g1eUJrRSsreVpCZVpDCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0KLS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJkekNDQVIyZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWpNU0V3SHdZRFZRUUREQmhyTTNNdFkyeHAKWlc1MExXTmhRREUyT1RBeU1qRTBNVFl3SGhjTk1qTXdOekkwTVRjMU5qVTJXaGNOTXpNd056SXhNVGMxTmpVMgpXakFqTVNFd0h3WURWUVFEREJock0zTXRZMnhwWlc1MExXTmhRREUyT1RBeU1qRTBNVFl3V1RBVEJnY3Foa2pPClBRSUJCZ2dxaGtqT1BRTUJCd05DQUFUTmxRRXZLaDdDRjJVU0JISkpyV01tTzNscTA4Y05CV3hFT2VwZDlTMEMKMGJ4dHNlMWRteDNqNDk1a1VkTFVMSDdXM0FJNjRSb2N3L3Zta1IyNGtnQ1BvMEl3UURBT0JnTlZIUThCQWY4RQpCQU1DQXFRd0R3WURWUjBUQVFIL0JBVXdBd0VCL3pBZEJnTlZIUTRFRmdRVVhTSklkdnd6bzJQb3BURzZwOUIyCnNkaUJXdWd3Q2dZSUtvWkl6ajBFQXdJRFNBQXdSUUlnUDREUjUyNzViWVYybWZ0dWpsVWN2WStMckxGRHJENzcKcC94ZEowbjVodU1DSVFDd3dEN0lUTGI0ZEhGMXlscEZRWEsvZ2VCWCtkWUJUNXY2STBIK05aV0FWdz09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
+    client-key-data: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUtmTnh5VVM2S3c4QmZqekdBM0dyb0FPNXBuODFOdDYrN3BVQ24wZ3YyK1dvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFdTJUaktERVBKTWhWV3JtdXkrN3lFTjVQRWtyVXVBWnFpZEZTcHlrK2FiNGxBUTNiajk1Tgp0SGhUeFRpSDZ2aHpvdXZzVllBRW53SUdKaUhmblB6ODdnPT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo=
--- a/services2/docker-compose.yml
+++ b/services2/docker-compose.yml
@ -2,36 +2,36 @@ version: '3'

 services:

-  # master2:
-  #   image: "rancher/k3s:${K3S_VERSION:-latest}"
-  #   container_name: master2
-  #   command: server --node-name=master2 --kube-proxy-arg iptables-sync-period=2s --kube-proxy-arg masquerade-all=true --disable-network-policy --disable servicelb --disable traefik --disable local-storage --disable-helm-controller --disable metrics-server --disable coredns --node-external-ip="${K3S_MASTER2}" --advertise-address ${K3S_MASTER} --tls-san ${K3S_MASTER} --tls-san ${K3S_MASTER2} --node-label "node-type=master" --node-label "node-name=master2" --node-label "k8s=egate"
-  #   tmpfs:
-  #   - /run
-  #   - /var/run
-  #   ulimits:
-  #     nproc: 65535
-  #     nofile:
-  #       soft: 65535
-  #       hard: 65535
-  #   privileged: true
-  #   restart: always
-  #   environment:
-  #   - K3S_URL=https://${K3S_MASTER}:6443
-  #   - K3S_TOKEN=${K3S_TOKEN:?err}
-  #   - K3S_DATASTORE_ENDPOINT=http://${K3S_MASTER}:2379
-  #   ports:
-  #   - 6443:6443
-  #   # volumes:
-  #   # - ../registry/registries.yaml:/etc/rancher/k3s/registries.yaml
-  #   networks:
-  #     vv:
-  #       ipv4_address: 172.25.0.36
+  master2:
+    image: "rancher/k3s:${K3S_VERSION:-latest}"
+    container_name: master2
+    command: server --node-name=master2 --flannel-backend=wireguard-native --flannel-external-ip --disable-network-policy --disable traefik --disable local-storage --disable-helm-controller --disable metrics-server --disable coredns --node-external-ip="${K3S_MASTER2}" --advertise-address ${K3S_MASTER} --tls-san ${K3S_MASTER} --tls-san ${K3S_MASTER2} --node-label "node-type=master" --node-label "node-name=master2" --node-label "k8s=egate"
+    tmpfs:
+    - /run
+    - /var/run
+    ulimits:
+      nproc: 65535
+      nofile:
+        soft: 65535
+        hard: 65535
+    privileged: true
+    restart: always
+    environment:
+    - K3S_URL=https://${K3S_MASTER}:6443
+    - K3S_TOKEN=${K3S_TOKEN:?err}
+    - K3S_DATASTORE_ENDPOINT=http://${K3S_MASTER}:2379
+    ports:
+    - 6443:6443
+    # volumes:
+    # - ../registry/registries.yaml:/etc/rancher/k3s/registries.yaml
+    networks:
+      vv:
+        ipv4_address: 172.25.0.36

  worker2:
    image: "rancher/k3s:${K3S_VERSION:-latest}"
    container_name: worker2
-    command: agent --node-name=worker2 --kube-proxy-arg iptables-sync-period=2s --kube-proxy-arg masquerade-all=true --node-external-ip="${K3S_MASTER2}" --node-label "node-type=worker" --node-label "node-name=worker2" --node-label "k8s=egate"
+    command: agent --node-name=worker2 --node-external-ip="${K3S_MASTER2}" --node-label "node-type=worker" --node-label "node-name=worker2" --node-label "k8s=egate"
    tmpfs:
    - /run
    - /var/run
@ -47,6 +47,10 @@ services:
    - K3S_TOKEN=${K3S_TOKEN:?err}
    # volumes:
    # - ../registry/registries.yaml:/etc/rancher/k3s/registries.yaml
+    ports:
+    - 8472:8472/udp
+    - 51820:51820/udp
+    - 51821:51821/udp
    networks:
      vv:
        ipv4_address: 172.25.0.37
--- a/services3/.env
+++ b/services3/.env
@ -1,5 +1,5 @@
 REGISTRY=192.168.221.14:5000
 K3S_VERSION=v1.26.5-k3s1
 K3S_TOKEN=vv
-K3S_MASTER=192.168.221.11
+K3S_MASTER=192.168.221.12
 K3S_WORKER=192.168.221.13
--- a/services3/docker-compose.yml
+++ b/services3/docker-compose.yml
@ -5,7 +5,7 @@ services:
  worker3:
    image: "rancher/k3s:${K3S_VERSION:-latest}"
    container_name: worker3
-    command: agent --node-name=worker3 --kube-proxy-arg iptables-sync-period=2s --kube-proxy-arg masquerade-all=true --node-external-ip ${K3S_WORKER} --node-label "node-type=worker" --node-label "node-name=worker3" --node-label "k8s=egate"
+    command: agent --node-name=worker3 --node-external-ip ${K3S_WORKER} --node-label "node-type=worker" --node-label "node-name=worker3" --node-label "k8s=egate"
    tmpfs:
    - /run
    - /var/run
@ -21,6 +21,10 @@ services:
    - K3S_TOKEN=${K3S_TOKEN:?err}
    # volumes:
    # - ../registry/registries.yaml:/etc/rancher/k3s/registries.yaml
+    ports:
+    - 8472:8472/udp
+    - 51820:51820/udp
+    - 51821:51821/udp
    networks:
      vv:
        ipv4_address: 172.25.0.38
--- a/services4/.env
+++ b/services4/.env
@ -0,0 +1,5 @@
+REGISTRY=192.168.221.14:5000
+K3S_VERSION=v1.26.5-k3s1
+K3S_TOKEN=vv
+K3S_MASTER=192.168.221.12
+K3S_WORKER=192.168.221.14
--- a/services4/docker-compose.yml
+++ b/services4/docker-compose.yml
@ -0,0 +1,57 @@
+version: '3'
+
+services:
+
+  worker4:
+    image: "rancher/k3s:${K3S_VERSION:-latest}"
+    container_name: worker4
+    command: agent --node-name=worker4 --node-external-ip ${K3S_WORKER} --node-label "node-type=worker" --node-label "node-name=worker4" --node-label "k8s=egate"
+    tmpfs:
+    - /run
+    - /var/run
+    ulimits:
+      nproc: 65535
+      nofile:
+        soft: 65535
+        hard: 65535
+    privileged: true
+    restart: always
+    environment:
+    - K3S_URL=https://${K3S_MASTER}:6443
+    - K3S_TOKEN=${K3S_TOKEN:?err}
+    # volumes:
+    # - ../registry/registries.yaml:/etc/rancher/k3s/registries.yaml
+    ports:
+    - 8472:8472/udp
+    - 51820:51820/udp
+    - 51821:51821/udp
+    networks:
+      vv:
+        ipv4_address: 172.25.0.39
+
+  nginx:
+    image: "nginx"
+    container_name: nginx
+    restart: always
+    ports:
+    - 3001:3001
+    - 3002:3002
+    - 3003:3003
+    - 3004:3004
+    - 3005:3005
+    - 3006:3006
+    - 3007:3007
+    volumes:
+      - ./nginx/default.conf:/etc/nginx/conf.d/default.conf
+    networks:
+      vv:
+        ipv4_address: 172.25.0.20
+
+networks:
+  vv:
+    driver: bridge
+    ipam:
+      driver: default
+      config:
+          - subnet: 172.25.0.0/24
+            gateway: 172.25.0.1
--- a/services4/nginx/default.conf
+++ b/services4/nginx/default.conf
@ -0,0 +1,83 @@
+upstream gos {
+    server 172.25.0.39:30001;
+}
+
+upstream goscore {
+    server 172.25.0.39:30002;
+}
+
+upstream prebooking {
+    server 172.25.0.39:30003;
+}
+
+upstream prebookingcore {
+    server 172.25.0.39:30004;
+}
+
+upstream streamer {
+    server 172.25.0.39:30005;
+}
+
+upstream nginx {
+    server 172.25.0.39:30013;
+}
+
+upstream nginx1 {
+    server 172.25.0.39:30014;
+}
+
+server {
+    listen       3001;
+
+    location / {
+        proxy_pass http://gos;
+    }
+}
+
+server {
+    listen       3002;
+
+    location / {
+        proxy_pass http://goscore;
+    }
+}
+
+server {
+    listen       3003;
+
+    location / {
+        proxy_pass http://prebooking;
+    }
+}
+
+server {
+    listen       3004;
+
+    location / {
+        proxy_pass http://prebookingcore;
+    }
+}
+
+server {
+    listen       3005;
+
+    location / {
+        proxy_pass http://streamer;
+    }
+}
+
+server {
+    listen       3006;
+
+    location / {
+        proxy_pass http://nginx;
+    }
+}
+
+server {
+    listen       3007;
+
+    location / {
+        proxy_pass http://nginx1;
+    }
+}