Help to implement Docker Swarm TCP TLS Client Authentication

Hi,

I'm trying to implement mutual authentication on MQTTS TCP connection but the Traefik keeps letting pass all connections, I'm missing something.

Docker Traefik configuration:

version: "3.7"

services:
  traefik:
    image: traefik:v2.2
    command:
      - "--log.level=INFO"
      - "--api=true"
      - "--providers.docker.exposedbydefault=false"
      # enable swarm mode
      - "--providers.docker.endpoint=unix:///var/run/docker.sock"
      - "--providers.docker.swarmMode=true"
      - "--providers.docker.network=local-network"
      # dynamic configs
      - "--providers.file.directory=/etc/traefik/conf/"
      # entrypoints
      - "--entrypoints.web.address=:80"
      - "--entrypoints.websecure.address=:443"
      - "--entrypoints.mqtt-backend.address=:1883"
      - "--entrypoints.mqtts-backend.address=:8883"
      - "--entrypoints.mqtt-frontend.address=:1884"
      - "--entrypoints.mqtts-frontend.address=:8884"
      # acme
      - "--certificatesresolvers.myresolver.acme.httpchallenge=true"
      - "--certificatesresolvers.myresolver.acme.httpchallenge.entrypoint=web"
      - "--certificatesresolvers.myresolver.acme.email=hidden.email@email.com"
      - "--certificatesresolvers.myresolver.acme.storage=/letsencrypt/acme.json"
    ports:
      - 80:80
      - 443:443
      - 1883:1883
      - 8883:8883
      - 1884:1884
      - 8884:8884
    volumes:
      - ./letsencrypt:/letsencrypt
      - ./certs:/certs:ro
      - ./keys:/keys:ro
      - ./configs/emqxbackend.toml:/etc/traefik/conf/emqxbackend.toml:ro
      - /var/run/docker.sock:/var/run/docker.sock:ro
    networks:
      - traefik-public
      - local-network
    deploy:
      replicas: 1
      placement:
        constraints:
          - node.role == manager
      restart_policy:
        condition: on-failure
      labels:
        - "traefik.enable=true"
        - "traefik.http.routers.traefik-dashboard.entrypoints=web"
        - "traefik.http.routers.traefik-dashboard.rule=Host(`hidden.dashboard.domain`)"
        # - "traefik.http.routers.traefik-dashboard.tls.certresolver=myresolver"
        - "traefik.http.routers.traefik-dashboard.service=api@internal"
        - "traefik.http.routers.traefik-dashboard.middlewares=auth"
        - "traefik.http.middlewares.auth.basicauth.users=test:$$apr1$$H6uskkkW$$IgXLP6ewTrSuBkTrqE8wj/"
        # Dummy service for Swarm port detection. The port can be any valid integer value.
        - "traefik.http.services.dummy-svc.loadbalancer.server.port=9999"
        - "traefik.http.middlewares.testIPwhitelist.ipwhitelist.sourcerange=127.0.0.1/32, 192.168.8.135"
        # - "traefik.http.middlewares.testIPwhitelist.ipwhitelist.ipstrategy.depth=2"

networks:
  traefik-public:
    external: true
  local-network:
    external: true

/etc/traefik/conf/emqxbackend.toml:

[tls.options]
  [tls.options.emqxbackend]
    [tls.options.emqxbackend.clientAuth]
      caFiles = ["/certs/client.crt"]
      clientAuthType = "RequireAndVerifyClientCert"

Docker EMQX backend:

version: "3.7"

services:
  emqx-backend01:
    image: emqx-backend:1.0.0
    environment:
      - "EMQX_LOADED_PLUGINS=\"emqx_management,emqx_auth_http,emqx_recon,emqx_retainer,emqx_dashboard\""
      - "EMQX_NAME=emqx"
      - "EMQX_HOST=emqx.backend01"
      - "EMQX_CLUSTER__DISCOVERY=static"
      - "EMQX_CLUSTER__STATIC__SEEDS=emqx@emqx.backend01, emqx@emqx.backend02, emqx@emqx.backend03"
      - "EMQX_NODE__COOKIE=cookie"
    volumes:
      - emqx-backend01-logs:/opt/emqx/log
    deploy:
      # replicas: 1
      placement:
        constraints:
          - node.labels.worker == 1
      restart_policy:
        condition: none
      labels:
        - "traefik.enable=true"
        - "traefik.docker.network=local-network"
        # http with redirection
        - "traefik.http.routers.emqx-backend-dashboard.entrypoints=web"
        - "traefik.http.routers.emqx-backend-dashboard.rule=Host(`hidden.domain`)"
        - "traefik.http.middlewares.redirect-middleware.redirectscheme.scheme=https"
        - "traefik.http.routers.emqx-backend-dashboard.middlewares=redirect-middleware"
        # https
        - "traefik.http.routers.emqx-backend-dashboard-secure.rule=Host(`hidden.domain`)"
        - "traefik.http.routers.emqx-backend-dashboard-secure.entrypoints=websecure"
        - "traefik.http.routers.emqx-backend-dashboard-secure.tls=true"
        # - "traefik.http.routers.emqx-backend-dashboard-secure.tls.certresolver=myresolver"
        - "traefik.http.routers.emqx-backend-dashboard-secure.service=service-emqx-backend-dashboard"
        - "traefik.http.services.service-emqx-backend-dashboard.loadbalancer.server.port=18083"
        # mqtts
        - "traefik.tcp.routers.emqx-backend-mqtts.entrypoints=mqtts-backend"
        - "traefik.tcp.routers.emqx-backend-mqtts.rule=HostSNI(`domain`)"
        - "traefik.tcp.routers.emqx-backend-mqtts.tls=true"
        # - "traefik.tcp.routers.emqx-backend-mqtts.tls.certresolver=myresolver"
        - "traefik.tcp.routers.emqx-backend-mqtts.tls.options=emqxbackend@file"
        - "traefik.tcp.routers.emqx-backend-mqtts.service=service-mqtts-backend"
        - "traefik.tcp.services.service-mqtts-backend.loadbalancer.server.port=1883"
    networks:
      local-network:
        aliases:
        - emqx.backend01
        - emqx.backend

networks:
  local-network:
    external: true

volumes:
  emqx-backend01-logs:

docker logs traefik

time="2020-07-31T08:42:37Z" level=info msg="Configuration loaded from flags."
time="2020-07-31T08:42:37Z" level=info msg="Traefik version 2.2.8 built on 2020-07-28T15:46:03Z"
time="2020-07-31T08:42:37Z" level=info msg="\nStats collection is disabled.\nHelp us improve Traefik by turning this feature on :)\nMore details on: https://docs.traefik.io/contributing/data-collection/\n"
time="2020-07-31T08:42:37Z" level=info msg="Starting provider aggregator.ProviderAggregator {}"
time="2020-07-31T08:42:37Z" level=info msg="Starting provider *file.Provider {\"directory\":\"/etc/traefik/conf/\",\"watch\":true}"
time="2020-07-31T08:42:37Z" level=info msg="Starting provider *acme.Provider {\"email\":\"hidden.email@email.com\",\"caServer\":\"https://acme-v02.api.letsencrypt.org/directory\",\"storage\":\"/letsencrypt/acme.json\",\"keyType\":\"RSA4096\",\"httpChallenge\":{\"entryPoint\":\"web\"},\"ResolverName\":\"myresolver\",\"store\":{},\"ChallengeStore\":{}}"
time="2020-07-31T08:42:37Z" level=info msg="Testing certificate renew..." providerName=myresolver.acme
time="2020-07-31T08:42:37Z" level=info msg="Starting provider *docker.Provider {\"watch\":true,\"endpoint\":\"unix:///var/run/docker.sock\",\"defaultRule\":\"Host(`{{ normalize .Name }}`)\",\"swarmMode\":true,\"network\":\"local-network\",\"swarmModeRefreshSeconds\":15000000000}"
time="2020-07-31T08:42:37Z" level=info msg="Starting provider *traefik.Provider {}"

Problem solved, the problem was outside the Traefik.