cannot reach grafana loki port with http using traefik - docker-compose

Question

I have been trying to find solutions to this but no luck. all services work internally. I am able to access grafana from browser with tls enabled but I am not able to reach loki port in any way(browser/postman etc.) but I can.
I can access to loki api with curl localy if I open port on for the service. but as I understand you need to expose ports from traefik to do that.
My compose file:
version: "3"
services:
grafana:
labels:
- "traefik.http.routers.grafana.entryPoints=port80"
- "traefik.http.routers.grafana.rule=host(`${DOMAIN}`)"
- "traefik.http.middlewares.grafana-redirect.redirectScheme.scheme=https"
- "traefik.http.middlewares.grafana-redirect.redirectScheme.permanent=true"
- "traefik.http.routers.grafana.middlewares=grafana-redirect"
# SSL endpoint
- "traefik.http.routers.grafana-ssl.entryPoints=port443"
- "traefik.http.routers.grafana-ssl.rule=host(`${DOMAIN}`)"
- "traefik.http.routers.grafana-ssl.tls=true"
- "traefik.http.routers.grafana-ssl.tls.certResolver=le-ssl"
- "traefik.http.routers.grafana-ssl.service=grafana-ssl"
- "traefik.http.services.grafana-ssl.loadBalancer.server.port=3000"
image: grafana/grafana:latest # or probably any other version
volumes:
- grafana-data:/var/lib/grafana
environment:
- GF_SERVER_ROOT_URL=https://${DOMAIN}
- GF_SERVER_DOMAIN=${DOMAIN}
- GF_USERS_ALLOW_SIGN_UP=false
- GF_SECURITY_ADMIN_USER=${GRAFANAUSER}
- GF_SECURITY_ADMIN_PASSWORD=${GRAFANAPASS}
networks:
- traefik-net
loki:
image: grafana/loki
labels:
- "traefik.http.routers.loki-ssl.entryPoints=port3100"
- "traefik.http.routers.loki-ssl.rule=host(`${DOMAIN}`)"
- "traefik.http.routers.loki-ssl.tls=true"
- "traefik.http.routers.loki-ssl.tls.certResolver=le-ssl"
- "traefik.http.routers.loki-ssl.service=loki-ssl"
- "traefik.http.services.loki-ssl.loadBalancer.server.port=3100"
command: -config.file=/etc/loki/config.yaml
volumes:
- ./loki/config.yml:/etc/loki/config.yaml
- loki:/data/loki
networks:
- traefik-net
promtail:
image: grafana/promtail:2.3.0
volumes:
- /var/log:/var/log
- ./promtail:/etc/promtail-config/
command: -config.file=/etc/promtail-config/promtail.yml
networks:
- traefik-net
influx:
image: influxdb:1.7 # or any other recent version
labels:
# SSL endpoint
- "traefik.http.routers.influx-ssl.entryPoints=port8086"
- "traefik.http.routers.influx-ssl.rule=host(`${DOMAIN}`)"
- "traefik.http.routers.influx-ssl.tls=true"
- "traefik.http.routers.influx-ssl.tls.certResolver=le-ssl"
- "traefik.http.routers.influx-ssl.service=influx-ssl"
- "traefik.http.services.influx-ssl.loadBalancer.server.port=8086"
restart: always
volumes:
- influx-data:/var/lib/influxdb
environment:
- INFLUXDB_DB=grafana # set any other to create database on initialization
- INFLUXDB_HTTP_ENABLED=true
- INFLUXDB_HTTP_AUTH_ENABLED=true
- INFLUXDB_ADMIN_USER=&{DB_USER}
- INFLUXDB_ADMIN_PASSWORD=&{DB_PASS}
networks:
- traefik-net
traefik:
image: traefik:v2.9.1
ports:
- "80:80"
- "443:443"
- "3100:3100"
# expose port below only if you need access to the Traefik API
- "8080:8080"
command:
# - "--log.level=DEBUG"
- "--api=true"
- "--api.dashboard=true"
- "--providers.docker=true"
- "--entryPoints.port443.address=:443"
- "--entryPoints.port80.address=:80"
- "--entryPoints.port8086.address=:8086"
- "--entryPoints.port3100.address=:3100"
- "--certificatesResolvers.le-ssl.acme.tlsChallenge=true"
- "--certificatesResolvers.le-ssl.acme.email=${TLS_MAIL}"
- "--certificatesResolvers.le-ssl.acme.storage=/letsencrypt/acme.json"
volumes:
- traefik-data:/letsencrypt/
- /var/run/docker.sock:/var/run/docker.sock
networks:
- traefik-net
volumes:
traefik-data:
grafana-data:
influx-data:
loki:
networks:
traefik-net:
loki conf
# (default configuration)
auth_enabled: false
server:
http_listen_port: 3100
ingester:
lifecycler:
address: 127.0.0.1
ring:
kvstore:
store: inmemory
replication_factor: 1
final_sleep: 0s
chunk_idle_period: 1h # Any chunk not receiving new logs in this time will be flushed
max_chunk_age: 1h # All chunks will be flushed when they hit this age, default is 1h
chunk_target_size: 1048576 # Loki will attempt to build chunks up to 1.5MB, flushing first if chunk_idle_period or max_chunk_age is reached first
chunk_retain_period: 30s # Must be greater than index read cache TTL if using an index cache (Default index read cache TTL is 5m)
max_transfer_retries: 0 # Chunk transfers disabled
wal:
enabled: true
dir: /loki/wal
common:
ring:
instance_addr: 0.0.0.0
kvstore:
store: inmemory
schema_config:
configs:
- from: 2020-10-24
store: boltdb-shipper
object_store: filesystem
schema: v11
index:
prefix: index_
period: 24h
storage_config:
boltdb_shipper:
active_index_directory: /loki/boltdb-shipper-active
cache_location: /loki/boltdb-shipper-cache
cache_ttl: 24h # Can be increased for faster performance over longer query periods, uses more disk space
shared_store: filesystem
filesystem:
directory: /loki/chunks
compactor:
working_directory: /loki/boltdb-shipper-compactor
shared_store: filesystem
limits_config:
reject_old_samples: true
reject_old_samples_max_age: 168h
ingestion_burst_size_mb: 16
ingestion_rate_mb: 16
chunk_store_config:
max_look_back_period: 0s
table_manager:
retention_deletes_enabled: false
retention_period: 0s
ruler:
storage:
type: local
local:
directory: /loki/rules
rule_path: /loki/rules-temp
alertmanager_url: localhost
ring:
kvstore:
store: inmemory
enable_api: true