From 4685a8a6c17339e79d333928492eafb89ce46cf3 Mon Sep 17 00:00:00 2001 From: Ruiling Zhang Date: Mon, 29 Jul 2024 17:40:55 -0400 Subject: [PATCH 1/3] apply traefik for public vhosts --- docker-compose.yml | 83 +++++++++++++++++++++- env.template | 4 ++ traefik/config/certificates.yml | 17 +++++ traefik/config/tls-cert/sample-cert.pem | 18 +++++ traefik/config/tls-cert/sample-private.pem | 27 +++++++ 5 files changed, 147 insertions(+), 2 deletions(-) create mode 100644 traefik/config/certificates.yml create mode 100644 traefik/config/tls-cert/sample-cert.pem create mode 100644 traefik/config/tls-cert/sample-private.pem diff --git a/docker-compose.yml b/docker-compose.yml index 47237bb..7716ddc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,3 @@ -version: '3' services: pisces-db: image: postgres:14.4 @@ -127,6 +126,7 @@ services: depends_on: - pisces-web restart: "always" + scorpio-db: image: postgres:14.4 volumes: @@ -136,6 +136,7 @@ services: restart: "always" environment: - POSTGRES_PASSWORD=${SCORPIO_DB_PASS} + scorpio-web: build: context: ./scorpio @@ -205,6 +206,7 @@ services: depends_on: - scorpio-web restart: "always" + argo-db: image: postgres:14.4 volumes: @@ -214,6 +216,7 @@ services: restart: "always" environment: - POSTGRES_PASSWORD=${ARGO_DB_PASS} + argo-web: environment: - DJANGO_PORT=${ARGO_PORT} @@ -244,6 +247,16 @@ services: hostname: argo.library.pitt.edu volumes: - ./argo:/code + labels: + - traefik.port=8001 + - traefik.enable=true + - traefik.http.routers.argo-web-https.rule=Host(`${ARGO_DNS}`) + - traefik.http.routers.argo-web-https.entrypoints=websecure + - traefik.http.routers.argo-web-https.tls=true + ## testing to hit middlewares + #- traefik.http.routers.argo-web-https.middlewares=auth + #- "traefik.http.middlewares.auth.basicauth.users=testuser:$$2y$$05$$PnbpRUEDgzGksx3OAiydNuMblgbWJiKCYWYSYCMgYTf2rZ8tPwH0." + - traefik.http.services.argo-web-https.loadbalancer.server.port=8001 networks: - astraeus-interop ports: @@ -251,6 +264,7 @@ services: depends_on: - argo-db - elasticsearch + - traefik restart: "always" request-broker-db: image: postgres:14.4 @@ -260,6 +274,7 @@ services: - astraeus-interop environment: - POSTGRES_PASSWORD=${RB_DB_PASS} + request-broker-web: environment: - DJANGO_DEBUG=${DJANGO_DEBUG} @@ -276,7 +291,8 @@ services: - AS_USERNAME=${AS_USERNAME} - AS_PASSWORD=${AS_PASSWORD} - AS_REPO_ID=${AS_REPO_ID} - - AEON_API_KEY=${RB_AEON_API_KEY} + - AEON_APIKEY=${RB_AEON_API_KEY} + - AEON_BASEURL=${RB_AEON_BASEURL} - EMAIL_HOST=${EMAIL_HOST} - EMAIL_PORT=${EMAIL_PORT} - EMAIL_HOST_USER=${EMAIL_HOST_USER} @@ -302,13 +318,24 @@ services: hostname: requestbroker.library.pitt.edu volumes: - ./request_broker:/code + labels: + - traefik.port=8000 + - traefik.enable=true + - traefik.http.middlewares.add-statusapi.addprefix.prefix=/api/status + - traefik.http.routers.request-broker-web.rule=Host(`${RB_DNS}`) + - traefik.http.routers.request-broker-web.middlewares=add-statusapi@docker + - traefik.http.routers.request-broker-web.entrypoints=websecure + - traefik.http.routers.request-broker-web.tls=true + - traefik.http.services.request-broker-web.loadbalancer.server.port=8000 networks: - astraeus-interop ports: - "${RB_PORT:-8000}:${RB_PORT:-8000}" depends_on: - request-broker-db + - traefik restart: "always" + dimes-web: build: context: ./dimes @@ -324,12 +351,20 @@ services: - REACT_APP_AEON_URL=${REACT_APP_AEON_URL} networks: - astraeus-interop + labels: + - traefik.port=80 + - traefik.enable=true + - traefik.http.routers.dimes-web-https.rule=Host(`myreadingroom.library.pitt.edu`) + - traefik.http.routers.dimes-web-https.entrypoints=websecure + - traefik.http.routers.dimes-web-https.tls=true + - traefik.http.services.dimes-web-https.loadbalancer.server.port=80 ports: - 3000:80 stdin_open: true depends_on: - argo-web - request-broker-web + - traefik restart: "always" elasticsearch: @@ -357,6 +392,50 @@ services: - 9200:9200 restart: "always" + traefik: + image: traefik:v3.1 + container_name: traefik + hostname: "traefik" + command: + ##- --log.level=DEBUG + - --log.level=INFO + - --providers.docker + - --api + ##- --api.insecure # only for testing environment + - --providers.docker.exposedbydefault=false + #entrypoints + - --entryPoints.web.address=:80 + - --entryPoints.websecure.address=:443 + #cert + - --providers.file.directory=/etc/traefik/config #dynamic config + - --providers.file.watch=true ## reload any changes + networks: + - astraeus-interop + ports: + - "80:80" #encrypt uses this port + - "443:443" + - "8080:8080" + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + - ${CONFIG_PATH}/config/certificates.yml:/etc/traefik/config/certificates.yml + - ${CONFIG_PATH}/config/tls-cert/:/etc/tls-cert/ + + labels: + - "traefik.port=8080" + - "traefik.enable=true" + # redirect nonsecure request to https + - "traefik.http.routers.http-request.rule=hostregexp(`{host:.+}`)" + - "traefik.http.routers.http-request.entrypoints=web" + - "traefik.http.routers.http-request.middlewares=redirect-to-https" + - "traefik.http.middlewares.redirect-to-https.redirectscheme.scheme=https" + # dashboard + - traefik.http.routers.api.entrypoints=websecure + - traefik.http.routers.api.rule=Host(`${DASHBOARD_HOST}`) + - traefik.http.routers.api.tls=true + - traefik.http.routers.api.service=api@internal #forward requests to api service + - traefik.http.services.dashboard.loadbalancer.server.port=8080 + restart: always #always restart traefik + volumes: piscesdbvolume: scorpiodbvolume: diff --git a/env.template b/env.template index e8689c9..956fd91 100644 --- a/env.template +++ b/env.template @@ -6,4 +6,8 @@ PISCES_DB_PASS=piscespasswordhere # database password used in postgres container SCORPIO_DB_PASS=scorpiopasswordhere # database password used in postgres container, fed to scorpio and scorpio cron ARGO_DB_PASS=piscespasswordhere # database password used in postgres container, fed to argo and argo cron REQUEST_BROKER_DB_PASS=rbpasswordhere # database password used in postgres container, fed to request broker +RB_DJANGO_ALLOWED_HOSTS = ['request-broker-web','localhost','requestbroker.library.pitt.edu'] +#Traefik variables +CONFIG_PATH=./traefik +DASHBOARD_HOST=dashboard.docker.localhost diff --git a/traefik/config/certificates.yml b/traefik/config/certificates.yml new file mode 100644 index 0000000..e500682 --- /dev/null +++ b/traefik/config/certificates.yml @@ -0,0 +1,17 @@ +#Dynamic configuration +tls: + options: + default: + minVersion: VersionTLS12 + cipherSuites: + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + certificates: + - certFile: /etc/tls-cert/cert.pem + keyFile: /etc/tls-cert/privkey.pem + + diff --git a/traefik/config/tls-cert/sample-cert.pem b/traefik/config/tls-cert/sample-cert.pem new file mode 100644 index 0000000..2285c3a --- /dev/null +++ b/traefik/config/tls-cert/sample-cert.pem @@ -0,0 +1,18 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIIC3jCCAcYCAQAwgYAxCzAJBgNVBAYTAnVzMQswCQYDVQQIDAJwYTETMBEGA1UE +BwwKcGl0dHNidXJnaDENMAsGA1UECgwEcGl0dDEMMAoGA1UECwwDdWxzMREwDwYD +VQQDDAh0ZXN0aGVyZTEfMB0GCSqGSIb3DQEJARYQdGVzdDQ1NkBwaXR0LmVkdTCC +ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALpYqZxKG7UrEKzNzZcRrgzC +BgDc7roOGMwK7dY9kAOr1aZxXyTlSO1sgdXrzqK5n6IUjFyAsegkYjNzxpP7kkLd +6rhDy8eZF6VSQt5ERuYzg6WE3eHwlNS+rhTVbI7JXnjxpFVcLChFnuEt0d6nTNby +5pJfDjG5bxtaOXI1F2sxS/wW9nyqn5ApNBgzEYxDyrPbEtym305ioFun1exY/9AY +HJNYjJi5YlPVS8+S1+u2emr/2QcwVixD+rikgDXETHBlUzO9ncOV0eUVUhH0i0yt +QNIrM6wcNF8me9+y4i6WL/Pb8SWNG8Cr/5usFCCmlzbwBzW6HqFTYak5qO2OJX0C +AwEAAaAYMBYGCSqGSIb3DQEJBzEJDAd0ZXN0YWJjMA0GCSqGSIb3DQEBCwUAA4IB +AQBQtmJcd8ZQ6Tw6vesoLR3IrnWPlN6l9eoqhBJr7wxe549ufg4d4loIYN+VLZaK +hbOPQ+neBa1XT5p4X/mtnNdzeRc1zBO+YwfcsDnON30PSKjUYvjnmckS6857mNLt +zvD2tOlmvWTvfmYSZManSydyYAr595hCBglVGytyNazVcFpWLMNAJT27RsyUw5cs +KOX8X+nlxjkwnGLmxeBmCFDb+5W8fG70CxRs/J7OPD+xhXcU3J9RLkdQ+ty7n/qV +4QTKv/sADx67MBZ9e0La+yx4wvUoAVvUuDPuSnlaOa9HV2bm1R3ra7w1j8GmNYWQ +x8Hf38vGnJUCfpNKwfDfb3XX +-----END CERTIFICATE REQUEST----- diff --git a/traefik/config/tls-cert/sample-private.pem b/traefik/config/tls-cert/sample-private.pem new file mode 100644 index 0000000..26432bc --- /dev/null +++ b/traefik/config/tls-cert/sample-private.pem @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAulipnEobtSsQrM3NlxGuDMIGANzuug4YzArt1j2QA6vVpnFf +JOVI7WyB1evOormfohSMXICx6CRiM3PGk/uSQt3quEPLx5kXpVJC3kRG5jODpYTd +4fCU1L6uFNVsjsleePGkVVwsKEWe4S3R3qdM1vLmkl8OMblvG1o5cjUXazFL/Bb2 +fKqfkCk0GDMRjEPKs9sS3KbfTmKgW6fV7Fj/0Bgck1iMmLliU9VLz5LX67Z6av/Z +BzBWLEP6uKSANcRMcGVTM72dw5XR5RVSEfSLTK1A0iszrBw0XyZ737LiLpYv89vx +JY0bwKv/m6wUIKaXNvAHNboeoVNhqTmo7Y4lfQIDAQABAoIBAAwjbGD23tktRffL +rCG0EB9aoCN8QLyz4F+iMp3rAq+KiO8/oU/484grskVqB9rHtqNLLV11MKGLhS4O +05eeIofihsCcAfEtgsHNGvf5gJjMMD4e6okmL7uv9Az9XgXrDhFYxDifOW0iI9hN +MMeNJE84IVbVhEou5xLkDKvo026y+cutViODRERyfUvtaA4DCN9PXqkHSzpEW9Co +WYb6teeFjmQDffx4wb7uoi+6AL9ruzqsM8Pf4bJ9OuRXZCdufkB7gMeOzB8OeBYS +HOEgVDQ/39s/g8lEbXDd7CWcN+YMxFVZrzpLKUd3v35scz+JZpuNa6zUvwH31HAY +mTTS/+ECgYEA7ss68KDmReSVuPr2816/+Q0xyI5lvnhoZPkTVuUd10WtAE/OfTQl +bkLhTwTcWasbls8EUrA0OeyuESl0iZXOBUZaftXIDqcGTowh3xzzipE1H4fXDRuD +fqBkGLmF+96ogl1Uwdvg8SHHkSFxGimzDCOw1QHQvgmT20e9IDOgsFkCgYEAx8X9 +b4eb01xfvpy+MBU5C/p7HgaoBEIA0s0hxJIKeNKmuRqWtpVoGrQpOzI0dLEx2PiL +x5RaOX154oJHBWMtKr3BmAlPlsvPMI9zAJXSzuzB8X039si2h/j3N7NvrDAbHDhw +Phna925RNuCoYFRXYUfleLWJFGQ+/aj3FsNj2cUCgYBkbSAqluCBQHMfSpyVGaIO +8degCxMLGcR9wqq5fr4gDPOHEAk9arLbPlFXVCn/pBCESif9RpGQUtOZ8B9Mxa3R +Vhc1BF+QmfnzCsgr9xcNjagTzKNKpemVVYsDQvLwTGH+AZZluT1O6+/sP247nJHq +ZxA1ZQAPDCQcsnz9j/jicQKBgHPNm5nZPEULWRz/c2ggBU+iRVgkd6TwNdX8v0RZ +e+SKB8dpWFBCz3QbV4NPGQVD6idh/HUW1C5bRBo/drfyw63xDZX6X76EKnh1zy5Z +qzf0GoDIG3bc5qJvea86PtPLlwuG09nL1xhzRHTRSgl9GqHzsVuFsA64BaO5HHJ/ +lRQZAoGBAK7s2s3upUB9ooL805OhrKdK30wm4ieu7i6kPvwt1dedj+Nx96q+U4vN +JByokqIaCsaUiuOYV0jOJnefbZyklcBq2TNLmMlgg2dYuFDDeKYKTP5XSeHyG+ly +fz8GZDIqpKXS5oUF/mMr5NrTYVBGuK4fR3+AYHJ4G+ld0MwURigk +-----END RSA PRIVATE KEY----- From 0726f14daa37abdc78f0770d124047f7af8cbb30 Mon Sep 17 00:00:00 2001 From: Ruiling Zhang Date: Mon, 5 Aug 2024 14:56:18 -0400 Subject: [PATCH 2/3] modified per code review: add non secure routes for public hosts; RB routes updates based on RB api entries; include certs files in .gitignore --- .gitignore | 7 +++++++ docker-compose.yml | 36 ++++++++++++++++-------------------- env.template | 4 ++++ 3 files changed, 27 insertions(+), 20 deletions(-) diff --git a/.gitignore b/.gitignore index df9eb47..9a9e653 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,9 @@ ### Deploy ### .env + +# traefik specific +certificates.yml +*.pem +*.cert +.vscode +traefik/config/tls-cert/* diff --git a/docker-compose.yml b/docker-compose.yml index 7716ddc..4fe91fb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -250,13 +250,13 @@ services: labels: - traefik.port=8001 - traefik.enable=true + # Entry Point for https - traefik.http.routers.argo-web-https.rule=Host(`${ARGO_DNS}`) - - traefik.http.routers.argo-web-https.entrypoints=websecure + - traefik.http.routers.argo-web-https.entrypoints=web,websecure - traefik.http.routers.argo-web-https.tls=true - ## testing to hit middlewares - #- traefik.http.routers.argo-web-https.middlewares=auth - #- "traefik.http.middlewares.auth.basicauth.users=testuser:$$2y$$05$$PnbpRUEDgzGksx3OAiydNuMblgbWJiKCYWYSYCMgYTf2rZ8tPwH0." + - traefik.http.routers.argo-web-https.service=argo-web-https - traefik.http.services.argo-web-https.loadbalancer.server.port=8001 + networks: - astraeus-interop ports: @@ -264,7 +264,6 @@ services: depends_on: - argo-db - elasticsearch - - traefik restart: "always" request-broker-db: image: postgres:14.4 @@ -321,10 +320,10 @@ services: labels: - traefik.port=8000 - traefik.enable=true - - traefik.http.middlewares.add-statusapi.addprefix.prefix=/api/status - - traefik.http.routers.request-broker-web.rule=Host(`${RB_DNS}`) - - traefik.http.routers.request-broker-web.middlewares=add-statusapi@docker - - traefik.http.routers.request-broker-web.entrypoints=websecure + ## expose available RB Api end entries + - "traefik.http.routers.request-broker-web.rule=(Host(`${RB_DNS}`) && PathRegexp(`^/api/(status|reading-rooms)(/?)$`)) + || (Host(`${RB_DNS}`) && PathRegexp(`^/api/(download-csv)(/.*)$`)) || (Host(`${RB_DNS}`) && (PathPrefix(`${RB_DELV_API}`) || PathPrefix(`${RB_PROC_API}`)))" + - traefik.http.routers.request-broker-web.entrypoints=web,websecure - traefik.http.routers.request-broker-web.tls=true - traefik.http.services.request-broker-web.loadbalancer.server.port=8000 networks: @@ -333,7 +332,6 @@ services: - "${RB_PORT:-8000}:${RB_PORT:-8000}" depends_on: - request-broker-db - - traefik restart: "always" dimes-web: @@ -354,8 +352,8 @@ services: labels: - traefik.port=80 - traefik.enable=true - - traefik.http.routers.dimes-web-https.rule=Host(`myreadingroom.library.pitt.edu`) - - traefik.http.routers.dimes-web-https.entrypoints=websecure + - traefik.http.routers.dimes-web-https.rule=Host(`${RM_DNS}`) + - traefik.http.routers.dimes-web-https.entrypoints=web,websecure - traefik.http.routers.dimes-web-https.tls=true - traefik.http.services.dimes-web-https.loadbalancer.server.port=80 ports: @@ -364,7 +362,6 @@ services: depends_on: - argo-web - request-broker-web - - traefik restart: "always" elasticsearch: @@ -401,7 +398,7 @@ services: - --log.level=INFO - --providers.docker - --api - ##- --api.insecure # only for testing environment + - --api.insecure # only for testing environment - --providers.docker.exposedbydefault=false #entrypoints - --entryPoints.web.address=:80 @@ -409,6 +406,10 @@ services: #cert - --providers.file.directory=/etc/traefik/config #dynamic config - --providers.file.watch=true ## reload any changes + #just apply a generic redirect non-secure instead of configuring every container + - --entrypoints.web.http.redirections.entryPoint.to=websecure + - --entrypoints.web.http.redirections.entryPoint.scheme=https + - --entrypoints.web.http.redirections.entrypoint.permanent=true networks: - astraeus-interop ports: @@ -417,17 +418,12 @@ services: - "8080:8080" volumes: - /var/run/docker.sock:/var/run/docker.sock:ro - - ${CONFIG_PATH}/config/certificates.yml:/etc/traefik/config/certificates.yml + - ${CONFIG_PATH}/config/certificates.yml:/etc/traefik/config/certificates.yml:ro - ${CONFIG_PATH}/config/tls-cert/:/etc/tls-cert/ labels: - "traefik.port=8080" - "traefik.enable=true" - # redirect nonsecure request to https - - "traefik.http.routers.http-request.rule=hostregexp(`{host:.+}`)" - - "traefik.http.routers.http-request.entrypoints=web" - - "traefik.http.routers.http-request.middlewares=redirect-to-https" - - "traefik.http.middlewares.redirect-to-https.redirectscheme.scheme=https" # dashboard - traefik.http.routers.api.entrypoints=websecure - traefik.http.routers.api.rule=Host(`${DASHBOARD_HOST}`) diff --git a/env.template b/env.template index 956fd91..5f3c2f1 100644 --- a/env.template +++ b/env.template @@ -11,3 +11,7 @@ RB_DJANGO_ALLOWED_HOSTS = ['request-broker-web','localhost','requestbroker.libra #Traefik variables CONFIG_PATH=./traefik DASHBOARD_HOST=dashboard.docker.localhost +RM_DNS = 'myreadingroom.library.pitt.edu' +RB_API=/api +RB_DELV_API=/api/deliver-request +RB_PROC_API=/api/process-request From 9735dafd7366d546bded980102e70efe9071a297 Mon Sep 17 00:00:00 2001 From: Ruiling Zhang Date: Mon, 5 Aug 2024 16:59:16 -0400 Subject: [PATCH 3/3] simplied RequestBroker api entries path, corrected env var REACT_APP_CAPTCHA_SITE_KEY --- docker-compose.yml | 5 ++--- env.template | 4 +--- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 4fe91fb..cc34986 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -321,8 +321,7 @@ services: - traefik.port=8000 - traefik.enable=true ## expose available RB Api end entries - - "traefik.http.routers.request-broker-web.rule=(Host(`${RB_DNS}`) && PathRegexp(`^/api/(status|reading-rooms)(/?)$`)) - || (Host(`${RB_DNS}`) && PathRegexp(`^/api/(download-csv)(/.*)$`)) || (Host(`${RB_DNS}`) && (PathPrefix(`${RB_DELV_API}`) || PathPrefix(`${RB_PROC_API}`)))" + - "traefik.http.routers.request-broker-web.rule=(Host(`${RB_DNS}`) && PathRegexp(`^/api/(.*)$`))" - traefik.http.routers.request-broker-web.entrypoints=web,websecure - traefik.http.routers.request-broker-web.tls=true - traefik.http.services.request-broker-web.loadbalancer.server.port=8000 @@ -345,7 +344,7 @@ services: - REACT_APP_MINIMAP_KEY=${REACT_APP_MINIMAP_KEY} - REACT_APP_S3_BASEURL=${REACT_APP_S3_BASEURL} - REACT_APP_EMAIL=${REACT_APP_EMAIL} - - REACT_APP_RECAPCHA_SITE_KEY=${REACT_APP_RECAPCHA_SITE_KEY} + - REACT_APP_CAPTCHA_SITE_KEY=${REACT_APP_CAPTCHA_SITE_KEY} - REACT_APP_AEON_URL=${REACT_APP_AEON_URL} networks: - astraeus-interop diff --git a/env.template b/env.template index 5f3c2f1..af978a5 100644 --- a/env.template +++ b/env.template @@ -7,11 +7,9 @@ SCORPIO_DB_PASS=scorpiopasswordhere # database password used in postgres contain ARGO_DB_PASS=piscespasswordhere # database password used in postgres container, fed to argo and argo cron REQUEST_BROKER_DB_PASS=rbpasswordhere # database password used in postgres container, fed to request broker RB_DJANGO_ALLOWED_HOSTS = ['request-broker-web','localhost','requestbroker.library.pitt.edu'] +REACT_APP_CAPTCHA_SITE_KEY = captchasitekeyvalue # the correct name from old env varible REACT_APP_RECAPTCHA_SITE_KEY #Traefik variables CONFIG_PATH=./traefik DASHBOARD_HOST=dashboard.docker.localhost RM_DNS = 'myreadingroom.library.pitt.edu' -RB_API=/api -RB_DELV_API=/api/deliver-request -RB_PROC_API=/api/process-request