[fix] Improved Celery worker shutdown handling #694

.gitignore

@@ -82,3 +82,4 @@ local/
 customization/*
 !customization/configuration/django/__init__.py
 !customization/configuration/django/custom_django_settings.example.py
+docker-compose.override.yml

docker-compose.yml

@@ -75,6 +75,8 @@ services:
 
   celery:
     image: openwisp/openwisp-dashboard:${OPENWISP_VERSION:-edge}
+    # Keep this longer than Supervisor's stopwaitsecs to let it reap workers and exit.
+    stop_grace_period: 120s
     restart: always
     environment:
       - MODULE_NAME=celery
@@ -90,6 +92,8 @@ services:
 
   celery_monitoring:
     image: openwisp/openwisp-dashboard:${OPENWISP_VERSION:-edge}
+    # Keep this longer than Supervisor's stopwaitsecs to let it reap workers and exit.
+    stop_grace_period: 120s
     restart: always
     environment:
       - MODULE_NAME=celery_monitoring

images/common/celery_supervisor.sh

@@ -0,0 +1,49 @@
+#!/bin/sh
+
+add_celery_worker() {
+	local name=$1 queue=$2 flags=$3
+	local command="/usr/local/bin/celery -A openwisp worker -l ${DJANGO_LOG_LEVEL} --queues ${queue} -n ${name}@%h ${flags}"
+	command=$(printf '%s' "$command" | sed 's/%/%%/g')
+	CELERY_SUPERVISOR_WORKERS="${CELERY_SUPERVISOR_WORKERS:+${CELERY_SUPERVISOR_WORKERS},}${name}"
+	cat >>"$CELERY_SUPERVISOR_WORKERS_CONF" <<EOF
+[program:${name}]
+command=${command}
+directory=/opt/openwisp
+autostart=true
+autorestart=true
+stopsignal=TERM
+; Leave time for Supervisor to exit before Docker's 120-second grace period ends.
+stopwaitsecs=90
+stopasgroup=false
+killasgroup=true
+redirect_stderr=true
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+EOF
+}
+
+generate_celery_supervisor_config() {
+	CELERY_SUPERVISOR_WORKERS_CONF=${1:-celery_supervisord.d/workers.conf}
+	mkdir -p "$(dirname "$CELERY_SUPERVISOR_WORKERS_CONF")"
+	: >"$CELERY_SUPERVISOR_WORKERS_CONF"
+	CELERY_SUPERVISOR_WORKERS=
+	if [ "$MODULE_NAME" = 'celery' ]; then
+		add_celery_worker celery celery "${OPENWISP_CELERY_COMMAND_FLAGS}"
+		if [ "$USE_OPENWISP_CELERY_NETWORK" = 'True' ]; then
+			add_celery_worker network network "${OPENWISP_CELERY_NETWORK_COMMAND_FLAGS}"
+		fi
+		if [[ "$USE_OPENWISP_FIRMWARE" == 'True' && "$USE_OPENWISP_CELERY_FIRMWARE" == 'True' ]]; then
+			add_celery_worker firmware_upgrader firmware_upgrader "${OPENWISP_CELERY_FIRMWARE_COMMAND_FLAGS}"
+		fi
+	else
+		add_celery_worker monitoring monitoring "${OPENWISP_CELERY_MONITORING_COMMAND_FLAGS}"
+		add_celery_worker monitoring_checks monitoring_checks "${OPENWISP_CELERY_MONITORING_CHECKS_COMMAND_FLAGS}"
+	fi
+	cat >>"$CELERY_SUPERVISOR_WORKERS_CONF" <<EOF
+
+; Stop workers together so each receives TERM in the same Supervisor shutdown pass.
+[group:workers]
+programs=${CELERY_SUPERVISOR_WORKERS}
+priority=100
+EOF
+}

images/common/celery_supervisord.conf

@@ -0,0 +1,17 @@
+[unix_http_server]
+file=/opt/openwisp/supervisor/supervisor.sock
+chmod=0700
+
+[supervisorctl]
+serverurl=unix:///opt/openwisp/supervisor/supervisor.sock
+
+[rpcinterface:supervisor]
+supervisor.rpcinterface_factory=supervisor.rpcinterface:make_main_rpcinterface
+
+[supervisord]
+logfile=/dev/stdout
+logfile_maxbytes=0
+pidfile=/opt/openwisp/supervisor/supervisord.pid
+
+[include]
+files=/opt/openwisp/supervisor/conf.d/*.conf

images/common/init_command.sh

@@ -76,51 +76,24 @@ elif [ "$MODULE_NAME" = 'nginx' ]; then
 	nginx -g 'daemon off;'
 elif [ "$MODULE_NAME" = 'celery' ]; then
 	python services.py database redis dashboard
-	echo "Starting the 'default' celery worker"
-	celery -A openwisp worker -l ${DJANGO_LOG_LEVEL} --queues celery \
-		-n celery@%h --logfile /opt/openwisp/logs/celery.log \
-		--pidfile /opt/openwisp/celery.pid --detach \
-		${OPENWISP_CELERY_COMMAND_FLAGS}
-
-	if [ "$USE_OPENWISP_CELERY_NETWORK" = "True" ]; then
-		echo "Starting the 'network' celery worker"
-		celery -A openwisp worker -l ${DJANGO_LOG_LEVEL} --queues network \
-			-n network@%h --logfile /opt/openwisp/logs/celery_network.log \
-			--pidfile /opt/openwisp/celery_network.pid --detach \
-			${OPENWISP_CELERY_NETWORK_COMMAND_FLAGS}
-	fi
-
-	if [[ "$USE_OPENWISP_FIRMWARE" == "True" && "$USE_OPENWISP_CELERY_FIRMWARE" == "True" ]]; then
-		echo "Starting the 'firmware_upgrader' celery worker"
-		celery -A openwisp worker -l ${DJANGO_LOG_LEVEL} --queues firmware_upgrader \
-			-n firmware_upgrader@%h --logfile /opt/openwisp/logs/celery_firmware_upgrader.log \
-			--pidfile /opt/openwisp/celery_firmware_upgrader.pid --detach \
-			${OPENWISP_CELERY_FIRMWARE_COMMAND_FLAGS}
-	fi
-	sleep 1s
-	tail -f /opt/openwisp/logs/*
+	source celery_supervisor.sh
+	generate_celery_supervisor_config /opt/openwisp/supervisor/conf.d/workers.conf
+	# Keep Supervisor as PID 1 so it receives Docker stop signals.
+	exec supervisord --nodaemon
 elif [ "$MODULE_NAME" = 'celery_monitoring' ]; then
 	python services.py database redis dashboard
 	if [[ "$USE_OPENWISP_MONITORING" == "True" && "$USE_OPENWISP_CELERY_MONITORING" == 'True' ]]; then
-		echo "Starting the 'monitoring' celery worker"
-		celery -A openwisp worker -l ${DJANGO_LOG_LEVEL} --queues monitoring \
-			-n monitoring@%h --logfile /opt/openwisp/logs/celery_monitoring.log \
-			--pidfile /opt/openwisp/celery_monitoring.pid --detach \
-			${OPENWISP_CELERY_MONITORING_COMMAND_FLAGS}
-		echo "Starting the 'monitoring_checks' celery worker"
-		celery -A openwisp worker -l ${DJANGO_LOG_LEVEL} --queues monitoring_checks \
-			-n monitoring_checks@%h --logfile /opt/openwisp/logs/celery_monitoring_checks.log \
-			--pidfile /opt/openwisp/celery_monitoring_checks.pid --detach \
-			${OPENWISP_CELERY_MONITORING_CHECKS_COMMAND_FLAGS}
-		sleep 1s
-		tail -f /opt/openwisp/logs/*
+		source celery_supervisor.sh
+		generate_celery_supervisor_config /opt/openwisp/supervisor/conf.d/workers.conf
+		# Keep Supervisor as PID 1 so it receives Docker stop signals.
+		exec supervisord --nodaemon
 	else
 		echo "Monitoring queues are not activated, exiting."
 	fi
 elif [ "$MODULE_NAME" = 'celerybeat' ]; then
 	rm -rf celerybeat.pid
 	python services.py database redis dashboard
-	celery -A openwisp beat -l ${DJANGO_LOG_LEVEL}
+	exec celery -A openwisp beat -l ${DJANGO_LOG_LEVEL}
 else
 	python services.py database redis dashboard
 	start_uwsgi

images/openwisp_base/Dockerfile

@@ -89,6 +89,7 @@ FROM system
 
 COPY --from=openwisp_python --chown=openwisp:root /home/openwisp/.local/ /usr/local
 COPY --chown=openwisp:root ./common/ /opt/openwisp/
+COPY ./common/celery_supervisord.conf /etc/supervisor/supervisord.conf
 RUN mkdir /opt/openwisp/static && \
     mkdir /opt/openwisp/media && \
     mkdir /opt/openwisp/private && \