使用 CeleryExecutor 时 Airflow Worker 不工作

Question

我正在尝试使用 Docker 和 Redis 从 Airflow 2.1.3 中的 LocalExecutor 迁移到 CeleryExecutor。我为网络服务器、调度程序、worker、redis 和数据库制作了单独的容器。问题：任务已排队但未执行。

docker-compose.yml：

version: "3.3"

services:
  redis:
    image: redis:6.0.9-alpine
    healthcheck:
    test: ["CMD", "redis-cli", "ping"]
    interval: 5s
    timeout: 30s
    retries: 50
    restart: always
    ports:
      - "6793:6793"

  database:
    image: postgres:12-alpine
    restart: always
    environment:
      POSTGRES_DB: airflow
      POSTGRES_USER: airflow
      POSTGRES_PASSWORD: airflow
    volumes:
      - airflow_database:/var/lib/postgresql/data/

  webserver:
    image: airflow:latest
    restart: always
    depends_on:
      - database
      - redis
    environment:
      AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@database/airflow
      GUNICORN_CMD_ARGS: --log-level WARNING
      EXECUTOR: Celery
    volumes:
      - airflow_logs:/var/log/airflow
      - airflow_data:/var/spool/airflow
      - ./airflow/dags:/usr/local/airflow/dags
      - ./airflow/plugins:/usr/local/airflow/plugins
    ports:
      - 8080:8080
      - 8888:8888
    command: webserver
    healthcheck:
      test: ["CMD-SHELL", "[ -f /usr/local/airflow/airflow-webserver.pid ]"]
      interval: 30s
      timeout: 30s
      retries: 3

  flower:
    image: airflow:latest
    restart: always
    depends_on:
      - redis
    environment:
      AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@database/airflow
      EXECUTOR: Celery
    ports:
      - "5555:5555"
    command: celery flower -b "redis://redis:6379/1"

  scheduler:
    image: airflow:latest
    restart: always
    depends_on:
      - webserver
    volumes:
      - airflow_logs:/var/log/airflow
      - airflow_data:/var/spool/airflow
      - ./airflow/dags:/usr/local/airflow/dags
      - ./airflow/plugins:/usr/local/airflow/plugins
    environment:
      AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@database/airflow
      EXECUTOR: Celery
    command: scheduler

  worker:
    image: airflow:latest
    restart: always
    depends_on:
      - scheduler
    volumes:
      - airflow_logs:/var/log/airflow
      - airflow_data:/var/spool/airflow
      - ./airflow/dags:/usr/local/airflow/dags
      - ./airflow/plugins:/usr/local/airflow/plugins
    environment:
      AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@database/airflow
      EXECUTOR: Celery
    healthcheck:
    test:
      - "CMD-SHELL"
      - 'celery --app airflow.executors.celery_executor.app inspect ping -d "celery@$${HOSTNAME}"'
    interval: 10s
    timeout: 10s
    retries: 5
    command: celery worker -b "redis://redis:6379/1" --result-backend "db+postgresql://airflow:airflow@database/airflow"

volumes:
  airflow_database:
  airflow_data:
  airflow_logs:
  staging_database:

Dockerfile，airflow.cfg，entrypoint.sh

所有容器均正常加载。我尝试执行 celery_result_backend == Broker_url == 'redis://redis:6379/1' 但无济于事。花显示了工作人员本身创建的内容，但工作人员容器不显示一行日志。我还尝试单独使用工作容器 - 它没有帮助。

花

Answer 1

正如我所看到的，Redis 存在明显的端口号不一致问题。
上方为“6793”，下方为“6379”