docker-compose.yml

x-airflow-common: &airflow-common
  build:
    context: ./airflow
  image: sagerx_airflow:v0.0.7 # versioning allows a rebuild of docker image where necessary
  networks:
      - airflow-dbt-network
  env_file:
      - .env
  environment: &airflow-common-env
    AIRFLOW__CORE__EXECUTOR: LocalExecutor
    AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: "true"
    AIRFLOW__CORE__LOAD_EXAMPLES: "false"
    #AIRFLOW__WEBSERVER__WEB_SERVER_MASTER_TIMEOUT: "300" # un-comment if gunicorn timeout reached
    AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres:5432/airflow
    AIRFLOW_CONN_POSTGRES_DEFAULT: postgresql://sagerx:sagerx@postgres:5432/sagerx
    AIRFLOW_VAR_UMLS_API: ${UMLS_API}
    AIRFLOW_VAR_SLACK_API: ${SLACK_API:-}
    AIRFLOW_CONN_SLACK: http://:@https%3A%2F%2Fhooks.slack.com%2Fservices%2F${SLACK_API:-}
    _PIP_ADDITIONAL_REQUIREMENTS: ""
    DBT_PROFILES_DIR: /dbt
    AIRFLOW_CONN_GOOGLE_CLOUD_DEFAULT: google-cloud-platform://?key_path=/opt/gcp.json
  user: "${AIRFLOW_UID:-50000}:${AIRFLOW_GID:-0}"
  volumes:
    - ./airflow/dags:/opt/airflow/dags
    - ./airflow/logs:/opt/airflow/logs
    - ./airflow/plugins:/opt/airflow/plugins
    - ./airflow/data:/opt/airflow/data
    - ./airflow/config/airflow.cfg:/opt/airflow/airflow.cfg
    - ./dbt:/dbt
    - ./gcp.json:/opt/gcp.json
    - /var/run/docker.sock:/var/run/docker.sock #not the most secure way of connecting
services:
  postgres:
    image: postgres:14-alpine
    networks:
      - airflow-dbt-network
    container_name: postgres
    environment:
      POSTGRES_USER: sagerx
      PGUSER: sagerx
      POSTGRES_PASSWORD: sagerx
      POSTGRES_DB: sagerx
    ports:
      - 5432:5432
    volumes:
      - ./postgres:/docker-entrypoint-initdb.d
      - ./airflow/data:/opt/airflow/data
      - ./airflow/extracts:/opt/airflow/extracts
    build:
      context: .
      shm_size: "4gb"

  pgadmin:
    image: dpage/pgadmin4:6.15
    networks:
      - airflow-dbt-network
    container_name: pgadmin
    environment:
      PGADMIN_DEFAULT_EMAIL: pgadmin@pgadmin.org
      PGADMIN_DEFAULT_PASSWORD: pgadmin
      PGADMIN_CONFIG_SERVER_MODE: "False"
      PGADMIN_CONFIG_MASTER_PASSWORD_REQUIRED: "False"
    ports:
      - 8002:80
    volumes:
      - ./pgadmin/servers.json:/pgadmin4/servers.json

  dbt:
    build:
      context: ./dbt
    env_file:
      - .env
    environment:
      AIRFLOW_CONN_GOOGLE_CLOUD_DEFAULT: google-cloud-platform://?key_path=/opt/gcp.json
    image: dbt:v0.0.5
    networks:
      - airflow-dbt-network
    container_name: dbt
    user: "${AIRFLOW_UID:-50000}:${AIRFLOW_GID:-0}"
    ports:
      - 8081:8081
    volumes:
      - ./dbt:/dbt
      - ./gcp.json:/opt/gcp.json
    command: tail -f /dev/null

  airflow-init:
    <<: *airflow-common
    container_name: airflow-init
    entrypoint: /bin/bash
    command:
      - -c
      - |
        mkdir -p /sources/logs /sources/dags /sources/plugins /sources/data /sources/extracts
        chown -R "${AIRFLOW_UID}:${AIRFLOW_GID-0}" /sources/{logs,dags,plugins,data,extracts}
        chmod -R o+rw /sources/extracts
        exec /entrypoint airflow version
    environment:
      <<: *airflow-common-env
      _AIRFLOW_DB_UPGRADE: "true"
      _AIRFLOW_WWW_USER_CREATE: "true"
      _AIRFLOW_WWW_USER_USERNAME: ${_AIRFLOW_WWW_USER_USERNAME:-airflow}
      _AIRFLOW_WWW_USER_PASSWORD: ${_AIRFLOW_WWW_USER_PASSWORD:-airflow}
    user: "0:0"
    volumes:
      - ./airflow:/sources
    depends_on:
      - postgres

  airflow-webserver:
    <<: *airflow-common
    container_name: airflow-webserver
    command: webserver
    environment:
      <<: *airflow-common-env
      AWS_ACCESS_KEY_ID: ${ACCESS_KEY}
      AWS_SECRET_ACCESS_KEY: ${SECRET_ACCESS_KEY}
      AWS_DEST_BUCKET: ${DEST_BUCKET}
    ports:
      - 8001:8080

  airflow-scheduler:
    <<: *airflow-common
    container_name: airflow-scheduler
    command: scheduler
    environment:
      <<: *airflow-common-env
      AWS_ACCESS_KEY_ID: ${ACCESS_KEY}
      AWS_SECRET_ACCESS_KEY: ${SECRET_ACCESS_KEY}
      AWS_DEST_BUCKET: ${DEST_BUCKET}

networks:
  airflow-dbt-network:
    driver: bridge