docker-compose.yml 3.8 KB
Newer Older
1
2
version: '3.7'
services:
3
    airflow_webserver:
Taylor Murphy's avatar
Taylor Murphy committed
4
        image: registry.gitlab.com/gitlab-data/data-image/airflow-image:v0.0.9
5
        restart: always
6
7
        ports:
          - "8080:8080"
8
        depends_on:
9
10
          - airflow_db
          - airflow_scheduler
11
12
13
        environment:
          AIRFLOW__CORE__DAGS_FOLDER: /usr/local/airflow/analytics/dags/
          NAMESPACE: testing
14
15
          IN_CLUSTER: "False"
          GOOGLE_APPLICATION_CREDENTIALS: /root/gcp_service_creds.json
16
          GIT_BRANCH: ${GIT_BRANCH}
17
          AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql://postgres:postgres@airflow_db
18
19
        command: bash -c "airflow webserver"
        volumes:
20
          - type: bind
21
            source: ${HOME}/.kube/config
22
23
24
25
26
            target: /root/.kube/config
          - type: bind
            source: ${GOOGLE_APPLICATION_CREDENTIALS}
            target: /root/gcp_service_creds.json
            read_only: True
27
28
29
          - type: volume
            source: airflow_logs
            target: /usr/local/airflow/logs
30
31
32
33
          - type: bind
            source: .
            target: /usr/local/airflow/analytics
            read_only: True
34

35
    airflow_scheduler:
Taylor Murphy's avatar
Taylor Murphy committed
36
        image: registry.gitlab.com/gitlab-data/data-image/airflow-image:v0.0.9
37
        restart: always
38
39
        ports:
          - "8793:8793"
40
        depends_on:
41
          - airflow_db
42
43
44
        environment:
          AIRFLOW__CORE__DAGS_FOLDER: /usr/local/airflow/analytics/dags/
          NAMESPACE: testing
45
46
          IN_CLUSTER: "False"
          GOOGLE_APPLICATION_CREDENTIALS: /root/gcp_service_creds.json
47
          GIT_BRANCH: ${GIT_BRANCH}
48
          AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql://postgres:postgres@airflow_db
49
        command: bash -c "airflow scheduler"
50
        volumes:
51
          - type: bind
52
            source: ${HOME}/.kube/config
53
54
55
56
57
            target: /root/.kube/config
          - type: bind
            source: ${GOOGLE_APPLICATION_CREDENTIALS}
            target: /root/gcp_service_creds.json
            read_only: True
58
59
60
          - type: volume
            source: airflow_logs
            target: /usr/local/airflow/logs
61
62
63
64
          - type: bind
            source: .
            target: /usr/local/airflow/analytics
            read_only: True
65

66
    airflow_db:
67
68
69
70
71
72
73
74
75
      image: postgres:9.6
      environment:
        POSTGRES_USER: postgres
        POSTGRES_PASSWORD: postgres
        POSTGRES_PORT: 5432
      ports:
        - "5432:5432"
      volumes:
        - pgdata:/var/lib/postgresql/data
76

77
    data_image:
Paul Armstrong's avatar
Paul Armstrong committed
78
      image: registry.gitlab.com/gitlab-data/data-image/data-image:v0.0.15
79
80
      restart: always
      command: bash -c "/bin/bash"
81
      working_dir: /usr/local/analytics/
82
83
84
85
86
      volumes:
        - type: bind
          source: .
          target: /usr/local/analytics

87
    dbt_image:
Justin Stark's avatar
Justin Stark committed
88
      image: registry.gitlab.com/gitlab-data/data-image/dbt-image:v0.0.15
89
      environment:
90
        DBT_PROFILES_DIR: /usr/local/dbt_profiles/
91
        DATA_TEST_BRANCH: "main"
92
        DATA_SIREN_BRANCH: "master"
93
        SNOWFLAKE_SNAPSHOT_DATABASE: "SNOWFLAKE"
94
        SNOWFLAKE_LOAD_DATABASE: "RAW"
Taylor Murphy's avatar
Taylor Murphy committed
95
96
97
        SNOWFLAKE_PREP_DATABASE: "PREP"
        SNOWFLAKE_PREP_SCHEMA: "preparation"
        SNOWFLAKE_PROD_DATABASE: "PROD"
98
        SNOWFLAKE_TRANSFORM_WAREHOUSE: "ANALYST_XS"
99
100
101
102
        SALT: "pizza"
        SALT_IP: "pie"
        SALT_NAME: "pepperoni"
        SALT_EMAIL: "cheese"
103
        SALT_PASSWORD: "416C736F4E6F745365637265FFFFFFAB"
104
105
106
107
108
109
110
      restart: always
      command: bash -c "/bin/bash"
      working_dir: /usr/local/analytics/transform/snowflake-dbt/
      volumes:
        - type: bind
          source: .
          target: /usr/local/analytics
111
        - type: bind
112
          source: ${HOME}/.dbt
Taylor A. Murphy's avatar
Taylor A. Murphy committed
113
          target: /usr/local/dbt_profiles/
114
          read_only: True
Taylor Murphy's avatar
Taylor Murphy committed
115
116
117
        - type: bind
          source: ${HOME}/.ssh
          target: /root/.ssh
118

119
120
volumes:
  airflow_logs:
121
  pgdata: