Dagster Instance#

$DAGSTER_HOME
├── dagster.yaml
├── history
│   ├── runs
│   │   ├── 00636713-98a9-461c-a9ac-d049407059cd.db
│   │   └── ...
│   └── runs.db
└── storage
    ├── 00636713-98a9-461c-a9ac-d049407059cd
    │   └── compute_logs
    │       ├── my_solid.compute.complete
    │       ├── my_solid.compute.err
    │       ├── my_solid.compute.out
    │       └── ...
    └── ...

# there are two ways to set storage to SqliteStorage

# this config manually sets the directory (`base_dir`) for Sqlite to store data in:
storage:
  sqlite:
    base_dir: /path/to/dir

# and this config grabs the directory from an environment variable
storage:
  sqlite:
    base_dir:
      env: SQLITE_STORAGE_BASE_DIR

# Postgres storage can be set using either credentials or a connection string.  This requires that
# the `dagster-postgres` library be installed.

# this config manually sets the Postgres credentials
storage:
  postgres:
    postgres_db:
      username: { DAGSTER_PG_USERNAME }
      password: { DAGSTER_PG_PASSWORD }
      hostname: { DAGSTER_PG_HOSTNAME }
      db_name: { DAGSTER_PG_DB }
      port: 5432

# and this config grabs the database credentials from environment variables
storage:
  postgres:
    postgres_db:
      username:
        env: DAGSTER_PG_USERNAME
      password:
        env: DAGSTER_PG_PASSWORD
      hostname:
        env: DAGSTER_PG_HOST
      db_name:
        env: DAGSTER_PG_DB
      port: 5432

# and this config sets the credentials via DB connection string / url:
storage:
  postgres:
    postgres_url: { PG_DB_CONN_STRING }

# This config gets the DB connection string / url via environment variables:
storage:
  postgres:
    postgres_url:
      env: PG_DB_CONN_STRING

# MySQL storage can be set using either credentials or a connection string.  This requires that the
# `dagster-mysql` library be installed.

# this config manually sets the MySQL credentials
storage:
  mysql:
    mysql_db:
      username: { DAGSTER_MYSQL_USERNAME }
      password: { DAGSTER_MYSQL_PASSWORD }
      hostname: { DAGSTER_MYSQL_HOSTNAME }
      db_name: { DAGSTER_MYSQL_DB }
      port: 3306


# and this config grabs the database credentials from environment variables
storage:
  mysql:
    mysql_db:
      username:
        env: DAGSTER_MYSQL_USERNAME
      password:
        env: DAGSTER_MYSQL_PASSWORD
      hostname:
        env: DAGSTER_MYSQL_HOSTNAME
      db_name:
        env: DAGSTER_MYSQL_DB
      port: 3306

# and this config sets the credentials via DB connection string / url:
storage:
  mysql:
    mysql_url: { MYSQL_DB_CONN_STRING }

# this config grabs the MySQL connection string from environment variables
storage:
  mysql:
    mysql_url:
      env: MYSQL_DB_CONN_STRING

run_launcher:
  module: dagster.core.launcher
  class: DefaultRunLauncher

run_launcher:
  module: dagster_docker
  class: DockerRunLauncher

# there are multiple ways to configure the K8sRunLauncher

# you can set the follow configuration values directly
run_launcher:
  module: dagster_k8s.launcher
  class: K8sRunLauncher
  config:
    service_account_name: pipeline_run_service_account
    job_image: my_project/dagster_image:latest
    instance_config_map: dagster-instance
    postgres_password_secret: dagster-postgresql-secret

# alternatively, you can grab any of these config values from environment variables:
run_launcher:
  module: dagster_k8s.launcher
  class: K8sRunLauncher
  config:
    service_account_name:
      env: PIPELINE_RUN_SERVICE_ACCOUNT
    job_image:
      env: DAGSTER_IMAGE_NAME
    instance_config_map:
      env: DAGSTER_INSTANCE_CONFIG_MAP
    postgres_password_secret:
      env: DAGSTER_POSTGRES_SECRET

# Since DefaultRunCoordinator is the default option, omitting the `run_coordinator` key will also suffice,
# but if you would like to set it explicitly:
run_coordinator:
  module: dagster.core.run_coordinator
  class: DefaultRunCoordinator

# There are a few ways to configure the QueuedRunCoordinator:

# this first option has concurrency limits set to default values
run_coordinator:
  module: dagster.core.run_coordinator
  class: QueuedRunCoordinator

# this second option manually specifies limits:
run_coordinator:
  module: dagster.core.run_coordinator
  class: QueuedRunCoordinator
  config:
    max_concurrent_runs: 25
    tag_concurrency_limits:
      - key: "database"
        value: "redshift"
        limit: 4
      - key: "dagster/backfill"
        limit: 10

# as always, some or all of these values can be obtained from environment variables:
run_coordinator:
  module: dagster.core.run_coordinator
  class: QueuedRunCoordinator
  config:
    max_concurrent_runs:
      env: DAGSTER_OVERALL_CONCURRENCY_LIMIT
    tag_concurrency_limits:
      - key: "database"
        value: "redshift"
        limit:
          env: DAGSTER_REDSHIFT_CONCURRENCY_LIMIT
      - key: "dagster/backfill"
        limit:
          env: DAGSTER_BACKFILL_CONCURRENCY_LIMIT

# there are two ways to set the directory that the LocalComputeLogManager writes
# stdout & stderr logs to

# You could directly set the `base_dir` key
compute_logs:
  module: dagster.core.storage.local_compute_log_manager
  class: LocalComputeLogManager
  config:
    base_dir: /path/to/directory

# Alternatively, you could set the `base_dir` key to an environment variable
compute_logs:
  module: dagster.core.storage.local_compute_log_manager
  class: LocalComputeLogManager
  config:
    base_dir:
      env: LOCAL_COMPUTE_LOG_MANAGER_DIRECTORY

# there are multiple ways to configure the AzureBlobComputeLogManager

# you can set the necessary configuration values directly:
compute_logs:
  module: dagster_azure.blob.compute_log_manager
  class: AzureBlobComputeLogManager
  config:
    storage_account: mycorp-dagster
    container: compute-logs
    secret_key: foo
    local_dir: /tmp/bar
    prefix: dagster-test-

# alternatively, you can obtain any of these config values from environment variables
compute_logs:
  module: dagster_azure.blob.compute_log_manager
  class: AzureBlobComputeLogManager
  config:
    storage_account:
      env: MYCORP_DAGSTER_STORAGE_ACCOUNT_NAME
    container:
      env: CONTAINER_NAME
    secret_key:
      env: SECRET_KEY
    local_dir:
      env: LOCAL_DIR_PATH
    prefix:
      env: DAGSTER_COMPUTE_LOG_PREFIX

# there are multiple ways to configure the S3ComputeLogManager

# you can set the config values directly:
compute_logs:
  module: dagster_aws.s3.compute_log_manager
  class: S3ComputeLogManager
  config:
    bucket: "mycorp-dagster-compute-logs"
    prefix: "dagster-test-"

# or grab some or all of them from environment variables
compute_logs:
  module: dagster_aws.s3.compute_log_manager
  class: S3ComputeLogManager
  config:
    bucket:
      env: MYCORP_DAGSTER_COMPUTE_LOGS_BUCKET
    prefix:
      env: DAGSTER_COMPUTE_LOG_PREFIX

# there are two possible ways to configure LocalArtifactStorage

# example local_artifact_storage setup pointing to /var/shared/dagster directory
local_artifact_storage:
  module: dagster.core.storage.root
  class: LocalArtifactStorage
  config:
    base_dir: "/path/to/dir"

# alternatively, `base_dir` can be set to an environment variable
local_artifact_storage:
  module: dagster.core.storage.root
  class: LocalArtifactStorage
  config:
    base_dir:
      env: DAGSTER_LOCAL_ARTIFACT_STORAGE_DIR

# Allows opting out of Dagster collecting usage statistics.
telemetry:
  enabled: false

# Configures how long Dagster waits for repositories
# to load before timing out.
code_servers:
  local_startup_timeout: 120

# Configures how long Dagster keeps sensor / schedule tick data
retention:
  schedule:
    purge_after_days: 90 # sets retention policy for schedule ticks of all types
  sensor:
    purge_after_days:
      skipped: 7
      failure: 30
      success: -1 # keep success ticks indefinitely

sensors:
  use_threads: true
  num_workers: 8