Restore Airflow limits to defaults

Author: DouweM

plugins/airflow/airflow.cfg

@@ -27,7 +27,7 @@ executor = LocalExecutor
 # Airflow, regardless of the worker count. Generally this value, multiplied by the number of
 # schedulers in your cluster, is the maximum number of task instances with the running
 # state in the metadata database.
-parallelism = 16
+parallelism = 32
 
 # The maximum number of task instances allowed to run concurrently in each DAG. To calculate
 # the number of tasks that is running concurrently for a DAG, add up the number of running
@@ -76,7 +76,7 @@ dagbag_import_error_tracebacks = True
 dagbag_import_error_traceback_depth = 2
 
 # How long before timing out a DagFileProcessor, which processes a dag file
-dag_file_processor_timeout = 90
+dag_file_processor_timeout = 50
 
 # The class to use for running task instances in a subprocess.
 # Choices include StandardTaskRunner, CgroupTaskRunner or the full import path to the class
@@ -926,11 +926,11 @@ scheduler_idle_sleep_time = 1
 # Number of seconds after which a DAG file is parsed. The DAG file is parsed every
 # ``min_file_process_interval`` number of seconds. Updates to DAGs are reflected after
 # this interval. Keeping this number low will increase CPU usage.
-min_file_process_interval = 900
+min_file_process_interval = 30
 
 # How often (in seconds) to check for stale DAGs (DAGs which are no longer present in
 # the expected files) which should be deactivated.
-deactivate_stale_dags_interval = 900
+deactivate_stale_dags_interval = 60
 
 # How often (in seconds) to scan the DAGs directory for new files. Default to 5 minutes.
 dag_dir_list_interval = 300