update columnflow and law.cfg

law.cfg

@@ -35,9 +35,13 @@ skip_ensure_proxy: False
 
 chunked_io_chunk_size: 100000
 chunked_io_pool_size: 2
-##chunked_io_debug: True 
+chunked_io_debug: False
 
-time_task_array_functions: False
+# whether to log runtimes of array functions by default
+log_array_function_runtime: False
+
+# Tasks for which outputs will be checked for non-finite values before saving them to disk
+check_finite_output: cf.CalibrateEvents, cf.SelectEvents, cf.ProduceColumns, cf.PrepareMLEvents, cf.MLEvaluation, cf.UniteColumns
 
 [outputs]
 
@@ -68,20 +72,47 @@ lfn_sources: wlcg_fs_desy_store, wlcg_fs_infn_redirector, wlcg_fs_global_redirec
 # cf.MLTraining: wlcg
 # cf.MLEvaluation: wlcg
 
-# NOTE: the law.cfg can in theory be used to set defaults,
-# but this is currently not implemented
-[cf.ReduceEvents]
-no_poll: True
-workflow: htcondor
-dataset: tt_sl_powheg
+# To set defaults on a per-task basis
+# NOTE: this does override defaults defined in the config, but it does not overwrite parameters
+#       when the parameter has already been set e.g. by another task requiring this task
+
+# TODO: to share some outputs over multiple analyses
+# [luigi_cf.GetDatasetLFNs]
+
+# analysis: hbw.analysis.hbw_merged.hbw_merged
+
+
+# [luigi_cf.CalibrateEvents]
+
+# analysis: hbw.analysis.hbw_merged.hbw_merged
+
+
+[luigi_cf.ReduceEvents]
 pilot: True
 
-[cf.MLTraining]
+
+# TODO: it would be good if these parameters could be overwritten even when required from MLTraining
+[luigi_cf.PrepareMLEvents]
+
+htcondor-gpus: 0
+htcondor-memory: -1
+max-runtime: 4h
+
+
+[luigi_cf.MergeMLEvents]
+
+htcondor-gpus: 0
+htcondor-memory: -1
+max-runtime: 4h
+
+
+[luigi_cf.MLTraining]
+
 workflow: htcondor
 htcondor-gpus: 1
 htcondor-memory: 60000
 max-runtime: 72h
-pilot: True
+
 
 [job]
 
@@ -184,9 +215,3 @@ base: &::gsiftp_base
 xrootd_base: root://dcache-cms-xrootd.desy.de:1094/pnfs/desy.de/cms/tier2/store/user/bwieders/hbt_store
 gsiftp_base: gsiftp://dcache-door-cms04.desy.de:2811/pnfs/desy.de/cms/tier2/store/user/bwieders/hbt_store
 base: &::gsiftp_base
-
-
-# NOTE: we want to run the scheduler with multiple cores, but for some reason, this does not work
-[luigi_core]
-parallel_scheduling: False
-parallel_scheduling_processes: 2