Add a script to supervise template fitting in pycbc live (#4813)

* Move code which will be shared into a module

* getting more things ready for the trigger fits / dq supervision

* start work on fit_by, fit_over and DQ trigger supervision

* some work - bring in latest trigger file finding

* updates to use latest trigger file finding

* pass cuts through to the collation script

* symlink fit_over_multiparam output to variable file

* remove unused imports, argument ordering, plot titles

* CC

* Fix docstring, formatting

* Move the trigger collatiojn/fitting superviosor to use configparsert

* start moving significance fit supervision to configparser

* Start puttinf template fits into supervision code shared with significance fits

* Deal properly with whene there are no trigers, so we can't calculate the median_sigma

* Remove no-longer-valid efficiency savings

* Some minor fixes for when sending mail from supervision

* Mve to a single supervision file for both trigger and significance fits

* Remove FIXMEs

* Use check=True with subprocess.run

* Remove function checking fit coefficients - it is now unclear what the 'safe' ranges should be for the various different stages

* Fix minor bug

* typo

* Some minor fixes and TDC comments

* CC

bin/all_sky_search/pycbc_fit_sngls_by_template

@@ -352,6 +352,9 @@ sigma_regions = trigf[args.ifo + '/sigmasq_template'][:]
 median_sigma = []
 for reg in sigma_regions:
     strigs = trigf[args.ifo + '/sigmasq'][reg]
+    if len(strigs) == 0:
+        median_sigma.append(np.nan)
+        continue
     median_sigma.append(np.median(strigs) ** 0.5)
 
 outfile = HFile(args.output, 'w')

bin/all_sky_search/pycbc_fit_sngls_over_multiparam

@@ -342,14 +342,20 @@ if len(args.template_fit_file) > 1:
     nasum = nabove[tidsort].cumsum()
     invsum = invalphan[tidsort].cumsum()
     ntsum = ntotal[tidsort].cumsum()
-    mssum = median_sigma[tidsort].cumsum()
     num = right - left
 
     tid = tid_unique
     nabove = (nasum[right] - nasum[left]) / num
     invalphan = (invsum[right] - invsum[left]) / num
     ntotal = (ntsum[right] - ntsum[left]) / num
-    median_sigma = (mssum[right] - mssum[left]) / num
+    if median_sigma is not None:
+        # Median sigma is a special one - we need to make sure that
+        # we do not mess things up when nan values are given, so we
+        # can't use the special cumsum fast option
+        median_sigma = [
+            numpy.nanmean(median_sigma[tidsort[l:r]])
+            for l, r in zip(left, right)
+        ]
 
 if args.output_fits_by_template:
     # Store fit_by_template values for output file

bin/live/pycbc_live_plot_single_significance_fits

@@ -131,10 +131,10 @@ for ifo in all_ifos:
         continue
 
     # Keep track of some maxima for use in setting the plot limits
-    maxstat = stats[ifo].max()
+    maxstat = np.nanmax(stats[ifo])
     max_rate = 0
 
-    statrange = maxstat - max(stats[ifo].min(), fit_threshold[ifo])
+    statrange = maxstat - max(np.nanmin(stats[ifo]), fit_threshold[ifo])
     plotmax = maxstat + statrange * 0.05
 
     plotbins = np.linspace(fit_threshold[ifo], plotmax, 400)

bin/live/pycbc_live_single_significance_fits

@@ -98,13 +98,6 @@ args.trigger_cuts = args.trigger_cuts or []
 args.trigger_cuts.append(f"end_time:{args.gps_start_time}:lower_inc")
 args.trigger_cuts.append(f"end_time:{args.gps_end_time}:upper_inc")
 
-# Efficiency saving: add SNR cut before any others as sngl_ranking can
-# only be less than SNR.
-args.trigger_cuts.insert(0, f"snr:{args.fit_threshold}:lower_inc")
-
-# Cut triggers with sngl-ranking below threshold
-args.trigger_cuts.append(f"{args.sngl_ranking}:{args.fit_threshold}:lower_inc")
-
 trigger_cut_dict, template_cut_dict = cuts.ingest_cuts_option_group(args)
 
 logging.info("Setting up duration bins")