Merge pull request #287 from WireCell/spdir-metric

Spdir metric

aux/docs/frame-files.org

@@ -53,26 +53,36 @@ summary_<tagA>_<ident1>.npy
 frame_<tagB>_<ident1>.npy
 channels_<tagB>_<ident1>.npy
 tickinfo_<tagB>_<ident1>.npy
+frame__<ident1>.npy
+channels__<ident1>.npy
+tickinfo__<ident1>.npy
+frame_*_<ident1>.npy
+channels_*_<ident1>.npy
+tickinfo_*_<ident1>.npy
 chanmask_<cmtagC>_<ident1>.npy
 chanmask_<cmtagD>_<ident1>.npy
 ...
 #+end_example
 
 We will call "framelet" the trio of arrays of type ~frame~, ~channels~ and
 ~tickinfo~ and an optional fourth ~summary~ array that carry the same
-~<tag>~ and frame ~<ident>~.  To be valid, the size of the 1D ~channels~
-array, and the summary array if present, must be the same as the
-number of (channel) rows in the ~frame~ array.  The first two entries
-(~time~ and ~tick~) of all ~tickinfo~ are expected to be identical but their
-third (~tbin0~) may differ for each framelet.  See [[file:tickinfo.org]].
-
-The rows of the ~frame~ and ~channel~ arrays are added to the ~IFrame~
-traces collection in the order they are encountered in the file.
-Otherwise, order does not matter except that all arrays for a given
-frame ~<ident>~ must be contiguous.  The ~<tag>~ portion of the framelet
-array file names is used to add to the ~IFrame~ trace map map.  In the
-current version of the file, there is no support for representing
-tagged trace indices.
+~<tag>~ and frame ~<ident>~.
+
+To be valid, the size of the 1D ~channels~ array, and the summary array if
+present, must be the same as the number of (channel) rows in the ~frame~ array.
+The first two entries (~time~ and ~tick~) of all ~tickinfo~ must be be identical but
+their third entry (~tbin0~) may differ for each framelet.  See [[file:tickinfo.org]].
+
+The rows of the ~frame~ array and the corresponding elements of the ~channel~ array
+are added to the ~IFrame~ traces collection in the order they are encountered in
+the file.  Otherwise, order does not matter except that all arrays for a given
+frame ~<ident>~ must be contiguous.
+
+The ~<tag>~ portion of the framelet provides a *trace tag* (and *not* a frame tag).
+When the ~<tag>~ is unspecified (ie, there are a pair of subsequent underscores,
+eg ~frame__0~~) or when the ~<tag>~ takes a special value of ~*~ (eg ~frame_*_0~) then
+the arrays are interpreted to represent an portion of the frame consisting of
+untagged traces.  In such a case, any associated ~summary_~ is ignored.
 
 The ~chanmask~ arrays are optional and each provides one entry in the
 ~IFrame~ channel mask map.  The key is take as the ~<cmtag>~ part of the

cfg/layers/README.org

@@ -3,13 +3,13 @@
 
 * TODO
 
+- [ ] move the emerging common pattern of ~mids/*/variants.jsonnet~ files into ~mids.jsonnet~.
 - [ ] add ~mains~ section
 - [ ] add ~options~
 - [ ] generate dots by cross product of detector type, variants and mains
 - [ ] add ~uboone~
 - [ ] remove ~helpers~
 
-
 * Introduction
 
 The WCT configuration structure layers convention defines a Jsonnet

cfg/layers/gen-mids.sh

@@ -17,10 +17,16 @@ cat <<EOF > "$out"
 EOF
 
 # this loop should keep path relative
-for one in mids/*/mids.jsonnet
+for api in mids/*/api.jsonnet
 do
-    det=$(basename $(dirname $one))
-    echo "    $det : import \"$one\"," >> "$out"
+    det=$(basename $(dirname $api))
+    var="$(dirname $api)/variants.jsonnet"
+    cat <<EOF >> "$out"
+    $det : {
+        api: import "$api",
+        variants: import "$var",
+    },
+EOF
 done
 echo "}" >> "$out"
 

cfg/layers/gengeom.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/bash
+me="$(realpath "$BASH_SOURCE")"
+mydir="$(dirname "$me")"
+
+cd "$mydir"
+
+for det in base pdsp uboone
+do
+    wirecell-util wires-volumes $det \
+        | jsonnetfmt -n 4 --max-blank-lines 1 - \
+                     > $mydir/mids/$det/variants/geometry.jsonnet
+done

cfg/layers/high.jsonnet

@@ -4,12 +4,11 @@
 
 // A lookup by detector and variant.  See gen-mids.sh.
 local mids = import "mids.jsonnet";
+local base = import "mids/base/api.jsonnet";
 local svcs = import "high/svcs.jsonnet";
 
 local midapi = import "midapi.jsonnet";
 
-
-
 {
     // Forward the various "standard" wirecell utillities as a
     // convenience to end user.
@@ -30,9 +29,50 @@ local midapi = import "midapi.jsonnet";
     // the mid.
     services :: svcs,
 
-    // The mid-level API factory function.
-    mid :: function(detector, variant="nominal", services=svcs(), options={})
-        midapi + mids[detector](services, variant, options=options),
+    // All known mid-layer detectors
+    mids :: mids,
+
+    // The parameter object factory.
+    // 
+    // The parameter object is selected from the detector's mid-level
+    // variants.jsonnet structure based on the variant name.
+    //
+    // All three arguments are saved into the selected parameter object with a
+    // key formed by appending "_name", eg "detector_name".
+    //
+    // The detector mid API may use "params.structure_name" to supply an API
+    // that results in variant DFP graph structure independent from the chosen
+    // parameter variant represented by "params.variant_name".
+    //
+    // The following variant and structure names SHALL be implemented according
+    // to these guiding definitions:
+    //
+    // - nominal :: In name only.  This is for defining an idealized
+    // configuration lacking variance.  It shall avoid introducing magic numbers
+    // such as arbitrary time offsets, bad or per channel variance and shall
+    // consider a uniform detector and resulting DFP graph structure.
+    // 
+    // - actual :: As close to the needs of realism as can be.  This
+    // configuration shall endeavor to produce a configuration that model the
+    // real world detector to the extent possible.  Strive to construct 
+    // "actual" parameter variant and structure that derive from nominal.
+    //
+    // Other variant and structure names are allowed for special purposes.
+    params :: function(detector, variant="nominal", structure=null)
+        mids[detector].variants[variant] {
+            detector_name: detector,
+            variant_name: variant,
+            structure_name: if std.type(structure) == "null" then variant else structure,
+        },
+
+    // Return a mid-level API implementation inside a midapi API.
+    api :: function(detector, params, services=svcs(), options={})
+        local sv = svcs();      //  fixme: need a way to choose eg GPU svcs
+        //local base = midapi(sv, params, options=options);
+        local def = base(sv, params, options=options);
+        local det = mids[detector].api(sv, params, options=options);
+        local imp = std.mergePatch(def, det);
+        midapi(imp),
 }
 
 

cfg/layers/low.jsonnet

@@ -6,9 +6,15 @@
 
     util : import "low/util.jsonnet",
     gen : import "low/gen.jsonnet",
-    drifter : import "low/drifter.jsonnet",
+
     anodes : import "low/anodes.jsonnet",
+
+    drifter : import "low/drifter.jsonnet",
+    reframer : import "low/reframer.jsonnet",
     dnnroi : import "low/dnnroi.jsonnet",
 
     img : import "low/img.jsonnet",
+
+    resps : import "low/resps.jsonnet",
+    ssss : import "low/ssss.jsonnet",
 }

cfg/layers/low/drifter.jsonnet

@@ -27,15 +27,17 @@ function(random, xregions, lar, offset=0, fluctuate=true, name="")
     pg.pnode({
         type: 'DepoSetDrifter',
         name: name,
-        data: { drifter: "Drifter" }
-    }, nin=1, nout=1, uses=[pg.pnode({
-        type: 'Drifter',
-        name: name,
-        data: lar {
-            rng: wc.tn(random),
-            xregions: xregions, // for help, see: low.util.driftsToXregions(drifts)
-            time_offset: offset,
-            fluctuate: fluctuate,
-        },
-    }, nin=1, nout=1, uses=[random])])
+        data: { drifter: "Drifter:"+name }
+    }, nin=1, nout=1, uses = [
+        pg.pnode({
+            type: 'Drifter',
+            name: name,
+            data: lar {
+                rng: wc.tn(random),
+                xregions: xregions, // for help, see: low.util.driftsToXregions(drifts)
+                time_offset: offset,
+                fluctuate: fluctuate,
+            },
+        }, nin=1, nout=1, uses=[random])
+    ])
 

cfg/layers/low/img.jsonnet

@@ -5,10 +5,7 @@ local util = import "util.jsonnet";
 local pg = import "pgraph.jsonnet";
 local wc = import "wirecell.jsonnet";
 
-function (anode) {
-
-    local ident = util.idents(anode),
-
+function (anode, name) {
 
     // The recommended slicer.  There is almost no reasonable default
     // to each detector variant may as well create the MaskSlices
@@ -17,11 +14,11 @@ function (anode) {
                         span=4, active_planes=[0,1,2], masked_planes=[], dummy_planes=[]) 
         pg.pnode({
             type: "MaskSlices",
-            name: ident+ext,
+            name: name+ext,
             data: {
-                wiener_tag: "wiener"+ident,
-                charge_tag: "gauss"+ident,
-                error_tag: "gauss_error"+ident,
+                wiener_tag: "wiener"+ext,
+                charge_tag: "gauss"+ext,
+                error_tag: "gauss_error"+ext,
                 tick_span: span,
                 anode: wc.tn(anode),
                 min_tbin: min_tbin,
@@ -36,7 +33,7 @@ function (anode) {
     // This slicing does not handle charge uncertainty nor bad channels.
     simple_slicing :: function(span=4, tag="") pg.pnode({
         type: "SumSlices",
-        name: ident,
+        name: name,
         data: {
             tag: tag,
             tick_span: span,
@@ -49,7 +46,7 @@ function (anode) {
     // unique name extension "ext" must be given.
     single_tiling :: function(face, ext="") pg.pnode({
         type: "GridTiling",
-        name: "%s-%d%s"%[ident, face, ext],
+        name: "%s-%d%s"%[name, face, ext],
         data: {
             anode: wc.tn(anode),
             face: face,
@@ -63,15 +60,15 @@ function (anode) {
 
         local slice_fanout = pg.pnode({
             type: "SliceFanout",
-            name: ident+ext,
+            name: name+ext,
             data: { multiplicity: 2 },
         }, nin=1, nout=2),
 
         local tilings = [$.single_tiling(face, ext) for face in [0,1]],
 
         local blobsync = pg.pnode({
             type: "BlobSetSync",
-            name: ident+ext,
+            name: name+ext,
             data: { multiplicity: 2 }
         }, nin=2, nout=1),
 
@@ -82,7 +79,7 @@ function (anode) {
             edges=
                 [pg.edge(slice_fanout, tilings[n], n, 0) for n in [0,1]] +
                 [pg.edge(tilings[n], blobsync, 0, n) for n in [0,1]],
-            name=ident+ext),
+            name=name+ext),
     }.ret,
 
     // A tiling subgraph matching the anode faces.  The "uname" must
@@ -229,15 +226,15 @@ function (anode) {
     // of slice spans.  Makes blob-blob edges
     clustering :: function(spans=1.0) pg.pnode({
         type: "BlobClustering",
-        name: ident,
+        name: name,
         data:  { spans : spans }
     }, nin=1, nout=1),    
 
     // A node that groups wires+channels into "measurements".  Makes
     // blob-measure edges.
     grouping :: function() pg.pnode({
         type: "BlobGrouping",
-        name: ident,
+        name: name,
         data:  { }
     }, nin=1, nout=1),
 
@@ -248,7 +245,7 @@ function (anode) {
                                whiten=true)
         pg.pnode({
             type: "ChargeSolving",
-            name: ident,
+            name: name,
             data:  {
                 "meas_value_threshold": meas_val_thresh,
                 "meas_error_threshold": meas_err_thresh,
@@ -260,14 +257,14 @@ function (anode) {
     // This solver is simplistic, prefer charge solving.
     blob_solving :: function(threshold=0.0) pg.pnode({
         type: "BlobSolving",
-        name: ident,
+        name: name,
         data:  { threshold: threshold }
     }, nin=1, nout=1),    
 
     // A function that projects blobs back to frames.  
     reframing :: function(tag="") pg.pnode({
         type: "BlobReframer",
-        name: ident,
+        name: name,
         data: {
             frame_tag: tag,
         }

cfg/layers/low/reframer.jsonnet

@@ -0,0 +1,18 @@
+local wc = import "wirecell.jsonnet";
+local pg = import "pgraph.jsonnet";
+local util = import "util.jsonnet";
+
+function(params, anode, tags=[], name=null)
+    pg.pnode({
+        type: 'Reframer',
+        name: if std.type(name) == "null" then util.idents(anode) else name,
+        data: {
+            anode: wc.tn(anode),
+            tags: tags,
+            fill: 0.0,
+            toffset: 0,
+            tbin: params.ductor.tbin,
+            nticks: params.ductor.binning.nticks - self.tbin,
+        },
+    }, nin=1, nout=1, uses=[anode])
+

cfg/layers/low/ssss.jsonnet

@@ -0,0 +1,45 @@
+// The spdir and other tests make use of "ssss" (simple sim sig smeared splat something something)
+// This helps generate the variant configs.
+//
+// Run wirecell-gen morse-* to find long/tran smearing numbers that match the
+// extra spread the sigproc induces.  CAVEAT: these results are indirectly tied
+// to the field response used in that process.  Defaults here are from PDSP.
+
+function(nominal,
+         long=[
+             2.691862363980221,
+             2.6750200122535057,
+             2.7137567141154055
+         ],
+         tran=[
+             0.7377218875719689,
+             0.7157764520393882,
+             0.13980698710556544
+         ])
+{
+    local smeared = nominal + {
+        splat: super.splat + {
+            "smear_long": long,
+            "smear_tran": tran,
+        }
+    },
+
+    // Used for test-morse-pdsp
+    morse_nominal: nominal,
+
+    // Used for test-ssss-pdsp
+    ssss_nominal: nominal,
+    ssss_smeared: smeared,
+
+    // Used for test/scripts/spdir 
+    spdir_lofr: smeared {
+        detector_data: super.detector_data {
+            fields: "%s-fields-lo.json.bz2" % nominal.detector_name
+        },
+    },
+    spdir_hifr: smeared {
+        detector: super.detector_data {
+            fields: "%s-fields-hi.json.bz2" % nominal.detector_name
+        },
+    },
+}