From 9e5cc98f2e34e49a8a7a136137d250b723d7b4c3 Mon Sep 17 00:00:00 2001 From: Felice Pantaleo Date: Fri, 7 Nov 2025 10:23:14 +0100 Subject: [PATCH 001/317] pyTICL: validated, type-safe TICL configuration framework (v5 core) Add RecoHGCal/pyTICL, a fluent-builder framework that declares a TICL configuration, builds the cms modules by cloning the real _cfi defaults, checks the plumbing with type-aware rules derived from the producers' consumes/produces, and exports a standalone, loadable cff fragment. This commit covers the v5 default and is validated byte-for-byte against the live iterTICLTask: - catalog.py type-aware module registry (consumes/produces C++ types, instance-label rules, backends, plugin-type enums) - model.py fluent builder (TICLConfig: iteration/seeding/filter/ pattern/masks/backend, links/superclustering/candidate/pf) - presets.py standard v5 iterations and singletons - assembler.py model -> cms modules + Task hierarchy (computed plumbing) - validator.py type-aware connection checks; rejects type-incompatible, missing, or GPU-on-CPU-only connections - exporter.py to_cff(): self-contained, re-loadable cff fragment - compare.py per-module comparison against a baseline Task Tests (scram b runtests): - testPyTICLReproduceV5 generated v5 == live iterTICLTask (acceptance gate + primary drift detector) - testPyTICLCatalogSchema locks the catalog against the live cfi defaults so new/changed InputTag plumbing is detected - testPyTICLPlumbing positive + negative validator tests - testPyTICLExportCff exported cff round-trips to the baseline --- RecoTICL/Configuration/README.md | 84 ++++++ RecoTICL/Configuration/python/assembler.py | 240 ++++++++++++++++ RecoTICL/Configuration/python/catalog.py | 258 ++++++++++++++++++ RecoTICL/Configuration/python/compare.py | 45 +++ RecoTICL/Configuration/python/exporter.py | 36 +++ RecoTICL/Configuration/python/model.py | 231 ++++++++++++++++ RecoTICL/Configuration/python/presets.py | 249 +++++++++++++++++ RecoTICL/Configuration/python/validator.py | 124 +++++++++ RecoTICL/Configuration/test/BuildFile.xml | 9 + .../Configuration/test/test_catalog_schema.py | 65 +++++ .../Configuration/test/test_export_cff.py | 56 ++++ RecoTICL/Configuration/test/test_plumbing.py | 70 +++++ .../Configuration/test/test_reproduce_v5.py | 46 ++++ 13 files changed, 1513 insertions(+) create mode 100644 RecoTICL/Configuration/README.md create mode 100644 RecoTICL/Configuration/python/assembler.py create mode 100644 RecoTICL/Configuration/python/catalog.py create mode 100644 RecoTICL/Configuration/python/compare.py create mode 100644 RecoTICL/Configuration/python/exporter.py create mode 100644 RecoTICL/Configuration/python/model.py create mode 100644 RecoTICL/Configuration/python/presets.py create mode 100644 RecoTICL/Configuration/python/validator.py create mode 100644 RecoTICL/Configuration/test/BuildFile.xml create mode 100644 RecoTICL/Configuration/test/test_catalog_schema.py create mode 100644 RecoTICL/Configuration/test/test_export_cff.py create mode 100644 RecoTICL/Configuration/test/test_plumbing.py create mode 100644 RecoTICL/Configuration/test/test_reproduce_v5.py diff --git a/RecoTICL/Configuration/README.md b/RecoTICL/Configuration/README.md new file mode 100644 index 0000000000000..1bf2fb585a2c7 --- /dev/null +++ b/RecoTICL/Configuration/README.md @@ -0,0 +1,84 @@ + +# pyTICL + +A validated, type-safe configuration framework for TICL (the Phase-2 CMS HGCAL +reconstruction). Instead of editing large hand-written `_cff.py` fragments, +declare a configuration with a concise fluent builder; pyTICL builds the `cms` +modules, **checks the plumbing with type-aware rules**, lets you pick CPU/GPU per +module, and **exports a standalone cff** (useful for HLT). + +## Quick start + +```python +from RecoTICL.Configuration import presets + +cfg = presets.v5() # the default iterTICLTask, as a TICLConfig +cfg.validate() # type-aware plumbing checks (raises on problems) +cfg.to_cff('myTICL_cff.py') # write a self-contained, loadable cff fragment +``` + +Building one by hand with the fluent API: + +```python +from RecoTICL.Configuration.model import TICLConfig, Global +from RecoTICL.Configuration import presets + +cfg = (TICLConfig('v5') + .iteration('CLUE3DHigh') + .seeding(Global) + .filter_by_algo_and_size(min_size=2) + .pattern_clue3d(criticalDensity=[0.6, 0.6, 0.6], + criticalEtaPhiDistance=[0.025, 0.025, 0.025]) + .iteration('Recovery').preset() # standard Recovery fragment + .masks_from('CLUE3DHigh') + .links(['CLUE3DHigh', 'Recovery'], **presets.links_defaults()) + .superclustering_dnn(source='CLUE3DHigh', **presets.supercluster_dnn_defaults()) + .candidate(**presets.candidate_defaults()) + .pf(**presets.pf_defaults())) + +cfg.validate() +``` + +`assemble()` returns the built modules + tasks; `add_to_process(process)` +registers them on a `cms.Process`. + +## How it works + +| Module | Responsibility | +| --- | --- | +| `catalog.py` | type-aware registry: each producer's `consumes`/`produces` C++ types, instance-label rules, backends, and the valid plugin-type strings | +| `model.py` | the fluent builder (`TICLConfig`); records intent only | +| `presets.py` | the standard v5 iterations & singletons (algorithm PSets transcribed from the baseline; plumbing left to pyTICL) | +| `assembler.py` | clones the real `_cfi` defaults + applies algorithm overrides and the **computed** plumbing `InputTag`s; builds the `Task` hierarchy | +| `validator.py` | builds the product graph and rejects type-incompatible / missing / GPU-unsupported connections | +| `exporter.py` | `to_cff(path)` -- a self-contained, loadable cff fragment | +| `compare.py` | per-module comparison of an assembled config vs a baseline `Task` | + +Byte-for-byte reproduction is achievable because the assembler clones the *same* +`_cfi` defaults the baseline uses and re-applies the same overrides; the +framework's own contribution is the wiring, validation, backend selection, and +export. + +## Validation & drift detection (`test/`) + +* `test_reproduce_v5.py` -- **acceptance gate**: the generated v5 config equals + the live `iterTICLTask` byte-for-byte. Also the primary drift detector: if a + baseline cff changes in a way pyTICL doesn't mirror, this fails with a diff. +* `test_catalog_schema.py` -- locks the catalog against the live `_cfi` defaults; + fails if a producer gains/loses/retypes an `InputTag` parameter (new plumbing + pyTICL must learn about). +* `test_plumbing.py` -- positive + negative validator tests (e.g. GPU on a + CPU-only module is rejected). +* `test_export_cff.py` -- the exported cff reloads and reproduces the baseline. + +Run them with `scram b runtests` (from the package) or directly with `python3`. + +## Status / roadmap + +Implemented: the v5 TICL core (iterations, links, superclustering, candidate, +pf), type-aware validation, cff export, and drift tests. + +Planned (see the package design notes): auto-derived & scheduled +labels/validation/dumper/associators; the TICL barrel path; local reconstruction ++ layer clustering (HGCAL/ECAL/HCAL) with real CPU/GPU (alpaka) backend +selection; a Phase-2 HLT target. diff --git a/RecoTICL/Configuration/python/assembler.py b/RecoTICL/Configuration/python/assembler.py new file mode 100644 index 0000000000000..c29b7ebbb2241 --- /dev/null +++ b/RecoTICL/Configuration/python/assembler.py @@ -0,0 +1,240 @@ +# Original Author: Felice Pantaleo, CERN, felice.pantaleo@cern.ch +"""Assemble a :class:`~RecoTICL.Configuration.model.TICLConfig` into ``cms`` objects. + +Each node is built by cloning the *real* ``_cfi`` default (so values match the +baseline) and applying (a) the algorithm overrides recorded in the model and +(b) the *plumbing* ``InputTag``s computed here from the iteration graph. The +result is a set of labelled modules plus the nested ``Task`` structure +(``ticlStepTask`` -> ``ticlIterationsTask`` -> ``mergeTICLTask`` -> +``iterTICLTask``) matching ``iterativeTICL_cff``. +""" + +import FWCore.ParameterSet.Config as cms + +from RecoTICL.Configuration.catalog import CATALOG, SEEDING_MODULE_LABEL +from RecoTICL.Configuration.model import PyTICLError + + +# --------------------------------------------------------------------------- # +# helpers +# --------------------------------------------------------------------------- # + +def _cfi_default(key): + spec = CATALOG[key] + mod = __import__(spec.cfi_module, fromlist=[spec.cfi_symbol]) + return getattr(mod, spec.cfi_symbol) + + +def trackster_label(name): + return "ticlTracksters" + name + + +def filter_label(name): + return "filteredLayerClusters" + name + + +def step_task_name(name): + return "ticl" + name + "StepTask" + + +# --------------------------------------------------------------------------- # +# result container +# --------------------------------------------------------------------------- # + +class Assembled: + """The product of assembling a config: labelled modules + named tasks.""" + + def __init__(self, config): + self.config = config + self.modules = {} # label -> cms module (insertion order preserved) + self.tasks = {} # name -> cms.Task + self.task_children = {} # name -> [child identifier, ...] (labels/task names) + self.top = None # the iterTICLTask + + def add_to_process(self, process): + """Register every module (labelled) and task on ``process``.""" + for label, mod in self.modules.items(): + setattr(process, label, mod) + for name, task in self.tasks.items(): + setattr(process, name, task) + return process + + +# --------------------------------------------------------------------------- # +# per-node builders +# --------------------------------------------------------------------------- # + +def _build_seeding(seeding_type): + base = _cfi_default("TICLSeedingRegionProducer") + return base.clone(seedingPSet=base.seedingPSet.clone(type=seeding_type)) + + +def _build_layer_tile(): + return _cfi_default("TICLLayerTileProducer").clone() + + +def _build_filter(it, prev_trackster): + base = _cfi_default("FilteredLayerClustersProducer") + ov = dict(clusterFilter=it.filter_type, iteration_label=it.name) + ov.update(it.filter_params) + if prev_trackster: + ov["LayerClustersInputMask"] = cms.InputTag(prev_trackster) + return base.clone(**ov) + + +def _build_trackster(it, flabel, seed_label, prev_trackster): + base = _cfi_default("TrackstersProducer") + ov = dict( + filtered_mask=cms.InputTag(flabel, it.name), + seeding_regions=cms.InputTag(seed_label), + itername=it.name, + patternRecognitionBy=it.pattern_type, + ) + ov["pluginPatternRecognitionBy" + it.pattern_type] = dict(**it.pattern_params) + if prev_trackster: + ov["original_mask"] = cms.InputTag(prev_trackster) + ov.update(it.trackster_extra) + return base.clone(**ov) + + +def _build_links(collection_labels, overrides): + base = _cfi_default("TracksterLinksProducer") + # baseline builds this as cms.VInputTag(*string_labels) -- reproduce exactly + ov = dict(tracksters_collections=cms.VInputTag(*collection_labels)) + ov.update(overrides or {}) + return base.clone(**ov) + + +def _build_supercluster_dnn(source_label, overrides): + base = _cfi_default("TracksterLinksProducer") + # baseline builds this as a python list of cms.InputTag -- reproduce exactly + ov = dict(tracksters_collections=[cms.InputTag(source_label)]) + ov.update(overrides or {}) + return base.clone(**ov) + + +def _build_egamma(): + return _cfi_default("EGammaSuperclusterProducer").clone() + + +def _build_candidate(overrides): + return _cfi_default("TICLCandidateProducer").clone(**(overrides or {})) + + +def _build_mtd(): + return _cfi_default("MTDSoAProducer").clone() + + +def _build_pf(overrides): + base = _cfi_default("PFTICLProducer") + ov = dict(ticlCandidateSrc=cms.InputTag("ticlCandidate")) + ov.update(overrides or {}) + return base.clone(**ov) + + +# --------------------------------------------------------------------------- # +# top-level assembly +# --------------------------------------------------------------------------- # + +def assemble(cfg): + res = Assembled(cfg) + m = res.modules + t = res.tasks + + def mktask(name, *children): + """Create ``cms.Task(name)`` from child identifiers (module labels or + task names already built), recording the composition for export.""" + objs = [] + for ch in children: + if ch in m: + objs.append(m[ch]) + elif ch in t: + objs.append(t[ch]) + else: + raise PyTICLError("task %r references unknown child %r" % (name, ch)) + t[name] = cms.Task(*objs) + res.task_children[name] = list(children) + return t[name] + + def ensure_seeding(stype): + label = SEEDING_MODULE_LABEL.get(stype) + if label is None: + raise PyTICLError("no canonical seeding module for seeding type %r; " + "extend catalog.SEEDING_MODULE_LABEL" % stype) + if label not in m: + m[label] = _build_seeding(stype) + return label + + # shared infrastructure: layer tile + if cfg.include_layer_tile: + m["ticlLayerTileProducer"] = _build_layer_tile() + mktask("ticlLayerTileTask", "ticlLayerTileProducer") + + # iterations + step_tasks = [] + for it in cfg.iterations: + if it.seeding_type is None: + raise PyTICLError("iteration %r has no seeding region" % it.name) + if it.filter_type is None: + raise PyTICLError("iteration %r has no cluster filter" % it.name) + if it.pattern_type is None: + raise PyTICLError("iteration %r has no pattern recognition" % it.name) + seed_label = ensure_seeding(it.seeding_type) + flabel = filter_label(it.name) + tlabel = trackster_label(it.name) + prev = None + if it.masks_from: + if it.masks_from not in cfg._by_name: + raise PyTICLError("iteration %r masks_from unknown iteration %r" + % (it.name, it.masks_from)) + prev = trackster_label(it.masks_from) + m[flabel] = _build_filter(it, prev) + m[tlabel] = _build_trackster(it, flabel, seed_label, prev) + sname = step_task_name(it.name) + mktask(sname, seed_label, flabel, tlabel) + step_tasks.append(sname) + + if step_tasks: + mktask("ticlIterationsTask", *step_tasks) + + # links + superclustering + if cfg.links_spec: + labels = [trackster_label(n) for n in cfg.links_spec.collections] + m["ticlTracksterLinks"] = _build_links(labels, cfg.links_spec.overrides) + + if cfg.superclustering_spec: + sc = cfg.superclustering_spec + src = trackster_label(sc.source) + m["ticlTracksterLinksSuperclusteringDNN"] = _build_supercluster_dnn(src, sc.overrides) + m["ticlEGammaSuperClusterProducer"] = _build_egamma() + mktask("ticlSuperclusteringTask", + "ticlTracksterLinksSuperclusteringDNN", "ticlEGammaSuperClusterProducer") + + if "ticlTracksterLinks" in m: + links_children = ["ticlTracksterLinks"] + if "ticlSuperclusteringTask" in t: + links_children.append("ticlSuperclusteringTask") + mktask("ticlTracksterLinksTask", *links_children) + + # mergeTICLTask + merge = [name for name in ("ticlLayerTileTask", "ticlIterationsTask", + "ticlTracksterLinksTask") if name in t] + mktask("mergeTICLTask", *merge) + + # candidate / mtd / pf + if cfg.include_mtd: + m["mtdSoA"] = _build_mtd() + mktask("mtdSoATask", "mtdSoA") + if cfg.include_candidate: + m["ticlCandidate"] = _build_candidate(cfg.candidate_spec) + mktask("ticlCandidateTask", "ticlCandidate") + if cfg.include_pf: + m["pfTICL"] = _build_pf(cfg.pf_spec) + mktask("ticlPFTask", "pfTICL") + + # iterTICLTask + top = ["mergeTICLTask"] + [name for name in ("mtdSoATask", "ticlCandidateTask", + "ticlPFTask") if name in t] + mktask("iterTICLTask", *top) + res.top = t["iterTICLTask"] + return res diff --git a/RecoTICL/Configuration/python/catalog.py b/RecoTICL/Configuration/python/catalog.py new file mode 100644 index 0000000000000..e6919c9dd14f9 --- /dev/null +++ b/RecoTICL/Configuration/python/catalog.py @@ -0,0 +1,258 @@ +# Original Author: Felice Pantaleo, CERN, felice.pantaleo@cern.ch +"""Type-aware module registry for pyTICL. + +This is pyTICL's "type system". For every TICL producer it records: + +* ``cfi`` -- where to import the real default module from, so the + assembler can clone it (guaranteeing the generated config + matches the baseline byte-for-byte); +* ``produces`` -- the C++ products the module puts into the event, each with + an *instance-label rule* so the validator knows the exact + ``InputTag(module, instance)`` that resolves to it; +* ``consumes`` -- every ``InputTag`` / ``VInputTag`` parameter the module + reads, with the required C++ product type, so the validator + can reject type-incompatible connections; +* ``backends`` -- which compute backends the module supports (``cpu`` only, or + ``cpu`` + ``gpu``/alpaka). + +The data is transcribed from the C++ ``consumes<>``/``produces<>`` calls of the +producers in ``RecoHGCal/TICL/plugins`` (see the design notes in the package +README). ``test/test_catalog_schema.py`` locks this registry against the live +``_cfi`` defaults so that any drift in the raw configuration is detected. +""" + +from dataclasses import dataclass, field +from typing import Tuple + + +# --------------------------------------------------------------------------- # +# Product / consumption descriptors +# --------------------------------------------------------------------------- # + +# Instance-label rules for produced products: +# "" -> produced under the module label only (no instance) +# "fixed: