Skip to content

feat: more lookup optimizations in decoding #1215

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
baltzell wants to merge 10 commits into
base: development
Choose a base branch
from
+124 −130
Open

feat: more lookup optimizations in decoding #1215

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
21073eb
only once
baltzell Apr 17, 2026
7fa7e9a
improve variable names
baltzell Apr 17, 2026
c26e587
optimize fadc pulse table/detector lookup
baltzell Apr 17, 2026
2690a58
cleanup, in prep for faster lookup
baltzell Apr 17, 2026
f6e145b
switch to HashSets
baltzell Apr 17, 2026
658aa2f
bugfix
baltzell Apr 17, 2026
45ccb68
cleanup
baltzell Apr 18, 2026
56aa7d7
cleanup
baltzell Apr 18, 2026
49fcd30
remove unsupported options from decoder cli
baltzell May 14, 2026
e7a48c6
fix oops
baltzell May 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 1 addition & 18 deletions common-tools/clas-detector/src/main/java/org/jlab/detector/decode/CLASDecoder4.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,6 @@ public static void main(String[] args){

OptionParser parser = CLASDecoder4U.getOptionParser();
parser.addOption("-d", "0","debug mode, set >0 for more verbose output");
parser.addOption("-m", "run","translation tables source (use -m devel for development tables)");
parser.addOption("-b", "16","record buffer size in MB");
parser.parse(args);
List<String> inputList = parser.getInputList();

Expand All @@ -34,26 +32,11 @@ public static void main(String[] args){
System.exit(1);
}

String modeDevel = parser.getOption("-m").stringValue();
boolean developmentMode = false;

if(modeDevel.compareTo("run")!=0&&modeDevel.compareTo("devel")!=0){
parser.printUsage();
System.out.println("\n >>>> error : mode has to be set to \"run\" or \"devel\" ");
System.exit(1);
}

if(modeDevel.compareTo("devel")==0){
developmentMode = true;
}

String outputFile = parser.getOption("-o").stringValue();
int compression = parser.getOption("-c").intValue();
int recordsize = parser.getOption("-b").intValue();
int debug = parser.getOption("-d").intValue();

CLASDecoder decoder = new CLASDecoder(developmentMode);

CLASDecoder decoder = new CLASDecoder();
decoder.setDebugMode(debug);

HipoWriterSorted writer = new HipoWriterSorted();
Expand Down
235 changes: 123 additions & 112 deletions common-tools/clas-detector/src/main/java/org/jlab/detector/decode/DetectorEventDecoder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,10 @@
import java.util.Arrays;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import org.jlab.detector.banks.RawBank.OrderType;
import org.jlab.detector.base.DetectorType;
import org.jlab.detector.calib.utils.ConstantsManager;
Expand All @@ -22,27 +24,22 @@ public class DetectorEventDecoder {
ConstantsManager fitterManager = new ConstantsManager();
ConstantsManager scalerManager = new ConstantsManager();

List<String> tablesTrans = null;
List<String> tablesFitter = null;
HashSet<DetectorType> keysFilter;
HashSet<DetectorType> keysMicromega;

List<DetectorType> keysTrans = null;
List<DetectorType> keysFitter = null;
List<DetectorType> keysFilter = null;
List<DetectorType> keysMicromega= null;
HashMap<DetectorType,String> tableTrans;
HashMap<DetectorType,String> tableFitter;

private int runNumber = 10;
HashMap<DetectorType, IndexedTable> tablesFitter;
TranslationTable translator;

private ExtendedFADCFitter extendedFitter = new ExtendedFADCFitter();
private MVTFitter mvtFitter = new MVTFitter();

private TranslationTable translator = new TranslationTable();
ExtendedFADCFitter extendedFitter = new ExtendedFADCFitter();
MVTFitter mvtFitter = new MVTFitter();

int runNumber = 10;

public DetectorEventDecoder(boolean development){
if(development==true){
this.initDecoderDev();
} else {
this.initDecoder();
}
this.initDecoder();
Comment thread
baltzell marked this conversation as resolved.
}

public void setTimestamp(String timestamp) {
Expand All @@ -60,8 +57,11 @@ public void setVariation(String variation) {
public void setRunNumber(int run){
if (run != this.runNumber) {
translator = new TranslationTable();
for (int i=0; i<keysTrans.size(); i++)
translator.add(keysTrans.get(i), translationManager.getConstants(run, tablesTrans.get(i)));
tablesFitter = new HashMap<>();
for (DetectorType t : tableTrans.keySet())
translator.add(t, translationManager.getConstants(run, tableTrans.get(t)));
for (DetectorType t: tableFitter.keySet())
tablesFitter.put(t, fitterManager.getConstants(run, tableFitter.get(t)));
}
this.runNumber = run;
}
Expand All @@ -84,68 +84,79 @@ public DetectorEventDecoder(){
this.initDecoder();
}

public final void initDecoderDev(){
keysTrans = Arrays.asList(new DetectorType[]{ DetectorType.HTCC,DetectorType.BST,DetectorType.RTPC} );
tablesTrans = Arrays.asList(new String[]{ "/daq/tt/clasdev/htcc","/daq/tt/clasdev/svt","/daq/tt/clasdev/rtpc" });
keysFitter = Arrays.asList(new DetectorType[]{DetectorType.HTCC});
tablesFitter = Arrays.asList(new String[]{"/daq/fadc/clasdev/htcc"});
translationManager.init(tablesTrans);
fitterManager.init(tablesFitter);
scalerManager.init(Arrays.asList(new String[]{"/runcontrol/fcup","/runcontrol/slm","/runcontrol/hwp",
"/runcontrol/helicity","/daq/config/scalers/dsc1"}));
}

public final void initDecoder(){

// Detector translation table
keysTrans = Arrays.asList(new DetectorType[]{DetectorType.FTCAL,DetectorType.FTHODO,DetectorType.FTTRK,DetectorType.LTCC,DetectorType.ECAL,DetectorType.FTOF,
DetectorType.HTCC,DetectorType.DC,DetectorType.CTOF,DetectorType.CND,DetectorType.BST,DetectorType.RF,DetectorType.BMT,DetectorType.FMT,
DetectorType.RICH,DetectorType.HEL,DetectorType.BAND,DetectorType.RTPC,
DetectorType.RASTER,DetectorType.ATOF,DetectorType.AHDC
});
tablesTrans = Arrays.asList(new String[]{
"/daq/tt/ftcal","/daq/tt/fthodo","/daq/tt/fttrk","/daq/tt/ltcc",
"/daq/tt/ec","/daq/tt/ftof","/daq/tt/htcc","/daq/tt/dc","/daq/tt/ctof","/daq/tt/cnd","/daq/tt/svt",
"/daq/tt/rf","/daq/tt/bmt","/daq/tt/fmt","/daq/tt/rich2","/daq/tt/hel","/daq/tt/band","/daq/tt/rtpc",
"/daq/tt/raster","/daq/tt/atof","/daq/tt/ahdc"
});
translationManager.init(tablesTrans);
keysFilter = new HashSet<>();
keysMicromega= new HashSet<>();
tableTrans = new HashMap<>();
tableFitter = new HashMap<>();

keysFilter.add(DetectorType.DC);

// ADC waveform fitter translation table
keysFitter = Arrays.asList(new DetectorType[]{DetectorType.FTCAL,DetectorType.FTHODO,DetectorType.FTTRK,DetectorType.FTOF,DetectorType.LTCC,
DetectorType.ECAL,DetectorType.HTCC,DetectorType.CTOF,DetectorType.CND,DetectorType.BMT,
DetectorType.FMT,DetectorType.HEL,DetectorType.RF,DetectorType.BAND,DetectorType.RASTER,
DetectorType.AHDC});
tablesFitter = Arrays.asList(new String[]{
"/daq/fadc/ftcal","/daq/fadc/fthodo","/daq/config/fttrk","/daq/fadc/ftof","/daq/fadc/ltcc",
"/daq/fadc/ec", "/daq/fadc/htcc","/daq/fadc/ctof","/daq/fadc/cnd","/daq/config/bmt",
"/daq/config/fmt","/daq/fadc/hel","/daq/fadc/rf","/daq/fadc/band","/daq/fadc/raster",
"/daq/config/ahdc"
});
fitterManager.init(tablesFitter);

// Data filter list
keysFilter = Arrays.asList(new DetectorType[]{DetectorType.DC});

scalerManager.init(Arrays.asList(new String[]{"/runcontrol/fcup","/runcontrol/slm","/runcontrol/hwp",
"/runcontrol/helicity","/daq/config/scalers/dsc1"}));

keysMicromega = Arrays.asList(new DetectorType[]{DetectorType.BMT,DetectorType.FMT,DetectorType.FTTRK});
keysMicromega.add(DetectorType.BMT);
keysMicromega.add(DetectorType.FMT);
keysMicromega.add(DetectorType.FTTRK);

tableTrans.put(DetectorType.FTCAL, "/daq/tt/ftcal");
tableTrans.put(DetectorType.FTHODO, "/daq/tt/fthodo");
tableTrans.put(DetectorType.FTTRK, "/daq/tt/fttrk");
tableTrans.put(DetectorType.LTCC, "/daq/tt/ltcc");
tableTrans.put(DetectorType.ECAL, "/daq/tt/ec");
tableTrans.put(DetectorType.FTOF, "/daq/tt/ftof");
tableTrans.put(DetectorType.HTCC, "/daq/tt/htcc");
tableTrans.put(DetectorType.DC, "/daq/tt/dc");
tableTrans.put(DetectorType.CTOF, "/daq/tt/ctof");
tableTrans.put(DetectorType.CND, "/daq/tt/cnd");
tableTrans.put(DetectorType.BST, "/daq/tt/svt");
tableTrans.put(DetectorType.RF, "/daq/tt/rf");
tableTrans.put(DetectorType.BMT, "/daq/tt/bmt");
tableTrans.put(DetectorType.FMT, "/daq/tt/fmt");
tableTrans.put(DetectorType.RICH, "/daq/tt/rich2");
tableTrans.put(DetectorType.HEL, "/daq/tt/hel");
tableTrans.put(DetectorType.BAND, "/daq/tt/band");
tableTrans.put(DetectorType.RTPC, "/daq/tt/rtpc");
tableTrans.put(DetectorType.RASTER, "/daq/tt/raster");
tableTrans.put(DetectorType.ATOF, "/daq/tt/atof");
tableTrans.put(DetectorType.AHDC, "/daq/tt/ahdc");
translationManager.init(tableTrans.values().stream().collect(Collectors.toList()));

tableFitter.put(DetectorType.FTCAL, "/daq/fadc/ftcal");
tableFitter.put(DetectorType.FTHODO, "/daq/fadc/fthodo");
tableFitter.put(DetectorType.FTTRK, "/daq/fadc/fttrk");
tableFitter.put(DetectorType.FTOF, "/daq/fadc/ftof");
tableFitter.put(DetectorType.LTCC, "/daq/fadc/ltcc");
tableFitter.put(DetectorType.ECAL, "/daq/fadc/ec");
tableFitter.put(DetectorType.HTCC, "/daq/fadc/htcc");
tableFitter.put(DetectorType.CTOF, "/daq/fadc/ctof");
tableFitter.put(DetectorType.CND, "/daq/fadc/cnd");
tableFitter.put(DetectorType.BMT, "/daq/fadc/bmt");
tableFitter.put(DetectorType.FMT, "/daq/fadc/fmt");
tableFitter.put(DetectorType.HEL, "/daq/fadc/hel");
tableFitter.put(DetectorType.RF, "/daq/fadc/rf");
tableFitter.put(DetectorType.BAND, "/daq/fadc/band");
tableFitter.put(DetectorType.RASTER, "/daq/fadc/raster");
tableFitter.put(DetectorType.AHDC, "/daq/fadc/ahdc");
fitterManager.init(tableFitter.values().stream().collect(Collectors.toList()));

scalerManager.init(Arrays.asList(new String[]{"/runcontrol/fcup",
"/runcontrol/slm","/runcontrol/hwp","/runcontrol/helicity","/daq/config/scalers/dsc1"}));

checkTables();
}

public void checkTables() {
for (int i=0; i<tablesTrans.size(); i++) {
IndexedTable t = translationManager.getConstants(runNumber, tablesTrans.get(i));
void checkTables() {
List<String> tables = (List)tableTrans.values().stream().collect(Collectors.toList());
for (int i=0; i<tables.size(); i++) {
IndexedTable t = translationManager.getConstants(runNumber, tables.get(i));
for (int j=0; j<i; j++)
t.conflicts(translationManager.getConstants(runNumber, tablesTrans.get(j)));
t.conflicts(translationManager.getConstants(runNumber, tables.get(j)));
}
}

/**
* applies translation table to the digitized data to translate
* crate,slot channel to sector layer component.
* Apply CCDB /daq/tt translation tables to the digitized data to translate
* crate/slot/channel to sector/layer/component/order.
*
* @param detectorData
*/
public void translate(List<DetectorDataDgtz> detectorData){
Expand All @@ -169,57 +180,59 @@ public void translate(List<DetectorDataDgtz> detectorData){
}
}

public void fitPulses(List<DetectorDataDgtz> detectorData){
/**
* Use CCDB /daq/fadc tables to convert waveforms to pulses.
*
* @param detectorData
*/
public void fitPulses(List<DetectorDataDgtz> detectorData) {

// preload CCDB tables once:
ArrayList<IndexedTable> tables = new ArrayList<>();
for (String name : tablesFitter) {
tables.add(fitterManager.getConstants(runNumber, name));
}
final long hash0 = IndexedTable.DEFAULT_GENERATOR.hashCode(0,0,0);

for (DetectorDataDgtz data : detectorData) {

for(DetectorDataDgtz data : detectorData){
if (data.getADCSize() == 0) continue;
int crate = data.getDescriptor().getCrate();
int slot = data.getDescriptor().getSlot();
int channel = data.getDescriptor().getChannel();
long hash = IndexedTable.DEFAULT_GENERATOR.hashCode(crate,slot,channel);
long hash0 = IndexedTable.DEFAULT_GENERATOR.hashCode(0,0,0);
boolean ismm = keysMicromega.contains(data.getDescriptor().getType());

for (int j=0; j<keysFitter.size(); ++j) {
IndexedTable daq = tables.get(j);
DetectorType type = keysFitter.get(j);
//custom MM fitter
if (ismm && data.getDescriptor().getType() == type) {
short adcOffset = (short) daq.getDoubleValueByHash("adc_offset", hash0);
double fineTimeStampResolution = (byte) daq.getDoubleValueByHash("dream_clock", hash0);
double samplingTime = (byte) daq.getDoubleValueByHash("sampling_time", hash0);
int sparseSample = daq.getIntValueByHash("sparse", hash0);
ADCData adc = data.getADCData(0);
mvtFitter.fit(adcOffset, fineTimeStampResolution, samplingTime, adc.getPulseArray(), adc.getTimeStamp(), sparseSample);
adc.setHeight((short) (mvtFitter.adcMax));
adc.setTime((int) (mvtFitter.timeMax));
adc.setIntegral((int) (mvtFitter.integral));
adc.setTimeStamp(mvtFitter.timestamp);
// first one wins:
break;
}
else if(daq.hasEntryByHash(hash)==true){
int nsa = daq.getIntValueByHash("nsa", hash);
int nsb = daq.getIntValueByHash("nsb", hash);
int tet = daq.getIntValueByHash("tet", hash);

DetectorType type = data.getDescriptor().getType();
if (!tablesFitter.containsKey(type)) continue;
IndexedTable daqTable = tablesFitter.get(type);

// For MM, assume crate/slot/channel=0/0/0 for table lookup:
if (keysMicromega.contains(type)) {
short adcOffset = (short) daqTable.getDoubleValueByHash("adc_offset", hash0);
double fineTimeStampResolution = (byte) daqTable.getDoubleValueByHash("dream_clock", hash0);
double samplingTime = (byte) daqTable.getDoubleValueByHash("sampling_time", hash0);
int sparseSample = daqTable.getIntValueByHash("sparse", hash0);
ADCData adc = data.getADCData(0);
mvtFitter.fit(adcOffset, fineTimeStampResolution, samplingTime, adc.getPulseArray(), adc.getTimeStamp(), sparseSample);
adc.setHeight((short) (mvtFitter.adcMax));
adc.setTime((int) (mvtFitter.timeMax));
adc.setIntegral((int) (mvtFitter.integral));
adc.setTimeStamp(mvtFitter.timestamp);
}

// Otherwise, use crate/slot/channel to find the table entry:
else {
long hash = IndexedTable.DEFAULT_GENERATOR.hashCode(
data.getDescriptor().getCrate(),
data.getDescriptor().getSlot(),
data.getDescriptor().getChannel());
if (daqTable.hasEntryByHash(hash)) {
int nsa = daqTable.getIntValueByHash("nsa", hash);
int nsb = daqTable.getIntValueByHash("nsb", hash);
int tet = daqTable.getIntValueByHash("tet", hash);
int ped = 0;
if(data.getDescriptor().getType() == DetectorType.RF && type == DetectorType.RF) {
ped = daq.getIntValueByHash("pedestal", hash);
if (data.getDescriptor().getType() == DetectorType.RF && type == DetectorType.RF) {
ped = daqTable.getIntValueByHash("pedestal", hash);
}
for(int i = 0; i < data.getADCSize(); i++){
for (int i = 0; i < data.getADCSize(); i++) {
ADCData adc = data.getADCData(i);
if(adc.getPulseSize()>0){
try {
extendedFitter.fit(nsa, nsb, tet, ped, adc.getPulseArray());
} catch (Exception e) {
System.out.println(">>>> error : fitting pulse "
+ crate + " / " + slot + " / " + channel);
System.err.println(">>>> error : fitting pulse "+data.getDescriptor().getCrate()+
" / "+data.getDescriptor().getSlot()+" / "+data.getDescriptor().getChannel());
}
int adc_corrected = extendedFitter.adc + extendedFitter.ped*(nsa+nsb);
adc.setHeight((short) this.extendedFitter.pulsePeakValue);
Expand All @@ -229,14 +242,12 @@ else if(daq.hasEntryByHash(hash)==true){
}
data.getADCData(i).setADC(nsa, nsb);
}
// first one wins:
break;
}
}
}
}

public void filterTDCs(List<DetectorDataDgtz> detectorData){
void filterTDCs(List<DetectorDataDgtz> detectorData){
int maxMultiplicity = 1;
for(DetectorType type : keysFilter){
Map<Integer,List<DetectorDataDgtz>> filteredData = new HashMap<>();
Expand Down