From 31c56c66775645e18d9cb951b079dc1940951b8e Mon Sep 17 00:00:00 2001 From: Alexandra Rahlin Date: Mon, 31 Jul 2023 13:15:30 -0500 Subject: [PATCH] Updates to dfmux library for compatibility with hidfmux IceBoard firmware (#106) * Configurable internal clock rate to ensure that the IceBoard sub-second counter can be converted to a proper timestamp * IceBoard packets indexed by module and sub-module block * Add `center_frequency` and `bandwidth` attributes to `BolometerProperties` objects, to differentiate between the actual mm-wave observing frequency of the resonator and the band into which channels would be binned during mapmaking. --- .../include/calibration/BoloProperties.h | 7 +- calibration/src/BoloProperties.cxx | 12 +++- dfmux/include/dfmux/DfMuxBuilder.h | 6 +- dfmux/include/dfmux/DfMuxCollector.h | 11 ++- dfmux/src/DfMuxBuilder.cxx | 23 +++++-- dfmux/src/DfMuxCollator.cxx | 24 +++++-- dfmux/src/DfMuxCollector.cxx | 68 +++++++++++++------ dfmux/src/LegacyDfMuxCollector.cxx | 3 + gcp/src/GCPMuxDataDecoder.cxx | 5 +- 9 files changed, 121 insertions(+), 38 deletions(-) diff --git a/calibration/include/calibration/BoloProperties.h b/calibration/include/calibration/BoloProperties.h index fca6b3c7..ee440d34 100644 --- a/calibration/include/calibration/BoloProperties.h +++ b/calibration/include/calibration/BoloProperties.h @@ -13,7 +13,8 @@ class BolometerProperties : public G3FrameObject { public: BolometerProperties() : x_offset(NAN), y_offset(NAN), band(NAN), - pol_angle(NAN), pol_efficiency(NAN), coupling(Unknown) {} + center_frequency(NAN), bandwidth(NAN), pol_angle(NAN), + pol_efficiency(NAN), coupling(Unknown) {} std::string Description() const; std::string physical_name; /* e.g. D4.A2.3.Y */ @@ -23,6 +24,8 @@ class BolometerProperties : public G3FrameObject double y_offset; /* and el */ double band; /* Standard frequency units */ + double center_frequency; + double bandwidth; double pol_angle; /* Standard angular units */ double pol_efficiency; /* 0-1 */ @@ -44,7 +47,7 @@ class BolometerProperties : public G3FrameObject }; G3_POINTERS(BolometerProperties); -G3_SERIALIZABLE(BolometerProperties, 6); +G3_SERIALIZABLE(BolometerProperties, 7); G3MAP_OF(std::string, BolometerProperties, BolometerPropertiesMap); diff --git a/calibration/src/BoloProperties.cxx b/calibration/src/BoloProperties.cxx index ec6b1a8c..161bdbc7 100644 --- a/calibration/src/BoloProperties.cxx +++ b/calibration/src/BoloProperties.cxx @@ -40,6 +40,10 @@ template void BolometerProperties::serialize(A &ar, unsigned v) ar & make_nvp("pixel_type", pixel_type); } + if (v > 6) { + ar & make_nvp("center_frequency", center_frequency); + ar & make_nvp("bandwidth", bandwidth); + } } std::string BolometerProperties::Description() const @@ -63,7 +67,11 @@ PYBINDINGS("calibration") { .def_readwrite("y_offset", &BolometerProperties::y_offset, "Vertical pointing offset relative to boresight in angular units.") .def_readwrite("band", &BolometerProperties::band, - "Center of detector observing band in frequency units") + "Nominal center of detector observing band in frequency units") + .def_readwrite("center_frequency", &BolometerProperties::center_frequency, + "Measured center of detector observing band in frequency units") + .def_readwrite("bandwidth", &BolometerProperties::bandwidth, + "Measured bandwidth of detector observing band in frequency units") .def_readwrite("pol_angle", &BolometerProperties::pol_angle, "Polarization angle in angular units") .def_readwrite("pol_efficiency", &BolometerProperties::pol_efficiency, @@ -72,7 +80,7 @@ PYBINDINGS("calibration") { "Optical coupling type") .def_readwrite("wafer_id", &BolometerProperties::wafer_id, - "Name of the name this detector is on") + "Name of the wafer this detector is on") .def_readwrite("pixel_id", &BolometerProperties::pixel_id, "Name of the pixel of which this detector is a part") .def_readwrite("pixel_type", &BolometerProperties::pixel_type, diff --git a/dfmux/include/dfmux/DfMuxBuilder.h b/dfmux/include/dfmux/DfMuxBuilder.h index 1502ddf9..558acf77 100644 --- a/dfmux/include/dfmux/DfMuxBuilder.h +++ b/dfmux/include/dfmux/DfMuxBuilder.h @@ -15,13 +15,15 @@ */ struct DfMuxBoardSamples : public G3FrameObject, public std::map { size_t nmodules; // Total number of modules expected from this board + size_t nblocks; // Total number of sub-module blocks expected from this board + size_t nchannels; // Total number of channels per block expected from this board - bool Complete() const { return (size() == nmodules); }; + bool Complete() const { return (size() == nmodules * nblocks); }; template void serialize(A &ar, unsigned v); }; G3_POINTERS(DfMuxBoardSamples); -G3_SERIALIZABLE(DfMuxBoardSamples, 1); +G3_SERIALIZABLE(DfMuxBoardSamples, 2); /* * DfMuxMetaSample: collection of DfMuxBoardSamples, indexed by board serial diff --git a/dfmux/include/dfmux/DfMuxCollector.h b/dfmux/include/dfmux/DfMuxCollector.h index bc87b12e..34b79d61 100644 --- a/dfmux/include/dfmux/DfMuxCollector.h +++ b/dfmux/include/dfmux/DfMuxCollector.h @@ -16,7 +16,10 @@ class DfMuxSamplePacket : public G3FrameObject { public: int32_t board; /* Board serial number */ int32_t module; /* Module number (0-7) */ - int32_t nmodules; /* Total number of modules on board (8) */ + int32_t block; /* Sub-module block number (0-7 for hidfmux, 0 for dfmux) */ + int32_t nmodules; /* Total number of modules on board (8 for dfmux) */ + int32_t nblocks; /* Total number of blocks per module (8 for hidfmux, 1 for dfmux) */ + int32_t nchannels; /* Total number of channels per block (128) */ DfMuxSamplePtr sample; /* Pointer to the DfMuxSample */ }; @@ -50,6 +53,10 @@ class DfMuxCollector { int Start(); // Start listening thread int Stop(); // Stop listening thread + // Configure iceboard clock rate (defaults to 100 MHz) + void SetClockRate(double rate); + double GetClockRate() { return clock_rate_; }; + private: int SetupUDPSocket(const char *listenaddr); int SetupSCTPSocket(std::vector hosts); @@ -67,6 +74,8 @@ class DfMuxCollector { std::vector board_list_; int fd_; in_addr_t listenaddr_; + double clock_rate_; + double clock_scale_; SET_LOGGER("DfMuxCollector"); }; diff --git a/dfmux/src/DfMuxBuilder.cxx b/dfmux/src/DfMuxBuilder.cxx index b56e53a1..02a0b575 100644 --- a/dfmux/src/DfMuxBuilder.cxx +++ b/dfmux/src/DfMuxBuilder.cxx @@ -19,6 +19,14 @@ template void DfMuxBoardSamples::serialize(A &ar, const unsigned v) ar & make_nvp("G3FrameObject", base_class(this)); ar & make_nvp("samples", base_class >(this)); ar & make_nvp("nmodules", nmodules); + + if (v > 1) { + ar & make_nvp("nblocks", nblocks); + ar & make_nvp("nchannels", nchannels); + } else { + nblocks = 1; + nchannels = 128; + } } G3_SERIALIZABLE_CODE(DfMuxBoardSamples); @@ -124,10 +132,13 @@ void DfMuxBuilder::ProcessNewData() } // Add to meta sample - g3_assert((*sample->sample)[pkt->board].find(pkt->module) == + int idx = pkt->module * pkt->nblocks + pkt->block; + g3_assert((*sample->sample)[pkt->board].find(idx) == (*sample->sample)[pkt->board].end()); - (*sample->sample)[pkt->board][pkt->module] = pkt->sample; + (*sample->sample)[pkt->board][idx] = pkt->sample; (*sample->sample)[pkt->board].nmodules = pkt->nmodules; + (*sample->sample)[pkt->board].nblocks = pkt->nblocks; + (*sample->sample)[pkt->board].nchannels = pkt->nchannels; while (oqueue_.size() > 0 && oqueue_.front().sample->size() == nboards_) { @@ -203,12 +214,16 @@ PYBINDINGS("dfmux") class_, DfMuxBoardSamplesPtr>("DfMuxBoardSamples", "Container structure for samples from modules on one board, mapping " - "0-indexed module IDs to a dfmux.DfMuxSample.") + "0-indexed module and block IDs to a dfmux.DfMuxSample.") .def(std_map_indexing_suite()) .def_readwrite("nmodules", &DfMuxBoardSamples::nmodules, "Number of modules expected to report from this board") + .def_readwrite("nblocks", &DfMuxBoardSamples::nblocks, + "Number of sub-module blocks expected to report from this board") + .def_readwrite("nchannels", &DfMuxBoardSamples::nchannels, + "Number of channels per block expected to report from this board") .def("Complete", &DfMuxBoardSamples::Complete, - "True if this structure contains data from all expected modules") + "True if this structure contains data from all expected modules and blocks") .def_pickle(g3frameobject_picklesuite()) ; register_pointer_conversions(); diff --git a/dfmux/src/DfMuxCollator.cxx b/dfmux/src/DfMuxCollator.cxx index 7f703c1e..7aa03052 100644 --- a/dfmux/src/DfMuxCollator.cxx +++ b/dfmux/src/DfMuxCollator.cxx @@ -141,24 +141,34 @@ void DfMuxCollator::Process(G3FramePtr frame, std::deque &out) if (board == metasamp->end()) continue; - auto module = board->second.find(chan->mapping->module); + int mod_idx, chan_idx; + // slight hackery for indexing timepoint frames with modules split into blocks (e.g. with hidfmux) + if (board->second.nblocks != 1) { + mod_idx = chan->mapping->module * board->second.nblocks + (int)(chan->mapping->channel / board->second.nchannels); + chan_idx = chan->mapping->channel % board->second.nchannels; + } else { + mod_idx = chan->mapping->module; + chan_idx = chan->mapping->channel; + } + + auto module = board->second.find(mod_idx); if (module == board->second.end()) continue; if (module->second->size()/2 < - size_t(chan->mapping->channel)) { - log_fatal("Board %d, module %d only has %zd " + size_t(chan_idx)) { + log_fatal("Board %d, module block %d only has %zd " "channels, but trying to read %d", chan->mapping->board_serial, - chan->mapping->module, + mod_idx, module->second->size()/2, - chan->mapping->channel); + chan_idx); } (*chan->i)[sample] = - (*module->second)[chan->mapping->channel*2]; + (*module->second)[chan_idx*2]; (*chan->q)[sample] = - (*module->second)[chan->mapping->channel*2 + 1]; + (*module->second)[chan_idx*2 + 1]; } // Next run through aux data. Missing points get filled in diff --git a/dfmux/src/DfMuxCollector.cxx b/dfmux/src/DfMuxCollector.cxx index da35ba32..54af8186 100644 --- a/dfmux/src/DfMuxCollector.cxx +++ b/dfmux/src/DfMuxCollector.cxx @@ -38,6 +38,7 @@ #include #include +#include struct RawTimestamp { /* SIGNED types are used here to permit negative numbers during @@ -55,8 +56,8 @@ struct DfmuxPacket { uint16_t serial; /* zero for V2, filled for V3 */ - uint8_t num_modules; - uint8_t channels_per_module; + uint8_t num_modules; /* correct for V4 */ + uint8_t channels_per_module; /* sub-module block in V4 */ uint8_t fir_stage; uint8_t module; /* linear; 0-7. don't like it much. */ @@ -68,11 +69,13 @@ struct DfmuxPacket { // Convert time stamp to a code in IRIG-B ticks (10 ns intervals) static int64_t -RawTimestampToTimeCode(RawTimestamp stamp) +RawTimestampToTimeCode(RawTimestamp stamp, double clock_scale) { - static __thread int64_t last_code = -1; + static __thread double last_code = -1; static __thread RawTimestamp last_stamp; + static __thread double last_ss; struct tm tm; + double ss; // Some IRIG generators don't fill in year, which is annoying. This // algorithm works unless time goes backwards, the computer clock's year @@ -96,6 +99,10 @@ RawTimestampToTimeCode(RawTimestamp stamp) stamp.y = last_stamp.y; } } + + // fix the subsecond counter if the iceboard is using + // an internal clock with a rate other than 100 MHz + ss = le32toh(stamp.ss) * clock_scale; tm.tm_year = le32toh(stamp.y) + 100 /* tm_year starts in 1900 */; tm.tm_yday = le32toh(stamp.d); @@ -108,16 +115,16 @@ RawTimestampToTimeCode(RawTimestamp stamp) stamp.m == last_stamp.m && stamp.s == last_stamp.s) { // If all fields but sub-second agree, just apply the change // in the subsecond field as an offset - last_code = (last_code - le32toh(last_stamp.ss)) + - le32toh(stamp.ss); + last_code = (last_code - last_ss) + ss; } else { tm.tm_mon = 0; // Fake out timegm with the 274th of Jan. tm.tm_mday = tm.tm_yday; // since it ignores tm_yday otherwise last_code = 100000000LL * int64_t(timegm(&tm)); - last_code += (uint64_t)le32toh(stamp.ss); + last_code += ss; } last_stamp = stamp; + last_ss = ss; return last_code; } @@ -127,6 +134,7 @@ DfMuxCollector::DfMuxCollector(G3EventBuilderPtr builder, builder_(builder), success_(false), stop_listening_(false) { + SetClockRate(100 * G3Units::MHz); success_ = (SetupSCTPSocket(hosts) != 0); } @@ -135,6 +143,7 @@ DfMuxCollector::DfMuxCollector(const char *listenaddr, builder_(builder), success_(false), stop_listening_(false), board_list_(board_list) { + SetClockRate(100 * G3Units::MHz); success_ = (SetupUDPSocket(listenaddr) != 0); } @@ -146,9 +155,17 @@ DfMuxCollector::DfMuxCollector(const char *listenaddr, for (auto i : board_serials_) board_list_.push_back(i.second); + SetClockRate(100 * G3Units::MHz); success_ = (SetupUDPSocket(listenaddr) != 0); } +void DfMuxCollector::SetClockRate(double rate) +{ + + clock_rate_ = rate; + clock_scale_ = 100 * G3Units::MHz / clock_rate_; +} + int DfMuxCollector::SetupUDPSocket(const char *listenaddr) { struct sockaddr_in addr; @@ -256,6 +273,7 @@ void DfMuxCollector::Listen(DfMuxCollector *collector) struct DfmuxPacket buf; ssize_t len; size_t target_size; + int nchan; size_t base_size = sizeof(buf) - sizeof(buf.s); @@ -264,12 +282,12 @@ void DfMuxCollector::Listen(DfMuxCollector *collector) while (!collector->stop_listening_) { len = recvfrom(collector->fd_, &buf, sizeof(buf), 0, (struct sockaddr *)&addr, &addrlen); - target_size = base_size + - buf.channels_per_module*sizeof(buf.s[0])*2; + nchan = (le16toh(buf.version) == 4) ? 128 : buf.channels_per_module; + target_size = base_size + nchan*sizeof(buf.s[0])*2; if (len != target_size) { log_error("Badly-sized packet with %d channels from %s " "(%zd bytes should be %zd)", - buf.channels_per_module, inet_ntoa(addr.sin_addr), + nchan, inet_ntoa(addr.sin_addr), len, target_size); continue; } @@ -288,6 +306,7 @@ int DfMuxCollector::BookPacket(struct DfmuxPacket *packet, struct in_addr src) std::map::iterator seq; int64_t timecode; int board_id; + int mod_id; if (le32toh(packet->magic) != FAST_MAGIC) { log_error("Corrupted packet from %s begins with %#x " @@ -304,15 +323,16 @@ int DfMuxCollector::BookPacket(struct DfmuxPacket *packet, struct in_addr src) return (-1); } board_id = id_it->second; - } else if (le16toh(packet->version) == 3) { + } else if (le16toh(packet->version) == 3 || le16toh(packet->version) == 4) { board_id = le16toh(packet->serial); if (board_list_.size() > 0 && std::find(board_list_.begin(), board_list_.end(), board_id) == board_list_.end()) { struct in_addr i; i.s_addr = listenaddr_; - log_debug("Received V3 data for board %d not " + log_debug("Received V%d data for board %d not " "enumerated on listener for interface %s", + le16toh(packet->version), board_id, inet_ntoa(i)); return (-1); } @@ -323,30 +343,33 @@ int DfMuxCollector::BookPacket(struct DfmuxPacket *packet, struct in_addr src) } modseq = &sequence_[board_id]; - seq = modseq->find(le32toh(packet->module)); + mod_id = (le16toh(packet->version) == 4) ? + (le32toh(packet->module) * 8 + le32toh(packet->channels_per_module)) : + le32toh(packet->module); + seq = modseq->find(mod_id); if (seq != modseq->end()) { seq->second++; if ((uint32_t)seq->second != le32toh(packet->seq)) { log_warn("Out-of-order packet from %d/%d (%d " - "instead of %d)", board_id, le32toh(packet->module), + "instead of %d)", board_id, mod_id, le32toh(packet->seq), seq->second); seq->second = le32toh(packet->seq); } } else { // New board we haven't seen before - (*modseq)[le32toh(packet->module)] = le32toh(packet->seq); + (*modseq)[mod_id] = le32toh(packet->seq); } // Decode packet - timecode = RawTimestampToTimeCode(packet->ts); + timecode = RawTimestampToTimeCode(packet->ts, clock_scale_); // All times reported by the readout are exactly one second behind, // likely due to a misparsing of which IRIG code the time marker refers // to (before or after). Shift them all 1 second forward. timecode += 100000000LL; - DfMuxSamplePtr sample(new DfMuxSample(timecode, - le32toh(packet->channels_per_module)*2)); + int nchan = (le16toh(packet->version) == 4) ? 128 : le32toh(packet->channels_per_module); + DfMuxSamplePtr sample(new DfMuxSample(timecode, nchan*2)); // NB: Bottom 8 bits are zero. Divide by 256 rather than >> 8 to // guarantee sign preservation. @@ -356,11 +379,16 @@ int DfMuxCollector::BookPacket(struct DfmuxPacket *packet, struct in_addr src) DfMuxSamplePacketPtr outpacket(new DfMuxSamplePacket); outpacket->board = board_id; outpacket->module = le32toh(packet->module); + outpacket->block = (le16toh(packet->version) == 4) ? + le32toh(packet->channels_per_module) : 0; outpacket->sample = sample; // Work around bug in IceBoard firmware. You always get 8 modules' // worth of packets, no matter what packet->num_modules says. - outpacket->nmodules = 8; + // (firmware V3 and below) + outpacket->nmodules = (le16toh(packet->version) == 4) ? le32toh(packet->num_modules) : 8; + outpacket->nblocks = (le16toh(packet->version) == 4) ? 8 : 1; + outpacket->nchannels = nchan; builder_->AsyncDatum(timecode, outpacket); @@ -420,6 +448,8 @@ PYBINDINGS("dfmux") .def("__init__", bp::make_constructor(make_dfmux_collector_v2_from_dict, bp::default_call_policies(), (bp::arg("interface"), bp::arg("builder"), bp::arg("board_serial_map"))), "Crate a DfMuxCollector that can parse V2 (64x) data. Pass a mapping from board IP address (strings or integers) to serial numbers as the last argument") .def("Start", &DfMuxCollector::Start) .def("Stop", &DfMuxCollector::Stop) + .add_property("clock_rate", &DfMuxCollector::GetClockRate, &DfMuxCollector::SetClockRate, + "Set the clock rate for the iceboard subseconds counter, e.g. for hidfmux. Values should be in G3Units of frequency. Defaults to 100*core.G3Units.MHz.") ; } diff --git a/dfmux/src/LegacyDfMuxCollector.cxx b/dfmux/src/LegacyDfMuxCollector.cxx index e789fb47..86abd6c1 100644 --- a/dfmux/src/LegacyDfMuxCollector.cxx +++ b/dfmux/src/LegacyDfMuxCollector.cxx @@ -291,7 +291,10 @@ int LegacyDfMuxCollector::BookPacket(struct DfmuxPacket *packet, outpacket->sample = sample; outpacket->module = m; + outpacket->block = 0; outpacket->nmodules = 4; + outpacket->nblocks = 1; + outpacket->nchannels = 16; builder_->AsyncDatum(timecode, outpacket); } diff --git a/gcp/src/GCPMuxDataDecoder.cxx b/gcp/src/GCPMuxDataDecoder.cxx index da3549ac..089a2a17 100644 --- a/gcp/src/GCPMuxDataDecoder.cxx +++ b/gcp/src/GCPMuxDataDecoder.cxx @@ -322,8 +322,11 @@ GCPMuxDataDecoder::Process(G3FramePtr frame, std::deque &out_queue) } // Set nmodules to avoid triggering warnings - for (auto j = metasample->begin(); j != metasample->end(); j++) + for (auto j = metasample->begin(); j != metasample->end(); j++) { j->second.nmodules = j->second.size(); + j->second.nblocks = 1; + j->second.nchannels = max_channel_count_; + } // Process boardValid: delete boards that are invalid. This // corresponds to the standard missing board signalling in