diff --git a/js/WsprSearch.js b/js/WsprSearch.js new file mode 100644 index 0000000..17c9fa8 --- /dev/null +++ b/js/WsprSearch.js @@ -0,0 +1,1120 @@ +/* +Copyright (c) 2023-forever Douglas Malnati. All rights reserved. + +See the /faq/tos page for details. + +(If this generated header is stamped on a file which is a 3rd party file or under a different license or copyright, then ignore this copyright statement and use that file's terms.) +*/ + +import * as utl from '/js/Utl.js'; + +import { Base } from './Base.js'; +import { CandidateFilterByBadTelemetry } from './CandidateFilterByBadTelemetry.js'; +import { CandidateFilterHeartbeat } from './CandidateFilterHeartbeat.js'; +import { CandidateFilterHighResLocation } from './CandidateFilterHighResLocation.js'; +import { CandidateFilterConfirmed } from './CandidateFilterConfirmed.js'; +import { CandidateFilterBySpec } from './CandidateFilterBySpec.js'; +import { CandidateFilterByFingerprinting } from './CandidateFilterByFingerprinting.js'; +import { CodecHeartbeat } from './CodecHeartbeat.js'; +import { CodecExpandedBasicTelemetry } from './CodecExpandedBasicTelemetry.js'; +import { CodecHighResLocation } from './CodecHighResLocation.js'; +import { QuerierWsprLive } from './QuerierWsprLive.js'; +import { WSPR } from '/js/WSPR.js'; +import { WsprCodecMaker } from '/pro/codec/WsprCodec.js'; +import { WSPREncoded } from '/js/WSPREncoded.js'; +import { WsprMessageCandidate, NonRejectedOnlyFilter } from './WsprMessageCandidate.js'; +import { WsprSearchResultDataTableBuilder } from './WsprSearchResultDataTableBuilder.js'; + + + +class Stats +{ + constructor() + { + // query stats + // duration + // row count + this.query = { + slot0Regular: { + durationMs: 0, + rowCount: 0, + uniqueMsgCount: 0, + }, + slot0Telemetry: { + durationMs: 0, + rowCount: 0, + uniqueMsgCount: 0, + }, + slot1Telemetry: { + durationMs: 0, + rowCount: 0, + uniqueMsgCount: 0, + }, + slot2Telemetry: { + durationMs: 0, + rowCount: 0, + uniqueMsgCount: 0, + }, + slot3Telemetry: { + durationMs: 0, + rowCount: 0, + uniqueMsgCount: 0, + }, + slot4Telemetry: { + durationMs: 0, + rowCount: 0, + uniqueMsgCount: 0, + }, + }; + + // processing stats + // duration + // elimination per stage + this.processing = { + decodeMs: 0, + filterMs: 0, + dataTableBuildMs: 0, + uiRenderMs: 0, + searchTotalMs: 0, + statsGatherMs: 0, + }; + + // result stats + // good results + // ambiguous results + this.results = { + windowCount: 0, + + slot0: { + // relative to windowCount, what pct, in these slots, have any data? + haveAnyMsgsPct: 0, + + // relative to 100% that do have data + noCandidatePct: 0, + oneCandidatePct: 0, + multiCandidatePct: 0, + }, + // ... for slot1-4 also + }; + } +} + + +/////////////////////////////////////////////////////////////////////////// +// +// WsprSearch class +// +// Handles the complete task of finding, filtering, and decoding +// of wspr messages in a given search window, according to +// the rules of Extended Telemetry. +// +// Fully asynchronous non-blocking callback-based interface. +// +/////////////////////////////////////////////////////////////////////////// + +export class WsprSearch +extends Base +{ + constructor() + { + super(); + + // stats + this.stats = new Stats(); + + // query interface + this.q = new QuerierWsprLive(); + + // get a blank codec just for reading header fields + this.codecMakerHeaderOnly = new WsprCodecMaker(); + this.codecHeaderOnly = this.codecMakerHeaderOnly.GetCodecInstance(); + this.codecHeartbeat = new CodecHeartbeat(); + this.codecExpandedBasicTelemetry = new CodecExpandedBasicTelemetry(); + this.codecHighResLocation = new CodecHighResLocation(); + this.band = ""; + this.channel = ""; + + // keep track of data by time + this.time__windowData = new Map(); + + // data table builder + this.dataTableBuilder = new WsprSearchResultDataTableBuilder(); + this.td = null; + + // event handler registration + this.onSearchCompleteFnList = []; + + // field definition lists + this.msgDefinitionUserDefinedList = null; + this.codecMakerUserDefinedList = null; + this.SetMsgDefinitionUserDefinedList(new Array(5).fill("")); + + this.msgDefinitionVendorDefinedList = null; + this.codecMakerVendorDefinedList = null; + this.SetMsgDefinitionVendorDefinedList(new Array(5).fill("")); + } + + SetDebug(tf) + { + super.SetDebug(tf); + + this.t.SetCcGlobal(tf); + + this.dataTableBuilder.SetDebug(tf); + } + + Reset() + { + this.t.Reset(); + this.time__windowData = new Map(); + } + + AddOnSearchCompleteEventHandler(fn) + { + this.onSearchCompleteFnList.push(fn); + } + + SetMsgDefinitionUserDefinedList(msgDefinitionUserDefinedList) + { + if (msgDefinitionUserDefinedList.length == 5) + { + this.msgDefinitionUserDefinedList = msgDefinitionUserDefinedList; + this.codecMakerUserDefinedList = []; + + // pre-calculate codecs for each of the field defs. + // it is a very expensive operation to create + for (let slot = 0; slot < 5; ++slot) + { + // look up field def by slot number + let msgDef = this.msgDefinitionUserDefinedList[slot]; + + // create decoder instance with that field def + let codecMaker = new WsprCodecMaker(); + codecMaker.SetCodecDefFragment("MyMessageType", msgDef); + + // store in slot + this.codecMakerUserDefinedList.push(codecMaker); + } + } + } + + SetMsgDefinitionVendorDefinedList(msgDefinitionVendorDefinedList) + { + if (msgDefinitionVendorDefinedList.length == 5) + { + this.msgDefinitionVendorDefinedList = msgDefinitionVendorDefinedList; + this.codecMakerVendorDefinedList = []; + + // pre-calculate codecs for each of the field defs. + // it is a very expensive operation to create + for (let slot = 0; slot < 5; ++slot) + { + // look up field def by slot number + let msgDef = this.msgDefinitionVendorDefinedList[slot]; + + // create decoder instance with that field def + let codecMaker = new WsprCodecMaker(); + codecMaker.SetCodecDefFragment("MyMessageType", msgDef); + + // store in slot + this.codecMakerVendorDefinedList.push(codecMaker); + } + } + } + + GetCodecMakerUserDefinedList() + { + return this.codecMakerUserDefinedList; + } + + GetCodecMakerVendorDefinedList() + { + return this.codecMakerVendorDefinedList; + } + + async Search(band, channel, callsign, gte, lte) + { + this.band = band; + this.channel = channel; + + if (channel != "") + { + await this.#SearchWithChannel(band, channel, callsign, gte, lte); + } + else + { + await this.#SearchNoChannel(band, callsign, gte, lte); + } + } + + async #SearchWithChannel(band, channel, callsign, gte, lte) + { + this.Reset(); + + let t1 = this.t.Event("WsprSearch::Search Start"); + let progress = this.#MakeWeightedProgressTracker({ + totalQueries: 6, + queryShare: 0.90, + postQueryStageCount: 4, + }); + progress.UpdateQueryProgress("Starting search", 0); + + // Calculate slot details + let cd = WSPR.GetChannelDetails(band, channel); + + let slot0Min = (cd.min + 0) % 10; + let slot1Min = (cd.min + 2) % 10; + let slot2Min = (cd.min + 4) % 10; + let slot3Min = (cd.min + 6) % 10; + let slot4Min = (cd.min + 8) % 10; + + // Search in slot 0 for Regular Type 1 messages + let pSlot0Reg = (async () => { + let t1 = this.t.Event("WsprSearch::Search Query Slot 1 RegularType1 Start"); + let p = await this.q.SearchRegularType1(band, slot0Min, callsign, gte, lte); + let t2 = this.t.Event("WsprSearch::Search Query Slot 1 RegularType1 Complete"); + + this.stats.query.slot0Regular.durationMs = Math.round(t2 - t1); + + return p; + })(); + + // Search in slot 0 for Extended Telemetry messages + let pSlot0Tel = (async () => { + let t1 = this.t.Event("WsprSearch::Search Query Slot 1 Telemetry Start"); + let p = await this.q.SearchTelemetry(band, slot0Min, cd.id1, cd.id3, gte, lte); + let t2 = this.t.Event("WsprSearch::Search Query Slot 1 Telemetry Complete"); + + this.stats.query.slot0Telemetry.durationMs = Math.round(t2 - t1); + + return p; + })(); + + // Search in slot 1 for Extended Telemetry messages. + // the telemetry search for Basic vs Extended is exactly the same, + // decoding will determine which is which. + let pSlot1Tel = (async () => { + let t1 = this.t.Event("WsprSearch::Search Query Slot 2 Telemetry Start"); + let p = await this.q.SearchTelemetry(band, slot1Min, cd.id1, cd.id3, gte, lte); + let t2 = this.t.Event("WsprSearch::Search Query Slot 2 Telemetry Complete"); + + this.stats.query.slot1Telemetry.durationMs = Math.round(t2 - t1); + + return p; + })(); + + // Search in slot 2 for Extended Telemetry messages + let pSlot2Tel = (async () => { + let t1 = this.t.Event("WsprSearch::Search Query Slot 3 Telemetry Start"); + let p = await this.q.SearchTelemetry(band, slot2Min, cd.id1, cd.id3, gte, lte); + let t2 = this.t.Event("WsprSearch::Search Query Slot 3 Telemetry Complete"); + + this.stats.query.slot2Telemetry.durationMs = Math.round(t2 - t1); + + return p; + })(); + + // Search in slot 3 for Extended Telemetry messages + let pSlot3Tel = (async () => { + let t1 = this.t.Event("WsprSearch::Search Query Slot 4 Telemetry Start"); + let p = await this.q.SearchTelemetry(band, slot3Min, cd.id1, cd.id3, gte, lte); + let t2 = this.t.Event("WsprSearch::Search Query Slot 4 Telemetry Complete"); + + this.stats.query.slot3Telemetry.durationMs = Math.round(t2 - t1); + + return p; + })(); + + // Search in slot 4 for Extended Telemetry messages + let pSlot4Tel = (async () => { + let t1 = this.t.Event("WsprSearch::Search Query Slot 5 Telemetry Start"); + let p = await this.q.SearchTelemetry(band, slot4Min, cd.id1, cd.id3, gte, lte); + let t2 = this.t.Event("WsprSearch::Search Query Slot 5 Telemetry Complete"); + + this.stats.query.slot4Telemetry.durationMs = Math.round(t2 - t1); + + return p; + })(); + + // Make sure we handle results as they come in, without blocking + pSlot0Reg.then(rxRecordList => { + this.stats.query.slot0Regular.rowCount = rxRecordList.length; + this.HandleSlotResults(0, "regular", rxRecordList); + progress.IncrementQuery(`Slot 0 RegularType1 query returned`); + }); + pSlot0Tel.then(rxRecordList => { + this.stats.query.slot0Telemetry.rowCount = rxRecordList.length; + this.HandleSlotResults(0, "telemetry", rxRecordList); + progress.IncrementQuery(`Slot 0 Telemetry query returned`); + }); + pSlot1Tel.then(rxRecordList => { + this.stats.query.slot1Telemetry.rowCount = rxRecordList.length; + this.HandleSlotResults(1, "telemetry", rxRecordList); + progress.IncrementQuery(`Slot 1 Telemetry query returned`); + }); + pSlot2Tel.then(rxRecordList => { + this.stats.query.slot2Telemetry.rowCount = rxRecordList.length; + this.HandleSlotResults(2, "telemetry", rxRecordList); + progress.IncrementQuery(`Slot 2 Telemetry query returned`); + }); + pSlot3Tel.then(rxRecordList => { + this.stats.query.slot3Telemetry.rowCount = rxRecordList.length; + this.HandleSlotResults(3, "telemetry", rxRecordList); + progress.IncrementQuery(`Slot 3 Telemetry query returned`); + }); + pSlot4Tel.then(rxRecordList => { + this.stats.query.slot4Telemetry.rowCount = rxRecordList.length; + this.HandleSlotResults(4, "telemetry", rxRecordList); + progress.IncrementQuery(`Slot 4 Telemetry query returned`); + }); + + // Wait for all results to be returned before moving on + let promiseList = []; + promiseList.push(pSlot0Reg); + promiseList.push(pSlot0Tel); + promiseList.push(pSlot1Tel); + promiseList.push(pSlot2Tel); + promiseList.push(pSlot3Tel); + promiseList.push(pSlot4Tel); + + await Promise.all(promiseList); + + // set up linkage + this.#OptimizeStructureLinkage(); + + // End of data sourcing + this.t.Event("WsprSearch::Query Results Complete"); + progress.UpdatePostQueryStage("Decoding messages", 0); + + // Do data processing + this.#Decode(); + progress.UpdatePostQueryStage("Filtering candidates", 1); + this.#CandidateFilter(); + progress.UpdatePostQueryStage("Building data table", 2); + + // optimize internal data structure for later use + this.#OptimizeIteration(); + + // debug + this.Debug(this.time__windowData); + + // build data table + let tdBuildT1 = this.t.Event("WsprSearch::BuildDataTable Start"); + this.td = this.dataTableBuilder.BuildDataTable(this); + let tdBuildT2 = this.t.Event("WsprSearch::BuildDataTable End"); + this.stats.processing.dataTableBuildMs = Math.round(tdBuildT2 - tdBuildT1); + progress.UpdatePostQueryStage("Gathering stats", 3); + + // End of search + let t2 = this.t.Event("WsprSearch::Search Complete"); + + // stats + this.stats.processing.searchTotalMs = Math.round(t2 - t1); + this.#GatherStats(); + this.Debug(this.stats); + + // Fire completed event + progress.UpdateComplete("Firing events"); + for (let fn of this.onSearchCompleteFnList) + { + fn(); + } + + this.t.Event("WsprSearch::Search Events Fired"); + + // this.t.Report("Final") + } + + async #SearchNoChannel(band, callsign, gte, lte) + { + this.Reset(); + + let t1 = this.t.Event("WsprSearch::Search Start"); + let progress = this.#MakeWeightedProgressTracker({ + totalQueries: 1, + queryShare: 0.90, + postQueryStageCount: 3, + }); + progress.UpdateQueryProgress("Starting search", 0); + + // Search in slot 0 for Regular Type 1 messages + let pSlot0Reg = (async () => { + let t1 = this.t.Event("WsprSearch::Search Query Slot 1 RegularType1 Start"); + let p = await this.q.SearchRegularType1AnyMinute(band, callsign, gte, lte); + let t2 = this.t.Event("WsprSearch::Search Query Slot 1 RegularType1 Complete"); + + this.stats.query.slot0Regular.durationMs = Math.round(t2 - t1); + + return p; + })(); + + // Make sure we handle results as they come in, without blocking + pSlot0Reg.then(rxRecordList => { + this.stats.query.slot0Regular.rowCount = rxRecordList.length; + this.HandleSlotResults(0, "regular", rxRecordList); + progress.IncrementQuery("Slot 0 RegularType1 query returned"); + }); + + // Wait for all results to be returned before moving on + let promiseList = []; + promiseList.push(pSlot0Reg); + + await Promise.all(promiseList); + + // set up linkage + this.#OptimizeStructureLinkage(); + + // End of data sourcing + this.t.Event("WsprSearch::Query Results Complete"); + progress.UpdatePostQueryStage("Building data table", 0); + + // optimize internal data structure for later use + this.#OptimizeIteration(); + + // debug + this.Debug(this.time__windowData); + + // build data table + let tdBuildT1 = this.t.Event("WsprSearch::BuildDataTable Start"); + this.td = this.dataTableBuilder.BuildDataTable(this); + let tdBuildT2 = this.t.Event("WsprSearch::BuildDataTable End"); + this.stats.processing.dataTableBuildMs = Math.round(tdBuildT2 - tdBuildT1); + progress.UpdatePostQueryStage("Gathering stats", 1); + + // End of search + let t2 = this.t.Event("WsprSearch::Search Complete"); + + // stats + this.stats.processing.searchTotalMs = Math.round(t2 - t1); + this.#GatherStats(); + this.Debug(this.stats); + + // Fire completed event + progress.UpdateComplete("Firing events"); + for (let fn of this.onSearchCompleteFnList) + { + fn(); + } + } + + #MakeWeightedProgressTracker({ totalQueries, queryShare, postQueryStageCount }) + { + let completedQueries = 0; + let postQueryStageIdx = -1; + let postQueryShare = Math.max(0, 1 - queryShare); + + let getRatio = () => { + let queryRatio = totalQueries > 0 ? (completedQueries / totalQueries) : 1; + queryRatio = Math.max(0, Math.min(1, queryRatio)); + + let postQueryRatio = 0; + if (postQueryStageCount > 0 && postQueryStageIdx >= 0) + { + postQueryRatio = (postQueryStageIdx + 1) / postQueryStageCount; + } + postQueryRatio = Math.max(0, Math.min(1, postQueryRatio)); + + return (queryShare * queryRatio) + (postQueryShare * postQueryRatio); + }; + + let emit = (label) => { + this.Emit({ + type: "SEARCH_PROGRESS", + label, + completedQueries, + totalQueries, + ratio: getRatio(), + }); + }; + + return { + UpdateQueryProgress: (label, completedQueriesNew) => { + completedQueries = Math.max(0, Math.min(totalQueries, completedQueriesNew)); + emit(label); + }, + IncrementQuery: (label) => { + completedQueries = Math.max(0, Math.min(totalQueries, completedQueries + 1)); + emit(label); + }, + UpdatePostQueryStage: (label, stageIdx) => { + postQueryStageIdx = Math.max(-1, Math.min(postQueryStageCount - 1, stageIdx)); + emit(label); + }, + UpdateComplete: (label) => { + completedQueries = totalQueries; + postQueryStageIdx = postQueryStageCount - 1; + emit(label); + }, + }; + } + + GetStats() + { + return this.stats; + } + + GetDataTable() + { + return this.td; + } + + // Allow iteration of every 10-minute window, in time-ascending order. + // + // The function argument is called back with: + // - time - time of window + // - slotMsgList - a single-dimensional array of messages, where the index + // corresponds to the slot it was from. + // each msg will either be a msg, or null if not present. + // + // The msgList is constructed by extracting single-candidate entries + // from the slot in the wider dataset, where available. + // + // Windows where no slot has a single-candidate entry will not be + // iterated here. + // + // Callback functions which return false immediately stop iteration. + ForEachWindow(fn) + { + for (const [time, windowData] of this.time__windowData) + { + let msgListList = []; + + for (let slotData of windowData.slotDataList) + { + msgListList.push(slotData.msgList); + } + + const [ok, slotMsgList] = + this.GetMsgListListWithOnlySingleCandidateEntries(msgListList); + + if (ok) + { + let retVal = fn(time, slotMsgList); + + if (retVal == false) + { + break; + } + } + } + } + + +// private + + + + /////////////////////////////////////////////////////////////////////////// + // Data Structures + /////////////////////////////////////////////////////////////////////////// + + // Window Structure + // + // Represents a given 10-minute window. + // Has data object for each of the 5 slots. + CreateWindow() + { + let obj = { + // the time associated with slot 0 + time: "", + + // message data for each of the 5 slots + // (see definition below) + slotDataList: [], + + // Audit/debug data populated by later processing stages. + // Members are optional, but the container is always present. + fingerprintingData: {}, + }; + + for (let slot = 0; slot < 5; ++slot) + { + obj.slotDataList.push({ + msgList: [], + }); + + // (convenience member that aids in debugging) + obj[`slot${slot}msgListShortcut`] = obj.slotDataList[slot].msgList; + } + + return obj; + } + + /////////////////////////////////////////////////////////////////////////// + // Data Structure Iterators + /////////////////////////////////////////////////////////////////////////// + + // Iterate over every message, across all slots, across all times + ForEachMsg(fn) + { + for (let [time, windowData] of this.time__windowData) + { + for (let slotData of windowData.slotDataList) + { + for (let msg of slotData.msgList) + { + fn(msg); + } + } + } + } + + ForEachWindowMsgListList(fn) + { + for (let [time, windowData] of this.time__windowData) + { + let msgListList = []; + + for (let slotData of windowData.slotDataList) + { + msgListList.push(slotData.msgList); + } + + fn(msgListList); + } + } + + /////////////////////////////////////////////////////////////////////////// + // Data Structure Iterator Utility Functions + /////////////////////////////////////////////////////////////////////////// + + // A msgListList has, in each slot, a collection of messages that + // can be candidates or rejected. + // + // This function returns a tuple: + // - ok + // - true if the returned set has at least one slot with a + // single candidate message. + // - slotMsgList + // - the now-filtered list. in each slot is either: + // - null + // - a single msg, which was the only candidate msg + // in the slot to begin with. + // (this therefore excludes slots with 0 or 2+ candidates) + // (as in, we think this message is ours) + // + // This is expected to be the common form of extracted data. + GetMsgListListWithOnlySingleCandidateEntries(msgListList) + { + let atLeastOne = false; + let slotMsgList = []; + + for (const msgList of msgListList) + { + const msgListFiltered = NonRejectedOnlyFilter(msgList); + + if (msgListFiltered.length == 1) + { + atLeastOne = true; + + slotMsgList.push(msgListFiltered[0]); + } + else + { + slotMsgList.push(null); + } + } + + return [atLeastOne, slotMsgList]; + } + + + /////////////////////////////////////////////////////////////////////////// + // Data Structure Filling + /////////////////////////////////////////////////////////////////////////// + + // Store in local data structure + HandleSlotResults(slot, type, rxRecordList) + { + this.Debug(`WsprSearch::HandleSlotResults ${slot} ${type} ${utl.Commas(rxRecordList.length)} records`); + + this.t.Event(`WsprSearch::HandleSlotResults Start ${slot} ${type}`); + + // collect into different time buckets + let timeSlot0UsedSet = new Set(); + let minuteOffset = slot * 2; + for (const rxRecord of rxRecordList) + { + // based on the slot the results are from, what would the time be for slot0? + let msThis = utl.ParseTimeToMs(rxRecord.time); + let slot0Ms = (msThis - (minuteOffset * 60 * 1000)); + let slot0Time = utl.MakeDateTimeFromMs(slot0Ms); + + // keep track of the times that actually were seen for this dataset + timeSlot0UsedSet.add(slot0Time); + + // look up window based on slot 0 time + if (this.time__windowData.has(slot0Time) == false) + { + // not found, init entry + let windowData = this.CreateWindow(); + + windowData.time = slot0Time; + + this.time__windowData.set(slot0Time, windowData); + } + + // get handle to entry + let windowData = this.time__windowData.get(slot0Time); + + // create temporary place to hold slot results associated with time + // without creating another hash table. pure convenience. + if (windowData.tmpRxRecordList == undefined) + { + windowData.tmpRxRecordList = []; + } + + // store rxRecord in appropriate bin + windowData.tmpRxRecordList.push(rxRecord); + } + + // create rxRecord groups + let Group = (msgList, rxRecordList) => { + let key__msg = new Map(); + + for (const rxRecord of rxRecordList) + { + let key = `${rxRecord.callsign}_${rxRecord.grid4}_${rxRecord.powerDbm}`; + + if (key__msg.has(key) == false) + { + let msg = new WsprMessageCandidate(); + + msg.type = type; + + msg.fields.callsign = rxRecord.callsign; + msg.fields.grid4 = rxRecord.grid4; + msg.fields.powerDbm = rxRecord.powerDbm; + + key__msg.set(key, msg); + } + + let msg = key__msg.get(key); + + msg.rxRecordList.push(rxRecord); + } + + // get keys in sorted order for nicer storage + let keyList = Array.from(key__msg.keys()).sort(); + + // store the object that has been built up + for (const key of keyList) + { + msgList.push(key__msg.get(key)); + } + }; + + for (const timeSlot0 of timeSlot0UsedSet) + { + let windowData = this.time__windowData.get(timeSlot0); + + let slotData = windowData.slotDataList[slot]; + let msgList = slotData.msgList; + + Group(msgList, windowData.tmpRxRecordList); + + // destroy temporary list + delete windowData.tmpRxRecordList; + } + + this.t.Event(`WsprSearch::HandleSlotResults End ${slot} ${type}`); + }; + + + /////////////////////////////////////////////////////////////////////////// + // Decode + /////////////////////////////////////////////////////////////////////////// + + #Decode() + { + let t1 = this.t.Event(`WsprSearch::Decode Start`); + + let count = 0; + + this.ForEachMsg(msg => { + if (msg.type != "regular") + { + ++count; + + let fields = msg.fields; + + let ret = WSPREncoded.DecodeU4BGridPower(fields.grid4, fields.powerDbm); + if (ret.msgType == "standard") + { + let [grid56, altitudeMeters] = WSPREncoded.DecodeU4BCall(fields.callsign); + let [temperatureCelsius, voltageVolts, speedKnots, gpsIsValid] = ret.data; + + let decSpot = { + grid56, + altitudeMeters, + temperatureCelsius, + voltageVolts, + speedKnots, + gpsIsValid, + }; + + msg.decodeDetails.type = "basic"; + msg.decodeDetails.decodeOk = true; + + msg.decodeDetails.basic = decSpot; + } + else + { + msg.decodeDetails.type = "extended"; + + // use blank codec to read headers + this.codecHeaderOnly.Reset(); + this.codecHeaderOnly.SetCall(fields.callsign); + this.codecHeaderOnly.SetGrid(fields.grid4); + this.codecHeaderOnly.SetPowerDbm(fields.powerDbm); + this.codecHeaderOnly.Decode(); + + // check type to know how to decode + let type = this.codecHeaderOnly.GetHdrTypeEnum(); + + let codec = null; + let prettyType = ""; + + if (type == 0) // user-defined + { + // look at slot identifier of message + let slot = this.codecHeaderOnly.GetHdrSlotEnum(); + + // look up codec maker by slot + let codecMaker = this.codecMakerUserDefinedList[slot]; + + // get codec instance + codec = codecMaker.GetCodecInstance(); + prettyType = "UserDefined"; + } + else if (type == 1) // Heartbeat + { + codec = this.codecHeartbeat.GetCodecInstance(); + prettyType = "Heartbeat"; + } + else if (type == 2) // ExpandedBasicTelemetry + { + codec = this.codecExpandedBasicTelemetry.GetCodecInstance(); + prettyType = "ExpBasicTelemetry"; + } + else if (type == 3) // HighResLocation + { + codec = this.codecHighResLocation.GetCodecInstance(); + prettyType = "HighResLocation"; + } + else if (type == 15) // vendor-defined + { + // look at slot identifier of message + let slot = this.codecHeaderOnly.GetHdrSlotEnum(); + + // look up codec maker by slot + let codecMaker = this.codecMakerVendorDefinedList[slot]; + + // get codec instance + codec = codecMaker.GetCodecInstance(); + prettyType = "VendorDefined"; + } + else + { + // we want a codec for every extended telemetry message, so if somehow + // an unidentified message appears (eg people sending the wrong msg type) + // then we should be able to capture that here. + codec = this.codecMakerHeaderOnly.GetCodecInstance(); + prettyType = "Unknown"; + } + + // actually decode + codec.SetCall(fields.callsign); + codec.SetGrid(fields.grid4); + codec.SetPowerDbm(fields.powerDbm); + codec.Decode(); + + // store codec instance + msg.decodeDetails.extended.prettyType = prettyType; + msg.decodeDetails.extended.codec = codec; + } + } + }); + + let t2 = this.t.Event(`WsprSearch::Decode End (${count} decoded)`); + + this.stats.processing.decodeMs = Math.round(t2 - t1); + } + + /////////////////////////////////////////////////////////////////////////// + // Candidate Filter + /////////////////////////////////////////////////////////////////////////// + + #CandidateFilter() + { + let t1 = this.t.Event(`WsprSearch::CandidateFilter Start`); + + // get list of filters to run + let candidateFilterList = [ + new CandidateFilterByBadTelemetry(this.t), + new CandidateFilterBySpec(this.t), + new CandidateFilterHeartbeat(this.t, this.band, this.channel), + new CandidateFilterHighResLocation(this.t), + new CandidateFilterConfirmed(this.t), + new CandidateFilterByFingerprinting(this.t), + ]; + + // create ForEach object + let forEachAble = { + ForEach: (fn) => { + this.ForEachWindowMsgListList(fn); + }, + }; + + // run filters + for (let candidateFilter of candidateFilterList) + { + candidateFilter.SetDebug(this.debug); + + candidateFilter.Filter(forEachAble); + } + + let t2 = this.t.Event(`WsprSearch::CandidateFilter End`); + + this.stats.processing.filterMs = Math.round(t2 - t1); + } + + + /////////////////////////////////////////////////////////////////////////// + // Optimizing + /////////////////////////////////////////////////////////////////////////// + + #OptimizeStructureLinkage() + { + // every MessageCandidate should be easy to link back to its place in the window + for (const [time, windowData] of this.time__windowData) + { + for (let slot = 0; slot < 5; ++slot) + { + let slotData = windowData.slotDataList[slot]; + + for (let idx = 0; idx < slotData.msgList.length; ++idx) + { + let msg = slotData.msgList[idx]; + + msg.windowShortcut = windowData; + msg.windowSlotName = `slot${slot}`; + msg.windowSlotShortcut = slotData; + msg.windowSlotShortcutIdx = idx; + } + } + } + } + + #OptimizeIteration() + { + // put time key in reverse-time-order + + let keyList = Array.from(this.time__windowData.keys()); + + keyList.sort(); + keyList.reverse(); + + let time__windowData2 = new Map(); + + for (let key of keyList) + { + time__windowData2.set(key, this.time__windowData.get(key)); + } + + this.time__windowData = time__windowData2; + } + + + /////////////////////////////////////////////////////////////////////////// + // Stats Gathering + /////////////////////////////////////////////////////////////////////////// + + #GatherStats() + { + let t1 = this.t.Event(`WsprSearch::GatherStats Start`); + + this.GatherQueryStats(); + this.GatherResultsStats(); + + let t2 = this.t.Event(`WsprSearch::GatherStats End`); + + // calculate processing stat + this.stats.processing.statsGatherMs = Math.round(t2 - t1); + } + + GatherQueryStats() + { + // calculate query stats + let GetUnique = (slot, type) => { + let count = 0; + + this.ForEachWindowMsgListList(msgListList => { + for (let msg of msgListList[slot]) + { + count += msg.IsType(type) ? 1 : 0; + } + }); + + return count; + }; + + this.stats.query.slot0Regular.uniqueMsgCount = GetUnique(0, "regular"); + this.stats.query.slot0Telemetry.uniqueMsgCount = GetUnique(0, "telemetry"); + this.stats.query.slot1Telemetry.uniqueMsgCount = GetUnique(1, "telemetry"); + this.stats.query.slot2Telemetry.uniqueMsgCount = GetUnique(2, "telemetry"); + this.stats.query.slot3Telemetry.uniqueMsgCount = GetUnique(3, "telemetry"); + this.stats.query.slot4Telemetry.uniqueMsgCount = GetUnique(4, "telemetry"); + } + + GatherResultsStats() + { + // calculate results stats + this.stats.results.windowCount = this.time__windowData.size; + + let GetResultsSlotStats = (slot) => { + let haveAnyMsgsCount = 0; + + let noCandidateCount = 0; + let oneCandidateCount = 0; + let multiCandidateCount = 0; + + this.ForEachWindowMsgListList(msgListList => { + let msgList = msgListList[slot]; + + haveAnyMsgsCount += msgList.length ? 1 : 0; + + let msgCandidateList = NonRejectedOnlyFilter(msgList); + + noCandidateCount += msgCandidateList.length == 0 ? 1 : 0; + oneCandidateCount += msgCandidateList.length == 1 ? 1 : 0; + multiCandidateCount += msgCandidateList.length > 1 ? 1 : 0; + }); + + // total count with data is the sum of each of these, since only one gets + // incremented per-window + let totalCount = noCandidateCount + oneCandidateCount + multiCandidateCount; + + let haveAnyMsgsPct = Math.round(haveAnyMsgsCount / this.stats.results.windowCount * 100); + let noCandidatePct = Math.round(noCandidateCount / totalCount * 100); + let oneCandidatePct = Math.round(oneCandidateCount / totalCount * 100); + let multiCandidatePct = Math.round(multiCandidateCount / totalCount * 100); + + return { + haveAnyMsgsPct, + noCandidatePct, + oneCandidatePct, + multiCandidatePct, + }; + }; + + this.stats.results.slot0 = GetResultsSlotStats(0); + this.stats.results.slot1 = GetResultsSlotStats(1); + this.stats.results.slot2 = GetResultsSlotStats(2); + this.stats.results.slot3 = GetResultsSlotStats(3); + this.stats.results.slot4 = GetResultsSlotStats(4); + } +} + + +