1121 lines
38 KiB
JavaScript
1121 lines
38 KiB
JavaScript
/*
|
|
Copyright (c) 2023-forever Douglas Malnati. All rights reserved.
|
|
|
|
See the /faq/tos page for details.
|
|
|
|
(If this generated header is stamped on a file which is a 3rd party file or under a different license or copyright, then ignore this copyright statement and use that file's terms.)
|
|
*/
|
|
|
|
import * as utl from '/js/Utl.js';
|
|
|
|
import { Base } from './Base.js';
|
|
import { CandidateFilterByBadTelemetry } from './CandidateFilterByBadTelemetry.js';
|
|
import { CandidateFilterHeartbeat } from './CandidateFilterHeartbeat.js';
|
|
import { CandidateFilterHighResLocation } from './CandidateFilterHighResLocation.js';
|
|
import { CandidateFilterConfirmed } from './CandidateFilterConfirmed.js';
|
|
import { CandidateFilterBySpec } from './CandidateFilterBySpec.js';
|
|
import { CandidateFilterByFingerprinting } from './CandidateFilterByFingerprinting.js';
|
|
import { CodecHeartbeat } from './CodecHeartbeat.js';
|
|
import { CodecExpandedBasicTelemetry } from './CodecExpandedBasicTelemetry.js';
|
|
import { CodecHighResLocation } from './CodecHighResLocation.js';
|
|
import { QuerierWsprLive } from './QuerierWsprLive.js';
|
|
import { WSPR } from '/js/WSPR.js';
|
|
import { WsprCodecMaker } from '/pro/codec/WsprCodec.js';
|
|
import { WSPREncoded } from '/js/WSPREncoded.js';
|
|
import { WsprMessageCandidate, NonRejectedOnlyFilter } from './WsprMessageCandidate.js';
|
|
import { WsprSearchResultDataTableBuilder } from './WsprSearchResultDataTableBuilder.js';
|
|
|
|
|
|
|
|
class Stats
|
|
{
|
|
constructor()
|
|
{
|
|
// query stats
|
|
// duration
|
|
// row count
|
|
this.query = {
|
|
slot0Regular: {
|
|
durationMs: 0,
|
|
rowCount: 0,
|
|
uniqueMsgCount: 0,
|
|
},
|
|
slot0Telemetry: {
|
|
durationMs: 0,
|
|
rowCount: 0,
|
|
uniqueMsgCount: 0,
|
|
},
|
|
slot1Telemetry: {
|
|
durationMs: 0,
|
|
rowCount: 0,
|
|
uniqueMsgCount: 0,
|
|
},
|
|
slot2Telemetry: {
|
|
durationMs: 0,
|
|
rowCount: 0,
|
|
uniqueMsgCount: 0,
|
|
},
|
|
slot3Telemetry: {
|
|
durationMs: 0,
|
|
rowCount: 0,
|
|
uniqueMsgCount: 0,
|
|
},
|
|
slot4Telemetry: {
|
|
durationMs: 0,
|
|
rowCount: 0,
|
|
uniqueMsgCount: 0,
|
|
},
|
|
};
|
|
|
|
// processing stats
|
|
// duration
|
|
// elimination per stage
|
|
this.processing = {
|
|
decodeMs: 0,
|
|
filterMs: 0,
|
|
dataTableBuildMs: 0,
|
|
uiRenderMs: 0,
|
|
searchTotalMs: 0,
|
|
statsGatherMs: 0,
|
|
};
|
|
|
|
// result stats
|
|
// good results
|
|
// ambiguous results
|
|
this.results = {
|
|
windowCount: 0,
|
|
|
|
slot0: {
|
|
// relative to windowCount, what pct, in these slots, have any data?
|
|
haveAnyMsgsPct: 0,
|
|
|
|
// relative to 100% that do have data
|
|
noCandidatePct: 0,
|
|
oneCandidatePct: 0,
|
|
multiCandidatePct: 0,
|
|
},
|
|
// ... for slot1-4 also
|
|
};
|
|
}
|
|
}
|
|
|
|
|
|
///////////////////////////////////////////////////////////////////////////
|
|
//
|
|
// WsprSearch class
|
|
//
|
|
// Handles the complete task of finding, filtering, and decoding
|
|
// of wspr messages in a given search window, according to
|
|
// the rules of Extended Telemetry.
|
|
//
|
|
// Fully asynchronous non-blocking callback-based interface.
|
|
//
|
|
///////////////////////////////////////////////////////////////////////////
|
|
|
|
export class WsprSearch
|
|
extends Base
|
|
{
|
|
constructor()
|
|
{
|
|
super();
|
|
|
|
// stats
|
|
this.stats = new Stats();
|
|
|
|
// query interface
|
|
this.q = new QuerierWsprLive();
|
|
|
|
// get a blank codec just for reading header fields
|
|
this.codecMakerHeaderOnly = new WsprCodecMaker();
|
|
this.codecHeaderOnly = this.codecMakerHeaderOnly.GetCodecInstance();
|
|
this.codecHeartbeat = new CodecHeartbeat();
|
|
this.codecExpandedBasicTelemetry = new CodecExpandedBasicTelemetry();
|
|
this.codecHighResLocation = new CodecHighResLocation();
|
|
this.band = "";
|
|
this.channel = "";
|
|
|
|
// keep track of data by time
|
|
this.time__windowData = new Map();
|
|
|
|
// data table builder
|
|
this.dataTableBuilder = new WsprSearchResultDataTableBuilder();
|
|
this.td = null;
|
|
|
|
// event handler registration
|
|
this.onSearchCompleteFnList = [];
|
|
|
|
// field definition lists
|
|
this.msgDefinitionUserDefinedList = null;
|
|
this.codecMakerUserDefinedList = null;
|
|
this.SetMsgDefinitionUserDefinedList(new Array(5).fill(""));
|
|
|
|
this.msgDefinitionVendorDefinedList = null;
|
|
this.codecMakerVendorDefinedList = null;
|
|
this.SetMsgDefinitionVendorDefinedList(new Array(5).fill(""));
|
|
}
|
|
|
|
SetDebug(tf)
|
|
{
|
|
super.SetDebug(tf);
|
|
|
|
this.t.SetCcGlobal(tf);
|
|
|
|
this.dataTableBuilder.SetDebug(tf);
|
|
}
|
|
|
|
Reset()
|
|
{
|
|
this.t.Reset();
|
|
this.time__windowData = new Map();
|
|
}
|
|
|
|
AddOnSearchCompleteEventHandler(fn)
|
|
{
|
|
this.onSearchCompleteFnList.push(fn);
|
|
}
|
|
|
|
SetMsgDefinitionUserDefinedList(msgDefinitionUserDefinedList)
|
|
{
|
|
if (msgDefinitionUserDefinedList.length == 5)
|
|
{
|
|
this.msgDefinitionUserDefinedList = msgDefinitionUserDefinedList;
|
|
this.codecMakerUserDefinedList = [];
|
|
|
|
// pre-calculate codecs for each of the field defs.
|
|
// it is a very expensive operation to create
|
|
for (let slot = 0; slot < 5; ++slot)
|
|
{
|
|
// look up field def by slot number
|
|
let msgDef = this.msgDefinitionUserDefinedList[slot];
|
|
|
|
// create decoder instance with that field def
|
|
let codecMaker = new WsprCodecMaker();
|
|
codecMaker.SetCodecDefFragment("MyMessageType", msgDef);
|
|
|
|
// store in slot
|
|
this.codecMakerUserDefinedList.push(codecMaker);
|
|
}
|
|
}
|
|
}
|
|
|
|
SetMsgDefinitionVendorDefinedList(msgDefinitionVendorDefinedList)
|
|
{
|
|
if (msgDefinitionVendorDefinedList.length == 5)
|
|
{
|
|
this.msgDefinitionVendorDefinedList = msgDefinitionVendorDefinedList;
|
|
this.codecMakerVendorDefinedList = [];
|
|
|
|
// pre-calculate codecs for each of the field defs.
|
|
// it is a very expensive operation to create
|
|
for (let slot = 0; slot < 5; ++slot)
|
|
{
|
|
// look up field def by slot number
|
|
let msgDef = this.msgDefinitionVendorDefinedList[slot];
|
|
|
|
// create decoder instance with that field def
|
|
let codecMaker = new WsprCodecMaker();
|
|
codecMaker.SetCodecDefFragment("MyMessageType", msgDef);
|
|
|
|
// store in slot
|
|
this.codecMakerVendorDefinedList.push(codecMaker);
|
|
}
|
|
}
|
|
}
|
|
|
|
GetCodecMakerUserDefinedList()
|
|
{
|
|
return this.codecMakerUserDefinedList;
|
|
}
|
|
|
|
GetCodecMakerVendorDefinedList()
|
|
{
|
|
return this.codecMakerVendorDefinedList;
|
|
}
|
|
|
|
async Search(band, channel, callsign, gte, lte)
|
|
{
|
|
this.band = band;
|
|
this.channel = channel;
|
|
|
|
if (channel != "")
|
|
{
|
|
await this.#SearchWithChannel(band, channel, callsign, gte, lte);
|
|
}
|
|
else
|
|
{
|
|
await this.#SearchNoChannel(band, callsign, gte, lte);
|
|
}
|
|
}
|
|
|
|
async #SearchWithChannel(band, channel, callsign, gte, lte)
|
|
{
|
|
this.Reset();
|
|
|
|
let t1 = this.t.Event("WsprSearch::Search Start");
|
|
let progress = this.#MakeWeightedProgressTracker({
|
|
totalQueries: 6,
|
|
queryShare: 0.90,
|
|
postQueryStageCount: 4,
|
|
});
|
|
progress.UpdateQueryProgress("Starting search", 0);
|
|
|
|
// Calculate slot details
|
|
let cd = WSPR.GetChannelDetails(band, channel);
|
|
|
|
let slot0Min = (cd.min + 0) % 10;
|
|
let slot1Min = (cd.min + 2) % 10;
|
|
let slot2Min = (cd.min + 4) % 10;
|
|
let slot3Min = (cd.min + 6) % 10;
|
|
let slot4Min = (cd.min + 8) % 10;
|
|
|
|
// Search in slot 0 for Regular Type 1 messages
|
|
let pSlot0Reg = (async () => {
|
|
let t1 = this.t.Event("WsprSearch::Search Query Slot 1 RegularType1 Start");
|
|
let p = await this.q.SearchRegularType1(band, slot0Min, callsign, gte, lte);
|
|
let t2 = this.t.Event("WsprSearch::Search Query Slot 1 RegularType1 Complete");
|
|
|
|
this.stats.query.slot0Regular.durationMs = Math.round(t2 - t1);
|
|
|
|
return p;
|
|
})();
|
|
|
|
// Search in slot 0 for Extended Telemetry messages
|
|
let pSlot0Tel = (async () => {
|
|
let t1 = this.t.Event("WsprSearch::Search Query Slot 1 Telemetry Start");
|
|
let p = await this.q.SearchTelemetry(band, slot0Min, cd.id1, cd.id3, gte, lte);
|
|
let t2 = this.t.Event("WsprSearch::Search Query Slot 1 Telemetry Complete");
|
|
|
|
this.stats.query.slot0Telemetry.durationMs = Math.round(t2 - t1);
|
|
|
|
return p;
|
|
})();
|
|
|
|
// Search in slot 1 for Extended Telemetry messages.
|
|
// the telemetry search for Basic vs Extended is exactly the same,
|
|
// decoding will determine which is which.
|
|
let pSlot1Tel = (async () => {
|
|
let t1 = this.t.Event("WsprSearch::Search Query Slot 2 Telemetry Start");
|
|
let p = await this.q.SearchTelemetry(band, slot1Min, cd.id1, cd.id3, gte, lte);
|
|
let t2 = this.t.Event("WsprSearch::Search Query Slot 2 Telemetry Complete");
|
|
|
|
this.stats.query.slot1Telemetry.durationMs = Math.round(t2 - t1);
|
|
|
|
return p;
|
|
})();
|
|
|
|
// Search in slot 2 for Extended Telemetry messages
|
|
let pSlot2Tel = (async () => {
|
|
let t1 = this.t.Event("WsprSearch::Search Query Slot 3 Telemetry Start");
|
|
let p = await this.q.SearchTelemetry(band, slot2Min, cd.id1, cd.id3, gte, lte);
|
|
let t2 = this.t.Event("WsprSearch::Search Query Slot 3 Telemetry Complete");
|
|
|
|
this.stats.query.slot2Telemetry.durationMs = Math.round(t2 - t1);
|
|
|
|
return p;
|
|
})();
|
|
|
|
// Search in slot 3 for Extended Telemetry messages
|
|
let pSlot3Tel = (async () => {
|
|
let t1 = this.t.Event("WsprSearch::Search Query Slot 4 Telemetry Start");
|
|
let p = await this.q.SearchTelemetry(band, slot3Min, cd.id1, cd.id3, gte, lte);
|
|
let t2 = this.t.Event("WsprSearch::Search Query Slot 4 Telemetry Complete");
|
|
|
|
this.stats.query.slot3Telemetry.durationMs = Math.round(t2 - t1);
|
|
|
|
return p;
|
|
})();
|
|
|
|
// Search in slot 4 for Extended Telemetry messages
|
|
let pSlot4Tel = (async () => {
|
|
let t1 = this.t.Event("WsprSearch::Search Query Slot 5 Telemetry Start");
|
|
let p = await this.q.SearchTelemetry(band, slot4Min, cd.id1, cd.id3, gte, lte);
|
|
let t2 = this.t.Event("WsprSearch::Search Query Slot 5 Telemetry Complete");
|
|
|
|
this.stats.query.slot4Telemetry.durationMs = Math.round(t2 - t1);
|
|
|
|
return p;
|
|
})();
|
|
|
|
// Make sure we handle results as they come in, without blocking
|
|
pSlot0Reg.then(rxRecordList => {
|
|
this.stats.query.slot0Regular.rowCount = rxRecordList.length;
|
|
this.HandleSlotResults(0, "regular", rxRecordList);
|
|
progress.IncrementQuery(`Slot 0 RegularType1 query returned`);
|
|
});
|
|
pSlot0Tel.then(rxRecordList => {
|
|
this.stats.query.slot0Telemetry.rowCount = rxRecordList.length;
|
|
this.HandleSlotResults(0, "telemetry", rxRecordList);
|
|
progress.IncrementQuery(`Slot 0 Telemetry query returned`);
|
|
});
|
|
pSlot1Tel.then(rxRecordList => {
|
|
this.stats.query.slot1Telemetry.rowCount = rxRecordList.length;
|
|
this.HandleSlotResults(1, "telemetry", rxRecordList);
|
|
progress.IncrementQuery(`Slot 1 Telemetry query returned`);
|
|
});
|
|
pSlot2Tel.then(rxRecordList => {
|
|
this.stats.query.slot2Telemetry.rowCount = rxRecordList.length;
|
|
this.HandleSlotResults(2, "telemetry", rxRecordList);
|
|
progress.IncrementQuery(`Slot 2 Telemetry query returned`);
|
|
});
|
|
pSlot3Tel.then(rxRecordList => {
|
|
this.stats.query.slot3Telemetry.rowCount = rxRecordList.length;
|
|
this.HandleSlotResults(3, "telemetry", rxRecordList);
|
|
progress.IncrementQuery(`Slot 3 Telemetry query returned`);
|
|
});
|
|
pSlot4Tel.then(rxRecordList => {
|
|
this.stats.query.slot4Telemetry.rowCount = rxRecordList.length;
|
|
this.HandleSlotResults(4, "telemetry", rxRecordList);
|
|
progress.IncrementQuery(`Slot 4 Telemetry query returned`);
|
|
});
|
|
|
|
// Wait for all results to be returned before moving on
|
|
let promiseList = [];
|
|
promiseList.push(pSlot0Reg);
|
|
promiseList.push(pSlot0Tel);
|
|
promiseList.push(pSlot1Tel);
|
|
promiseList.push(pSlot2Tel);
|
|
promiseList.push(pSlot3Tel);
|
|
promiseList.push(pSlot4Tel);
|
|
|
|
await Promise.all(promiseList);
|
|
|
|
// set up linkage
|
|
this.#OptimizeStructureLinkage();
|
|
|
|
// End of data sourcing
|
|
this.t.Event("WsprSearch::Query Results Complete");
|
|
progress.UpdatePostQueryStage("Decoding messages", 0);
|
|
|
|
// Do data processing
|
|
this.#Decode();
|
|
progress.UpdatePostQueryStage("Filtering candidates", 1);
|
|
this.#CandidateFilter();
|
|
progress.UpdatePostQueryStage("Building data table", 2);
|
|
|
|
// optimize internal data structure for later use
|
|
this.#OptimizeIteration();
|
|
|
|
// debug
|
|
this.Debug(this.time__windowData);
|
|
|
|
// build data table
|
|
let tdBuildT1 = this.t.Event("WsprSearch::BuildDataTable Start");
|
|
this.td = this.dataTableBuilder.BuildDataTable(this);
|
|
let tdBuildT2 = this.t.Event("WsprSearch::BuildDataTable End");
|
|
this.stats.processing.dataTableBuildMs = Math.round(tdBuildT2 - tdBuildT1);
|
|
progress.UpdatePostQueryStage("Gathering stats", 3);
|
|
|
|
// End of search
|
|
let t2 = this.t.Event("WsprSearch::Search Complete");
|
|
|
|
// stats
|
|
this.stats.processing.searchTotalMs = Math.round(t2 - t1);
|
|
this.#GatherStats();
|
|
this.Debug(this.stats);
|
|
|
|
// Fire completed event
|
|
progress.UpdateComplete("Firing events");
|
|
for (let fn of this.onSearchCompleteFnList)
|
|
{
|
|
fn();
|
|
}
|
|
|
|
this.t.Event("WsprSearch::Search Events Fired");
|
|
|
|
// this.t.Report("Final")
|
|
}
|
|
|
|
async #SearchNoChannel(band, callsign, gte, lte)
|
|
{
|
|
this.Reset();
|
|
|
|
let t1 = this.t.Event("WsprSearch::Search Start");
|
|
let progress = this.#MakeWeightedProgressTracker({
|
|
totalQueries: 1,
|
|
queryShare: 0.90,
|
|
postQueryStageCount: 3,
|
|
});
|
|
progress.UpdateQueryProgress("Starting search", 0);
|
|
|
|
// Search in slot 0 for Regular Type 1 messages
|
|
let pSlot0Reg = (async () => {
|
|
let t1 = this.t.Event("WsprSearch::Search Query Slot 1 RegularType1 Start");
|
|
let p = await this.q.SearchRegularType1AnyMinute(band, callsign, gte, lte);
|
|
let t2 = this.t.Event("WsprSearch::Search Query Slot 1 RegularType1 Complete");
|
|
|
|
this.stats.query.slot0Regular.durationMs = Math.round(t2 - t1);
|
|
|
|
return p;
|
|
})();
|
|
|
|
// Make sure we handle results as they come in, without blocking
|
|
pSlot0Reg.then(rxRecordList => {
|
|
this.stats.query.slot0Regular.rowCount = rxRecordList.length;
|
|
this.HandleSlotResults(0, "regular", rxRecordList);
|
|
progress.IncrementQuery("Slot 0 RegularType1 query returned");
|
|
});
|
|
|
|
// Wait for all results to be returned before moving on
|
|
let promiseList = [];
|
|
promiseList.push(pSlot0Reg);
|
|
|
|
await Promise.all(promiseList);
|
|
|
|
// set up linkage
|
|
this.#OptimizeStructureLinkage();
|
|
|
|
// End of data sourcing
|
|
this.t.Event("WsprSearch::Query Results Complete");
|
|
progress.UpdatePostQueryStage("Building data table", 0);
|
|
|
|
// optimize internal data structure for later use
|
|
this.#OptimizeIteration();
|
|
|
|
// debug
|
|
this.Debug(this.time__windowData);
|
|
|
|
// build data table
|
|
let tdBuildT1 = this.t.Event("WsprSearch::BuildDataTable Start");
|
|
this.td = this.dataTableBuilder.BuildDataTable(this);
|
|
let tdBuildT2 = this.t.Event("WsprSearch::BuildDataTable End");
|
|
this.stats.processing.dataTableBuildMs = Math.round(tdBuildT2 - tdBuildT1);
|
|
progress.UpdatePostQueryStage("Gathering stats", 1);
|
|
|
|
// End of search
|
|
let t2 = this.t.Event("WsprSearch::Search Complete");
|
|
|
|
// stats
|
|
this.stats.processing.searchTotalMs = Math.round(t2 - t1);
|
|
this.#GatherStats();
|
|
this.Debug(this.stats);
|
|
|
|
// Fire completed event
|
|
progress.UpdateComplete("Firing events");
|
|
for (let fn of this.onSearchCompleteFnList)
|
|
{
|
|
fn();
|
|
}
|
|
}
|
|
|
|
#MakeWeightedProgressTracker({ totalQueries, queryShare, postQueryStageCount })
|
|
{
|
|
let completedQueries = 0;
|
|
let postQueryStageIdx = -1;
|
|
let postQueryShare = Math.max(0, 1 - queryShare);
|
|
|
|
let getRatio = () => {
|
|
let queryRatio = totalQueries > 0 ? (completedQueries / totalQueries) : 1;
|
|
queryRatio = Math.max(0, Math.min(1, queryRatio));
|
|
|
|
let postQueryRatio = 0;
|
|
if (postQueryStageCount > 0 && postQueryStageIdx >= 0)
|
|
{
|
|
postQueryRatio = (postQueryStageIdx + 1) / postQueryStageCount;
|
|
}
|
|
postQueryRatio = Math.max(0, Math.min(1, postQueryRatio));
|
|
|
|
return (queryShare * queryRatio) + (postQueryShare * postQueryRatio);
|
|
};
|
|
|
|
let emit = (label) => {
|
|
this.Emit({
|
|
type: "SEARCH_PROGRESS",
|
|
label,
|
|
completedQueries,
|
|
totalQueries,
|
|
ratio: getRatio(),
|
|
});
|
|
};
|
|
|
|
return {
|
|
UpdateQueryProgress: (label, completedQueriesNew) => {
|
|
completedQueries = Math.max(0, Math.min(totalQueries, completedQueriesNew));
|
|
emit(label);
|
|
},
|
|
IncrementQuery: (label) => {
|
|
completedQueries = Math.max(0, Math.min(totalQueries, completedQueries + 1));
|
|
emit(label);
|
|
},
|
|
UpdatePostQueryStage: (label, stageIdx) => {
|
|
postQueryStageIdx = Math.max(-1, Math.min(postQueryStageCount - 1, stageIdx));
|
|
emit(label);
|
|
},
|
|
UpdateComplete: (label) => {
|
|
completedQueries = totalQueries;
|
|
postQueryStageIdx = postQueryStageCount - 1;
|
|
emit(label);
|
|
},
|
|
};
|
|
}
|
|
|
|
GetStats()
|
|
{
|
|
return this.stats;
|
|
}
|
|
|
|
GetDataTable()
|
|
{
|
|
return this.td;
|
|
}
|
|
|
|
// Allow iteration of every 10-minute window, in time-ascending order.
|
|
//
|
|
// The function argument is called back with:
|
|
// - time - time of window
|
|
// - slotMsgList - a single-dimensional array of messages, where the index
|
|
// corresponds to the slot it was from.
|
|
// each msg will either be a msg, or null if not present.
|
|
//
|
|
// The msgList is constructed by extracting single-candidate entries
|
|
// from the slot in the wider dataset, where available.
|
|
//
|
|
// Windows where no slot has a single-candidate entry will not be
|
|
// iterated here.
|
|
//
|
|
// Callback functions which return false immediately stop iteration.
|
|
ForEachWindow(fn)
|
|
{
|
|
for (const [time, windowData] of this.time__windowData)
|
|
{
|
|
let msgListList = [];
|
|
|
|
for (let slotData of windowData.slotDataList)
|
|
{
|
|
msgListList.push(slotData.msgList);
|
|
}
|
|
|
|
const [ok, slotMsgList] =
|
|
this.GetMsgListListWithOnlySingleCandidateEntries(msgListList);
|
|
|
|
if (ok)
|
|
{
|
|
let retVal = fn(time, slotMsgList);
|
|
|
|
if (retVal == false)
|
|
{
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
// private
|
|
|
|
|
|
|
|
///////////////////////////////////////////////////////////////////////////
|
|
// Data Structures
|
|
///////////////////////////////////////////////////////////////////////////
|
|
|
|
// Window Structure
|
|
//
|
|
// Represents a given 10-minute window.
|
|
// Has data object for each of the 5 slots.
|
|
CreateWindow()
|
|
{
|
|
let obj = {
|
|
// the time associated with slot 0
|
|
time: "",
|
|
|
|
// message data for each of the 5 slots
|
|
// (see definition below)
|
|
slotDataList: [],
|
|
|
|
// Audit/debug data populated by later processing stages.
|
|
// Members are optional, but the container is always present.
|
|
fingerprintingData: {},
|
|
};
|
|
|
|
for (let slot = 0; slot < 5; ++slot)
|
|
{
|
|
obj.slotDataList.push({
|
|
msgList: [],
|
|
});
|
|
|
|
// (convenience member that aids in debugging)
|
|
obj[`slot${slot}msgListShortcut`] = obj.slotDataList[slot].msgList;
|
|
}
|
|
|
|
return obj;
|
|
}
|
|
|
|
///////////////////////////////////////////////////////////////////////////
|
|
// Data Structure Iterators
|
|
///////////////////////////////////////////////////////////////////////////
|
|
|
|
// Iterate over every message, across all slots, across all times
|
|
ForEachMsg(fn)
|
|
{
|
|
for (let [time, windowData] of this.time__windowData)
|
|
{
|
|
for (let slotData of windowData.slotDataList)
|
|
{
|
|
for (let msg of slotData.msgList)
|
|
{
|
|
fn(msg);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
ForEachWindowMsgListList(fn)
|
|
{
|
|
for (let [time, windowData] of this.time__windowData)
|
|
{
|
|
let msgListList = [];
|
|
|
|
for (let slotData of windowData.slotDataList)
|
|
{
|
|
msgListList.push(slotData.msgList);
|
|
}
|
|
|
|
fn(msgListList);
|
|
}
|
|
}
|
|
|
|
///////////////////////////////////////////////////////////////////////////
|
|
// Data Structure Iterator Utility Functions
|
|
///////////////////////////////////////////////////////////////////////////
|
|
|
|
// A msgListList has, in each slot, a collection of messages that
|
|
// can be candidates or rejected.
|
|
//
|
|
// This function returns a tuple:
|
|
// - ok
|
|
// - true if the returned set has at least one slot with a
|
|
// single candidate message.
|
|
// - slotMsgList
|
|
// - the now-filtered list. in each slot is either:
|
|
// - null
|
|
// - a single msg, which was the only candidate msg
|
|
// in the slot to begin with.
|
|
// (this therefore excludes slots with 0 or 2+ candidates)
|
|
// (as in, we think this message is ours)
|
|
//
|
|
// This is expected to be the common form of extracted data.
|
|
GetMsgListListWithOnlySingleCandidateEntries(msgListList)
|
|
{
|
|
let atLeastOne = false;
|
|
let slotMsgList = [];
|
|
|
|
for (const msgList of msgListList)
|
|
{
|
|
const msgListFiltered = NonRejectedOnlyFilter(msgList);
|
|
|
|
if (msgListFiltered.length == 1)
|
|
{
|
|
atLeastOne = true;
|
|
|
|
slotMsgList.push(msgListFiltered[0]);
|
|
}
|
|
else
|
|
{
|
|
slotMsgList.push(null);
|
|
}
|
|
}
|
|
|
|
return [atLeastOne, slotMsgList];
|
|
}
|
|
|
|
|
|
///////////////////////////////////////////////////////////////////////////
|
|
// Data Structure Filling
|
|
///////////////////////////////////////////////////////////////////////////
|
|
|
|
// Store in local data structure
|
|
HandleSlotResults(slot, type, rxRecordList)
|
|
{
|
|
this.Debug(`WsprSearch::HandleSlotResults ${slot} ${type} ${utl.Commas(rxRecordList.length)} records`);
|
|
|
|
this.t.Event(`WsprSearch::HandleSlotResults Start ${slot} ${type}`);
|
|
|
|
// collect into different time buckets
|
|
let timeSlot0UsedSet = new Set();
|
|
let minuteOffset = slot * 2;
|
|
for (const rxRecord of rxRecordList)
|
|
{
|
|
// based on the slot the results are from, what would the time be for slot0?
|
|
let msThis = utl.ParseTimeToMs(rxRecord.time);
|
|
let slot0Ms = (msThis - (minuteOffset * 60 * 1000));
|
|
let slot0Time = utl.MakeDateTimeFromMs(slot0Ms);
|
|
|
|
// keep track of the times that actually were seen for this dataset
|
|
timeSlot0UsedSet.add(slot0Time);
|
|
|
|
// look up window based on slot 0 time
|
|
if (this.time__windowData.has(slot0Time) == false)
|
|
{
|
|
// not found, init entry
|
|
let windowData = this.CreateWindow();
|
|
|
|
windowData.time = slot0Time;
|
|
|
|
this.time__windowData.set(slot0Time, windowData);
|
|
}
|
|
|
|
// get handle to entry
|
|
let windowData = this.time__windowData.get(slot0Time);
|
|
|
|
// create temporary place to hold slot results associated with time
|
|
// without creating another hash table. pure convenience.
|
|
if (windowData.tmpRxRecordList == undefined)
|
|
{
|
|
windowData.tmpRxRecordList = [];
|
|
}
|
|
|
|
// store rxRecord in appropriate bin
|
|
windowData.tmpRxRecordList.push(rxRecord);
|
|
}
|
|
|
|
// create rxRecord groups
|
|
let Group = (msgList, rxRecordList) => {
|
|
let key__msg = new Map();
|
|
|
|
for (const rxRecord of rxRecordList)
|
|
{
|
|
let key = `${rxRecord.callsign}_${rxRecord.grid4}_${rxRecord.powerDbm}`;
|
|
|
|
if (key__msg.has(key) == false)
|
|
{
|
|
let msg = new WsprMessageCandidate();
|
|
|
|
msg.type = type;
|
|
|
|
msg.fields.callsign = rxRecord.callsign;
|
|
msg.fields.grid4 = rxRecord.grid4;
|
|
msg.fields.powerDbm = rxRecord.powerDbm;
|
|
|
|
key__msg.set(key, msg);
|
|
}
|
|
|
|
let msg = key__msg.get(key);
|
|
|
|
msg.rxRecordList.push(rxRecord);
|
|
}
|
|
|
|
// get keys in sorted order for nicer storage
|
|
let keyList = Array.from(key__msg.keys()).sort();
|
|
|
|
// store the object that has been built up
|
|
for (const key of keyList)
|
|
{
|
|
msgList.push(key__msg.get(key));
|
|
}
|
|
};
|
|
|
|
for (const timeSlot0 of timeSlot0UsedSet)
|
|
{
|
|
let windowData = this.time__windowData.get(timeSlot0);
|
|
|
|
let slotData = windowData.slotDataList[slot];
|
|
let msgList = slotData.msgList;
|
|
|
|
Group(msgList, windowData.tmpRxRecordList);
|
|
|
|
// destroy temporary list
|
|
delete windowData.tmpRxRecordList;
|
|
}
|
|
|
|
this.t.Event(`WsprSearch::HandleSlotResults End ${slot} ${type}`);
|
|
};
|
|
|
|
|
|
///////////////////////////////////////////////////////////////////////////
|
|
// Decode
|
|
///////////////////////////////////////////////////////////////////////////
|
|
|
|
#Decode()
|
|
{
|
|
let t1 = this.t.Event(`WsprSearch::Decode Start`);
|
|
|
|
let count = 0;
|
|
|
|
this.ForEachMsg(msg => {
|
|
if (msg.type != "regular")
|
|
{
|
|
++count;
|
|
|
|
let fields = msg.fields;
|
|
|
|
let ret = WSPREncoded.DecodeU4BGridPower(fields.grid4, fields.powerDbm);
|
|
if (ret.msgType == "standard")
|
|
{
|
|
let [grid56, altitudeMeters] = WSPREncoded.DecodeU4BCall(fields.callsign);
|
|
let [temperatureCelsius, voltageVolts, speedKnots, gpsIsValid] = ret.data;
|
|
|
|
let decSpot = {
|
|
grid56,
|
|
altitudeMeters,
|
|
temperatureCelsius,
|
|
voltageVolts,
|
|
speedKnots,
|
|
gpsIsValid,
|
|
};
|
|
|
|
msg.decodeDetails.type = "basic";
|
|
msg.decodeDetails.decodeOk = true;
|
|
|
|
msg.decodeDetails.basic = decSpot;
|
|
}
|
|
else
|
|
{
|
|
msg.decodeDetails.type = "extended";
|
|
|
|
// use blank codec to read headers
|
|
this.codecHeaderOnly.Reset();
|
|
this.codecHeaderOnly.SetCall(fields.callsign);
|
|
this.codecHeaderOnly.SetGrid(fields.grid4);
|
|
this.codecHeaderOnly.SetPowerDbm(fields.powerDbm);
|
|
this.codecHeaderOnly.Decode();
|
|
|
|
// check type to know how to decode
|
|
let type = this.codecHeaderOnly.GetHdrTypeEnum();
|
|
|
|
let codec = null;
|
|
let prettyType = "";
|
|
|
|
if (type == 0) // user-defined
|
|
{
|
|
// look at slot identifier of message
|
|
let slot = this.codecHeaderOnly.GetHdrSlotEnum();
|
|
|
|
// look up codec maker by slot
|
|
let codecMaker = this.codecMakerUserDefinedList[slot];
|
|
|
|
// get codec instance
|
|
codec = codecMaker.GetCodecInstance();
|
|
prettyType = "UserDefined";
|
|
}
|
|
else if (type == 1) // Heartbeat
|
|
{
|
|
codec = this.codecHeartbeat.GetCodecInstance();
|
|
prettyType = "Heartbeat";
|
|
}
|
|
else if (type == 2) // ExpandedBasicTelemetry
|
|
{
|
|
codec = this.codecExpandedBasicTelemetry.GetCodecInstance();
|
|
prettyType = "ExpBasicTelemetry";
|
|
}
|
|
else if (type == 3) // HighResLocation
|
|
{
|
|
codec = this.codecHighResLocation.GetCodecInstance();
|
|
prettyType = "HighResLocation";
|
|
}
|
|
else if (type == 15) // vendor-defined
|
|
{
|
|
// look at slot identifier of message
|
|
let slot = this.codecHeaderOnly.GetHdrSlotEnum();
|
|
|
|
// look up codec maker by slot
|
|
let codecMaker = this.codecMakerVendorDefinedList[slot];
|
|
|
|
// get codec instance
|
|
codec = codecMaker.GetCodecInstance();
|
|
prettyType = "VendorDefined";
|
|
}
|
|
else
|
|
{
|
|
// we want a codec for every extended telemetry message, so if somehow
|
|
// an unidentified message appears (eg people sending the wrong msg type)
|
|
// then we should be able to capture that here.
|
|
codec = this.codecMakerHeaderOnly.GetCodecInstance();
|
|
prettyType = "Unknown";
|
|
}
|
|
|
|
// actually decode
|
|
codec.SetCall(fields.callsign);
|
|
codec.SetGrid(fields.grid4);
|
|
codec.SetPowerDbm(fields.powerDbm);
|
|
codec.Decode();
|
|
|
|
// store codec instance
|
|
msg.decodeDetails.extended.prettyType = prettyType;
|
|
msg.decodeDetails.extended.codec = codec;
|
|
}
|
|
}
|
|
});
|
|
|
|
let t2 = this.t.Event(`WsprSearch::Decode End (${count} decoded)`);
|
|
|
|
this.stats.processing.decodeMs = Math.round(t2 - t1);
|
|
}
|
|
|
|
///////////////////////////////////////////////////////////////////////////
|
|
// Candidate Filter
|
|
///////////////////////////////////////////////////////////////////////////
|
|
|
|
#CandidateFilter()
|
|
{
|
|
let t1 = this.t.Event(`WsprSearch::CandidateFilter Start`);
|
|
|
|
// get list of filters to run
|
|
let candidateFilterList = [
|
|
new CandidateFilterByBadTelemetry(this.t),
|
|
new CandidateFilterBySpec(this.t),
|
|
new CandidateFilterHeartbeat(this.t, this.band, this.channel),
|
|
new CandidateFilterHighResLocation(this.t),
|
|
new CandidateFilterConfirmed(this.t),
|
|
new CandidateFilterByFingerprinting(this.t),
|
|
];
|
|
|
|
// create ForEach object
|
|
let forEachAble = {
|
|
ForEach: (fn) => {
|
|
this.ForEachWindowMsgListList(fn);
|
|
},
|
|
};
|
|
|
|
// run filters
|
|
for (let candidateFilter of candidateFilterList)
|
|
{
|
|
candidateFilter.SetDebug(this.debug);
|
|
|
|
candidateFilter.Filter(forEachAble);
|
|
}
|
|
|
|
let t2 = this.t.Event(`WsprSearch::CandidateFilter End`);
|
|
|
|
this.stats.processing.filterMs = Math.round(t2 - t1);
|
|
}
|
|
|
|
|
|
///////////////////////////////////////////////////////////////////////////
|
|
// Optimizing
|
|
///////////////////////////////////////////////////////////////////////////
|
|
|
|
#OptimizeStructureLinkage()
|
|
{
|
|
// every MessageCandidate should be easy to link back to its place in the window
|
|
for (const [time, windowData] of this.time__windowData)
|
|
{
|
|
for (let slot = 0; slot < 5; ++slot)
|
|
{
|
|
let slotData = windowData.slotDataList[slot];
|
|
|
|
for (let idx = 0; idx < slotData.msgList.length; ++idx)
|
|
{
|
|
let msg = slotData.msgList[idx];
|
|
|
|
msg.windowShortcut = windowData;
|
|
msg.windowSlotName = `slot${slot}`;
|
|
msg.windowSlotShortcut = slotData;
|
|
msg.windowSlotShortcutIdx = idx;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
#OptimizeIteration()
|
|
{
|
|
// put time key in reverse-time-order
|
|
|
|
let keyList = Array.from(this.time__windowData.keys());
|
|
|
|
keyList.sort();
|
|
keyList.reverse();
|
|
|
|
let time__windowData2 = new Map();
|
|
|
|
for (let key of keyList)
|
|
{
|
|
time__windowData2.set(key, this.time__windowData.get(key));
|
|
}
|
|
|
|
this.time__windowData = time__windowData2;
|
|
}
|
|
|
|
|
|
///////////////////////////////////////////////////////////////////////////
|
|
// Stats Gathering
|
|
///////////////////////////////////////////////////////////////////////////
|
|
|
|
#GatherStats()
|
|
{
|
|
let t1 = this.t.Event(`WsprSearch::GatherStats Start`);
|
|
|
|
this.GatherQueryStats();
|
|
this.GatherResultsStats();
|
|
|
|
let t2 = this.t.Event(`WsprSearch::GatherStats End`);
|
|
|
|
// calculate processing stat
|
|
this.stats.processing.statsGatherMs = Math.round(t2 - t1);
|
|
}
|
|
|
|
GatherQueryStats()
|
|
{
|
|
// calculate query stats
|
|
let GetUnique = (slot, type) => {
|
|
let count = 0;
|
|
|
|
this.ForEachWindowMsgListList(msgListList => {
|
|
for (let msg of msgListList[slot])
|
|
{
|
|
count += msg.IsType(type) ? 1 : 0;
|
|
}
|
|
});
|
|
|
|
return count;
|
|
};
|
|
|
|
this.stats.query.slot0Regular.uniqueMsgCount = GetUnique(0, "regular");
|
|
this.stats.query.slot0Telemetry.uniqueMsgCount = GetUnique(0, "telemetry");
|
|
this.stats.query.slot1Telemetry.uniqueMsgCount = GetUnique(1, "telemetry");
|
|
this.stats.query.slot2Telemetry.uniqueMsgCount = GetUnique(2, "telemetry");
|
|
this.stats.query.slot3Telemetry.uniqueMsgCount = GetUnique(3, "telemetry");
|
|
this.stats.query.slot4Telemetry.uniqueMsgCount = GetUnique(4, "telemetry");
|
|
}
|
|
|
|
GatherResultsStats()
|
|
{
|
|
// calculate results stats
|
|
this.stats.results.windowCount = this.time__windowData.size;
|
|
|
|
let GetResultsSlotStats = (slot) => {
|
|
let haveAnyMsgsCount = 0;
|
|
|
|
let noCandidateCount = 0;
|
|
let oneCandidateCount = 0;
|
|
let multiCandidateCount = 0;
|
|
|
|
this.ForEachWindowMsgListList(msgListList => {
|
|
let msgList = msgListList[slot];
|
|
|
|
haveAnyMsgsCount += msgList.length ? 1 : 0;
|
|
|
|
let msgCandidateList = NonRejectedOnlyFilter(msgList);
|
|
|
|
noCandidateCount += msgCandidateList.length == 0 ? 1 : 0;
|
|
oneCandidateCount += msgCandidateList.length == 1 ? 1 : 0;
|
|
multiCandidateCount += msgCandidateList.length > 1 ? 1 : 0;
|
|
});
|
|
|
|
// total count with data is the sum of each of these, since only one gets
|
|
// incremented per-window
|
|
let totalCount = noCandidateCount + oneCandidateCount + multiCandidateCount;
|
|
|
|
let haveAnyMsgsPct = Math.round(haveAnyMsgsCount / this.stats.results.windowCount * 100);
|
|
let noCandidatePct = Math.round(noCandidateCount / totalCount * 100);
|
|
let oneCandidatePct = Math.round(oneCandidateCount / totalCount * 100);
|
|
let multiCandidatePct = Math.round(multiCandidateCount / totalCount * 100);
|
|
|
|
return {
|
|
haveAnyMsgsPct,
|
|
noCandidatePct,
|
|
oneCandidatePct,
|
|
multiCandidatePct,
|
|
};
|
|
};
|
|
|
|
this.stats.results.slot0 = GetResultsSlotStats(0);
|
|
this.stats.results.slot1 = GetResultsSlotStats(1);
|
|
this.stats.results.slot2 = GetResultsSlotStats(2);
|
|
this.stats.results.slot3 = GetResultsSlotStats(3);
|
|
this.stats.results.slot4 = GetResultsSlotStats(4);
|
|
}
|
|
}
|
|
|
|
|
|
|