set stats [dict get $state stats]
set result [list server string [ns_info server] from number [dict get $options from] to number [dict get $options to] timezone string [dict get $options timezone] pathLiteral string [dict get $options pathLiteral] pathMatching string [expr {[dict get $options pathLiteral] eq "" ? "all parsed request paths; no path filter" : "case-sensitive literal substring of logged queryless path; no URL decoding"}]]
foreach key {requestCount statusBelow400Records responseBytes timingCount fallbackCount runTotal runMax acceptTotal queueTotal filterTotal unknownUserAgentRecords truncatedUserAgentRecords unknownPeerRecords} {
lappend result $key number [dict get $stats $key]
}
lappend result {*}[::nsmcp::accessFailureFields $stats $options $redactor]
set sql [dict exists $state aggregateResult]
if {$sql} {
set aggregate [dict get $state aggregateResult]
}
set omissions {}
set full true
foreach field {paths methods statuses pools hours agents peers} {
set overflow 0
if {[dict exists $stats omittedRecords $field]} {
set overflow [dict get $stats omittedRecords $field]
}
if {$overflow>0} {
set full false
}
if {$sql && $field in {peers agents}} {
set stored [dict get $aggregate $field]
set totals [dict get $aggregate ${field}Totals]
set omitted [dict get $aggregate [expr {$field eq "peers" ? "peerOmitted" : "agentOmitted"}]]
set rows {}
set i 0
foreach row $stored {
set fields [list count number [dict get $row count]]
if {$field eq "agents"} {
lappend fields value string [dict get $row value] valueTruncated boolean [dict get $row valueTruncated] valueRedacted boolean [dict get $row valueRedacted]
}
lappend rows $i object $fields
incr i
}
set unknown [dict get $stats [expr {$field eq "peers" ? "unknownPeerRecords" : "unknownUserAgentRecords"}]]
set complete [expr {$omitted==0 && $unknown==0 && [dict get $aggregate status] eq "active"}]
lappend result $field array $rows
lappend omissions $field object [list capacityOmittedRecords number 0 storageOmittedRecords number $omitted rankedOmittedRecords number [expr {[dict get $totals records]-[dict get $totals rankedRecords]}] rankedOmittedGroups number [expr {[dict get $totals distinct]-$i}] aggregationComplete boolean $complete rankingComplete boolean [expr {$complete && [dict get $totals distinct]==$i}]]
continue
}
set ranking {}
set i 0
set omittedRank 0
set omittedGroups 0
dict for {key count} [dict get $stats $field] {
lappend ranking [list $count $key]
}
set rows {}
foreach entry [lsort -integer -decreasing -index 0 $ranking] {
lassign $entry count key
if {$i >= [dict get $options maxGroups]} {
incr omittedRank $count
incr omittedGroups
continue
}
if {$field eq "peers"} {
set fields [list count number $count]
} else {
set fields [list value [expr {$field in {hours statuses} ? "number" : "string"}] $key count number $count]
}
lappend rows $i object $fields
incr i
}
set complete [expr {$overflow==0}]
if {$field eq "agents" && ([dict get $stats unknownUserAgentRecords]>0 || [dict get $stats truncatedUserAgentRecords]>0)} {
set complete false
}
if {$field eq "peers" && [dict get $stats unknownPeerRecords]>0} {
set complete false
}
lappend result $field array $rows
lappend omissions $field object [list capacityOmittedRecords number $overflow rankedOmittedRecords number $omittedRank rankedOmittedGroups number $omittedGroups aggregationComplete boolean $complete rankingComplete boolean [expr {$complete && $omittedRank==0}]]
}
lappend result breakdowns object $omissions
if {$sql} {
set exact [expr {[dict get $aggregate status] eq "active" && [dict get $aggregate peerOmitted]==0 && [dict get $stats unknownPeerRecords]==0}]
lappend result observedDistinctPeers number [dict get $aggregate peersTotals distinct] distinctPeersIsLowerBound boolean [expr {!$exact}] observedDistinctUserAgents number [dict get $aggregate agentsTotals distinct] distinctUserAgentsIsLowerBound boolean [expr {[dict get $aggregate agentOmitted]>0 || [dict get $stats unknownUserAgentRecords]>0 || [dict get $aggregate status] ne "active"}]
set known [dict get $aggregate peersTotals records]
set omitted [dict get $aggregate peerOmitted]
set unknown [dict get $stats unknownPeerRecords]
set storedComplete [expr {[dict get $aggregate status] eq "active" && $omitted==0}]
set fields [list available boolean [expr {$storedComplete && $known>0}] complete boolean $exact lowerBound boolean [expr {!$exact}] reason string [expr {[dict get $stats requestCount]==0 ? "no-matching-records" : (!$storedComplete ? "storage-incomplete" : ($unknown>0 ? "unknown-peer-records-known-subset-only" : ""))}] scope string known-logged-peer-values knownPeerRecords number $known unknownPeerRecords number $unknown]
foreach n {1 10 100} {
set count [dict get $aggregate peersTotals top$n]
lappend fields top${n}Records number $count top${n}KnownShare [expr {$known ? "number" : "null"}] [expr {$known ? $count/double($known) : ""}] top${n}Share [expr {[dict get $stats requestCount] ? "number" : "null"}] [expr {[dict get $stats requestCount] ? $count/double([dict get $stats requestCount]) : ""}] top${n}ShareUpperBound [expr {[dict get $stats requestCount] ? "number" : "null"}] [expr {[dict get $stats requestCount] ? min(1.0,($count+$unknown+$omitted)/double([dict get $stats requestCount])) : ""}]
}
lappend result peerConcentration object $fields
set fields {}
foreach key {backend status reason} {
set value [dict get $aggregate $key]
if {$key eq "status" && $value eq "active"} {
set value complete
}
lappend fields $key string $value
}
foreach key {maxBytes diskBytes peerOmitted agentOmitted peakBufferedKeys labelTruncatedRecords labelRedactedRecords} {
lappend fields $key number [dict get $aggregate $key]
}
lappend fields {*}[accessAggregationFields $aggregate]
lappend fields batchRecordLimit number 512 cacheKiB number [expr {[dict exists $aggregate cacheKiB] ? [dict get $aggregate cacheKiB] : 1024}] mainFileBudgetPercent number 40 groupingKey string sha256-of-full-logged-value
lappend result aggregateStorage object $fields
} else {
lappend result observedDistinctPeers number [dict size [dict get $stats peers]] distinctPeersIsLowerBound boolean [expr {[dict get $stats unknownPeerRecords]>0 || [dict exists $stats omittedRecords peers]}] aggregateStorage object [list backend string bounded-memory status string fallback reason string [dict get $state aggregate reason]]
}
set auth {}
dict for {key count} [dict get $stats authenticationStates] {
lappend auth $key number $count
}
lappend result authenticationStates object $auth
foreach field {samples successSamples} {
set rows {}
set i 0
foreach pair [dict get $stats $field] {
lappend rows $i object [lindex $pair 1]
incr i
}
lappend result $field array $rows
}
lappend result sampleLimit number 5 omittedSampleRecords number [expr {[dict get $stats requestCount]-[llength [dict get $stats samples]]}] omittedSuccessSampleRecords number [expr {[dict get $stats statusBelow400Records]-[llength [dict get $stats successSamples]]}]
set files {}
set i 0
set all true
set resolved true
set skipped 0
set skippedFiles 0
set rotationsExamined 0
set indexSkipped 0
set indexValidated 0
foreach file [dict get $state files] {
set status [dict get $file status]
if {$status ne "complete" || [dict exists $file orderedSelection] || ([dict exists $file timeIndexSkippedBytes] && [dict get $file timeIndexSkippedBytes]>0)} {
set all false
}
if {$status eq "outside-window"} {
incr skippedFiles
if {[dict get $file rotation]} {
incr skipped
}
}
if {$status ni {complete outside-window}} {
set resolved false
}
if {[dict get $file rotation] && $status eq "complete"} {
incr rotationsExamined
}
set fields [list fileIndex number [dict get $file fileIndex] sourceIndex number [dict get $file source] rotation boolean [dict get $file rotation] compressed boolean [dict get $file compressed] offsetBasis string [expr {[dict get $file compressed] ? "decompressed" : "file"}] snapshotBytes number [dict get $file size] examinedBytes number [expr {([dict exists $file scanReadBytes] ? [dict get $file scanReadBytes] : [dict get $file offset]-([dict exists $file scanStart] ? [dict get $file scanStart] : 0)-([dict exists $file timeIndexSkippedBytes] && ![dict exists $file timeIndexWholeFile] ? [dict get $file timeIndexSkippedBytes] : 0))+([dict exists $file timeIndexValidationBytes] ? [dict get $file timeIndexValidationBytes] : 0)}] finalOffset number [dict get $file offset] status string $status]
if {[dict exists $file orderedReason]} {
set selected [list method string endpoint-records-and-byte-bisection reason string [dict get $file orderedReason]]
if {[dict exists $file orderedSelection]} {
dict for {key value} [dict get $file orderedSelection] {
lappend selected $key [expr {$key in {pendingTail orderingAssumed} ? "boolean" : "number"}] $value
}
}
if {[dict exists $file selectionProbeBytes]} {
lappend selected probeBytes number [dict get $file selectionProbeBytes]
}
lappend fields intervalSelection object $selected
}
if {[dict exists $file timeIndexHit] || [dict exists $file timeIndexValidationBytes]} {
set skippedBytes [expr {[dict exists $file timeIndexSkippedBytes] ? [dict get $file timeIndexSkippedBytes] : 0}]
set validationBytes [expr {[dict exists $file timeIndexValidationBytes] ? [dict get $file timeIndexValidationBytes] : 0}]
incr indexSkipped $skippedBytes
incr indexValidated $validationBytes
set indexFields [list used boolean [dict exists $file timeIndexHit] scope string all-parsed-access-record-start-times skippedBytes number $skippedBytes validationBytes number $validationBytes]
set evidence {}
set j 0
if {[dict exists $file timeIndexEvidence]} {
foreach row [dict get $file timeIndexEvidence] {
set ef {}
dict for {key value} $row {
lappend ef $key [expr {$value eq "" ? "null" : "number"}] $value
}
lappend evidence $j object $ef
incr j
}
}
lappend indexFields excludedRanges array $evidence omittedRangeEvidence number [expr {[dict exists $file timeIndexExcludedRanges] ? [dict get $file timeIndexExcludedRanges]-$j : 0}]
lappend fields timeIndex object $indexFields
}
if {[dict exists $file decodedSize]} {
lappend fields decodedBytes number [dict get $file decodedSize]
}
if {[dict exists $file rangeMin]} {
set rangeFields [list verified boolean [expr {![dict get $file rangeUncertain] && $status in {complete outside-window}}] scope string path-filter-matching-records cacheHit boolean [expr {[dict exists $file timeRangeCacheHit] && [dict get $file timeRangeCacheHit]}]]
foreach {key field} {rangeMin minimum rangeMax maximum} {
set value [dict get $file $key]
lappend rangeFields $field [expr {$value eq "" ? "null" : "number"}] $value
}
lappend fields timeRange object $rangeFields
}
lappend files $i object $fields
incr i
}
set limitReasons {}
if {[dict get $state omittedFiles]>0} {
lappend limitReasons file-count-limit
}
foreach file [dict get $state files] {
set status [dict get $file status]
if {$status in {scan-byte-limit gzip-compressed-byte-limit gzip-decoded-byte-limit gzip-time-limit evidence-record-limit}} {
if {$status ni $limitReasons} {
lappend limitReasons $status
}
}
}
if {[dict get $stats oversizedRecords]>0} {
lappend limitReasons line-byte-limit
}
if {$sql && [dict get $aggregate reason] in {aggregation-disk-limit aggregation-pool-wait-limit}} {
lappend limitReasons [dict get $aggregate reason]
}
set truncated [expr {[llength $limitReasons]>0}]
set reasons {retention-and-rotation-history-unverified}
foreach key {omittedFiles unavailableFiles} {
if {[dict get $state $key]>0} {
set all false
set resolved false
lappend reasons $key
}
}
if {!$resolved} {
lappend reasons files-not-fully-examined
}
foreach key {parseErrors oversizedRecords pendingRecords} {
if {[dict get $stats $key]>0} {
lappend reasons $key
}
}
set complete [expr {$resolved && [dict get $stats parseErrors]==0 && [dict get $stats oversizedRecords]==0 && [dict get $stats pendingRecords]==0}]
set diagnosticCountsComplete [expr {![dict exists $stats diagnosticCountsComplete] || [dict get $stats diagnosticCountsComplete]}]
if {!$diagnosticCountsComplete} {
set complete false
lappend reasons parsing-counts-partial-scope
}
set recordsComplete $complete
if {$sql && [dict get $aggregate status] ne "active"} {
set complete false
lappend reasons aggregation-incomplete
}
set scanStatus [expr {$complete ? "complete" : "partial"}]
set summary "Scan finished. Log retention coverage remains unverified."
if {!$complete} {
set summary "Finished with incomplete coverage: some selected records or files could not be examined."
}
if {$truncated} {
set labels {}
foreach reason $limitReasons {
switch $reason {
aggregation-disk-limit {
lappend labels "temporary peer/User-Agent aggregation disk limit reached"
}
evidence-record-limit {
lappend labels "bounded origin evidence record limit reached"
}
scan-byte-limit {
lappend labels "scan byte limit reached"
}
file-count-limit {
lappend labels "file count limit reached"
}
gzip-compressed-byte-limit {
lappend labels "compressed file byte limit reached"
}
gzip-decoded-byte-limit {
lappend labels "decoded file byte limit reached"
}
gzip-time-limit {
lappend labels "gzip time limit reached"
}
line-byte-limit {
lappend labels "record byte limit reached"
}
}
if {$reason ni $reasons} {
lappend reasons $reason
}
}
set summary "Finished with incomplete coverage: [join $labels {; }]."
}
set limitFields {}
set i 0
foreach reason $limitReasons {
lappend limitFields $i string $reason
incr i
}
if {$recordsComplete && $sql && [dict get $aggregate status] ne "active"} {
set summary "File scanning finished; temporary SQLite aggregation is incomplete ([dict get $aggregate reason]). Peer/User-Agent omissions are reported separately; other aggregates cover the scanned records."
}
if {[accessOrderedEnabled $options]} {
append summary " Selection assumes approximate request-start-time ordering (completion fallback), with five seconds of selection padding; unsampled inversions and delayed completions can be omitted."
}
lappend result requestCountScope string [expr {[accessEvidenceLimit $options] ? "bounded-candidate-prefix" : ([accessOrderedEnabled $options] ? "ordered-selected-interval" : "exhaustively-scanned-window")}]
lappend result mode string [expr {[accessEvidenceLimit $options] ? "evidence" : "summary"}] evidenceOnly boolean [expr {[accessEvidenceLimit $options]>0}] evidenceRecordLimit number [accessEvidenceLimit $options] evidenceCandidateLimit number [expr {[accessEvidenceLimit $options] ? max(50000,10*[accessEvidenceLimit $options]) : 0}] selection string [expr {[accessOrderedEnabled $options] ? "ordered" : "exhaustive"}] orderingAssumed boolean [accessOrderedEnabled $options]
lappend result scanStatus string $scanStatus scanComplete boolean $complete truncatedByLimits boolean $truncated limitReasons array $limitFields completionSummary string $summary
set coverage [list source string configured-access-files files array $files sourceFullyExamined boolean $all rotationsRequested boolean [dict get $options includeRotations] rotationsExamined number $rotationsExamined filesSkippedOutsideWindow number $skippedFiles rotationsSkippedOutsideWindow number $skipped eligibleFilesFullyExamined boolean $resolved scanComplete boolean $complete truncatedByLimits boolean $truncated limitReasons array $limitFields timeIndexSkippedBytes number $indexSkipped timeIndexValidationBytes number $indexValidated timeIndexRetention string server-process-lifetime nativeLiteralSearch boolean [dict get $state nativeSearch] parseDiagnosticsScope string [expr {!$diagnosticCountsComplete ? "complete-checkpoint-spans-only" : ([dict get $state nativeSearch] ? "literal-candidates-and-conservative-fallbacks" : "all-examined-records")}] diagnosticCountsComplete boolean $diagnosticCountsComplete windowComplete boolean false limits object [list maxFiles number [dict get $limits files] maxScanBytes number [dict get $limits bytes] maxDecodedFileBytes number [dict get $limits decoded] maxLineBytes number [dict get $limits line] maxInflateSeconds number 15]]
lappend coverage recordsFullyExamined boolean $recordsComplete
lappend coverage selectionProbeBytes number [expr {[dict exists $state selectionProbeBytes] ? [dict get $state selectionProbeBytes] : 0}]
foreach key {matchedFirstAt matchedLastAt} {
set value [expr {[dict exists $stats $key] ? [dict get $stats $key] : ""}]
lappend coverage $key [expr {$value eq "" ? "null" : "number"}] $value
}
foreach key {omittedFiles unavailableFiles ignoredCandidates examinedBytes} {
lappend coverage $key number [dict get $state $key]
}
foreach key {examinedRecords parseErrors oversizedRecords pendingRecords earliestAt latestAt} {
set value [dict get $stats $key]
lappend coverage $key [expr {$value eq "" ? "null" : "number"}] $value
}
set items {}
set i 0
foreach reason $reasons {
lappend items $i string $reason
incr i
}
lappend coverage incompleteReasons array $items
lappend result coverage object $coverage limitations string {Evidence-mode counts and rankings concern a bounded prefix of parsed literal candidates and its matched time span, not an unbiased sample or full-window distribution. Ordered selection assumes start-time ordering; endpoint/probe observations do not certify unseen ordering, and delayed completions can be omitted. Counts are physical access records in fixed file snapshots, not unique incidents. Configured source aliases/hardlinks are counted once; copied or overlapping rotation contents are not deduplicated. Observed timestamp bounds do not certify continuous coverage; deleted rotations, custom naming and unlogged requests are unknown. Samples omit query strings and identities. User-Agents are self-reported, not verified bot identities. Peer rankings disclose counts only. Timing totals are logged phases, not CPU or database time; successful HTTP status alone does not establish PHP execution.}
return [list {} object $result]
