Class ::nsmcp::AccessDiagnosticTool (public)
::nx::Class ::nsmcp::AccessDiagnosticTool![[i]](/resources/acs-subsite/ZoomIn16.gif)
Defined in /usr/local/ns/tcl/nsmcp/traffic-tools.tcl
- Testcases:
- No testcase defined.
Source code: :property authorizer:required :property redactor:required :property peerRedactor:required :property routing:required :property {service {}} :property {defaultMode scan} :property {operation bot} :property {permission diagnostics.logs.read} :property {maxReadBytes:integer 4194304} :property {seekMinimumBytes:integer 8388608} :property {maxAggregateSources:integer 100000} :property {maxAggregateUserAgents:integer 4096} :property {inputSchema { { "type": "object", "properties": { "mode": { "type": "string", "enum": [ "scan", "page" ], "description": "Bot routing: aggregateTraffic true adds address-free source concentration and User-Agent rankings; path selects an exact queryless path. scan aggregates all pages in a background job, polled with log_results. page returns one bounded byte page. Slow requests remain paged." }, "authenticationState": { "type": "string", "enum": [ "anonymous", "authenticated", "unknown" ], "description": "Filter by logged authentication state; no identity is disclosed." }, "aggregateTraffic": { "type": "boolean", "default": false, "description": "Scan mode only: address-free source concentration and User-Agent rankings over all selected records, independent of evidence limits. No peer-disclosure permission required." }, "path": { "type": "string", "description": "Exact queryless path, e.g. /bugtracker/openacs/" }, "scanStrategy": { "type": "string", "enum": [ "window", "full" ], "default": "window", "description": "Background scans: window seeks by completion timestamps with a 300s margin and explicit ordering assumptions; full examines the entire current file." }, "lastMinutes": { "type": "integer", "minimum": 1, "maximum": 1440, "default": 60 }, "from": { "type": "number" }, "to": { "type": "number" }, "limit": { "type": "integer", "minimum": 1, "maximum": 100, "default": 20 }, "pool": { "type": "string", "default": "bots" }, "module": { "type": "string", "default": "nslog" }, "category": { "type": "string", "enum": [ "matching-rule-in-target", "matching-rule-other-pool-candidate", "bot-like-unmatched-heuristic", "no-target-rule-match", "unknown", "native-target-in-target", "native-target-other-pool-candidate", "bot-like-routed-elsewhere-heuristic", "native-other-pool" ], "description": "Bot routing only: retain evidence for this exact category. categoryTotals still covers all matching records; selectedMatches and filteredOutRecords distinguish selection from omitted evidence." }, "includePeerAddresses": { "type": "boolean", "default": false }, "beforeOffset": { "type": "integer", "minimum": 0 }, "sourceBytes": { "type": "integer", "minimum": 0 }, "sourceIdentity": { "type": "string" }, "mappingIdentity": { "type": "string" } }, "additionalProperties": false } }} :method init {} { next if {${:maxReadBytes}<1 || ${:maxReadBytes}>16777216} {error "Diagnostic read budget must be 1..16 MiB"} } :method modules {server} {return [ns_server -server $server modules]} :method logDirectory {server} {return [ns_server -server $server logdir]} :method configuredSlowPath {server module} {return [ns_config ns/server/$server/module/$module slowlogfile {}]} :method options {arguments} { if {[lindex $arguments 1] ne "object"} {error "Expected an arguments object"} set options {lastMinutes 60 limit 20 pool bots module nslog includePeerAddresses false category {} scanStrategy window aggregateTraffic false} dict set options mode [expr {${:operation} eq "bot" ? ${:defaultMode} : "page"}] set seen {} foreach {key type value} [lindex $arguments 2] { if {$key in $seen} {error "Duplicate argument: $key"} lappend seen $key switch -- $key { lastMinutes - limit - beforeOffset - sourceBytes { if {$type ne "number" || ![string is entier -strict $value] || $value<0} {error "Invalid $key"} if {$key eq "lastMinutes" && ($value<1 || $value>1440)} {error "lastMinutes must be 1..1440"} if {$key eq "limit" && ($value<1 || $value>100)} {error "limit must be 1..100"} } from - to { if {$type ne "number" || ![string is double -strict $value] || $value<0 || $value>[clock seconds]} {error "Invalid $key"} } pool - module { if {$type ne "string" || ![regexp {^[A-Za-z0-9_.-]+$} $value]} {error "Invalid $key"} } mode { if {$type ne "string" || $value ni {scan page} || (${:operation} ne "bot" && $value ne "page")} {error "Invalid diagnostic mode"} } scanStrategy { if {$type ne "string" || $value ni {window full}} {error "Invalid scanStrategy"} } category { if {${:operation} ne "bot" || $type ne "string" || $value ni { matching-rule-in-target matching-rule-other-pool-candidate bot-like-unmatched-heuristic no-target-rule-match unknown native-target-in-target native-target-other-pool-candidate bot-like-routed-elsewhere-heuristic native-other-pool }} {error "Invalid routing category"} } sourceIdentity - mappingIdentity { if {$type ne "string" || ![regexp {^[a-f0-9]{64}$} $value]} {error "Invalid $key"} } authenticationState { if {$type ne "string" || $value ni {anonymous authenticated unknown}} {error "Invalid authenticationState"} } path { if {$type ne "string" || ![string match /* $value] || [string length $value]>2048 || [string first ? $value]>=0 || [string first \x00 $value]>=0} {error "Expected exact queryless path"} } aggregateTraffic - includePeerAddresses {if {$type ne "boolean"} {error "Expected a boolean"}} default {error "Unknown argument: $key"} } dict set options $key $value } if {[dict exists $options from] != [dict exists $options to]} {error "Specify both from and to"} if {[dict exists $options from]} { if {"lastMinutes" in $seen || [dict get $options to]<=[dict get $options from] || [dict get $options to]-[dict get $options from]>86400} {error "Invalid or conflicting time selectors"} } else { dict set options to [clock seconds] dict set options from [expr {[dict get $options to]-60*[dict get $options lastMinutes]}] } set paging 0 foreach key {beforeOffset sourceBytes sourceIdentity} {incr paging [dict exists $options $key]} if {$paging ni {0 3}} {error "Paging requires beforeOffset, sourceBytes and sourceIdentity"} if {$paging && [dict get $options beforeOffset]>[dict get $options sourceBytes]} {error "Invalid cursor"} return $options } :public method validate {arguments} { if {[catch {:options $arguments} message]} {return -code error -errorcode {NSMCP RPC -32602} $message} } :method source {server options} { if {${:operation} eq "bot"} {return [dict get [::nsmcp::logDiscovery discover] access path]} set module [dict get $options module] set modules [:modules $server] if {![dict exists $modules $module name] || [dict get $modules $module name] ne "nslog"} { error "Selected nslog module is not loaded for this server" } set path [:configuredSlowPath $server $module] if {$path ne "" && [file pathtype $path] ne "absolute"} {set path [file join [:logDirectory $server] $path]} return $path } # Evaluate only fully observable root URL mappings with a User-Agent # constraint. Other headers, address predicates and nested constraints are # unknown. This compares rule membership, not urlspace precedence replay. :public method classify {record maps target} { set matches {}; set unknown 0; set index 0 foreach map $maps { set id $index; incr index if {[llength $map] ni {4 5}} {incr unknown; continue} lassign $map method base glob inherit condition if {$method ne [dict get $record method]} continue if {$base ne "/" || $glob ne "*" || $inherit ne "inherit" || [llength $map]!=5} {incr unknown; continue} if {[llength $condition]!=2 || ![string equal -nocase [lindex $condition 0] user-agent]} {incr unknown; continue} if {![dict exists $record userAgent]} {incr unknown; continue} if {[string match [lindex $condition 1] [dict get $record userAgent]]} {lappend matches $id} } if {[dict get $record pool] eq "unknown"} { set category unknown } elseif {[llength $matches]} { set category [expr {[dict get $record pool] eq $target ? "matching-rule-in-target" : "matching-rule-other-pool-candidate"}] } elseif {$unknown || ![dict exists $record userAgent]} { set category unknown } elseif {[regexp -nocase {bot|crawl|spider|baidu|knowledge} [dict get $record userAgent]] && [dict get $record pool] ne $target} { set category bot-like-unmatched-heuristic } else {set category no-target-rule-match} return [dict create category $category matchingRuleIndexes $matches unknownRules $unknown] } # Native lookup sees only logged User-Agent and peer address. A missing # predicate input can change precedence, so do not present a partial context # as a fully determined route. Inspect all pools, not just the bot pool. :public method contextComplete {record snapshot} { if {[dict size [dict get $snapshot availability]]>0} {return false} dict for {pool maps} [dict get $snapshot pools] { foreach map $maps { if {[llength $map] ni {4 5}} {return false} if {[lindex $map 0] ne [dict get $record method]} {continue} if {[llength $map]==4} {continue} set condition [lindex $map 4] if {[llength $condition]%2} {return false} foreach {field pattern} $condition { switch -nocase -- $field { user-agent { if {![dict exists $record userAgent] || [dict get $record userAgent] in {{} - {[REDACTED]}}} {return false} } x-ns-ip { if {![ns_ip valid [dict get $record peerAddress]]} {return false} } default {return false} } } } } return true } :public method nativeClassify {server record snapshot target} { set result [dict create lookupMode native-context expectedPool unknown contextComplete [:contextComplete $record $snapshot] category unknown matchingRuleIndexes {} unknownRules 0 lookupError {}] if {![dict get $result contextComplete]} {return $result} if {[catch {${:routing} mapped $server $record} expected]} { # A row-specific lookup error must not silently become fallback # rule membership or a claim that the default pool is correct. dict set result lookupError {Native route lookup failed for this record.} return $result } dict set result expectedPool $expected set actual [dict get $record pool] if {$actual eq "unknown"} {return $result} if {$expected eq $target} { dict set result category [expr {$actual eq $target ? "native-target-in-target" : "native-target-other-pool-candidate"}] } elseif {[dict exists $record userAgent] && [regexp -nocase {bot|crawl|spider|baidu|knowledge} [dict get $record userAgent]] && $actual ne $target} { dict set result category bot-like-routed-elsewhere-heuristic } else { dict set result category native-other-pool } return $result } # Preserve evidence for routing discrepancies before routine traffic can # consume the bounded result. Equal-priority groups retain first encounter. :method evidencePriority {category} { switch -- $category { matching-rule-other-pool-candidate - native-target-other-pool-candidate {return 0} unknown {return 1} default {return 2} } } :method recordFields {record} { set fields {} dict for {key value} $record { if {$key eq "matchingRuleIndexes"} { set array {}; set i 0; foreach item $value {lappend array $i number $item; incr i} lappend fields $key array $array } elseif {$key in {startTime status responseBytes offset endOffset acceptTime queueTime filterTime runTime unknownRules count runTotal timingCount firstAt lastAt runMax}} { lappend fields $key number $value } elseif {$key in {redacted contextComplete}} {lappend fields $key boolean $value } else {lappend fields $key string $value} } return $fields } :method argumentTriples {options} { set fields {} dict for {key value} $options { if {$key eq "category" && $value eq ""} {continue} set type string if {$key in {includePeerAddresses aggregateTraffic}} {set type boolean} if {$key in {from to limit beforeOffset sourceBytes lastMinutes}} {set type number} lappend fields $key $type $value } return [list {} object $fields] } :method aggregateKey {record} { set key {} foreach field {category method path pool status userAgent expectedPool contextComplete lookupError matchingRuleIndexes peerAddress} { if {[dict exists $record $field]} {lappend key $field [dict get $record $field]} } return $key } # These source keys stay in private job state; only numeric distributions # leave this method. Never serialize the accumulator itself. :method accumulateTraffic {stateVar record} { upvar 1 $stateVar state set auth [dict get $record authenticationState] if {![dict exists $state authenticationStates $auth]} { dict set state authenticationStates $auth {requestCount 0 status503Count 0 successfulCount 0 successfulTimingCount 0 successfulRunTotal 0.0} } dict incr state requestCount set status [dict get $record status] set success [expr {$status>=200 && $status<300}] set timed [expr {$success && [dict exists $record runTime]}] set run [expr {$timed ? [dict get $record runTime] : 0.0}] if {$status==503} {dict incr state status503Count} if {$success} {dict incr state successfulCount} if {$timed} {dict incr state successfulTimingCount} dict set state successfulRunTotal [expr {[dict get $state successfulRunTotal]+$run}] set authRow [dict get $state authenticationStates $auth] dict incr authRow requestCount if {$status==503} {dict incr authRow status503Count} if {$success} {dict incr authRow successfulCount} if {$timed} {dict incr authRow successfulTimingCount} dict set authRow successfulRunTotal [expr {[dict get $authRow successfulRunTotal]+$run}] dict set state authenticationStates $auth $authRow set peer [dict get $record peerAddress] set valid [ns_ip valid $peer] if {$valid} { set peer [ns_crypto::md string -digest sha256 -- [list [dict get $state salt] $peer]] } if {!$valid} { dict incr state unknownPeerCount } elseif {![dict exists $state sources $peer] && [dict size [dict get $state sources]]>=${:maxAggregateSources}} { dict incr state untrackedSourceRecords } else { set count 0 if {[dict exists $state sources $peer]} {set count [dict get $state sources $peer]} dict set state sources $peer [expr {$count+1}] } set ua {} if {[dict exists $record userAgent]} {set ua [dict get $record userAgent]} if {$ua in {{} - {[REDACTED]}}} { dict incr state unknownUserAgentCount return } set ua [${:redactor} message $ua] if {[string length $ua]>2048 || (![dict exists $state agents $ua] && [dict size [dict get $state agents]]>=${:maxAggregateUserAgents})} { dict incr state untrackedUserAgentRecords return } if {![dict exists $state agents $ua]} { dict set state agents $ua {requestCount 0 status503Count 0 successfulCount 0 successfulTimingCount 0 successfulRunTotal 0.0} } set row [dict get $state agents $ua] dict incr row requestCount if {$status==503} {dict incr row status503Count} if {$success} {dict incr row successfulCount} if {$timed} {dict incr row successfulTimingCount} dict set row successfulRunTotal [expr {[dict get $row successfulRunTotal]+$run}] dict set state agents $ua $row } :method trafficTriples {state duration complete} { set fields [list scope string examined-selected-records windowComplete boolean false scanComplete boolean $complete] foreach key {requestCount status503Count successfulCount successfulTimingCount successfulRunTotal unknownPeerCount untrackedSourceRecords unknownUserAgentCount untrackedUserAgentRecords} { lappend fields $key number [dict get $state $key] } set authentication {} if {[dict exists $state authenticationStates]} { dict for {name row} [dict get $state authenticationStates] { set entry {}; dict for {key value} $row {lappend entry $key number $value} lappend authentication $name object $entry } } lappend fields authenticationStates object $authentication set requests [dict get $state requestCount] lappend fields durationSeconds number $duration requestRate number [expr {$requests/double($duration)}] successfulTimingMissingCount number [expr {[dict get $state successfulCount]-[dict get $state successfulTimingCount]}] set peerReasons {} if {[dict get $state unknownPeerCount]} {lappend peerReasons 0 string missing-or-invalid-peer-fields} if {[dict get $state untrackedSourceRecords]} {lappend peerReasons 1 string source-cardinality-limit} set uaReasons {} if {[dict get $state unknownUserAgentCount]} {lappend uaReasons 0 string missing-user-agent-fields} if {[dict get $state untrackedUserAgentRecords]} {lappend uaReasons 1 string user-agent-cardinality-or-length-limit} set counts [lsort -integer -decreasing [dict values [dict get $state sources]]] set known [dict size [dict get $state sources]] lappend fields distinctKnownSources number $known observedPeerCountsComplete boolean [expr {$peerReasons eq ""}] peerCountsIncompleteReasons array $peerReasons observedUserAgentCountsComplete boolean [expr {$uaReasons eq ""}] userAgentCountsIncompleteReasons array $uaReasons sourceLimit number ${:maxAggregateSources} userAgentLimit number ${:maxAggregateUserAgents} if {$peerReasons eq ""} {lappend fields distinctSources number $known} else {lappend fields distinctSources null {}} set buckets {1 0 2-5 0 6-10 0 11-100 0 101-1000 0 1001+ 0} foreach count $counts { set bucket [expr {$count==1 ? "1" : ($count<=5 ? "2-5" : ($count<=10 ? "6-10" : ($count<=100 ? "11-100" : ($count<=1000 ? "101-1000" : "1001+"))))}] dict incr buckets $bucket } set histogram {}; dict for {key count} $buckets {lappend histogram $key number $count} lappend fields sourcesByRequestCount object $histogram set top {}; set rank 0; set sum 0 foreach count [lrange $counts 0 9] { incr sum $count lappend top $rank object [list rank number [expr {$rank+1}] requestCount number $count requestRate number [expr {$count/double($duration)}] share number [expr {$requests ? $count/double($requests) : 0.0}]] incr rank } lappend fields topSources array $top topTenSourceShare number [expr {$requests ? $sum/double($requests) : 0.0}] set rankings {} foreach {label metric} {byRequests requestCount by503 status503Count bySuccessfulRuntime successfulRunTotal} { set ordered {} dict for {ua row} [dict get $state agents] {lappend ordered [list [dict get $row $metric] $ua $row]} set output {}; set i 0 foreach item [lrange [lsort -real -decreasing -index 0 $ordered] 0 19] { lassign $item value ua row set entry [list userAgent string $ua] dict for {key count} $row {lappend entry $key number $count} lappend output $i object $entry incr i } lappend rankings $label array $output } lappend fields userAgents object $rankings distinctKnownUserAgents number [dict size [dict get $state agents]] rankingLimit number 20 omittedRankedUserAgents number [expr {max(0,[dict size [dict get $state agents]]-20)}] limitations string {Counts cover examined records selected by window, path and category, before evidence truncation. Rankings cover tracked redacted User-Agents, not verified bots. Source keys and addresses are never disclosed. Unknown fields and cardinality limits make observed distributions partial. Seek assumptions and rotations still limit time-window coverage; observed completeness is not window completeness. Rates use the requested window duration, not a rolling burst. Successful runtime is summed 2xx handler time, not CPU time.} return $fields } # Seeking never certifies chronological order: concurrent requests and # clock changes can reorder completions. Probe failures fall back to full. :method seekSample {channel position size samplesVar bytesVar} { upvar 1 $samplesVar samples $bytesVar probeBytes seek $channel $position set block [read $channel [expr {min(65536,$size-$position)}]] incr probeBytes [string length $block] set offset $position if {$position>0} { set cut [string first \n $block] if {$cut<0} {error "No complete timestamped line at seek probe"} incr offset [expr {$cut+1}] set block [string range $block [expr {$cut+1}] end] } foreach line [lrange [split $block \n] 0 end-1] { set end [expr {$offset+[string length $line]+1}] set at [::nsmcp::accessRecordParser completionTime [encoding convertfrom utf-8 $line]] if {$at ne ""} { dict set samples $offset $at set previous -1 foreach key [lsort -integer [dict keys $samples]] { set time [dict get $samples $key] if {$time<$previous} {error "Completion timestamps are not ordered at seek probes"} set previous $time } return [list $offset $end $at] } set offset $end } error "No timestamp at seek probe" } :method scanBounds {context options} { set path [:source [$context server] $options] file stat $path before set size $before(size) set bounds [dict create from 0 to $size sourceBytes $size strategy full reason {Full scan requested or source below seek threshold.} probeBytes 0] if {[dict get $options scanStrategy] eq "full" || $size<${:seekMinimumBytes}} {return $bounds} set f [open $path rb] set samples {}; set probeBytes 0 try { set prefix [read $f [expr {min(256,$size)}]] set identity [ns_crypto::md string -digest sha256 -- [list [file normalize $path] $before(dev) $before(ino) $prefix]] set first [:seekSample $f 0 $size samples probeBytes] set last [:seekSample $f [expr {max(0,$size-65536)}] $size samples probeBytes] # Spread probes across the file before assuming monotonicity. for {set i 1} {$i<16} {incr i} { :seekSample $f [expr {$size*$i/16}] $size samples probeBytes } foreach {key target} [list from [expr {[dict get $options from]-300}] to [expr {[dict get $options to]+300}]] { if {$target<=[lindex $first 2]} {dict set bounds $key 0; continue} if {$target>=[lindex $last 2]} {dict set bounds $key $size; continue} set low $first; set high $last for {set i 0} {$i<40 && [lindex $high 0]-[lindex $low 0]>131072} {incr i} { set middle [:seekSample $f [expr {([lindex $low 0]+[lindex $high 0])/2}] $size samples probeBytes] if {[lindex $middle 2]<$target} {set low $middle} else {set high $middle} } dict set bounds $key [expr {$key eq "from" ? [lindex $low 0] : [lindex $high 1]}] } # A lower bound beyond the final sample still examines its tail. if {[dict get $bounds from]==$size} {dict set bounds from [lindex $last 0]} file stat $path after if {$before(dev)!=$after(dev) || $before(ino)!=$after(ino) || $after(size)<$size} {error "Source changed during seek"} dict set bounds identity $identity dict set bounds strategy completion-window dict set bounds reason {Sampled completion timestamps were ordered; unsampled disorder and requests completing more than 300s outside the start-time window may be missed. Use scanStrategy full to avoid seek assumptions.} } on error {message} { dict set bounds from 0 dict set bounds to $size dict set bounds reason "Seek unavailable; full-file fallback: $message" } finally {close $f} dict set bounds probeBytes $probeBytes return $bounds } # One bounded page per ns_job invocation. Only bounded aggregate evidence # and counters cross worker boundaries, never the raw log contents. :public method scanStep {context state} { set options [dict get $state options] if {![dict exists $state bounds]} { if {![dict exists $options scanStrategy]} {dict set options scanStrategy window} set bounds [:scanBounds $context $options] dict set state bounds $bounds if {[dict get $bounds strategy] eq "completion-window"} { dict set options beforeOffset [dict get $bounds to] dict set options sourceBytes [dict get $bounds sourceBytes] dict set options sourceIdentity [dict get $bounds identity] } } set bounds [dict get $state bounds] set aggregateTraffic [expr {[dict exists $options aggregateTraffic] && [dict get $options aggregateTraffic]}] set traffic {} if {$aggregateTraffic} { if {[dict exists $state traffic]} {set traffic [dict get $state traffic]} else { set traffic {sources {} agents {} authenticationStates {} requestCount 0 status503Count 0 successfulCount 0 successfulTimingCount 0 successfulRunTotal 0.0 unknownPeerCount 0 untrackedSourceRecords 0 unknownUserAgentCount 0 untrackedUserAgentRecords 0} dict set traffic salt [ns_crypto::randombytes -encoding hex 32] } } set page [:page $context [:argumentTriples $options] [dict get $bounds from] [expr {$aggregateTraffic ? "traffic" : ""}]] if {$aggregateTraffic} {dict set state traffic $traffic} set value [ns_json parse [ns_json value -- $page]] if {![dict get $value available]} {return [list $page {} {}]} if {![dict exists $state fields]} { dict set state fields [lindex $page 2] dict set state groups {} dict set state totals {} dict set state matches 0 dict set state selected 0 dict set state pages 0 dict set state coverage [dict get $value coverage] } else { set initial [ns_json parse [ns_json value -- [list {} object [dict get $state fields]]]] foreach key {sourceIdentity mappingIdentity from to lookupMode selectedCategory} { if {[dict get $initial $key] ne [dict get $value $key]} {error "Routing scan identity changed; start a new query"} } set coverage [dict get $state coverage] set nextCoverage [dict get $value coverage] foreach key {unrecognizedRecords oversizedRecords missingUserAgent missingPool} { dict incr coverage $key [dict get $nextCoverage $key] } dict set coverage pendingFinalRecord [expr {[dict get $coverage pendingFinalRecord] || [dict get $nextCoverage pendingFinalRecord]}] dict set coverage examinedFromOffset [dict get $nextCoverage examinedFromOffset] dict set coverage unreadEarlierBytes [dict get $nextCoverage unreadEarlierBytes] dict set state coverage $coverage } dict incr state pages dict incr state matches [dict get $value matches] dict incr state selected [dict get $value selectedMatches] set totals [dict get $state totals] dict for {category count} [dict get $value categoryTotals] {dict incr totals $category $count} dict set state totals $totals set groups [dict get $state groups] foreach record [dict get $value records] { set key [:aggregateKey $record] if {[dict exists $groups $key]} { set group [dict get $groups $key] foreach field {count timingCount} {dict incr group $field [dict get $record $field]} dict set group runTotal [expr {[dict get $group runTotal]+[dict get $record runTotal]}] dict set group runMax [expr {max([dict get $group runMax],[dict get $record runMax])}] dict set group firstAt [expr {min([dict get $group firstAt],[dict get $record firstAt])}] dict set group lastAt [expr {max([dict get $group lastAt],[dict get $record lastAt])}] dict set groups $key $group } else { if {[dict size $groups]>=[dict get $options limit]} { set worst [:evidencePriority [dict get $record category]] set victim {} dict for {otherKey otherRecord} $groups { set rank [:evidencePriority [dict get $otherRecord category]] if {$rank>$worst} {set worst $rank; set victim $otherKey} } if {$victim eq ""} {continue} dict unset groups $victim } dict set groups $key $record } } # Bound state by the same evidence byte budget used for individual pages. set ordered {} dict for {key group} $groups {lappend ordered [list [:evidencePriority [dict get $group category]] $key $group]} set groups {}; set entries {}; set index 0; set bytes 0; set returned 0 set omitted {} dict for {category count} $totals { if {[dict get $value selectedCategory] eq "" || [dict get $value selectedCategory] eq $category} {dict set omitted $category $count} } foreach pair [lsort -integer -index 0 $ordered] { lassign $pair priority key record set fields [:recordFields $record] set size [string bytelength [ns_json value -- [list {} object $fields]]] if {$bytes+$size>65536} {continue} incr bytes $size dict set groups $key $record lappend entries $index object $fields incr index incr returned [dict get $record count] dict incr omitted [dict get $record category] [expr {-[dict get $record count]}] } dict set state groups $groups set counts {}; dict for {category count} $totals {lappend counts $category number $count} set omittedCounts {}; dict for {category count} $omitted {lappend omittedCounts $category number $count} set coverage [dict get $state coverage] set coverageFields {} foreach {key type original} [ns_json triples getvalue -output triples -path coverage -- $page] { lappend coverageFields $key $type [dict get $coverage $key] } lappend coverageFields scanStrategy string [dict get $bounds strategy] seekReason string [dict get $bounds reason] seekMarginSeconds number 300 seekProbeBytes number [dict get $bounds probeBytes] skippedEarlierBytes number [dict get $bounds from] skippedLaterBytes number [expr {[dict get $bounds sourceBytes]-[dict get $bounds to]}] completionOrderAssumed boolean [expr {[dict get $bounds strategy] eq "completion-window"}] lappend coverageFields pages number [dict get $state pages] aggregationScope string all-examined-pages sourceFullyExamined boolean [expr {[dict get $coverage unreadEarlierBytes]==0 && [dict get $coverage examinedToOffset]==[dict get $coverage sourceBytes]}] set fields {} foreach {key type original} [dict get $state fields] { switch -- $key { matches {set original [dict get $state matches]} selectedMatches {set original [dict get $state selected]} filteredOutRecords {set original [expr {[dict get $state matches]-[dict get $state selected]}]} categoryTotals {set original $counts} omittedRecords {set original [expr {[dict get $state selected]-$returned}]} omittedCategoryTotals {set original $omittedCounts} records {set original $entries} more {set original false} nextArguments {set original {}} coverage {set original $coverageFields} } lappend fields $key $type $original } if {$aggregateTraffic} { lappend fields trafficAggregation object [:trafficTriples $traffic [expr {[dict get $options to]-[dict get $options from]}] [expr {![dict get $value more]}]] } lappend fields scanComplete boolean [expr {![dict get $value more]}] set continuation {} if {[dict get $value more]} { set nextOptions [dict get $value nextArguments] dict set nextOptions aggregateTraffic $aggregateTraffic dict set state options $nextOptions set continuation $state } set progress [list examinedBytes number [expr {[dict get $bounds to]-[dict get $coverage unreadEarlierBytes]}] sourceBytes number [expr {[dict get $bounds to]-[dict get $bounds from]}]] return [list [list {} object $fields] $continuation $progress] } :public method execute {context arguments} { set options [:options $arguments] if {${:operation} eq "bot" && [dict get $options mode] eq "scan"} { ${:authorizer} require $context diagnostics.runtime.read if {[dict get $options includePeerAddresses]} {${:authorizer} require $context diagnostics.logs.peers.read} if {${:service} eq ""} {error "Routing background service is not configured"} dict set options mode page dict unset options lastMinutes return [${:service} start $context routingSummary [dict create tool [self] options $options]] } if {[dict get $options aggregateTraffic]} {error "aggregateTraffic requires bot routing mode scan"} return [:page $context $arguments] } :public method page {context arguments {floor 0} {aggregationVar {}}} { if {$aggregationVar ne ""} {upvar 1 $aggregationVar traffic} set options [:options $arguments] set server [$context server] set peers [dict get $options includePeerAddresses] if {$peers} {${:authorizer} require $context diagnostics.logs.peers.read} set redactor [expr {$peers ? ${:peerRedactor} : ${:redactor}}] set result [list server string $server from number [dict get $options from] to number [dict get $options to]] foreach key {path authenticationState} {if {[dict exists $options $key]} {lappend result $key string [dict get $options $key]}} set snapshot {}; set maps {}; set mappingIdentity {} set lookupMode fallback-rule-membership set routeCache {} if {${:operation} eq "bot"} { ${:authorizer} require $context diagnostics.runtime.read set snapshot [${:routing} snapshot $server] set target [dict get $options pool] if {![dict exists $snapshot pools $target]} { return [list {} object [concat $result [list available boolean false reason string {Effective target-pool mappings unavailable.}]]] } set maps [dict get $snapshot pools $target] if {[${:routing} contextSupported $server]} {set lookupMode native-context} set category [dict get $options category] if {$category ne "" && $category ne "unknown"} { set nativeCategory [expr {$category in { native-target-in-target native-target-other-pool-candidate bot-like-routed-elsewhere-heuristic native-other-pool }}] if {$nativeCategory != ($lookupMode eq "native-context")} { return -code error -errorcode {NSMCP RPC -32602} "Category '$category' is not produced by lookupMode '$lookupMode'; select a category for the current mode." } } set mappingIdentity [ns_crypto::md string -digest sha256 -- [list [dict get $snapshot pools] $lookupMode]] if {[dict exists $options mappingIdentity] && [dict get $options mappingIdentity] ne $mappingIdentity} {error "Effective mappings changed; start a new analysis"} lappend result pool string $target lookupMode string $lookupMode mappingIdentity string $mappingIdentity {*}[${:routing} encodeSnapshot $snapshot] } if {${:operation} eq "slow"} { set thresholds {} foreach key {slowlogconntime slowlogfiltertime slowlogqueuetime} { set value [ns_config ns/server/$server/module/[dict get $options module] $key {}] lappend thresholds $key string $value } lappend result configuredThresholds object $thresholds } set path [:source $server $options] if {$path eq "" || ![file isfile $path] || ![file readable $path]} { return [list {} object [concat $result [list available boolean false reason string {Configured log is missing, disabled or unreadable.}]]] } set path [file normalize $path] file stat $path before set channel [open $path rb] try { set prefix [read $channel [expr {min(256,$before(size))}]] set identity [ns_crypto::md string -digest sha256 -- [list $path $before(dev) $before(ino) $prefix]] set sourceBytes $before(size); set end $sourceBytes if {[dict exists $options beforeOffset]} { if {$identity ne [dict get $options sourceIdentity] || $sourceBytes<[dict get $options sourceBytes]} {error "Log rotated or truncated; start a new query"} set sourceBytes [dict get $options sourceBytes] set end [dict get $options beforeOffset] } set start [expr {max($floor,$end-${:maxReadBytes})}] seek $channel $start set bytes [read $channel [expr {$end-$start}]] set discarded 0 if {$start>0} { seek $channel [expr {$start-1}] if {[read $channel 1] ne "\n"} { set newline [string first \n $bytes] if {$newline>=0} {set discarded [expr {$newline+1}]} else {set discarded [string length $bytes]} } } file stat $path after if {[list $before(dev) $before(ino)] ne [list $after(dev) $after(ino)] || $after(size)<$sourceBytes} {error "Log changed during read; start a new query"} } finally {close $channel} set position [expr {$start+$discarded}] set cursor $position if {$cursor==$end && $start>0} {set cursor $start} set bytes [string range $bytes $discarded end] set pending [expr {$bytes ne "" && [string index $bytes end] ne "\n"}] set records {}; set groups {}; set totals {}; set matches 0; set unrecognized 0; set oversized 0; set missingUA 0; set missingPool 0 set selectedMatches 0 foreach line [lrange [split $bytes \n] 0 end-1] { set offset $position incr position [expr {[string length $line]+1}] if {[string length $line]>65536} {incr oversized; continue} set record [::nsmcp::accessRecordParser parse [encoding convertfrom utf-8 [string trimright $line \r]]] if {$record eq ""} {incr unrecognized; continue} set at [dict get $record startTime] if {$at<[dict get $options from] || $at>=[dict get $options to]} continue if {[dict exists $options path] && [dict get $record path] ne [dict get $options path]} continue if {[dict exists $options authenticationState] && [dict get $record authenticationState] ne [dict get $options authenticationState]} continue incr matches dict set record offset $offset; dict set record endOffset $position if {![dict exists $record userAgent]} {incr missingUA} if {[dict get $record pool] eq "unknown"} {incr missingPool} if {${:operation} eq "bot"} { if {$lookupMode eq "native-context"} { set routeKey [list [dict get $record method] [dict get $record path] [dict get $record peerAddress] [dict get $record pool] [dict exists $record userAgent] [expr {[dict exists $record userAgent] ? [dict get $record userAgent] : ""}]] if {[dict exists $routeCache $routeKey]} { set classification [dict get $routeCache $routeKey] } else { set classification [:nativeClassify $server $record $snapshot [dict get $options pool]] if {[dict size $routeCache]<4096} {dict set routeCache $routeKey $classification} } } else { set classification [:classify $record $maps [dict get $options pool]] dict set classification lookupMode fallback-rule-membership dict set classification expectedPool unknown dict set classification contextComplete false dict set classification lookupError {} } set category [dict get $classification category] dict incr totals $category if {[dict get $options category] ne "" && [dict get $options category] ne $category} {continue} incr selectedMatches if {$aggregationVar ne ""} {:accumulateTraffic traffic $record} # Group only after peer and URL redaction. Omitted groups still # contribute to category totals and omittedRecords. set record [$redactor record $record] if {[dict exists $record userAgent]} {dict set record userAgent [$redactor message [dict get $record userAgent]]} set record [dict merge $record $classification] set key [list [dict get $record category] [dict get $record method] [dict get $record path] [dict get $record pool] [dict get $record status]] lappend key [expr {[dict exists $record userAgent] ? [dict get $record userAgent] : "unknown"}] lappend key [dict get $record expectedPool] [dict get $record contextComplete] [dict get $record lookupError] if {$peers} {lappend key [dict get $record peerAddress]} if {![dict exists $groups $key]} { if {[dict size $groups]>=[dict get $options limit]} { set priority [:evidencePriority $category] set victim {} set worst $priority dict for {existingKey existingGroup} $groups { set rank [:evidencePriority [dict get $existingGroup category]] if {$rank>$worst} { set worst $rank set victim $existingKey } } if {$victim eq ""} {continue} dict unset groups $victim } set group [dict create method [dict get $record method] path [$redactor queryValues [dict get $record path]] pool [dict get $record pool] status [dict get $record status] userAgent [lindex $key 5] category [dict get $record category] matchingRuleIndexes [dict get $record matchingRuleIndexes] unknownRules [dict get $record unknownRules] sampleRequestIdentifier [dict get $record requestIdentifier] firstAt $at lastAt $at count 0 runTotal 0 runMax 0 timingCount 0] foreach field {lookupMode expectedPool contextComplete lookupError} { dict set group $field [dict get $record $field] } if {$peers} {dict set group peerAddress [dict get $record peerAddress]} dict set groups $key $group } set group [dict get $groups $key]; dict incr group count dict set group firstAt [expr {min([dict get $group firstAt],$at)}] dict set group lastAt [expr {max([dict get $group lastAt],$at)}] if {[dict exists $record runTime]} {dict incr group timingCount; dict set group runMax [expr {max([dict get $group runMax],[dict get $record runTime])}]; dict set group runTotal [expr {[dict get $group runTotal]+[dict get $record runTime]}]} dict set groups $key $group } else { set record [$redactor record $record] if {[dict exists $record userAgent]} {dict set record userAgent [$redactor message [dict get $record userAgent]]} lappend records $record set records [lrange $records end-[expr {[dict get $options limit]-1}] end] } } if {${:operation} eq "bot"} { set latest [${:routing} snapshot $server] set latestIdentity [ns_crypto::md string -digest sha256 -- [list [dict get $latest pools] $lookupMode]] if {$latestIdentity ne $mappingIdentity} {error "Effective mappings changed during analysis; start a new query"} } set entries {}; set i 0 if {${:operation} eq "bot"} { set ordered {} dict for {key group} $groups { lappend ordered [list [:evidencePriority [dict get $group category]] $group] } # The common output loop reverses records for slow-log chronology. set records {} foreach pair [lsort -integer -index 0 $ordered] {lappend records [lindex $pair 1]} set records [lreverse $records] } else { set selectedMatches $matches } set returnedRecords {}; set resultBytes 0 foreach record [lreverse $records] { set fields [:recordFields $record] set bytes [string length [ns_json value -- [list {} object $fields]]] if {$resultBytes+$bytes>65536} { continue } incr resultBytes $bytes lappend returnedRecords $record lappend entries $i object $fields incr i } set records $returnedRecords set counts {}; dict for {key value} $totals {lappend counts $key number $value} set omittedCounts {} set returnedMatches [llength $records] if {${:operation} eq "bot"} { set remaining {} dict for {category value} $totals { if {[dict get $options category] eq "" || [dict get $options category] eq $category} { dict set remaining $category $value } } set returnedMatches 0 foreach record $records { set count [dict get $record count] incr returnedMatches $count dict incr remaining [dict get $record category] [expr {-$count}] } dict for {category value} $remaining {lappend omittedCounts $category number $value} } set next {} if {$cursor>$floor} { foreach key {from to limit pool module includePeerAddresses mode} { lappend next $key [expr {$key eq "includePeerAddresses" ? "boolean" : ($key in {from to limit} ? "number" : "string")}] [dict get $options $key] } if {${:operation} eq "bot" && [dict get $options category] ne ""} { lappend next category string [dict get $options category] } foreach key {path authenticationState} {if {[dict exists $options $key]} {lappend next $key string [dict get $options $key]}} lappend next beforeOffset number $cursor sourceBytes number $sourceBytes sourceIdentity string $identity if {$mappingIdentity ne ""} {lappend next mappingIdentity string $mappingIdentity} } set evidencePolicy {Newest matching slow-request records within evidence limits.} if {${:operation} eq "bot"} { set evidencePolicy {Routing candidates first, then unknowns, then routine groups; first encounter within priority. Category filtering selects evidence only; categoryTotals still includes all observed records.} } lappend result available boolean true sourcePath string $path sourceIdentity string $identity matches number $matches categoryTotals object $counts records array $entries selectedCategory string [dict get $options category] selectedMatches number $selectedMatches filteredOutRecords number [expr {$matches-$selectedMatches}] omittedRecords number [expr {$selectedMatches-$returnedMatches}] omittedCategoryTotals object $omittedCounts evidencePolicy string $evidencePolicy more boolean [expr {$cursor>$floor}] nextArguments object $next coverage object [list sourceBytes number $sourceBytes examinedFromOffset number [expr {$start+$discarded}] examinedToOffset number $end unreadEarlierBytes number $cursor pendingFinalRecord boolean $pending unrecognizedRecords number $unrecognized oversizedRecords number $oversized missingUserAgent number $missingUA missingPool number $missingPool rotationsExamined boolean false windowComplete boolean false limitations string {Page-local counts; sum disjoint pages. Current file only. Full-window coverage is not certified. Mappings reflect query time, not historical configuration. lookupMode identifies native-context or fallback-rule-membership. Native lookup uses current URL/path precedence and logged User-Agent and valid peer IP, without disclosing peers by default. Unlogged context predicates remain unknown; completeness checks conservatively inspect all same-method rules. Older versions retain root wildcard User-Agent membership analysis, not precedence replay. Bot-like unmatched uses a separate case-insensitive heuristic, not proof of a bot. Slow-log timings are access-log timings, not SQL profiles.}] return [list {} object $result] }XQL Not present: Generic, PostgreSQL, Oracle
