From 79a006caee700f9714220cfb48721dca66b040d3 Mon Sep 17 00:00:00 2001 From: Ties de Kock Date: Mon, 21 Jun 2021 15:53:50 +0200 Subject: [PATCH 1/2] Track objects that differ for a longer period separately * Track when the VRP was first seen for each VRP * Add a metric which tracks how many objects that were first seen [duration] ago are not present in the other endpoint. * Use current naming convention for the metric to be consistent (i.e. no prefix with application name). --- cmd/rtrmon/rtrmon.go | 80 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 68 insertions(+), 12 deletions(-) diff --git a/cmd/rtrmon/rtrmon.go b/cmd/rtrmon/rtrmon.go index 6948190..671434d 100644 --- a/cmd/rtrmon/rtrmon.go +++ b/cmd/rtrmon/rtrmon.go @@ -13,6 +13,7 @@ import ( "net/url" "os" "runtime" + "strconv" "sync" "time" @@ -90,6 +91,13 @@ var ( }, []string{"server", "url", "type"}, ) + VRPDifferenceForDuration = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "vrp_diff", + Help: "Number of VRPS in [lhs_url] that are not in [rhs_url] that were first seen [visibility_seconds] ago in lhs.", + }, + []string{"lhs_url", "rhs_url", "visibility_seconds"}, + ) RTRState = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: "rtr_state", @@ -124,10 +132,13 @@ var ( 1: "primary", 2: "secondary", } + + VisibilityThresholds = []int64{0, 56, 256, 596, 851, 1024, 1706, 3411} ) func init() { prometheus.MustRegister(VRPCount) + prometheus.MustRegister(VRPDifferenceForDuration) prometheus.MustRegister(RTRState) prometheus.MustRegister(RTRSerial) prometheus.MustRegister(RTRSession) @@ -279,6 +290,7 @@ func (c *Client) Start(id int, ch chan int) { } c.lastUpdate = time.Now().UTC() + tCurrentUpdate := time.Now().UTC().Unix() tmpVrpMap := make(map[string]*VRPJsonSimple) for _, vrp := range decoded.Data { @@ -296,10 +308,17 @@ func (c *Client) Start(id int, ch chan int) { maxlen := vrp.GetMaxLen() key := fmt.Sprintf("%s-%d-%d", prefix.String(), maxlen, asn) + firstSeen := tCurrentUpdate + currentEntry, ok := c.vrps[key] + if ok { + firstSeen = currentEntry.FirstSeen + } + vrpSimple := VRPJsonSimple{ - Prefix: prefix.String(), - ASN: asn, - Length: uint8(maxlen), + Prefix: prefix.String(), + ASN: asn, + Length: uint8(maxlen), + FirstSeen: firstSeen, } tmpVrpMap[key] = &vrpSimple } @@ -321,9 +340,10 @@ func (c *Client) HandlePDU(cs *rtr.ClientSession, pdu rtr.PDU) { switch pdu := pdu.(type) { case *rtr.PDUIPv4Prefix: vrp := VRPJsonSimple{ - Prefix: pdu.Prefix.String(), - ASN: pdu.ASN, - Length: pdu.MaxLen, + Prefix: pdu.Prefix.String(), + ASN: pdu.ASN, + Length: pdu.MaxLen, + FirstSeen: time.Now().Unix(), } key := fmt.Sprintf("%s-%d-%d", pdu.Prefix.String(), pdu.MaxLen, pdu.ASN) @@ -338,9 +358,10 @@ func (c *Client) HandlePDU(cs *rtr.ClientSession, pdu rtr.PDU) { c.compRtrLock.Unlock() case *rtr.PDUIPv6Prefix: vrp := VRPJsonSimple{ - Prefix: pdu.Prefix.String(), - ASN: pdu.ASN, - Length: pdu.MaxLen, + Prefix: pdu.Prefix.String(), + ASN: pdu.ASN, + Length: pdu.MaxLen, + FirstSeen: time.Now().Unix(), } key := fmt.Sprintf("%s-%d-%d", pdu.Prefix.String(), pdu.MaxLen, pdu.ASN) @@ -487,6 +508,18 @@ func NewComparator(c1, c2 *Client) *Comparator { } } +func countFirstSeenOnOrBefore(vrps []*VRPJsonSimple, thresholdTimestamp int64) float64 { + count := 0 + + for _, vrp := range vrps { + if vrp.FirstSeen <= thresholdTimestamp { + count++ + } + } + + return float64(count) +} + func Diff(a, b map[string]*VRPJsonSimple) []*VRPJsonSimple { onlyInA := make([]*VRPJsonSimple, 0) for key, vrp := range a { @@ -510,9 +543,10 @@ type diffMetadata struct { } type VRPJsonSimple struct { - ASN uint32 `json:"asn"` - Length uint8 `json:"max-length"` - Prefix string `json:"prefix"` + ASN uint32 `json:"asn"` + Length uint8 `json:"max-length"` + Prefix string `json:"prefix"` + FirstSeen int64 `json:"first-seen"` } type diffExport struct { @@ -547,6 +581,7 @@ func (c *Comparator) ServeDiff(wr http.ResponseWriter, req *http.Request) { func (c *Comparator) Compare() { var donePrimary, doneSecondary bool var stop bool + startedAt := time.Now().Unix() for !stop { select { case <-c.q: @@ -596,6 +631,27 @@ func (c *Comparator) Compare() { "type": "diff", }).Set(float64(len(onlyIn2))) + for _, visibleFor := range VisibilityThresholds { + thresholdTimestamp := time.Now().Unix() - visibleFor + // Prevent differences with value 0 appearing if the process has not + // been running long enough for them to exist. + if thresholdTimestamp >= startedAt { + VRPDifferenceForDuration.With( + prometheus.Labels{ + "lhs_url": md1.URL, + "rhs_url": md2.URL, + "visibility_seconds": strconv.FormatInt(visibleFor, 10), + }).Set(countFirstSeenOnOrBefore(onlyIn1, thresholdTimestamp)) + + VRPDifferenceForDuration.With( + prometheus.Labels{ + "lhs_url": md2.URL, + "rhs_url": md1.URL, + "visibility_seconds": strconv.FormatInt(visibleFor, 10), + }).Set(countFirstSeenOnOrBefore(onlyIn2, thresholdTimestamp)) + } + } + RTRSerial.With( prometheus.Labels{ "server": "primary", From 71a76480ee282a88d69d4813b7067257a6f66259 Mon Sep 17 00:00:00 2001 From: Ties de Kock Date: Tue, 22 Jun 2021 10:29:35 +0200 Subject: [PATCH 2/2] Visibility thresholds can be overridden via args * Visibility thresholds can be overridden via command line arguments. * Format the default thresholds in help output --- cmd/rtrmon/rtrmon.go | 40 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/cmd/rtrmon/rtrmon.go b/cmd/rtrmon/rtrmon.go index 671434d..4230952 100644 --- a/cmd/rtrmon/rtrmon.go +++ b/cmd/rtrmon/rtrmon.go @@ -14,6 +14,7 @@ import ( "os" "runtime" "strconv" + "strings" "sync" "time" @@ -35,6 +36,8 @@ const ( METHOD_KEY ) +type thresholds []int64 + var ( version = "" buildinfos = "" @@ -133,7 +136,7 @@ var ( 2: "secondary", } - VisibilityThresholds = []int64{0, 56, 256, 596, 851, 1024, 1706, 3411} + visibilityThresholds = thresholds{0, 56, 256, 596, 851, 1024, 1706, 3411} ) func init() { @@ -143,6 +146,39 @@ func init() { prometheus.MustRegister(RTRSerial) prometheus.MustRegister(RTRSession) prometheus.MustRegister(LastUpdate) + + flag.Var(&visibilityThresholds, "visibility.thresholds", "comma-separated list of visibility thresholds to override the default") +} + +// String formats an array of thresholds as a comma separated string. +func (t *thresholds) String() string { + res := []byte("") + for idx, tr := range *t { + res = strconv.AppendInt(res, tr, 10) + if idx < len(*t)-1 { + res = append(res, ","...) + } + } + return string(res) +} + +func (t *thresholds) Set(value string) error { + // Setting overrides current values + if len(*t) > 0 { + *t = thresholds{} + } + + for _, tr := range strings.Split(value, ",") { + threshold, err := strconv.ParseInt(tr, 10, 32) + + if err != nil { + return err + } + + *t = append(*t, threshold) + } + + return nil } func decodeJSON(data []byte) (*prefixfile.VRPList, error) { @@ -631,7 +667,7 @@ func (c *Comparator) Compare() { "type": "diff", }).Set(float64(len(onlyIn2))) - for _, visibleFor := range VisibilityThresholds { + for _, visibleFor := range visibilityThresholds { thresholdTimestamp := time.Now().Unix() - visibleFor // Prevent differences with value 0 appearing if the process has not // been running long enough for them to exist.