1321 lines
34 KiB
JSON
1321 lines
34 KiB
JSON
{
|
|
"annotations": {
|
|
"list": [
|
|
{
|
|
"builtIn": 1,
|
|
"datasource": {
|
|
"type": "grafana",
|
|
"uid": "-- Grafana --"
|
|
},
|
|
"enable": true,
|
|
"hide": true,
|
|
"iconColor": "rgba(0, 211, 255, 1)",
|
|
"name": "Annotations & Alerts",
|
|
"type": "dashboard"
|
|
}
|
|
]
|
|
},
|
|
"editable": true,
|
|
"fiscalYearStartMonth": 0,
|
|
"graphTooltip": 1,
|
|
"id": null,
|
|
"links": [],
|
|
"liveNow": false,
|
|
"panels": [
|
|
{
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 0
|
|
},
|
|
"id": 1,
|
|
"panels": [],
|
|
"title": "Current health",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "victoriametrics-logs-datasource",
|
|
"uid": "victorialogs"
|
|
},
|
|
"description": "Age of the newest guest telemetry record where admission cache, nftables, and WireGuard are all healthy. The panel turns red after three minutes.",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": 0
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 180000000000
|
|
}
|
|
]
|
|
},
|
|
"unit": "ns"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 6,
|
|
"x": 0,
|
|
"y": 1
|
|
},
|
|
"id": 2,
|
|
"options": {
|
|
"colorMode": "background",
|
|
"graphMode": "none",
|
|
"justifyMode": "center",
|
|
"orientation": "auto",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"textMode": "auto",
|
|
"wideLayout": true
|
|
},
|
|
"pluginVersion": "12.3.0",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "victoriametrics-logs-datasource",
|
|
"uid": "victorialogs"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "_HOSTNAME:\"oci-l4-relay\" \"oci_l4_relay_telemetry timestamp=\" | extract_regexp \"admission_cached=(?P<admission_cached>[^ ]+) nftables_active=(?P<nftables_active>[^ ]+) wireguard_active=(?P<wireguard_active>[^ ]+)\" from _msg | filter admission_cached:true nftables_active:true wireguard_active:true | first 1 by (_time desc) | math (now() - _time) default 0 as lag_ns | max(lag_ns) as lag_ns",
|
|
"queryType": "stats",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Healthy telemetry age",
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "victoriametrics-logs-datasource",
|
|
"uid": "victorialogs"
|
|
},
|
|
"description": "Newest active-owner event from the home relay. Node and pod fields identify the Lease holder observed in logs.",
|
|
"fieldConfig": {
|
|
"defaults": {},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 9,
|
|
"x": 6,
|
|
"y": 1
|
|
},
|
|
"id": 3,
|
|
"options": {
|
|
"cellHeight": "sm",
|
|
"footer": {
|
|
"countRows": false,
|
|
"fields": "",
|
|
"reducer": [
|
|
"sum"
|
|
],
|
|
"show": false
|
|
},
|
|
"showHeader": true
|
|
},
|
|
"pluginVersion": "12.3.0",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "victoriametrics-logs-datasource",
|
|
"uid": "victorialogs"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "kubernetes.container_name:\"relay-agent\" \"relay ownership acquired\" | fields _time, kubernetes.pod_node_name, kubernetes.pod_name, _msg | sort by (_time) desc limit 1",
|
|
"queryType": "instant",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Latest owner",
|
|
"type": "table"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "victoriametrics-logs-datasource",
|
|
"uid": "victorialogs"
|
|
},
|
|
"description": "Distinct home relay container IDs in the selected range. This covers replacements and container restarts without depending on Kubernetes inventory metrics.",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": 0
|
|
},
|
|
{
|
|
"color": "yellow",
|
|
"value": 3
|
|
}
|
|
]
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 4,
|
|
"x": 15,
|
|
"y": 1
|
|
},
|
|
"id": 4,
|
|
"options": {
|
|
"colorMode": "value",
|
|
"graphMode": "area",
|
|
"justifyMode": "center",
|
|
"orientation": "auto",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"textMode": "auto",
|
|
"wideLayout": true
|
|
},
|
|
"pluginVersion": "12.3.0",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "victoriametrics-logs-datasource",
|
|
"uid": "victorialogs"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "kubernetes.container_name:\"relay-agent\" | stats count_uniq(kubernetes.container_id) as incarnations",
|
|
"queryType": "stats",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Container incarnations",
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "victoriametrics-logs-datasource",
|
|
"uid": "victorialogs"
|
|
},
|
|
"description": "Guest and home relay error records in the selected range.",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": 0
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 1
|
|
}
|
|
]
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 5,
|
|
"x": 19,
|
|
"y": 1
|
|
},
|
|
"id": 5,
|
|
"options": {
|
|
"colorMode": "background",
|
|
"graphMode": "none",
|
|
"justifyMode": "center",
|
|
"orientation": "horizontal",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"textMode": "auto",
|
|
"wideLayout": true
|
|
},
|
|
"pluginVersion": "12.3.0",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "victoriametrics-logs-datasource",
|
|
"uid": "victorialogs"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "_HOSTNAME:\"oci-l4-relay\" | filter level:error | count()",
|
|
"legendFormat": "guest",
|
|
"queryType": "stats",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "victoriametrics-logs-datasource",
|
|
"uid": "victorialogs"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "kubernetes.container_name:\"relay-agent\" | filter _msg:~\" ERROR |error=\" | count()",
|
|
"legendFormat": "home",
|
|
"queryType": "stats",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"title": "Errors in range",
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 7
|
|
},
|
|
"id": 6,
|
|
"panels": [],
|
|
"title": "Guest telemetry",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "victoriametrics-logs-datasource",
|
|
"uid": "victorialogs"
|
|
},
|
|
"description": "Conntrack occupancy derived from the guest's bounded telemetry record.",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 2,
|
|
"pointSize": 5,
|
|
"showPoints": "auto",
|
|
"spanNulls": false
|
|
},
|
|
"mappings": [],
|
|
"max": 100,
|
|
"min": 0,
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": 0
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 70
|
|
}
|
|
]
|
|
},
|
|
"unit": "percent"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 8
|
|
},
|
|
"id": 7,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"lastNotNull",
|
|
"max"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"pluginVersion": "12.3.0",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "victoriametrics-logs-datasource",
|
|
"uid": "victorialogs"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "_HOSTNAME:\"oci-l4-relay\" \"oci_l4_relay_telemetry timestamp=\" | extract_regexp \"conntrack_count=(?P<conntrack_count>[0-9]+) conntrack_max=(?P<conntrack_max>[0-9]+)\" from _msg | math (conntrack_count * 100 / conntrack_max) as conntrack_percent | stats by (_time:$__interval) max(conntrack_percent) as conntrack_percent",
|
|
"legendFormat": "conntrack",
|
|
"queryType": "statsRange",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Conntrack utilization",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "victoriametrics-logs-datasource",
|
|
"uid": "victorialogs"
|
|
},
|
|
"description": "WireGuard handshake age. Values over 180 seconds violate guest health acceptance.",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 2,
|
|
"pointSize": 5,
|
|
"showPoints": "auto",
|
|
"spanNulls": false
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": 0
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 180
|
|
}
|
|
]
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 8
|
|
},
|
|
"id": 8,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"lastNotNull",
|
|
"max"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"pluginVersion": "12.3.0",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "victoriametrics-logs-datasource",
|
|
"uid": "victorialogs"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "_HOSTNAME:\"oci-l4-relay\" \"oci_l4_relay_telemetry timestamp=\" | extract_regexp \"wireguard_handshake_age_seconds=(?P<handshake_seconds>[0-9]+)\" from _msg | stats by (_time:$__interval) max(handshake_seconds) as handshake_seconds",
|
|
"legendFormat": "handshake age",
|
|
"queryType": "statsRange",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "WireGuard handshake age",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "victoriametrics-logs-datasource",
|
|
"uid": "victorialogs"
|
|
},
|
|
"description": "Count of primary telemetry records per interval. Missing buckets expose ingestion or guest telemetry gaps.",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"drawStyle": "bars",
|
|
"fillOpacity": 30,
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"showPoints": "never",
|
|
"spanNulls": false
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": 0
|
|
}
|
|
]
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 8
|
|
},
|
|
"id": 9,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"pluginVersion": "12.3.0",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "victoriametrics-logs-datasource",
|
|
"uid": "victorialogs"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "_HOSTNAME:\"oci-l4-relay\" \"oci_l4_relay_telemetry timestamp=\" | stats by (_time:$__interval) count() as samples",
|
|
"legendFormat": "samples",
|
|
"queryType": "statsRange",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Telemetry samples",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "victoriametrics-logs-datasource",
|
|
"uid": "victorialogs"
|
|
},
|
|
"description": "Cumulative relay-owned nftables counters. Counter increases distinguish admission, per-source, quarantine, SYN aggregate, and packet aggregate drops.",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"fillOpacity": 0,
|
|
"lineInterpolation": "stepAfter",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"showPoints": "auto",
|
|
"spanNulls": false
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": 0
|
|
}
|
|
]
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 9,
|
|
"w": 16,
|
|
"x": 0,
|
|
"y": 16
|
|
},
|
|
"id": 10,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"pluginVersion": "12.3.0",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "victoriametrics-logs-datasource",
|
|
"uid": "victorialogs"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "_HOSTNAME:\"oci-l4-relay\" web_packet_aggregate_dropped | extract_regexp \"web_packet_aggregate_dropped.*?\\\"packets\\\": (?P<packet_drops>[0-9]+).*?web_syn_quarantine_dropped.*?\\\"packets\\\": (?P<quarantine_drops>[0-9]+).*?web_syn_source_dropped.*?\\\"packets\\\": (?P<source_drops>[0-9]+).*?web_syn_aggregate_dropped.*?\\\"packets\\\": (?P<syn_drops>[0-9]+).*?admission_denied.*?\\\"packets\\\": (?P<admission_drops>[0-9]+)\" from _msg | stats by (_time:$__interval) max(packet_drops) as packet_drops, max(quarantine_drops) as quarantine_drops, max(source_drops) as source_drops, max(syn_drops) as syn_drops, max(admission_drops) as admission_drops",
|
|
"queryType": "statsRange",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "nftables drop counters",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "victoriametrics-logs-datasource",
|
|
"uid": "victorialogs"
|
|
},
|
|
"description": "Cumulative traffic-control drops from the relay's TBF qdisc.",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"fillOpacity": 0,
|
|
"lineInterpolation": "stepAfter",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"showPoints": "auto",
|
|
"spanNulls": false
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": 0
|
|
}
|
|
]
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 9,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 16
|
|
},
|
|
"id": 11,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"pluginVersion": "12.3.0",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "victoriametrics-logs-datasource",
|
|
"uid": "victorialogs"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "_HOSTNAME:\"oci-l4-relay\" tbf | extract_regexp \"drops\\\":(?P<qdisc_drops>[0-9]+)\" from _msg | filter qdisc_drops:* | stats by (_time:$__interval) max(qdisc_drops) as qdisc_drops",
|
|
"legendFormat": "qdisc drops",
|
|
"queryType": "statsRange",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "qdisc drop counter",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 25
|
|
},
|
|
"id": 16,
|
|
"panels": [],
|
|
"title": "Transport evidence",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "victoriametrics"
|
|
},
|
|
"description": "Current VictoriaMetrics scrape status for the active relay metrics path.",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "red",
|
|
"value": 0
|
|
},
|
|
{
|
|
"color": "green",
|
|
"value": 1
|
|
}
|
|
]
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 6,
|
|
"x": 0,
|
|
"y": 26
|
|
},
|
|
"id": 17,
|
|
"options": {
|
|
"colorMode": "background",
|
|
"graphMode": "none",
|
|
"justifyMode": "center",
|
|
"orientation": "auto",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"textMode": "auto",
|
|
"wideLayout": true
|
|
},
|
|
"pluginVersion": "12.3.0",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "victoriametrics"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "up{job=\"oci-l4-relay\"}",
|
|
"instant": true,
|
|
"legendFormat": "scrape",
|
|
"range": false,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Metrics scrape",
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "victoriametrics"
|
|
},
|
|
"description": "Seconds since the relay generated its latest bounded evidence snapshot.",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"mappings": [],
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 6,
|
|
"x": 6,
|
|
"y": 26
|
|
},
|
|
"id": 18,
|
|
"options": {
|
|
"colorMode": "value",
|
|
"graphMode": "area",
|
|
"justifyMode": "center",
|
|
"orientation": "auto",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"textMode": "auto",
|
|
"wideLayout": true
|
|
},
|
|
"pluginVersion": "12.3.0",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "victoriametrics"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "time() - oci_l4_relay_tcp_evidence_generated_timestamp_seconds{job=\"oci-l4-relay\"}",
|
|
"instant": true,
|
|
"legendFormat": "age",
|
|
"range": false,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Evidence age",
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "victoriametrics"
|
|
},
|
|
"description": "Current bounded evidence records exposed for each service.",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"mappings": [],
|
|
"unit": "short"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 6,
|
|
"x": 12,
|
|
"y": 26
|
|
},
|
|
"id": 19,
|
|
"options": {
|
|
"colorMode": "value",
|
|
"graphMode": "area",
|
|
"justifyMode": "center",
|
|
"orientation": "auto",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"textMode": "auto",
|
|
"wideLayout": true
|
|
},
|
|
"pluginVersion": "12.3.0",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "victoriametrics"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "oci_l4_relay_tcp_evidence_records{job=\"oci-l4-relay\"}",
|
|
"instant": true,
|
|
"legendFormat": "{{service}}",
|
|
"range": false,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Evidence records",
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "victoriametrics"
|
|
},
|
|
"description": "Whether the configured evidence record bound truncated a service snapshot.",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": 0
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 1
|
|
}
|
|
]
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 6,
|
|
"x": 18,
|
|
"y": 26
|
|
},
|
|
"id": 20,
|
|
"options": {
|
|
"colorMode": "background",
|
|
"graphMode": "none",
|
|
"justifyMode": "center",
|
|
"orientation": "auto",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"textMode": "auto",
|
|
"wideLayout": true
|
|
},
|
|
"pluginVersion": "12.3.0",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "victoriametrics"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "oci_l4_relay_tcp_evidence_truncated{job=\"oci-l4-relay\"}",
|
|
"instant": true,
|
|
"legendFormat": "{{service}}",
|
|
"range": false,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Evidence truncated",
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "victoriametrics"
|
|
},
|
|
"description": "Per-second nftables drop rate by relay service and bounded drop reason.",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 2,
|
|
"pointSize": 5,
|
|
"showPoints": "auto",
|
|
"spanNulls": false
|
|
},
|
|
"mappings": [],
|
|
"min": 0,
|
|
"unit": "pps"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 32
|
|
},
|
|
"id": 21,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"lastNotNull",
|
|
"max"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"pluginVersion": "12.3.0",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "victoriametrics"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "rate(oci_l4_relay_nftables_dropped_packets_total{job=\"oci-l4-relay\"}[$__rate_interval])",
|
|
"legendFormat": "{{service}} / {{reason}}",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Transport drop rate",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "victoriametrics"
|
|
},
|
|
"description": "Current packet counts for up to twenty quarantined source addresses.",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 2,
|
|
"pointSize": 5,
|
|
"showPoints": "auto",
|
|
"spanNulls": false
|
|
},
|
|
"mappings": [],
|
|
"min": 0,
|
|
"unit": "short"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 6,
|
|
"x": 12,
|
|
"y": 32
|
|
},
|
|
"id": 22,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"lastNotNull",
|
|
"max"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"pluginVersion": "12.3.0",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "victoriametrics"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "topk(20, oci_l4_relay_tcp_quarantine_packets{job=\"oci-l4-relay\"})",
|
|
"legendFormat": "{{service}} / {{address}}",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Quarantined sources",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "victoriametrics"
|
|
},
|
|
"description": "Seconds remaining for up to twenty quarantined sources nearest expiry.",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 2,
|
|
"pointSize": 5,
|
|
"showPoints": "auto",
|
|
"spanNulls": false
|
|
},
|
|
"mappings": [],
|
|
"min": 0,
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 6,
|
|
"x": 18,
|
|
"y": 32
|
|
},
|
|
"id": 23,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"lastNotNull",
|
|
"min"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "asc"
|
|
}
|
|
},
|
|
"pluginVersion": "12.3.0",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "victoriametrics"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "bottomk(20, clamp_min(oci_l4_relay_tcp_quarantine_expires_timestamp_seconds{job=\"oci-l4-relay\"} - time(), 0))",
|
|
"legendFormat": "{{service}} / {{address}}",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Quarantines nearest expiry",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 40
|
|
},
|
|
"id": 12,
|
|
"panels": [],
|
|
"title": "Lifecycle and availability events",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "victoriametrics-logs-datasource",
|
|
"uid": "victorialogs"
|
|
},
|
|
"description": "Sealed-guest bootstrap, cached-state restoration, credential, and admission records.",
|
|
"fieldConfig": {
|
|
"defaults": {},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 10,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 41
|
|
},
|
|
"id": 13,
|
|
"options": {
|
|
"dedupStrategy": "none",
|
|
"detailsMode": "inline",
|
|
"enableInfiniteScrolling": false,
|
|
"enableLogDetails": true,
|
|
"prettifyLogMessage": false,
|
|
"showCommonLabels": false,
|
|
"showControls": true,
|
|
"showLabels": true,
|
|
"showTime": true,
|
|
"sortOrder": "Descending",
|
|
"syntaxHighlighting": false,
|
|
"wrapLogMessage": true
|
|
},
|
|
"pluginVersion": "12.3.0",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "victoriametrics-logs-datasource",
|
|
"uid": "victorialogs"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "_HOSTNAME:\"oci-l4-relay\" | filter _SYSTEMD_UNIT:in(oci-l4-relay-bootstrap.service,oci-l4-relay-admission.service,oci-l4-relay-log-credential.service) | sort by (_time) desc limit 200",
|
|
"queryType": "instant",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Guest bootstrap and admission",
|
|
"type": "logs"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "victoriametrics-logs-datasource",
|
|
"uid": "victorialogs"
|
|
},
|
|
"description": "Home relay ownership transitions, unhealthy-term backoff, and active configuration failures.",
|
|
"fieldConfig": {
|
|
"defaults": {},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 10,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 41
|
|
},
|
|
"id": 14,
|
|
"options": {
|
|
"dedupStrategy": "none",
|
|
"detailsMode": "inline",
|
|
"enableInfiniteScrolling": false,
|
|
"enableLogDetails": true,
|
|
"prettifyLogMessage": false,
|
|
"showCommonLabels": false,
|
|
"showControls": true,
|
|
"showLabels": true,
|
|
"showTime": true,
|
|
"sortOrder": "Descending",
|
|
"syntaxHighlighting": false,
|
|
"wrapLogMessage": true
|
|
},
|
|
"pluginVersion": "12.3.0",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "victoriametrics-logs-datasource",
|
|
"uid": "victorialogs"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "kubernetes.container_name:\"relay-agent\" | filter _msg:~\"relay ownership|retry relay election|configure active relay|refresh relay ownership fence\" | sort by (_time) desc limit 200",
|
|
"queryType": "instant",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Ownership and failover",
|
|
"type": "logs"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "victoriametrics-logs-datasource",
|
|
"uid": "victorialogs"
|
|
},
|
|
"description": "All guest and home relay errors retained for diagnosis. Console history and OCI Run Command remain break-glass surfaces rather than dashboard dependencies.",
|
|
"fieldConfig": {
|
|
"defaults": {},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 10,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 51
|
|
},
|
|
"id": 15,
|
|
"options": {
|
|
"dedupStrategy": "none",
|
|
"detailsMode": "inline",
|
|
"enableInfiniteScrolling": false,
|
|
"enableLogDetails": true,
|
|
"prettifyLogMessage": false,
|
|
"showCommonLabels": false,
|
|
"showControls": true,
|
|
"showLabels": true,
|
|
"showTime": true,
|
|
"sortOrder": "Descending",
|
|
"syntaxHighlighting": false,
|
|
"wrapLogMessage": true
|
|
},
|
|
"pluginVersion": "12.3.0",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "victoriametrics-logs-datasource",
|
|
"uid": "victorialogs"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "_HOSTNAME:\"oci-l4-relay\" | filter level:error | sort by (_time) desc limit 200",
|
|
"queryType": "instant",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "victoriametrics-logs-datasource",
|
|
"uid": "victorialogs"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "kubernetes.container_name:\"relay-agent\" | filter _msg:~\" ERROR |error=\" | sort by (_time) desc limit 200",
|
|
"queryType": "instant",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"title": "Relay errors",
|
|
"type": "logs"
|
|
}
|
|
],
|
|
"refresh": "30s",
|
|
"schemaVersion": 42,
|
|
"tags": [
|
|
"OCI",
|
|
"Tier R",
|
|
"relay"
|
|
],
|
|
"templating": {
|
|
"list": []
|
|
},
|
|
"time": {
|
|
"from": "now-6h",
|
|
"to": "now"
|
|
},
|
|
"timepicker": {},
|
|
"timezone": "browser",
|
|
"title": "OCI L4 Relay",
|
|
"uid": "oci-l4-relay",
|
|
"version": 1,
|
|
"weekStart": ""
|
|
}
|