{ "schema_version": 1, "generated_at": "2026-07-19T22:34:42+03:00", "weeks": [ { "week_key": "2026-03-14T07:00__2026-03-21T07:00", "week_label": "W11", "period_start": "2026-03-14T07:00:00+02:00", "period_end": "2026-03-21T07:00:00+02:00", "families": { "delivery": true, "tests": true, "alerts": true, "planning": true }, "has_any_report": true }, { "week_key": "2026-03-21T07:00__2026-03-28T07:00", "week_label": "W12", "period_start": "2026-03-21T07:00:00+02:00", "period_end": "2026-03-28T07:00:00+02:00", "families": { "delivery": true, "tests": true, "alerts": true, "planning": true }, "has_any_report": true }, { "week_key": "2026-03-28T07:00__2026-04-04T07:00", "week_label": "W13", "period_start": "2026-03-28T07:00:00+02:00", "period_end": "2026-04-04T07:00:00+03:00", "families": { "delivery": true, "tests": true, "alerts": true, "planning": true }, "has_any_report": true }, { "week_key": "2026-04-04T07:00__2026-04-11T07:00", "week_label": "W14", "period_start": "2026-04-04T07:00:00+03:00", "period_end": "2026-04-11T07:00:00+03:00", "families": { "delivery": true, "tests": true, "alerts": true, "planning": true }, "has_any_report": true }, { "week_key": "2026-04-11T07:00__2026-04-18T07:00", "week_label": "W15", "period_start": "2026-04-11T07:00:00+03:00", "period_end": "2026-04-18T07:00:00+03:00", "families": { "delivery": true, "tests": true, "alerts": true, "planning": true }, "has_any_report": true }, { "week_key": "2026-04-18T07:00__2026-04-25T07:00", "week_label": "W16", "period_start": "2026-04-18T07:00:00+03:00", "period_end": "2026-04-25T07:00:00+03:00", "families": { "delivery": true, "tests": true, "alerts": true, "planning": true }, "has_any_report": true }, { "week_key": "2026-04-25T07:00__2026-05-02T07:00", "week_label": "W17", "period_start": "2026-04-25T07:00:00+03:00", "period_end": "2026-05-02T07:00:00+03:00", "families": { "delivery": true, "tests": true, "alerts": true, "planning": true }, "has_any_report": true }, { "week_key": "2026-05-02T07:00__2026-05-09T07:00", "week_label": "W18", "period_start": "2026-05-02T07:00:00+03:00", "period_end": "2026-05-09T07:00:00+03:00", "families": { "delivery": true, "tests": true, "alerts": true, "planning": true }, "has_any_report": true }, { "week_key": "2026-05-09T07:00__2026-05-16T07:00", "week_label": "W19", "period_start": "2026-05-09T07:00:00+03:00", "period_end": "2026-05-16T07:00:00+03:00", "families": { "delivery": true, "tests": true, "alerts": true, "planning": true }, "has_any_report": true }, { "week_key": "2026-05-16T07:00__2026-05-23T07:00", "week_label": "W20", "period_start": "2026-05-16T07:00:00+03:00", "period_end": "2026-05-23T07:00:00+03:00", "families": { "delivery": true, "tests": true, "alerts": true, "planning": true }, "has_any_report": true }, { "week_key": "2026-05-23T07:00__2026-05-30T07:00", "week_label": "W21", "period_start": "2026-05-23T07:00:00+03:00", "period_end": "2026-05-30T07:00:00+03:00", "families": { "delivery": true, "tests": true, "alerts": true, "planning": true }, "has_any_report": true }, { "week_key": "2026-05-30T07:00__2026-06-06T07:00", "week_label": "W22", "period_start": "2026-05-30T07:00:00+03:00", "period_end": "2026-06-06T07:00:00+03:00", "families": { "delivery": true, "tests": true, "alerts": true, "planning": true }, "has_any_report": true }, { "week_key": "2026-06-06T07:00__2026-06-13T07:00", "week_label": "W23", "period_start": "2026-06-06T07:00:00+03:00", "period_end": "2026-06-13T07:00:00+03:00", "families": { "delivery": true, "tests": true, "alerts": true, "planning": true }, "has_any_report": true }, { "week_key": "2026-06-14T07:00__2026-06-21T07:00", "week_label": "W24", "period_start": "2026-06-14T07:00:00+03:00", "period_end": "2026-06-21T07:00:00+03:00", "families": { "delivery": true, "tests": true, "alerts": true, "planning": true }, "has_any_report": true }, { "week_key": "2026-06-21T07:00__2026-06-28T07:00", "week_label": "W25", "period_start": "2026-06-21T07:00:00+03:00", "period_end": "2026-06-28T07:00:00+03:00", "families": { "delivery": true, "tests": true, "alerts": true, "planning": true }, "has_any_report": true }, { "week_key": "2026-06-28T07:00__2026-07-05T07:00", "week_label": "W26", "period_start": "2026-06-28T07:00:00+03:00", "period_end": "2026-07-05T07:00:00+03:00", "families": { "delivery": true, "tests": true, "alerts": true, "planning": true }, "has_any_report": true }, { "week_key": "2026-07-04T07:00__2026-07-11T07:00", "week_label": "W27", "period_start": "2026-07-04T07:00:00+03:00", "period_end": "2026-07-11T07:00:00+03:00", "families": { "delivery": true, "tests": true, "alerts": true, "planning": true }, "has_any_report": true }, { "week_key": "2026-07-11T07:00__2026-07-18T07:00", "week_label": "W28", "period_start": "2026-07-11T07:00:00+03:00", "period_end": "2026-07-18T07:00:00+03:00", "families": { "delivery": true, "tests": true, "alerts": true, "planning": true }, "has_any_report": true } ], "items": [ { "period_start": "2026-03-14T07:00:00+02:00", "period_end": "2026-03-21T07:00:00+02:00", "week_key": "2026-03-14T07:00__2026-03-21T07:00", "week_label": "W11", "source_type": "application", "source_label": "Application alert", "item_key": "KubeHpaMaxedOut", "item_label": "KubeHpaMaxedOut", "severity": "warning", "status": "Seen today", "count": 9, "last_seen": "2026-03-20 11:11", "last_seen_ts": 1773997882.351599, "services": "docgen2-api (6), subscriptions-api (3)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1773997882351599", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1773997882351599", "source_href": "reports/production_reliability_dashboard_2026-03-14_0700_2026-03-21_0700.html", "source_json": "production_reliability_dashboard_2026-03-14_0700_2026-03-21_0700.json" }, { "period_start": "2026-03-14T07:00:00+02:00", "period_end": "2026-03-21T07:00:00+02:00", "week_key": "2026-03-14T07:00__2026-03-21T07:00", "week_label": "W11", "source_type": "application", "source_label": "Application alert", "item_key": "KubeJobFailed", "item_label": "KubeJobFailed", "severity": "warning", "status": "Recent (72h)", "count": 27, "last_seen": "2026-03-19 07:46", "last_seen_ts": 1773899212.809979, "services": "admission-end-session (27), update-recurenta (13)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1773899212809979", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1773899212809979", "source_href": "reports/production_reliability_dashboard_2026-03-14_0700_2026-03-21_0700.html", "source_json": "production_reliability_dashboard_2026-03-14_0700_2026-03-21_0700.json" }, { "period_start": "2026-03-14T07:00:00+02:00", "period_end": "2026-03-21T07:00:00+02:00", "week_key": "2026-03-14T07:00__2026-03-21T07:00", "week_label": "W11", "source_type": "application", "source_label": "Application alert", "item_key": "NodeCPUHighUsage", "item_label": "NodeCPUHighUsage", "severity": "warning", "status": "Recent (72h)", "count": 1, "last_seen": "2026-03-18 14:53", "last_seen_ts": 1773838409.742669, "services": "grafana (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1773838409742669", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1773838409742669", "source_href": "reports/production_reliability_dashboard_2026-03-14_0700_2026-03-21_0700.html", "source_json": "production_reliability_dashboard_2026-03-14_0700_2026-03-21_0700.json" }, { "period_start": "2026-03-14T07:00:00+02:00", "period_end": "2026-03-21T07:00:00+02:00", "week_key": "2026-03-14T07:00__2026-03-21T07:00", "week_label": "W11", "source_type": "application", "source_label": "Application alert", "item_key": "NodeSystemSaturation", "item_label": "NodeSystemSaturation", "severity": "warning", "status": "Recent (72h)", "count": 3, "last_seen": "2026-03-19 12:27", "last_seen_ts": 1773916039.689679, "services": "grafana (3)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1773916039689679", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1773916039689679", "source_href": "reports/production_reliability_dashboard_2026-03-14_0700_2026-03-21_0700.html", "source_json": "production_reliability_dashboard_2026-03-14_0700_2026-03-21_0700.json" }, { "period_start": "2026-03-14T07:00:00+02:00", "period_end": "2026-03-21T07:00:00+02:00", "week_key": "2026-03-14T07:00__2026-03-21T07:00", "week_label": "W11", "source_type": "application", "source_label": "Application alert", "item_key": "TraefikServiceHighErrorRate", "item_label": "TraefikServiceHighErrorRate", "severity": "critical", "status": "Seen today", "count": 17, "last_seen": "2026-03-20 21:47", "last_seen_ts": 1774036060.731799, "services": "uni-api-svc-4000 (12), accommodations-api-svc-4100 (5)", "signals": "General investigation, Observability storage", "latest_note": "SQL Integrity Constraint Violation: (conn=3570102) Cannot add or update a child row: a foreign key constraint fails (`ums_uni_catalog`.`dis… | Andrei Alexandru", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1774036060731799", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1774036060731799?thread_ts=1774036060.731799&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-03-14_0700_2026-03-21_0700.html", "source_json": "production_reliability_dashboard_2026-03-14_0700_2026-03-21_0700.json" }, { "period_start": "2026-03-14T07:00:00+02:00", "period_end": "2026-03-21T07:00:00+02:00", "week_key": "2026-03-14T07:00__2026-03-21T07:00", "week_label": "W11", "source_type": "application", "source_label": "Application alert", "item_key": "TraefikServiceHighLatency", "item_label": "TraefikServiceHighLatency", "severity": "warning", "status": "Seen today", "count": 5, "last_seen": "2026-03-20 16:35", "last_seen_ts": 1774017308.169829, "services": "ai-api-svc-3900 (4), web-80 (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1774017308169829", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1774017308169829", "source_href": "reports/production_reliability_dashboard_2026-03-14_0700_2026-03-21_0700.html", "source_json": "production_reliability_dashboard_2026-03-14_0700_2026-03-21_0700.json" }, { "period_start": "2026-03-14T07:00:00+02:00", "period_end": "2026-03-21T07:00:00+02:00", "week_key": "2026-03-14T07:00__2026-03-21T07:00", "week_label": "W11", "source_type": "customer", "source_label": "Customer warning", "item_key": "https://www.adservio.ro/api/v2/status", "item_label": "https://www.adservio.ro/api/v2/status", "severity": "warning", "status": "Recovered recently", "count": 2, "last_seen": "2026-03-21 00:00", "last_seen_ts": 1774044000.0, "services": "adservio-ro-api-v2-status", "signals": "adservio-rds-mysql-master-memory-low, adservio-rds-mysql-master-write-latency-high", "latest_note": "adservio-rds-mysql-master-memory-low, adservio-rds-mysql-master-write-latency-high", "alert_url": "https://www.adservio.ro/api/v2/status", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-03-14_0700_2026-03-21_0700.html", "source_json": "production_reliability_dashboard_2026-03-14_0700_2026-03-21_0700.json" }, { "period_start": "2026-03-14T07:00:00+02:00", "period_end": "2026-03-21T07:00:00+02:00", "week_key": "2026-03-14T07:00__2026-03-21T07:00", "week_label": "W11", "source_type": "infrastructure", "source_label": "Infrastructure alarm", "item_key": "adservio-rds-mysql-master-memory-low", "item_label": "adservio-rds-mysql-master-memory-low", "severity": "critical", "status": "Still alarming", "count": 63, "last_seen": "2026-03-18 07:09", "last_seen_ts": 1773810594.0, "services": "", "signals": "ALARM 32, OK 31, Flips 62", "latest_note": "ALARM", "alert_url": "", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-03-14_0700_2026-03-21_0700.html", "source_json": "production_reliability_dashboard_2026-03-14_0700_2026-03-21_0700.json" }, { "period_start": "2026-03-14T07:00:00+02:00", "period_end": "2026-03-21T07:00:00+02:00", "week_key": "2026-03-14T07:00__2026-03-21T07:00", "week_label": "W11", "source_type": "infrastructure", "source_label": "Infrastructure alarm", "item_key": "adservio-rds-mysql-master-write-latency-high", "item_label": "adservio-rds-mysql-master-write-latency-high", "severity": "warning", "status": "Latest OK", "count": 2, "last_seen": "2026-03-16 23:13", "last_seen_ts": 1773695602.0, "services": "", "signals": "ALARM 1, OK 1, Flips 1", "latest_note": "OK", "alert_url": "", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-03-14_0700_2026-03-21_0700.html", "source_json": "production_reliability_dashboard_2026-03-14_0700_2026-03-21_0700.json" }, { "period_start": "2026-03-21T07:00:00+02:00", "period_end": "2026-03-28T07:00:00+02:00", "week_key": "2026-03-21T07:00__2026-03-28T07:00", "week_label": "W12", "source_type": "application", "source_label": "Application alert", "item_key": "KubeHpaMaxedOut", "item_label": "KubeHpaMaxedOut", "severity": "warning", "status": "Seen this week", "count": 5, "last_seen": "2026-03-27 13:05", "last_seen_ts": 1774609552.342729, "services": "docgen2-api (5)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1774609552342729", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1774609552342729", "source_href": "reports/production_reliability_dashboard_2026-03-21_0700_2026-03-28_0700.html", "source_json": "production_reliability_dashboard_2026-03-21_0700_2026-03-28_0700.json" }, { "period_start": "2026-03-21T07:00:00+02:00", "period_end": "2026-03-28T07:00:00+02:00", "week_key": "2026-03-21T07:00__2026-03-28T07:00", "week_label": "W12", "source_type": "application", "source_label": "Application alert", "item_key": "KubeJobFailed", "item_label": "KubeJobFailed", "severity": "warning", "status": "Seen this week", "count": 32, "last_seen": "2026-03-28 05:12", "last_seen_ts": 1774667542.570389, "services": "accommodations-sync-users-29572740 (21), download-album-29576545 (6), accommodations-sync-users-29571300 (5)", "signals": "General investigation", "latest_note": "production.ERROR: Allowed memory size of 134217728 bytes exhausted (tried to allocate 200704 bytes) {\"exception\":\"[object] (Symfony\\\\Compon… | Ionut Ciolan a dat out of memory. E setat cumva in php.ini max memory la 128mb? Ai cum sa o cresti sa nu mai crape? | mai bine ar fi sa faci un pic refactor la cod. Pare ca inc…", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1774667542570389", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1774342311275009?thread_ts=1774342311.275009&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-03-21_0700_2026-03-28_0700.html", "source_json": "production_reliability_dashboard_2026-03-21_0700_2026-03-28_0700.json" }, { "period_start": "2026-03-21T07:00:00+02:00", "period_end": "2026-03-28T07:00:00+02:00", "week_key": "2026-03-21T07:00__2026-03-28T07:00", "week_label": "W12", "source_type": "application", "source_label": "Application alert", "item_key": "KubeNodeEviction", "item_label": "KubeNodeEviction", "severity": "warning", "status": "Seen this week", "count": 2, "last_seen": "2026-03-26 08:38", "last_seen_ts": 1774507123.429909, "services": "grafana (2)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1774507123429909", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1774507123429909", "source_href": "reports/production_reliability_dashboard_2026-03-21_0700_2026-03-28_0700.html", "source_json": "production_reliability_dashboard_2026-03-21_0700_2026-03-28_0700.json" }, { "period_start": "2026-03-21T07:00:00+02:00", "period_end": "2026-03-28T07:00:00+02:00", "week_key": "2026-03-21T07:00__2026-03-28T07:00", "week_label": "W12", "source_type": "application", "source_label": "Application alert", "item_key": "NodeCPUHighUsage", "item_label": "NodeCPUHighUsage", "severity": "warning", "status": "Seen this week", "count": 3, "last_seen": "2026-03-27 11:29", "last_seen_ts": 1774603799.695859, "services": "grafana (3)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1774603799695859", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1774603799695859", "source_href": "reports/production_reliability_dashboard_2026-03-21_0700_2026-03-28_0700.html", "source_json": "production_reliability_dashboard_2026-03-21_0700_2026-03-28_0700.json" }, { "period_start": "2026-03-21T07:00:00+02:00", "period_end": "2026-03-28T07:00:00+02:00", "week_key": "2026-03-21T07:00__2026-03-28T07:00", "week_label": "W12", "source_type": "application", "source_label": "Application alert", "item_key": "NodeSystemSaturation", "item_label": "NodeSystemSaturation", "severity": "warning", "status": "Seen this week", "count": 6, "last_seen": "2026-03-27 11:29", "last_seen_ts": 1774603789.674389, "services": "grafana (6)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1774603789674389", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1774603789674389", "source_href": "reports/production_reliability_dashboard_2026-03-21_0700_2026-03-28_0700.html", "source_json": "production_reliability_dashboard_2026-03-21_0700_2026-03-28_0700.json" }, { "period_start": "2026-03-21T07:00:00+02:00", "period_end": "2026-03-28T07:00:00+02:00", "week_key": "2026-03-21T07:00__2026-03-28T07:00", "week_label": "W12", "source_type": "application", "source_label": "Application alert", "item_key": "TraefikServiceHighErrorRate", "item_label": "TraefikServiceHighErrorRate", "severity": "critical", "status": "Seen this week", "count": 2, "last_seen": "2026-03-27 08:23", "last_seen_ts": 1774592623.169059, "services": "uni-api-svc-4000 (1), web-80 (1)", "signals": "General investigation", "latest_note": "Andrei Alexandru pare ca e o buba cu foreign keys pe tuiasi | Service method DisciplineServiceImpl.patchDisciplineByIdAndAcademicPlan failed after 12ms: could not execute statement [(conn=4451547) Cann… | ok, urmaresc si rezolv, am mai gasit ieri.o astfel de problema, o urmaresc si pe asta", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1774592623169059", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1774429093165839?thread_ts=1774429093.165839&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-03-21_0700_2026-03-28_0700.html", "source_json": "production_reliability_dashboard_2026-03-21_0700_2026-03-28_0700.json" }, { "period_start": "2026-03-21T07:00:00+02:00", "period_end": "2026-03-28T07:00:00+02:00", "week_key": "2026-03-21T07:00__2026-03-28T07:00", "week_label": "W12", "source_type": "application", "source_label": "Application alert", "item_key": "TraefikServiceHighLatency", "item_label": "TraefikServiceHighLatency", "severity": "warning", "status": "Seen this week", "count": 16, "last_seen": "2026-03-27 17:21", "last_seen_ts": 1774624868.180819, "services": "ai-api-svc-3900 (11), web-80 (5), subscriptions-api-svc-3400 (4), rooms-api-svc-3700 (1)", "signals": "General investigation", "latest_note": "era un query la catalog care mergea greu", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1774624868180819", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1774516778168249?thread_ts=1774516778.168249&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-03-21_0700_2026-03-28_0700.html", "source_json": "production_reliability_dashboard_2026-03-21_0700_2026-03-28_0700.json" }, { "period_start": "2026-03-21T07:00:00+02:00", "period_end": "2026-03-28T07:00:00+02:00", "week_key": "2026-03-21T07:00__2026-03-28T07:00", "week_label": "W12", "source_type": "infrastructure", "source_label": "Infrastructure alarm", "item_key": "adservio-rds-mysql-catalog-disk-queue-high", "item_label": "adservio-rds-mysql-catalog-disk-queue-high", "severity": "warning", "status": "Latest OK", "count": 2, "last_seen": "2026-03-27 08:41", "last_seen_ts": 1774593708.0, "services": "", "signals": "ALARM 1, OK 1, Flips 1", "latest_note": "OK", "alert_url": "", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-03-21_0700_2026-03-28_0700.html", "source_json": "production_reliability_dashboard_2026-03-21_0700_2026-03-28_0700.json" }, { "period_start": "2026-03-21T07:00:00+02:00", "period_end": "2026-03-28T07:00:00+02:00", "week_key": "2026-03-21T07:00__2026-03-28T07:00", "week_label": "W12", "source_type": "infrastructure", "source_label": "Infrastructure alarm", "item_key": "adservio-rds-mysql-catalog-memory-low", "item_label": "adservio-rds-mysql-catalog-memory-low", "severity": "critical", "status": "Still alarming", "count": 2, "last_seen": "2026-03-27 08:41", "last_seen_ts": 1774593678.0, "services": "", "signals": "ALARM 1, OK 1, Flips 1", "latest_note": "ALARM", "alert_url": "", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-03-21_0700_2026-03-28_0700.html", "source_json": "production_reliability_dashboard_2026-03-21_0700_2026-03-28_0700.json" }, { "period_start": "2026-03-21T07:00:00+02:00", "period_end": "2026-03-28T07:00:00+02:00", "week_key": "2026-03-21T07:00__2026-03-28T07:00", "week_label": "W12", "source_type": "infrastructure", "source_label": "Infrastructure alarm", "item_key": "adservio-rds-mysql-catalog-swap-high", "item_label": "adservio-rds-mysql-catalog-swap-high", "severity": "critical", "status": "Still alarming", "count": 2, "last_seen": "2026-03-27 12:14", "last_seen_ts": 1774606482.0, "services": "", "signals": "ALARM 1, OK 1, Flips 1", "latest_note": "ALARM", "alert_url": "", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-03-21_0700_2026-03-28_0700.html", "source_json": "production_reliability_dashboard_2026-03-21_0700_2026-03-28_0700.json" }, { "period_start": "2026-03-21T07:00:00+02:00", "period_end": "2026-03-28T07:00:00+02:00", "week_key": "2026-03-21T07:00__2026-03-28T07:00", "week_label": "W12", "source_type": "infrastructure", "source_label": "Infrastructure alarm", "item_key": "adservio-rds-mysql-catalog2-memory-low", "item_label": "adservio-rds-mysql-catalog2-memory-low", "severity": "critical", "status": "Still alarming", "count": 43, "last_seen": "2026-03-25 10:11", "last_seen_ts": 1774426269.0, "services": "", "signals": "ALARM 22, OK 21, Flips 42", "latest_note": "ALARM", "alert_url": "", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-03-21_0700_2026-03-28_0700.html", "source_json": "production_reliability_dashboard_2026-03-21_0700_2026-03-28_0700.json" }, { "period_start": "2026-03-21T07:00:00+02:00", "period_end": "2026-03-28T07:00:00+02:00", "week_key": "2026-03-21T07:00__2026-03-28T07:00", "week_label": "W12", "source_type": "infrastructure", "source_label": "Infrastructure alarm", "item_key": "adservio-rds-mysql-catalog2-swap-high", "item_label": "adservio-rds-mysql-catalog2-swap-high", "severity": "critical", "status": "Still alarming", "count": 1, "last_seen": "2026-03-27 13:43", "last_seen_ts": 1774611823.0, "services": "", "signals": "ALARM 1, OK 0, Flips 0", "latest_note": "ALARM", "alert_url": "", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-03-21_0700_2026-03-28_0700.html", "source_json": "production_reliability_dashboard_2026-03-21_0700_2026-03-28_0700.json" }, { "period_start": "2026-03-28T07:00:00+02:00", "period_end": "2026-04-04T07:00:00+03:00", "week_key": "2026-03-28T07:00__2026-04-04T07:00", "week_label": "W13", "source_type": "application", "source_label": "Application alert", "item_key": "KubeAggregatedAPIDown", "item_label": "KubeAggregatedAPIDown", "severity": "warning", "status": "Seen today", "count": 10, "last_seen": "2026-04-04 04:19", "last_seen_ts": 1775265587.844989, "services": "metrics-server (10)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775265587844989", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775265587844989", "source_href": "reports/production_reliability_dashboard_2026-03-28_0700_2026-04-04_0700.html", "source_json": "production_reliability_dashboard_2026-03-28_0700_2026-04-04_0700.json" }, { "period_start": "2026-03-28T07:00:00+02:00", "period_end": "2026-04-04T07:00:00+03:00", "week_key": "2026-03-28T07:00__2026-04-04T07:00", "week_label": "W13", "source_type": "application", "source_label": "Application alert", "item_key": "KubeCPUOvercommit", "item_label": "KubeCPUOvercommit", "severity": "warning", "status": "Seen today", "count": 1, "last_seen": "2026-04-03 10:18", "last_seen_ts": 1775200707.893329, "services": "grafana (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775200707893329", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775200707893329", "source_href": "reports/production_reliability_dashboard_2026-03-28_0700_2026-04-04_0700.html", "source_json": "production_reliability_dashboard_2026-03-28_0700_2026-04-04_0700.json" }, { "period_start": "2026-03-28T07:00:00+02:00", "period_end": "2026-04-04T07:00:00+03:00", "week_key": "2026-03-28T07:00__2026-04-04T07:00", "week_label": "W13", "source_type": "application", "source_label": "Application alert", "item_key": "KubeDeploymentReplicasMismatch", "item_label": "KubeDeploymentReplicasMismatch", "severity": "warning", "status": "Recent (72h)", "count": 26, "last_seen": "2026-04-02 15:44", "last_seen_ts": 1775133891.047499, "services": "accommodations-events (25), metrics-server (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775133891047499", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775133891047499", "source_href": "reports/production_reliability_dashboard_2026-03-28_0700_2026-04-04_0700.html", "source_json": "production_reliability_dashboard_2026-03-28_0700_2026-04-04_0700.json" }, { "period_start": "2026-03-28T07:00:00+02:00", "period_end": "2026-04-04T07:00:00+03:00", "week_key": "2026-03-28T07:00__2026-04-04T07:00", "week_label": "W13", "source_type": "application", "source_label": "Application alert", "item_key": "KubeHpaMaxedOut", "item_label": "KubeHpaMaxedOut", "severity": "warning", "status": "Seen today", "count": 7, "last_seen": "2026-04-03 13:55", "last_seen_ts": 1775213722.350009, "services": "docgen2-api (4), grafana (3)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775213722350009", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775213722350009", "source_href": "reports/production_reliability_dashboard_2026-03-28_0700_2026-04-04_0700.html", "source_json": "production_reliability_dashboard_2026-03-28_0700_2026-04-04_0700.json" }, { "period_start": "2026-03-28T07:00:00+02:00", "period_end": "2026-04-04T07:00:00+03:00", "week_key": "2026-03-28T07:00__2026-04-04T07:00", "week_label": "W13", "source_type": "application", "source_label": "Application alert", "item_key": "KubeJobFailed", "item_label": "KubeJobFailed", "severity": "warning", "status": "Seen today", "count": 54, "last_seen": "2026-04-04 04:36", "last_seen_ts": 1775266612.837159, "services": "accommodations-sync-users (31), download-album (13), update-recurenta (10), minicrm-sync (8)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775266612837159", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775266612837159", "source_href": "reports/production_reliability_dashboard_2026-03-28_0700_2026-04-04_0700.html", "source_json": "production_reliability_dashboard_2026-03-28_0700_2026-04-04_0700.json" }, { "period_start": "2026-03-28T07:00:00+02:00", "period_end": "2026-04-04T07:00:00+03:00", "week_key": "2026-03-28T07:00__2026-04-04T07:00", "week_label": "W13", "source_type": "application", "source_label": "Application alert", "item_key": "KubeJobNotCompleted", "item_label": "KubeJobNotCompleted", "severity": "warning", "status": "Recent (72h)", "count": 1, "last_seen": "2026-04-02 15:05", "last_seen_ts": 1775131552.349969, "services": "update-recurenta (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775131552349969", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775131552349969", "source_href": "reports/production_reliability_dashboard_2026-03-28_0700_2026-04-04_0700.html", "source_json": "production_reliability_dashboard_2026-03-28_0700_2026-04-04_0700.json" }, { "period_start": "2026-03-28T07:00:00+02:00", "period_end": "2026-04-04T07:00:00+03:00", "week_key": "2026-03-28T07:00__2026-04-04T07:00", "week_label": "W13", "source_type": "application", "source_label": "Application alert", "item_key": "KubePodCrashLooping", "item_label": "KubePodCrashLooping", "severity": "warning", "status": "Recent (72h)", "count": 12, "last_seen": "2026-04-02 14:29", "last_seen_ts": 1775129361.085209, "services": "accommodations-events (12)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775129361085209", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775129361085209", "source_href": "reports/production_reliability_dashboard_2026-03-28_0700_2026-04-04_0700.html", "source_json": "production_reliability_dashboard_2026-03-28_0700_2026-04-04_0700.json" }, { "period_start": "2026-03-28T07:00:00+02:00", "period_end": "2026-04-04T07:00:00+03:00", "week_key": "2026-03-28T07:00__2026-04-04T07:00", "week_label": "W13", "source_type": "application", "source_label": "Application alert", "item_key": "KubePodNotReady", "item_label": "KubePodNotReady", "severity": "warning", "status": "Recent (72h)", "count": 1, "last_seen": "2026-04-01 10:54", "last_seen_ts": 1775030092.344839, "services": "core-grafana (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775030092344839", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775030092344839", "source_href": "reports/production_reliability_dashboard_2026-03-28_0700_2026-04-04_0700.html", "source_json": "production_reliability_dashboard_2026-03-28_0700_2026-04-04_0700.json" }, { "period_start": "2026-03-28T07:00:00+02:00", "period_end": "2026-04-04T07:00:00+03:00", "week_key": "2026-03-28T07:00__2026-04-04T07:00", "week_label": "W13", "source_type": "application", "source_label": "Application alert", "item_key": "TraefikServiceHighErrorRate", "item_label": "TraefikServiceHighErrorRate", "severity": "critical", "status": "Seen today", "count": 1, "last_seen": "2026-04-03 09:24", "last_seen_ts": 1775197483.233569, "services": "docgen2-api (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775197483233569", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775197483233569", "source_href": "reports/production_reliability_dashboard_2026-03-28_0700_2026-04-04_0700.html", "source_json": "production_reliability_dashboard_2026-03-28_0700_2026-04-04_0700.json" }, { "period_start": "2026-03-28T07:00:00+02:00", "period_end": "2026-04-04T07:00:00+03:00", "week_key": "2026-03-28T07:00__2026-04-04T07:00", "week_label": "W13", "source_type": "application", "source_label": "Application alert", "item_key": "TraefikServiceHighLatency", "item_label": "TraefikServiceHighLatency", "severity": "warning", "status": "Seen today", "count": 6, "last_seen": "2026-04-03 19:02", "last_seen_ts": 1775232158.182759, "services": "ai-api (6)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775232158182759", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775232158182759", "source_href": "reports/production_reliability_dashboard_2026-03-28_0700_2026-04-04_0700.html", "source_json": "production_reliability_dashboard_2026-03-28_0700_2026-04-04_0700.json" }, { "period_start": "2026-03-28T07:00:00+02:00", "period_end": "2026-04-04T07:00:00+03:00", "week_key": "2026-03-28T07:00__2026-04-04T07:00", "week_label": "W13", "source_type": "customer", "source_label": "Customer warning", "item_key": "https://www.adservio.ro/api/v2/status", "item_label": "https://www.adservio.ro/api/v2/status", "severity": "warning", "status": "Recovered recently", "count": 2, "last_seen": "2026-04-04 00:00", "last_seen_ts": 1775250000.0, "services": "adservio-ro-api-v2-status", "signals": "Pingdom-only evidence so far", "latest_note": "Pingdom-only evidence so far", "alert_url": "https://www.adservio.ro/api/v2/status", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-03-28_0700_2026-04-04_0700.html", "source_json": "production_reliability_dashboard_2026-03-28_0700_2026-04-04_0700.json" }, { "period_start": "2026-03-28T07:00:00+02:00", "period_end": "2026-04-04T07:00:00+03:00", "week_key": "2026-03-28T07:00__2026-04-04T07:00", "week_label": "W13", "source_type": "infrastructure", "source_label": "Infrastructure alarm", "item_key": "adservio-rds-postgres-billing-cpu-high", "item_label": "adservio-rds-postgres-billing-cpu-high", "severity": "warning", "status": "Latest OK", "count": 4, "last_seen": "2026-04-02 10:22", "last_seen_ts": 1775114523.0, "services": "", "signals": "ALARM 2, OK 2, Flips 3", "latest_note": "OK", "alert_url": "", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-03-28_0700_2026-04-04_0700.html", "source_json": "production_reliability_dashboard_2026-03-28_0700_2026-04-04_0700.json" }, { "period_start": "2026-03-28T07:00:00+02:00", "period_end": "2026-04-04T07:00:00+03:00", "week_key": "2026-03-28T07:00__2026-04-04T07:00", "week_label": "W13", "source_type": "infrastructure", "source_label": "Infrastructure alarm", "item_key": "adservio-root-account-usage", "item_label": "adservio-root-account-usage", "severity": "warning", "status": "Latest OK", "count": 2, "last_seen": "2026-04-03 13:50", "last_seen_ts": 1775213434.0, "services": "", "signals": "ALARM 1, OK 1, Flips 1", "latest_note": "OK", "alert_url": "", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-03-28_0700_2026-04-04_0700.html", "source_json": "production_reliability_dashboard_2026-03-28_0700_2026-04-04_0700.json" }, { "period_start": "2026-04-04T07:00:00+03:00", "period_end": "2026-04-11T07:00:00+03:00", "week_key": "2026-04-04T07:00__2026-04-11T07:00", "week_label": "W14", "source_type": "application", "source_label": "Application alert", "item_key": "KubeAggregatedAPIDown", "item_label": "KubeAggregatedAPIDown", "severity": "warning", "status": "Seen this week", "count": 15, "last_seen": "2026-04-06 17:34", "last_seen_ts": 1775486089.032009, "services": "metrics-server (15)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775486089032009", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775486089032009", "source_href": "reports/production_reliability_dashboard_2026-04-04_0700_2026-04-11_0700.html", "source_json": "" }, { "period_start": "2026-04-04T07:00:00+03:00", "period_end": "2026-04-11T07:00:00+03:00", "week_key": "2026-04-04T07:00__2026-04-11T07:00", "week_label": "W14", "source_type": "application", "source_label": "Application alert", "item_key": "KubeHpaMaxedOut", "item_label": "KubeHpaMaxedOut", "severity": "warning", "status": "Seen this week", "count": 6, "last_seen": "2026-04-08 02:53", "last_seen_ts": 1775606002.531429, "services": "grafana (6)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775606002531429", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775606002531429", "source_href": "reports/production_reliability_dashboard_2026-04-04_0700_2026-04-11_0700.html", "source_json": "" }, { "period_start": "2026-04-04T07:00:00+03:00", "period_end": "2026-04-11T07:00:00+03:00", "week_key": "2026-04-04T07:00__2026-04-11T07:00", "week_label": "W14", "source_type": "application", "source_label": "Application alert", "item_key": "KubeJobFailed", "item_label": "KubeJobFailed", "severity": "warning", "status": "Seen today", "count": 44, "last_seen": "2026-04-11 03:30", "last_seen_ts": 1775867452.435479, "services": "update-recurenta (44)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775867452435479", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775867452435479", "source_href": "reports/production_reliability_dashboard_2026-04-04_0700_2026-04-11_0700.html", "source_json": "" }, { "period_start": "2026-04-04T07:00:00+03:00", "period_end": "2026-04-11T07:00:00+03:00", "week_key": "2026-04-04T07:00__2026-04-11T07:00", "week_label": "W14", "source_type": "application", "source_label": "Application alert", "item_key": "KubeJobNotCompleted", "item_label": "KubeJobNotCompleted", "severity": "warning", "status": "Recent (72h)", "count": 12, "last_seen": "2026-04-09 03:20", "last_seen_ts": 1775694052.569049, "services": "update-recurenta (12)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775694052569049", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775694052569049", "source_href": "reports/production_reliability_dashboard_2026-04-04_0700_2026-04-11_0700.html", "source_json": "" }, { "period_start": "2026-04-04T07:00:00+03:00", "period_end": "2026-04-11T07:00:00+03:00", "week_key": "2026-04-04T07:00__2026-04-11T07:00", "week_label": "W14", "source_type": "application", "source_label": "Application alert", "item_key": "TraefikServiceHighErrorRate", "item_label": "TraefikServiceHighErrorRate", "severity": "critical", "status": "Seen this week", "count": 2, "last_seen": "2026-04-06 00:44", "last_seen_ts": 1775425453.269459, "services": "core-grafana-80 (2)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775425453269459", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775425453269459", "source_href": "reports/production_reliability_dashboard_2026-04-04_0700_2026-04-11_0700.html", "source_json": "" }, { "period_start": "2026-04-04T07:00:00+03:00", "period_end": "2026-04-11T07:00:00+03:00", "week_key": "2026-04-04T07:00__2026-04-11T07:00", "week_label": "W14", "source_type": "application", "source_label": "Application alert", "item_key": "TraefikServiceHighLatency", "item_label": "TraefikServiceHighLatency", "severity": "warning", "status": "Seen today", "count": 3, "last_seen": "2026-04-10 17:25", "last_seen_ts": 1775831138.187949, "services": "ai-api (3)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775831138187949", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775831138187949", "source_href": "reports/production_reliability_dashboard_2026-04-04_0700_2026-04-11_0700.html", "source_json": "" }, { "period_start": "2026-04-11T07:00:00+03:00", "period_end": "2026-04-18T07:00:00+03:00", "week_key": "2026-04-11T07:00__2026-04-18T07:00", "week_label": "W15", "source_type": "application", "source_label": "Application alert", "item_key": "AlertmanagerFailedToSendAlerts", "item_label": "AlertmanagerFailedToSendAlerts", "severity": "warning", "status": "Recent (72h)", "count": 1, "last_seen": "2026-04-16 07:29", "last_seen_ts": 1776313798.456709, "services": "grafana (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776313798456709", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776313798456709", "source_href": "reports/production_reliability_dashboard_2026-04-11_0700_2026-04-18_0700.html", "source_json": "" }, { "period_start": "2026-04-11T07:00:00+03:00", "period_end": "2026-04-18T07:00:00+03:00", "week_key": "2026-04-11T07:00__2026-04-18T07:00", "week_label": "W15", "source_type": "application", "source_label": "Application alert", "item_key": "KubeCPUOvercommit", "item_label": "KubeCPUOvercommit", "severity": "warning", "status": "Seen today", "count": 6, "last_seen": "2026-04-17 10:18", "last_seen_ts": 1776410307.906639, "services": "grafana (6)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776410307906639", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776410307906639", "source_href": "reports/production_reliability_dashboard_2026-04-11_0700_2026-04-18_0700.html", "source_json": "" }, { "period_start": "2026-04-11T07:00:00+03:00", "period_end": "2026-04-18T07:00:00+03:00", "week_key": "2026-04-11T07:00__2026-04-18T07:00", "week_label": "W15", "source_type": "application", "source_label": "Application alert", "item_key": "KubeDeploymentReplicasMismatch", "item_label": "KubeDeploymentReplicasMismatch", "severity": "warning", "status": "Seen today", "count": 2, "last_seen": "2026-04-17 13:28", "last_seen_ts": 1776421732.339979, "services": "accommodations-api (1), library-api (1), rooms-api (1), web (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776421732339979", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776421732339979", "source_href": "reports/production_reliability_dashboard_2026-04-11_0700_2026-04-18_0700.html", "source_json": "" }, { "period_start": "2026-04-11T07:00:00+03:00", "period_end": "2026-04-18T07:00:00+03:00", "week_key": "2026-04-11T07:00__2026-04-18T07:00", "week_label": "W15", "source_type": "application", "source_label": "Application alert", "item_key": "KubeHpaMaxedOut", "item_label": "KubeHpaMaxedOut", "severity": "warning", "status": "Seen today", "count": 5, "last_seen": "2026-04-17 09:51", "last_seen_ts": 1776408682.407399, "services": "docgen2-api (5)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776408682407399", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776408682407399", "source_href": "reports/production_reliability_dashboard_2026-04-11_0700_2026-04-18_0700.html", "source_json": "" }, { "period_start": "2026-04-11T07:00:00+03:00", "period_end": "2026-04-18T07:00:00+03:00", "week_key": "2026-04-11T07:00__2026-04-18T07:00", "week_label": "W15", "source_type": "application", "source_label": "Application alert", "item_key": "KubeJobFailed", "item_label": "KubeJobFailed", "severity": "warning", "status": "Seen today", "count": 44, "last_seen": "2026-04-18 03:33", "last_seen_ts": 1776472432.595369, "services": "update-recurenta (39), colecteaza-sms-note-abs (6), download-album (5), rezumat (5)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776472432595369", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776472432595369", "source_href": "reports/production_reliability_dashboard_2026-04-11_0700_2026-04-18_0700.html", "source_json": "" }, { "period_start": "2026-04-11T07:00:00+03:00", "period_end": "2026-04-18T07:00:00+03:00", "week_key": "2026-04-11T07:00__2026-04-18T07:00", "week_label": "W15", "source_type": "application", "source_label": "Application alert", "item_key": "KubePodCrashLooping", "item_label": "KubePodCrashLooping", "severity": "warning", "status": "Seen today", "count": 4, "last_seen": "2026-04-17 10:19", "last_seen_ts": 1776410391.066099, "services": "accommodations-events (2), library-api (2), rooms-api (2), accommodations-api (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776410391066099", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776410391066099", "source_href": "reports/production_reliability_dashboard_2026-04-11_0700_2026-04-18_0700.html", "source_json": "" }, { "period_start": "2026-04-11T07:00:00+03:00", "period_end": "2026-04-18T07:00:00+03:00", "week_key": "2026-04-11T07:00__2026-04-18T07:00", "week_label": "W15", "source_type": "application", "source_label": "Application alert", "item_key": "KubePodNotReady", "item_label": "KubePodNotReady", "severity": "warning", "status": "Seen today", "count": 3, "last_seen": "2026-04-17 14:01", "last_seen_ts": 1776423682.323779, "services": "web (3)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776423682323779", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776423682323779", "source_href": "reports/production_reliability_dashboard_2026-04-11_0700_2026-04-18_0700.html", "source_json": "" }, { "period_start": "2026-04-11T07:00:00+03:00", "period_end": "2026-04-18T07:00:00+03:00", "week_key": "2026-04-11T07:00__2026-04-18T07:00", "week_label": "W15", "source_type": "application", "source_label": "Application alert", "item_key": "TargetDown", "item_label": "TargetDown", "severity": "warning", "status": "Seen today", "count": 1, "last_seen": "2026-04-17 13:24", "last_seen_ts": 1776421497.821429, "services": "grafana (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776421497821429", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776421497821429", "source_href": "reports/production_reliability_dashboard_2026-04-11_0700_2026-04-18_0700.html", "source_json": "" }, { "period_start": "2026-04-11T07:00:00+03:00", "period_end": "2026-04-18T07:00:00+03:00", "week_key": "2026-04-11T07:00__2026-04-18T07:00", "week_label": "W15", "source_type": "application", "source_label": "Application alert", "item_key": "TraefikServiceHighLatency", "item_label": "TraefikServiceHighLatency", "severity": "warning", "status": "Seen today", "count": 6, "last_seen": "2026-04-17 18:22", "last_seen_ts": 1776439328.183589, "services": "ai-api (6)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776439328183589", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776439328183589", "source_href": "reports/production_reliability_dashboard_2026-04-11_0700_2026-04-18_0700.html", "source_json": "" }, { "period_start": "2026-04-11T07:00:00+03:00", "period_end": "2026-04-18T07:00:00+03:00", "week_key": "2026-04-11T07:00__2026-04-18T07:00", "week_label": "W15", "source_type": "infrastructure", "source_label": "Infrastructure alarm", "item_key": "adservio-rds-mysql-catalog-disk-queue-high", "item_label": "adservio-rds-mysql-catalog-disk-queue-high", "severity": "warning", "status": "Latest OK", "count": 2, "last_seen": "2026-04-15 12:26", "last_seen_ts": 1776245205.0, "services": "", "signals": "ALARM 1, OK 1, Flips 1", "latest_note": "OK", "alert_url": "", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-04-11_0700_2026-04-18_0700.html", "source_json": "" }, { "period_start": "2026-04-18T07:00:00+03:00", "period_end": "2026-04-25T07:00:00+03:00", "week_key": "2026-04-18T07:00__2026-04-25T07:00", "week_label": "W16", "source_type": "application", "source_label": "Application alert", "item_key": "KubeContainerWaiting", "item_label": "KubeContainerWaiting", "severity": "warning", "status": "Seen this week", "count": 2, "last_seen": "2026-04-20 12:38", "last_seen_ts": 1776677902.384839, "services": "grafana (2)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776677902384839", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776677902384839", "source_href": "reports/production_reliability_dashboard_2026-04-18_0700_2026-04-25_0700.html", "source_json": "production_reliability_dashboard_2026-04-18_0700_2026-04-25_0700.json" }, { "period_start": "2026-04-18T07:00:00+03:00", "period_end": "2026-04-25T07:00:00+03:00", "week_key": "2026-04-18T07:00__2026-04-25T07:00", "week_label": "W16", "source_type": "application", "source_label": "Application alert", "item_key": "KubeCPUOvercommit", "item_label": "KubeCPUOvercommit", "severity": "warning", "status": "Seen today", "count": 24, "last_seen": "2026-04-24 12:29", "last_seen_ts": 1777022967.925309, "services": "grafana (24)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777022967925309", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777022967925309", "source_href": "reports/production_reliability_dashboard_2026-04-18_0700_2026-04-25_0700.html", "source_json": "production_reliability_dashboard_2026-04-18_0700_2026-04-25_0700.json" }, { "period_start": "2026-04-18T07:00:00+03:00", "period_end": "2026-04-25T07:00:00+03:00", "week_key": "2026-04-18T07:00__2026-04-25T07:00", "week_label": "W16", "source_type": "application", "source_label": "Application alert", "item_key": "KubeDeploymentReplicasMismatch", "item_label": "KubeDeploymentReplicasMismatch", "severity": "warning", "status": "Seen this week", "count": 2, "last_seen": "2026-04-20 11:48", "last_seen_ts": 1776674902.445639, "services": "core-getresponse-events-worker (2)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776674902445639", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776674902445639", "source_href": "reports/production_reliability_dashboard_2026-04-18_0700_2026-04-25_0700.html", "source_json": "production_reliability_dashboard_2026-04-18_0700_2026-04-25_0700.json" }, { "period_start": "2026-04-18T07:00:00+03:00", "period_end": "2026-04-25T07:00:00+03:00", "week_key": "2026-04-18T07:00__2026-04-25T07:00", "week_label": "W16", "source_type": "application", "source_label": "Application alert", "item_key": "KubeDeploymentRolloutStuck", "item_label": "KubeDeploymentRolloutStuck", "severity": "warning", "status": "Likely noise / resolved", "count": 2, "last_seen": "2026-04-20 17:27", "last_seen_ts": 1776695272.342619, "services": "forms-api (2)", "signals": "Alert tuning / noise", "latest_note": "Investigation summary for `forms-api` | | *What happened* | - The `forms-api` rollout stalled and triggered `KubeDeploymentRolloutStuck`. |… | , @U09JYAWCGLB @U633D9JBW I made this skill that can run automatically as soon as an alert appears in Slack, so it ca… | we can improve it further based o…", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776695272342619", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776695272342619?thread_ts=1776695272.342619&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-04-18_0700_2026-04-25_0700.html", "source_json": "production_reliability_dashboard_2026-04-18_0700_2026-04-25_0700.json" }, { "period_start": "2026-04-18T07:00:00+03:00", "period_end": "2026-04-25T07:00:00+03:00", "week_key": "2026-04-18T07:00__2026-04-25T07:00", "week_label": "W16", "source_type": "application", "source_label": "Application alert", "item_key": "KubeHpaMaxedOut", "item_label": "KubeHpaMaxedOut", "severity": "warning", "status": "Recent (72h)", "count": 4, "last_seen": "2026-04-22 09:47", "last_seen_ts": 1776840472.369839, "services": "docgen2-api (4)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776840472369839", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776840472369839", "source_href": "reports/production_reliability_dashboard_2026-04-18_0700_2026-04-25_0700.html", "source_json": "production_reliability_dashboard_2026-04-18_0700_2026-04-25_0700.json" }, { "period_start": "2026-04-18T07:00:00+03:00", "period_end": "2026-04-25T07:00:00+03:00", "week_key": "2026-04-18T07:00__2026-04-25T07:00", "week_label": "W16", "source_type": "application", "source_label": "Application alert", "item_key": "KubeJobFailed", "item_label": "KubeJobFailed", "severity": "warning", "status": "Recent (72h)", "count": 27, "last_seen": "2026-04-22 13:48", "last_seen_ts": 1776854934.134389, "services": "update-recurenta (27), colecteaza-sms-note-abs (14), download-album (14), rezumat (14)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776854934134389", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776854934134389", "source_href": "reports/production_reliability_dashboard_2026-04-18_0700_2026-04-25_0700.html", "source_json": "production_reliability_dashboard_2026-04-18_0700_2026-04-25_0700.json" }, { "period_start": "2026-04-18T07:00:00+03:00", "period_end": "2026-04-25T07:00:00+03:00", "week_key": "2026-04-18T07:00__2026-04-25T07:00", "week_label": "W16", "source_type": "application", "source_label": "Application alert", "item_key": "KubePodCrashLooping", "item_label": "KubePodCrashLooping", "severity": "warning", "status": "Seen this week", "count": 3, "last_seen": "2026-04-20 17:24", "last_seen_ts": 1776695062.359829, "services": "forms-api (2), update-recurenta (1)", "signals": "General investigation", "latest_note": "Warning Unhealthy 80s (x19 over 3m22s) kubelet spec.containers{forms-api-container}: Readiness probe failed: Get \"\": dial tcp 10.66.111.240… | one of the pod is healthy to keep the service alive . | Thats right", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776695062359829", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776692782380059?thread_ts=1776692782.380059&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-04-18_0700_2026-04-25_0700.html", "source_json": "production_reliability_dashboard_2026-04-18_0700_2026-04-25_0700.json" }, { "period_start": "2026-04-18T07:00:00+03:00", "period_end": "2026-04-25T07:00:00+03:00", "week_key": "2026-04-18T07:00__2026-04-25T07:00", "week_label": "W16", "source_type": "application", "source_label": "Application alert", "item_key": "TraefikServiceHighErrorRate", "item_label": "TraefikServiceHighErrorRate", "severity": "critical", "status": "Seen this week", "count": 7, "last_seen": "2026-04-21 12:41", "last_seen_ts": 1776764502.679159, "services": "core-grafana-80 (3), admission-api (3), uni-api (1)", "signals": "General investigation", "latest_note": "@U09JYAWCGLB we know about this, discussing in daily with the devs | Hi Raul Popovici sure thanks", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776764502679159", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776753193171579?thread_ts=1776753193.171579&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-04-18_0700_2026-04-25_0700.html", "source_json": "production_reliability_dashboard_2026-04-18_0700_2026-04-25_0700.json" }, { "period_start": "2026-04-18T07:00:00+03:00", "period_end": "2026-04-25T07:00:00+03:00", "week_key": "2026-04-18T07:00__2026-04-25T07:00", "week_label": "W16", "source_type": "application", "source_label": "Application alert", "item_key": "TraefikServiceHighLatency", "item_label": "TraefikServiceHighLatency", "severity": "warning", "status": "Seen today", "count": 7, "last_seen": "2026-04-24 17:35", "last_seen_ts": 1777041338.182049, "services": "ai-api (4), social-api (3)", "signals": "Resource limits, General investigation", "latest_note": "some POSTs to ai summaries seem to take 5s or more. But we don't have traces in ai-api pod to see if it is normal (summaries could indeed t…", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777041338182049", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777041338182049?thread_ts=1777041338.182049&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-04-18_0700_2026-04-25_0700.html", "source_json": "production_reliability_dashboard_2026-04-18_0700_2026-04-25_0700.json" }, { "period_start": "2026-04-18T07:00:00+03:00", "period_end": "2026-04-25T07:00:00+03:00", "week_key": "2026-04-18T07:00__2026-04-25T07:00", "week_label": "W16", "source_type": "infrastructure", "source_label": "Infrastructure alarm", "item_key": "adservio-rds-mysql-catalog-disk-queue-high", "item_label": "adservio-rds-mysql-catalog-disk-queue-high", "severity": "warning", "status": "Latest OK", "count": 2, "last_seen": "2026-04-20 10:14", "last_seen_ts": 1776669285.0, "services": "", "signals": "ALARM 1, OK 1, Flips 1", "latest_note": "OK", "alert_url": "", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-04-18_0700_2026-04-25_0700.html", "source_json": "production_reliability_dashboard_2026-04-18_0700_2026-04-25_0700.json" }, { "period_start": "2026-04-18T07:00:00+03:00", "period_end": "2026-04-25T07:00:00+03:00", "week_key": "2026-04-18T07:00__2026-04-25T07:00", "week_label": "W16", "source_type": "infrastructure", "source_label": "Infrastructure alarm", "item_key": "adservio-rds-mysql-catalog2-storage-low", "item_label": "adservio-rds-mysql-catalog2-storage-low", "severity": "warning", "status": "Latest OK", "count": 2, "last_seen": "2026-04-20 13:52", "last_seen_ts": 1776682338.0, "services": "", "signals": "ALARM 1, OK 1, Flips 1", "latest_note": "OK", "alert_url": "", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-04-18_0700_2026-04-25_0700.html", "source_json": "production_reliability_dashboard_2026-04-18_0700_2026-04-25_0700.json" }, { "period_start": "2026-04-18T07:00:00+03:00", "period_end": "2026-04-25T07:00:00+03:00", "week_key": "2026-04-18T07:00__2026-04-25T07:00", "week_label": "W16", "source_type": "infrastructure", "source_label": "Infrastructure alarm", "item_key": "adservio-root-account-usage", "item_label": "adservio-root-account-usage", "severity": "warning", "status": "Latest OK", "count": 4, "last_seen": "2026-04-23 15:32", "last_seen_ts": 1776947574.0, "services": "", "signals": "ALARM 2, OK 2, Flips 3", "latest_note": "OK", "alert_url": "", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-04-18_0700_2026-04-25_0700.html", "source_json": "production_reliability_dashboard_2026-04-18_0700_2026-04-25_0700.json" }, { "period_start": "2026-04-25T07:00:00+03:00", "period_end": "2026-05-02T07:00:00+03:00", "week_key": "2026-04-25T07:00__2026-05-02T07:00", "week_label": "W17", "source_type": "application", "source_label": "Application alert", "item_key": "KubeAggregatedAPIDown", "item_label": "KubeAggregatedAPIDown", "severity": "warning", "status": "Seen today", "count": 43, "last_seen": "2026-05-04 18:19", "last_seen_ts": 1777907988.501169, "services": "metrics-server (43)", "signals": "General investigation", "latest_note": "False alarm metrics-server pod is healthy (1/1 Running, 0 restarts) and kubectl top nodes works. Alert was a brief flap during a node rotat…", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777907988501169", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777273195154609?thread_ts=1777273195.154609&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-04-25_0700_2026-05-02_0700.html", "source_json": "production_reliability_dashboard_2026-04-25_0700_2026-05-02_0700.json" }, { "period_start": "2026-04-25T07:00:00+03:00", "period_end": "2026-05-02T07:00:00+03:00", "week_key": "2026-04-25T07:00__2026-05-02T07:00", "week_label": "W17", "source_type": "application", "source_label": "Application alert", "item_key": "KubeAPIErrorBudgetBurn", "item_label": "KubeAPIErrorBudgetBurn", "severity": "critical", "status": "Likely noise / resolved", "count": 3, "last_seen": "2026-04-27 12:48", "last_seen_ts": 1777283323.254389, "services": "grafana (4)", "signals": "General investigation, Alert tuning / noise", "latest_note": "Nothing is down apiserver healthy (/readyz ok), all pods Running, HPAs scaled back down (web 9, docgen2 4). Alert is the rolling 1h SLO win… | you can silence it in alertmanger | Okay i will try a bit later thanks for the info", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777293458219179", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777283323254389?thread_ts=1777283323.254389&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-04-25_0700_2026-05-02_0700.html", "source_json": "production_reliability_dashboard_2026-04-25_0700_2026-05-02_0700.json" }, { "period_start": "2026-04-25T07:00:00+03:00", "period_end": "2026-05-02T07:00:00+03:00", "week_key": "2026-04-25T07:00__2026-05-02T07:00", "week_label": "W17", "source_type": "application", "source_label": "Application alert", "item_key": "KubeClientErrors", "item_label": "KubeClientErrors", "severity": "warning", "status": "No recent signal", "count": 1, "last_seen": "2026-04-27 12:09", "last_seen_ts": 1777280977.725749, "services": "grafana (1)", "signals": "General investigation", "latest_note": "aiserver is healthy (/readyz ok). The 2% error rate is from admission webhook timeouts during the HPA scaling burst (web 11 replicas, docge…", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777280977725749", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777280977725749?thread_ts=1777280977.725749&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-04-25_0700_2026-05-02_0700.html", "source_json": "production_reliability_dashboard_2026-04-25_0700_2026-05-02_0700.json" }, { "period_start": "2026-04-25T07:00:00+03:00", "period_end": "2026-05-02T07:00:00+03:00", "week_key": "2026-04-25T07:00__2026-05-02T07:00", "week_label": "W17", "source_type": "application", "source_label": "Application alert", "item_key": "KubeCPUOvercommit", "item_label": "KubeCPUOvercommit", "severity": "warning", "status": "Seen today", "count": 20, "last_seen": "2026-05-04 13:12", "last_seen_ts": 1777889577.894069, "services": "grafana (20)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777889577894069", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777889577894069", "source_href": "reports/production_reliability_dashboard_2026-04-25_0700_2026-05-02_0700.html", "source_json": "production_reliability_dashboard_2026-04-25_0700_2026-05-02_0700.json" }, { "period_start": "2026-04-25T07:00:00+03:00", "period_end": "2026-05-02T07:00:00+03:00", "week_key": "2026-04-25T07:00__2026-05-02T07:00", "week_label": "W17", "source_type": "application", "source_label": "Application alert", "item_key": "KubeDeploymentReplicasMismatch", "item_label": "KubeDeploymentReplicasMismatch", "severity": "warning", "status": "Recent (72h)", "count": 13, "last_seen": "2026-05-03 07:09", "last_seen_ts": 1777781361.612109, "services": "rooms-api (13), core-scheduled-events-worker (2)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777781361612109", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777781361612109", "source_href": "reports/production_reliability_dashboard_2026-04-25_0700_2026-05-02_0700.html", "source_json": "production_reliability_dashboard_2026-04-25_0700_2026-05-02_0700.json" }, { "period_start": "2026-04-25T07:00:00+03:00", "period_end": "2026-05-02T07:00:00+03:00", "week_key": "2026-04-25T07:00__2026-05-02T07:00", "week_label": "W17", "source_type": "application", "source_label": "Application alert", "item_key": "KubeHpaMaxedOut", "item_label": "KubeHpaMaxedOut", "severity": "warning", "status": "Seen today", "count": 5, "last_seen": "2026-05-04 12:17", "last_seen_ts": 1777886272.344179, "services": "docgen2-api (5)", "signals": "Resource limits", "latest_note": "HPA already scaled back down (now 4/6). Pods are barely using CPU, so the scale-up was driven by a KEDA cron trigger, not real load.", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777886272344179", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777280212372279?thread_ts=1777280212.372279&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-04-25_0700_2026-05-02_0700.html", "source_json": "production_reliability_dashboard_2026-04-25_0700_2026-05-02_0700.json" }, { "period_start": "2026-04-25T07:00:00+03:00", "period_end": "2026-05-02T07:00:00+03:00", "week_key": "2026-04-25T07:00__2026-05-02T07:00", "week_label": "W17", "source_type": "application", "source_label": "Application alert", "item_key": "KubeJobFailed", "item_label": "KubeJobFailed", "severity": "warning", "status": "Seen today", "count": 23, "last_seen": "2026-05-04 19:21", "last_seen_ts": 1777911712.019249, "services": "minicrm-sync (23), colecteaza-sms-note-abs (21), accommodations-sync-users (19)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777911712019249", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777911712019249", "source_href": "reports/production_reliability_dashboard_2026-04-25_0700_2026-05-02_0700.html", "source_json": "production_reliability_dashboard_2026-04-25_0700_2026-05-02_0700.json" }, { "period_start": "2026-04-25T07:00:00+03:00", "period_end": "2026-05-02T07:00:00+03:00", "week_key": "2026-04-25T07:00__2026-05-02T07:00", "week_label": "W17", "source_type": "application", "source_label": "Application alert", "item_key": "KubeMemoryOvercommit", "item_label": "KubeMemoryOvercommit", "severity": "warning", "status": "Seen this week", "count": 1, "last_seen": "2026-04-29 09:26", "last_seen_ts": 1777443987.935159, "services": "grafana (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777443987935159", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777443987935159", "source_href": "reports/production_reliability_dashboard_2026-04-25_0700_2026-05-02_0700.html", "source_json": "production_reliability_dashboard_2026-04-25_0700_2026-05-02_0700.json" }, { "period_start": "2026-04-25T07:00:00+03:00", "period_end": "2026-05-02T07:00:00+03:00", "week_key": "2026-04-25T07:00__2026-05-02T07:00", "week_label": "W17", "source_type": "application", "source_label": "Application alert", "item_key": "KubePodCrashLooping", "item_label": "KubePodCrashLooping", "severity": "warning", "status": "Recent (72h)", "count": 14, "last_seen": "2026-05-03 07:14", "last_seen_ts": 1777781691.594009, "services": "rooms-api (13), core-scheduled-events-worker (3)", "signals": "General investigation", "latest_note": "danny Valentin Pal something is breaking on the Tuiasi cluster which i am not able to connect | | i would need a kubeconfig for tuiasi and…", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777781691594009", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777605291164119?thread_ts=1777605291.164119&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-04-25_0700_2026-05-02_0700.html", "source_json": "production_reliability_dashboard_2026-04-25_0700_2026-05-02_0700.json" }, { "period_start": "2026-04-25T07:00:00+03:00", "period_end": "2026-05-02T07:00:00+03:00", "week_key": "2026-04-25T07:00__2026-05-02T07:00", "week_label": "W17", "source_type": "application", "source_label": "Application alert", "item_key": "NodeSystemSaturation", "item_label": "NodeSystemSaturation", "severity": "warning", "status": "Seen this week", "count": 1, "last_seen": "2026-04-30 12:30", "last_seen_ts": 1777541449.731469, "services": "grafana (1)", "signals": "Resource limits", "latest_note": "Node 10.66.121.122 is at 79% CPU / 50% memory, no pressure conditions load is 2.27 because 5 of the web pods landed on the same node plus s… | Will clear when traffic dies down or HPA rebalances | Stabilized load on 10.66.121.122 dropped from 2.27 to 1.68 per core (15m avg), CPU from 79% to 55%. HPA scaled down + web…", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777541449731469", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777541449731469?thread_ts=1777541449.731469&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-04-25_0700_2026-05-02_0700.html", "source_json": "production_reliability_dashboard_2026-04-25_0700_2026-05-02_0700.json" }, { "period_start": "2026-04-25T07:00:00+03:00", "period_end": "2026-05-02T07:00:00+03:00", "week_key": "2026-04-25T07:00__2026-05-02T07:00", "week_label": "W17", "source_type": "application", "source_label": "Application alert", "item_key": "TraefikServiceHighErrorRate", "item_label": "TraefikServiceHighErrorRate", "severity": "critical", "status": "Seen today", "count": 26, "last_seen": "2026-05-04 15:06", "last_seen_ts": 1777896400.713119, "services": "accommodations-api (10), web-80 (8), uni-api (7), core-grafana-80 (1)", "signals": "General investigation, Observability storage", "latest_note": "uni-api is throwing 500s because the same academic-structure save is being submitted twice second insert hits a unique-key constraint (stud…", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777896400713119", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777883293167389?thread_ts=1777883293.167389&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-04-25_0700_2026-05-02_0700.html", "source_json": "production_reliability_dashboard_2026-04-25_0700_2026-05-02_0700.json" }, { "period_start": "2026-04-25T07:00:00+03:00", "period_end": "2026-05-02T07:00:00+03:00", "week_key": "2026-04-25T07:00__2026-05-02T07:00", "week_label": "W17", "source_type": "application", "source_label": "Application alert", "item_key": "TraefikServiceHighLatency", "item_label": "TraefikServiceHighLatency", "severity": "warning", "status": "Seen today", "count": 18, "last_seen": "2026-05-04 12:39", "last_seen_ts": 1777887548.184509, "services": "social-api (10), ai-api (7), subscriptions-api (2), web-80 (2)", "signals": "General investigation", "latest_note": "Two outbound calls to web dominated: GET /api/v2/utilizatori/profile?cuPermisiuni=1&withRoles=1 (6.6s) and GET /api/v2/ani/scolari (6.4s) —…", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777887548184509", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777887548184509?thread_ts=1777887548.184509&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-04-25_0700_2026-05-02_0700.html", "source_json": "production_reliability_dashboard_2026-04-25_0700_2026-05-02_0700.json" }, { "period_start": "2026-04-25T07:00:00+03:00", "period_end": "2026-05-02T07:00:00+03:00", "week_key": "2026-04-25T07:00__2026-05-02T07:00", "week_label": "W17", "source_type": "customer", "source_label": "Customer warning", "item_key": "https://www.adservio.ro/api/v2/status", "item_label": "https://www.adservio.ro/api/v2/status", "severity": "warning", "status": "Recovered recently", "count": 9, "last_seen": "2026-05-05 00:00", "last_seen_ts": 1777928400.0, "services": "adservio-ro-api-v2-status", "signals": "adservio-rds-mysql-catalog-disk-queue-high", "latest_note": "adservio-rds-mysql-catalog-disk-queue-high", "alert_url": "https://www.adservio.ro/api/v2/status", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-04-25_0700_2026-05-02_0700.html", "source_json": "production_reliability_dashboard_2026-04-25_0700_2026-05-02_0700.json" }, { "period_start": "2026-04-25T07:00:00+03:00", "period_end": "2026-05-02T07:00:00+03:00", "week_key": "2026-04-25T07:00__2026-05-02T07:00", "week_label": "W17", "source_type": "infrastructure", "source_label": "Infrastructure alarm", "item_key": "adservio-rds-mysql-catalog-disk-queue-high", "item_label": "adservio-rds-mysql-catalog-disk-queue-high", "severity": "warning", "status": "Flapping, latest OK", "count": 14, "last_seen": "2026-04-30 10:21", "last_seen_ts": 1777533704.0, "services": "", "signals": "ALARM 7, OK 7, Flips 13", "latest_note": "OK", "alert_url": "", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-04-25_0700_2026-05-02_0700.html", "source_json": "production_reliability_dashboard_2026-04-25_0700_2026-05-02_0700.json" }, { "period_start": "2026-04-25T07:00:00+03:00", "period_end": "2026-05-02T07:00:00+03:00", "week_key": "2026-04-25T07:00__2026-05-02T07:00", "week_label": "W17", "source_type": "infrastructure", "source_label": "Infrastructure alarm", "item_key": "adservio-rds-postgres-billing-cpu-high", "item_label": "adservio-rds-postgres-billing-cpu-high", "severity": "warning", "status": "Latest OK", "count": 2, "last_seen": "2026-05-01 10:13", "last_seen_ts": 1777619583.0, "services": "", "signals": "ALARM 1, OK 1, Flips 1", "latest_note": "OK", "alert_url": "", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-04-25_0700_2026-05-02_0700.html", "source_json": "production_reliability_dashboard_2026-04-25_0700_2026-05-02_0700.json" }, { "period_start": "2026-05-02T07:00:00+03:00", "period_end": "2026-05-09T07:00:00+03:00", "week_key": "2026-05-02T07:00__2026-05-09T07:00", "week_label": "W18", "source_type": "application", "source_label": "Application alert", "item_key": "KubeAggregatedAPIDown", "item_label": "KubeAggregatedAPIDown", "severity": "warning", "status": "Seen this week", "count": 15, "last_seen": "2026-05-04 18:19", "last_seen_ts": 1777907988.501169, "services": "metrics-server (15)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777907988501169", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777907988501169", "source_href": "reports/production_reliability_dashboard_2026-05-02_0700_2026-05-09_0700.html", "source_json": "production_reliability_dashboard_2026-05-02_0700_2026-05-09_0700.json" }, { "period_start": "2026-05-02T07:00:00+03:00", "period_end": "2026-05-09T07:00:00+03:00", "week_key": "2026-05-02T07:00__2026-05-09T07:00", "week_label": "W18", "source_type": "application", "source_label": "Application alert", "item_key": "KubeCPUOvercommit", "item_label": "KubeCPUOvercommit", "severity": "warning", "status": "Seen this week", "count": 1, "last_seen": "2026-05-04 13:12", "last_seen_ts": 1777889577.894069, "services": "grafana (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777889577894069", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777889577894069", "source_href": "reports/production_reliability_dashboard_2026-05-02_0700_2026-05-09_0700.html", "source_json": "production_reliability_dashboard_2026-05-02_0700_2026-05-09_0700.json" }, { "period_start": "2026-05-02T07:00:00+03:00", "period_end": "2026-05-09T07:00:00+03:00", "week_key": "2026-05-02T07:00__2026-05-09T07:00", "week_label": "W18", "source_type": "application", "source_label": "Application alert", "item_key": "KubeDeploymentReplicasMismatch", "item_label": "KubeDeploymentReplicasMismatch", "severity": "warning", "status": "Seen this week", "count": 6, "last_seen": "2026-05-03 07:09", "last_seen_ts": 1777781361.612109, "services": "rooms-api (6)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777781361612109", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777781361612109", "source_href": "reports/production_reliability_dashboard_2026-05-02_0700_2026-05-09_0700.html", "source_json": "production_reliability_dashboard_2026-05-02_0700_2026-05-09_0700.json" }, { "period_start": "2026-05-02T07:00:00+03:00", "period_end": "2026-05-09T07:00:00+03:00", "week_key": "2026-05-02T07:00__2026-05-09T07:00", "week_label": "W18", "source_type": "application", "source_label": "Application alert", "item_key": "KubeHpaMaxedOut", "item_label": "KubeHpaMaxedOut", "severity": "warning", "status": "Seen this week", "count": 2, "last_seen": "2026-05-04 12:17", "last_seen_ts": 1777886272.344179, "services": "docgen2-api (2)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777886272344179", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777886272344179", "source_href": "reports/production_reliability_dashboard_2026-05-02_0700_2026-05-09_0700.html", "source_json": "production_reliability_dashboard_2026-05-02_0700_2026-05-09_0700.json" }, { "period_start": "2026-05-02T07:00:00+03:00", "period_end": "2026-05-09T07:00:00+03:00", "week_key": "2026-05-02T07:00__2026-05-09T07:00", "week_label": "W18", "source_type": "application", "source_label": "Application alert", "item_key": "KubeJobFailed", "item_label": "KubeJobFailed", "severity": "warning", "status": "Seen this week", "count": 15, "last_seen": "2026-05-04 19:21", "last_seen_ts": 1777911712.019249, "services": "accommodations-sync-users (15), colecteaza-sms-note-abs (15), minicrm-sync (15)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777911712019249", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777911712019249", "source_href": "reports/production_reliability_dashboard_2026-05-02_0700_2026-05-09_0700.html", "source_json": "production_reliability_dashboard_2026-05-02_0700_2026-05-09_0700.json" }, { "period_start": "2026-05-02T07:00:00+03:00", "period_end": "2026-05-09T07:00:00+03:00", "week_key": "2026-05-02T07:00__2026-05-09T07:00", "week_label": "W18", "source_type": "application", "source_label": "Application alert", "item_key": "KubePodCrashLooping", "item_label": "KubePodCrashLooping", "severity": "warning", "status": "Seen this week", "count": 6, "last_seen": "2026-05-03 07:14", "last_seen_ts": 1777781691.594009, "services": "rooms-api (6)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777781691594009", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777781691594009", "source_href": "reports/production_reliability_dashboard_2026-05-02_0700_2026-05-09_0700.html", "source_json": "production_reliability_dashboard_2026-05-02_0700_2026-05-09_0700.json" }, { "period_start": "2026-05-02T07:00:00+03:00", "period_end": "2026-05-09T07:00:00+03:00", "week_key": "2026-05-02T07:00__2026-05-09T07:00", "week_label": "W18", "source_type": "application", "source_label": "Application alert", "item_key": "TraefikServiceHighErrorRate", "item_label": "TraefikServiceHighErrorRate", "severity": "critical", "status": "Seen this week", "count": 12, "last_seen": "2026-05-04 15:06", "last_seen_ts": 1777896400.713119, "services": "accommodations-api (8), uni-api (4)", "signals": "Observability storage", "latest_note": "uni-api is throwing 500s because the same academic-structure save is being submitted twice second insert hits a unique-key constraint (stud…", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777896400713119", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777883293167389?thread_ts=1777883293.167389&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-05-02_0700_2026-05-09_0700.html", "source_json": "production_reliability_dashboard_2026-05-02_0700_2026-05-09_0700.json" }, { "period_start": "2026-05-02T07:00:00+03:00", "period_end": "2026-05-09T07:00:00+03:00", "week_key": "2026-05-02T07:00__2026-05-09T07:00", "week_label": "W18", "source_type": "application", "source_label": "Application alert", "item_key": "TraefikServiceHighLatency", "item_label": "TraefikServiceHighLatency", "severity": "warning", "status": "Seen this week", "count": 7, "last_seen": "2026-05-04 12:39", "last_seen_ts": 1777887548.184509, "services": "social-api (6), ai-api (1)", "signals": "General investigation", "latest_note": "Two outbound calls to web dominated: GET /api/v2/utilizatori/profile?cuPermisiuni=1&withRoles=1 (6.6s) and GET /api/v2/ani/scolari (6.4s) —…", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777887548184509", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777887548184509?thread_ts=1777887548.184509&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-05-02_0700_2026-05-09_0700.html", "source_json": "production_reliability_dashboard_2026-05-02_0700_2026-05-09_0700.json" }, { "period_start": "2026-05-02T07:00:00+03:00", "period_end": "2026-05-09T07:00:00+03:00", "week_key": "2026-05-02T07:00__2026-05-09T07:00", "week_label": "W18", "source_type": "customer", "source_label": "Customer warning", "item_key": "https://www.adservio.ro/api/v2/status", "item_label": "https://www.adservio.ro/api/v2/status", "severity": "warning", "status": "Recovered recently", "count": 5, "last_seen": "2026-05-09 00:00", "last_seen_ts": 1778274000.0, "services": "adservio-ro-api-v2-status", "signals": "adservio-rds-mysql-catalog-disk-queue-high, adservio-rds-mysql-catalog-storage-low", "latest_note": "adservio-rds-mysql-catalog-disk-queue-high, adservio-rds-mysql-catalog-storage-low", "alert_url": "https://www.adservio.ro/api/v2/status", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-05-02_0700_2026-05-09_1000.html", "source_json": "" }, { "period_start": "2026-05-02T07:00:00+03:00", "period_end": "2026-05-09T07:00:00+03:00", "week_key": "2026-05-02T07:00__2026-05-09T07:00", "week_label": "W18", "source_type": "infrastructure", "source_label": "Infrastructure alarm", "item_key": "adservio-rds-mysql-catalog-disk-queue-high", "item_label": "adservio-rds-mysql-catalog-disk-queue-high", "severity": "warning", "status": "Latest OK", "count": 2, "last_seen": "2026-05-06 12:30", "last_seen_ts": 1778059814.0, "services": "", "signals": "ALARM 1, OK 1, Flips 1", "latest_note": "OK", "alert_url": "", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-05-02_0700_2026-05-09_0700.html", "source_json": "production_reliability_dashboard_2026-05-02_0700_2026-05-09_0700.json" }, { "period_start": "2026-05-02T07:00:00+03:00", "period_end": "2026-05-09T07:00:00+03:00", "week_key": "2026-05-02T07:00__2026-05-09T07:00", "week_label": "W18", "source_type": "infrastructure", "source_label": "Infrastructure alarm", "item_key": "adservio-rds-mysql-catalog-storage-low", "item_label": "adservio-rds-mysql-catalog-storage-low", "severity": "warning", "status": "Latest OK", "count": 2, "last_seen": "2026-05-06 12:26", "last_seen_ts": 1778059565.0, "services": "", "signals": "ALARM 1, OK 1, Flips 1", "latest_note": "OK", "alert_url": "", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-05-02_0700_2026-05-09_0700.html", "source_json": "production_reliability_dashboard_2026-05-02_0700_2026-05-09_0700.json" }, { "period_start": "2026-05-02T07:00:00+03:00", "period_end": "2026-05-09T07:00:00+03:00", "week_key": "2026-05-02T07:00__2026-05-09T07:00", "week_label": "W18", "source_type": "infrastructure", "source_label": "Infrastructure alarm", "item_key": "adservio-rds-mysql-catalog-write-latency-high", "item_label": "adservio-rds-mysql-catalog-write-latency-high", "severity": "warning", "status": "Latest OK", "count": 2, "last_seen": "2026-05-06 12:26", "last_seen_ts": 1778059618.0, "services": "", "signals": "ALARM 1, OK 1, Flips 1", "latest_note": "OK", "alert_url": "", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-05-02_0700_2026-05-09_0700.html", "source_json": "production_reliability_dashboard_2026-05-02_0700_2026-05-09_0700.json" }, { "period_start": "2026-05-09T07:00:00+03:00", "period_end": "2026-05-16T07:00:00+03:00", "week_key": "2026-05-09T07:00__2026-05-16T07:00", "week_label": "W19", "source_type": "application", "source_label": "Application alert", "item_key": "AlertmanagerFailedToSendAlerts", "item_label": "AlertmanagerFailedToSendAlerts", "severity": "warning", "status": "Seen this week", "count": 1, "last_seen": "2026-05-12 10:35", "last_seen_ts": 1778571353.424169, "services": "grafana (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1778571353424169", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1778571353424169", "source_href": "reports/production_reliability_dashboard_2026-05-09_0700_2026-05-16_0700.html", "source_json": "production_reliability_dashboard_2026-05-09_0700_2026-05-16_0700.json" }, { "period_start": "2026-05-09T07:00:00+03:00", "period_end": "2026-05-16T07:00:00+03:00", "week_key": "2026-05-09T07:00__2026-05-16T07:00", "week_label": "W19", "source_type": "application", "source_label": "Application alert", "item_key": "KubeCPUOvercommit", "item_label": "KubeCPUOvercommit", "severity": "warning", "status": "Seen today", "count": 5, "last_seen": "2026-05-15 15:02", "last_seen_ts": 1778846577.960959, "services": "grafana (5)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1778846577960959", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1778846577960959", "source_href": "reports/production_reliability_dashboard_2026-05-09_0700_2026-05-16_0700.html", "source_json": "production_reliability_dashboard_2026-05-09_0700_2026-05-16_0700.json" }, { "period_start": "2026-05-09T07:00:00+03:00", "period_end": "2026-05-16T07:00:00+03:00", "week_key": "2026-05-09T07:00__2026-05-16T07:00", "week_label": "W19", "source_type": "application", "source_label": "Application alert", "item_key": "KubeDeploymentReplicasMismatch", "item_label": "KubeDeploymentReplicasMismatch", "severity": "warning", "status": "Seen today", "count": 14, "last_seen": "2026-05-16 05:19", "last_seen_ts": 1778897962.434189, "services": "audit-worker (14)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1778897962434189", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1778897962434189", "source_href": "reports/production_reliability_dashboard_2026-05-09_0700_2026-05-16_0700.html", "source_json": "production_reliability_dashboard_2026-05-09_0700_2026-05-16_0700.json" }, { "period_start": "2026-05-09T07:00:00+03:00", "period_end": "2026-05-16T07:00:00+03:00", "week_key": "2026-05-09T07:00__2026-05-16T07:00", "week_label": "W19", "source_type": "application", "source_label": "Application alert", "item_key": "KubeHpaMaxedOut", "item_label": "KubeHpaMaxedOut", "severity": "warning", "status": "Seen this week", "count": 2, "last_seen": "2026-05-12 13:48", "last_seen_ts": 1778582902.353969, "services": "docgen2-api (2)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1778582902353969", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1778582902353969", "source_href": "reports/production_reliability_dashboard_2026-05-09_0700_2026-05-16_0700.html", "source_json": "production_reliability_dashboard_2026-05-09_0700_2026-05-16_0700.json" }, { "period_start": "2026-05-09T07:00:00+03:00", "period_end": "2026-05-16T07:00:00+03:00", "week_key": "2026-05-09T07:00__2026-05-16T07:00", "week_label": "W19", "source_type": "application", "source_label": "Application alert", "item_key": "KubePodCrashLooping", "item_label": "KubePodCrashLooping", "severity": "warning", "status": "Seen today", "count": 5, "last_seen": "2026-05-16 04:43", "last_seen_ts": 1778895802.601009, "services": "audit-worker (5)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1778895802601009", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1778895802601009", "source_href": "reports/production_reliability_dashboard_2026-05-09_0700_2026-05-16_0700.html", "source_json": "production_reliability_dashboard_2026-05-09_0700_2026-05-16_0700.json" }, { "period_start": "2026-05-09T07:00:00+03:00", "period_end": "2026-05-16T07:00:00+03:00", "week_key": "2026-05-09T07:00__2026-05-16T07:00", "week_label": "W19", "source_type": "application", "source_label": "Application alert", "item_key": "NodeCPUHighUsage", "item_label": "NodeCPUHighUsage", "severity": "warning", "status": "Recent (72h)", "count": 3, "last_seen": "2026-05-13 09:27", "last_seen_ts": 1778653649.689149, "services": "grafana (3)", "signals": "Resource limits", "latest_note": "Node running at 98 percent CPU checking | | Web pods are the most resource dominant on the cluster | this is resolved for now , rollout restarted the web pods to replace them to the other nodes that was sitting idle to balance the load", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1778653649689149", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1778651549707879?thread_ts=1778651549.707879&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-05-09_0700_2026-05-16_0700.html", "source_json": "production_reliability_dashboard_2026-05-09_0700_2026-05-16_0700.json" }, { "period_start": "2026-05-09T07:00:00+03:00", "period_end": "2026-05-16T07:00:00+03:00", "week_key": "2026-05-09T07:00__2026-05-16T07:00", "week_label": "W19", "source_type": "application", "source_label": "Application alert", "item_key": "NodeHighNumberConntrackEntriesUsed", "item_label": "NodeHighNumberConntrackEntriesUsed", "severity": "warning", "status": "Seen today", "count": 23, "last_seen": "2026-05-15 12:29", "last_seen_ts": 1778837389.695469, "services": "grafana (23)", "signals": "General investigation", "latest_note": "This has been resolved", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1778837389695469", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1778750239702649?thread_ts=1778750239.702649&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-05-09_0700_2026-05-16_0700.html", "source_json": "production_reliability_dashboard_2026-05-09_0700_2026-05-16_0700.json" }, { "period_start": "2026-05-09T07:00:00+03:00", "period_end": "2026-05-16T07:00:00+03:00", "week_key": "2026-05-09T07:00__2026-05-16T07:00", "week_label": "W19", "source_type": "application", "source_label": "Application alert", "item_key": "NodeSystemSaturation", "item_label": "NodeSystemSaturation", "severity": "warning", "status": "Seen today", "count": 6, "last_seen": "2026-05-15 12:33", "last_seen_ts": 1778837629.699669, "services": "grafana (6)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1778837629699669", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1778837629699669", "source_href": "reports/production_reliability_dashboard_2026-05-09_0700_2026-05-16_0700.html", "source_json": "production_reliability_dashboard_2026-05-09_0700_2026-05-16_0700.json" }, { "period_start": "2026-05-09T07:00:00+03:00", "period_end": "2026-05-16T07:00:00+03:00", "week_key": "2026-05-09T07:00__2026-05-16T07:00", "week_label": "W19", "source_type": "application", "source_label": "Application alert", "item_key": "TraefikServiceHighErrorRate", "item_label": "TraefikServiceHighErrorRate", "severity": "critical", "status": "Recent (72h)", "count": 1, "last_seen": "2026-05-13 11:55", "last_seen_ts": 1778662513.202819, "services": "uni-api (1)", "signals": "General investigation", "latest_note": "no errors now pod is healthy | | we saw this error | | Service method CareerServiceImpl.saveCareer failed after Nms: | Invalid employment t…", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1778662513202819", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1778662513202819?thread_ts=1778662513.202819&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-05-09_0700_2026-05-16_0700.html", "source_json": "production_reliability_dashboard_2026-05-09_0700_2026-05-16_0700.json" }, { "period_start": "2026-05-09T07:00:00+03:00", "period_end": "2026-05-16T07:00:00+03:00", "week_key": "2026-05-09T07:00__2026-05-16T07:00", "week_label": "W19", "source_type": "application", "source_label": "Application alert", "item_key": "TraefikServiceHighLatency", "item_label": "TraefikServiceHighLatency", "severity": "warning", "status": "Seen today", "count": 25, "last_seen": "2026-05-15 19:19", "last_seen_ts": 1778861978.209259, "services": "social-api (11), ai-api (11), web-80 (6), uni-api (2)", "signals": "Release / migration issue, Scaling config", "latest_note": "12 min ago, 15+ web pod probe failures fired simultaneously (readiness /ready timeouts + php-fpm-healthcheck 10s timeouts). All 6+ web pods… | Suggesting two quick config changes on web to stop the recurring p95 latency alerts | | 1. Loosen readiness probe: failureThreshold: 4, per… | understood, good point", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1778861978209259", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1778746838179519?thread_ts=1778746838.179519&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-05-09_0700_2026-05-16_0700.html", "source_json": "production_reliability_dashboard_2026-05-09_0700_2026-05-16_0700.json" }, { "period_start": "2026-05-09T07:00:00+03:00", "period_end": "2026-05-16T07:00:00+03:00", "week_key": "2026-05-09T07:00__2026-05-16T07:00", "week_label": "W19", "source_type": "customer", "source_label": "Customer warning", "item_key": "https://www.adservio.ro/api/v2/status", "item_label": "https://www.adservio.ro/api/v2/status", "severity": "warning", "status": "Recovered recently", "count": 15, "last_seen": "2026-05-15 11:49", "last_seen_ts": 1778834940.0, "services": "adservio-ro-api-v2-status", "signals": "Pingdom-only evidence so far", "latest_note": "Pingdom-only evidence so far", "alert_url": "https://www.adservio.ro/api/v2/status", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-05-09_0700_2026-05-16_0700.html", "source_json": "production_reliability_dashboard_2026-05-09_0700_2026-05-16_0700.json" }, { "period_start": "2026-05-16T07:00:00+03:00", "period_end": "2026-05-23T07:00:00+03:00", "week_key": "2026-05-16T07:00__2026-05-23T07:00", "week_label": "W20", "source_type": "application", "source_label": "Application alert", "item_key": "KubeCPUOvercommit", "item_label": "KubeCPUOvercommit", "severity": "warning", "status": "Seen this week", "count": 12, "last_seen": "2026-05-21 11:27", "last_seen_ts": 1779352077.869409, "services": "grafana (12)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1779352077869409", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1779352077869409", "source_href": "reports/production_reliability_dashboard_2026-05-16_0700_2026-05-24_2011.html", "source_json": "production_reliability_dashboard_2026-05-16_0700_2026-05-24_2011.json" }, { "period_start": "2026-05-16T07:00:00+03:00", "period_end": "2026-05-23T07:00:00+03:00", "week_key": "2026-05-16T07:00__2026-05-23T07:00", "week_label": "W20", "source_type": "application", "source_label": "Application alert", "item_key": "KubeDeploymentReplicasMismatch", "item_label": "KubeDeploymentReplicasMismatch", "severity": "warning", "status": "Seen this week", "count": 35, "last_seen": "2026-05-18 15:59", "last_seen_ts": 1779109162.382199, "services": "audit-worker (35)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1779109162382199", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1779109162382199", "source_href": "reports/production_reliability_dashboard_2026-05-16_0700_2026-05-24_2011.html", "source_json": "production_reliability_dashboard_2026-05-16_0700_2026-05-24_2011.json" }, { "period_start": "2026-05-16T07:00:00+03:00", "period_end": "2026-05-23T07:00:00+03:00", "week_key": "2026-05-16T07:00__2026-05-23T07:00", "week_label": "W20", "source_type": "application", "source_label": "Application alert", "item_key": "KubeHpaMaxedOut", "item_label": "KubeHpaMaxedOut", "severity": "warning", "status": "Seen this week", "count": 6, "last_seen": "2026-05-21 14:16", "last_seen_ts": 1779362212.405069, "services": "grafana (3), docgen2-api (3)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1779362212405069", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1779362212405069", "source_href": "reports/production_reliability_dashboard_2026-05-16_0700_2026-05-24_2011.html", "source_json": "production_reliability_dashboard_2026-05-16_0700_2026-05-24_2011.json" }, { "period_start": "2026-05-16T07:00:00+03:00", "period_end": "2026-05-23T07:00:00+03:00", "week_key": "2026-05-16T07:00__2026-05-23T07:00", "week_label": "W20", "source_type": "application", "source_label": "Application alert", "item_key": "KubeJobFailed", "item_label": "KubeJobFailed", "severity": "warning", "status": "Recent (72h)", "count": 3, "last_seen": "2026-05-22 07:27", "last_seen_ts": 1779424072.451799, "services": "download-album (3), rezumat (3)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1779424072451799", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1779424072451799", "source_href": "reports/production_reliability_dashboard_2026-05-16_0700_2026-05-24_2011.html", "source_json": "production_reliability_dashboard_2026-05-16_0700_2026-05-24_2011.json" }, { "period_start": "2026-05-16T07:00:00+03:00", "period_end": "2026-05-23T07:00:00+03:00", "week_key": "2026-05-16T07:00__2026-05-23T07:00", "week_label": "W20", "source_type": "application", "source_label": "Application alert", "item_key": "KubePodCrashLooping", "item_label": "KubePodCrashLooping", "severity": "warning", "status": "Seen this week", "count": 14, "last_seen": "2026-05-18 13:53", "last_seen_ts": 1779101603.202509, "services": "audit-worker (14)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1779101603202509", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1779101603202509", "source_href": "reports/production_reliability_dashboard_2026-05-16_0700_2026-05-24_2011.html", "source_json": "production_reliability_dashboard_2026-05-16_0700_2026-05-24_2011.json" }, { "period_start": "2026-05-16T07:00:00+03:00", "period_end": "2026-05-23T07:00:00+03:00", "week_key": "2026-05-16T07:00__2026-05-23T07:00", "week_label": "W20", "source_type": "application", "source_label": "Application alert", "item_key": "NodeHighNumberConntrackEntriesUsed", "item_label": "NodeHighNumberConntrackEntriesUsed", "severity": "warning", "status": "Seen this week", "count": 85, "last_seen": "2026-05-21 12:02", "last_seen_ts": 1779354169.715839, "services": "grafana (85)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1779354169715839", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1779354169715839", "source_href": "reports/production_reliability_dashboard_2026-05-16_0700_2026-05-24_2011.html", "source_json": "production_reliability_dashboard_2026-05-16_0700_2026-05-24_2011.json" }, { "period_start": "2026-05-16T07:00:00+03:00", "period_end": "2026-05-23T07:00:00+03:00", "week_key": "2026-05-16T07:00__2026-05-23T07:00", "week_label": "W20", "source_type": "application", "source_label": "Application alert", "item_key": "NodeSystemSaturation", "item_label": "NodeSystemSaturation", "severity": "warning", "status": "Seen this week", "count": 2, "last_seen": "2026-05-19 09:19", "last_seen_ts": 1779171589.690869, "services": "grafana (2)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1779171589690869", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1779171589690869", "source_href": "reports/production_reliability_dashboard_2026-05-16_0700_2026-05-24_2011.html", "source_json": "production_reliability_dashboard_2026-05-16_0700_2026-05-24_2011.json" }, { "period_start": "2026-05-16T07:00:00+03:00", "period_end": "2026-05-23T07:00:00+03:00", "week_key": "2026-05-16T07:00__2026-05-23T07:00", "week_label": "W20", "source_type": "application", "source_label": "Application alert", "item_key": "TraefikServiceHighErrorRate", "item_label": "TraefikServiceHighErrorRate", "severity": "critical", "status": "Seen this week", "count": 3, "last_seen": "2026-05-21 16:10", "last_seen_ts": 1779369043.275199, "services": "core-grafana-80 (2), web-80 (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1779369043275199", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1779369043275199", "source_href": "reports/production_reliability_dashboard_2026-05-16_0700_2026-05-24_2011.html", "source_json": "production_reliability_dashboard_2026-05-16_0700_2026-05-24_2011.json" }, { "period_start": "2026-05-16T07:00:00+03:00", "period_end": "2026-05-23T07:00:00+03:00", "week_key": "2026-05-16T07:00__2026-05-23T07:00", "week_label": "W20", "source_type": "application", "source_label": "Application alert", "item_key": "TraefikServiceHighLatency", "item_label": "TraefikServiceHighLatency", "severity": "warning", "status": "Recent (72h)", "count": 53, "last_seen": "2026-05-22 18:25", "last_seen_ts": 1779463508.176599, "services": "web-80 (37), social-api (33), ai-api (17), subscriptions-api (7)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1779463508176599", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1779463508176599", "source_href": "reports/production_reliability_dashboard_2026-05-16_0700_2026-05-24_2011.html", "source_json": "production_reliability_dashboard_2026-05-16_0700_2026-05-24_2011.json" }, { "period_start": "2026-05-16T07:00:00+03:00", "period_end": "2026-05-23T07:00:00+03:00", "week_key": "2026-05-16T07:00__2026-05-23T07:00", "week_label": "W20", "source_type": "customer", "source_label": "Customer warning", "item_key": "Adservio Ro", "item_label": "Adservio Ro", "severity": "warning", "status": "Seen this week", "count": 1, "last_seen": "2026-05-20 18:13", "last_seen_ts": 1779289980.0, "services": "adservio-ro", "signals": "Pingdom-only evidence so far", "latest_note": "Pingdom-only evidence so far", "alert_url": "https://www.adservio.ro", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-05-16_0700_2026-05-24_2011.html", "source_json": "production_reliability_dashboard_2026-05-16_0700_2026-05-24_2011.json" }, { "period_start": "2026-05-16T07:00:00+03:00", "period_end": "2026-05-23T07:00:00+03:00", "week_key": "2026-05-16T07:00__2026-05-23T07:00", "week_label": "W20", "source_type": "customer", "source_label": "Customer warning", "item_key": "https://www.adservio.ro/api/v2/status", "item_label": "https://www.adservio.ro/api/v2/status", "severity": "warning", "status": "Seen this week", "count": 13, "last_seen": "2026-05-20 19:04", "last_seen_ts": 1779293040.0, "services": "adservio-ro-api-v2-status", "signals": "Pingdom-only evidence so far", "latest_note": "Pingdom-only evidence so far", "alert_url": "https://www.adservio.ro/api/v2/status", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-05-16_0700_2026-05-24_2011.html", "source_json": "production_reliability_dashboard_2026-05-16_0700_2026-05-24_2011.json" }, { "period_start": "2026-05-23T07:00:00+03:00", "period_end": "2026-05-30T07:00:00+03:00", "week_key": "2026-05-23T07:00__2026-05-30T07:00", "week_label": "W21", "source_type": "application", "source_label": "Application alert", "item_key": "KubeHpaMaxedOut", "item_label": "KubeHpaMaxedOut", "severity": "warning", "status": "Recent (72h)", "count": 12, "last_seen": "2026-05-28 14:40", "last_seen_ts": 1779968422.512619, "services": "docgen2-api (9), grafana (3)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1779968422512619", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1779968422512619", "source_href": "reports/production_reliability_dashboard_2026-05-23_0700_2026-05-30_0700.html", "source_json": "production_reliability_dashboard_2026-05-23_0700_2026-05-30_0700.json" }, { "period_start": "2026-05-23T07:00:00+03:00", "period_end": "2026-05-30T07:00:00+03:00", "week_key": "2026-05-23T07:00__2026-05-30T07:00", "week_label": "W21", "source_type": "application", "source_label": "Application alert", "item_key": "KubeMemoryOvercommit", "item_label": "KubeMemoryOvercommit", "severity": "warning", "status": "Recent (72h)", "count": 4, "last_seen": "2026-05-28 19:09", "last_seen_ts": 1779984598.009709, "services": "grafana (4)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1779984598009709", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1779984598009709", "source_href": "reports/production_reliability_dashboard_2026-05-23_0700_2026-05-30_0700.html", "source_json": "production_reliability_dashboard_2026-05-23_0700_2026-05-30_0700.json" }, { "period_start": "2026-05-23T07:00:00+03:00", "period_end": "2026-05-30T07:00:00+03:00", "week_key": "2026-05-23T07:00__2026-05-30T07:00", "week_label": "W21", "source_type": "application", "source_label": "Application alert", "item_key": "KubePodNotReady", "item_label": "KubePodNotReady", "severity": "warning", "status": "Seen this week", "count": 1, "last_seen": "2026-05-26 13:51", "last_seen_ts": 1779792682.405959, "services": "biling-efactura-worker-ff45f6d-rxxlg (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1779792682405959", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1779792682405959", "source_href": "reports/production_reliability_dashboard_2026-05-23_0700_2026-05-30_0700.html", "source_json": "production_reliability_dashboard_2026-05-23_0700_2026-05-30_0700.json" }, { "period_start": "2026-05-23T07:00:00+03:00", "period_end": "2026-05-30T07:00:00+03:00", "week_key": "2026-05-23T07:00__2026-05-30T07:00", "week_label": "W21", "source_type": "application", "source_label": "Application alert", "item_key": "NodeSystemSaturation", "item_label": "NodeSystemSaturation", "severity": "warning", "status": "Seen today", "count": 4, "last_seen": "2026-05-29 12:26", "last_seen_ts": 1780046779.856509, "services": "grafana (4)", "signals": "Release / migration issue", "latest_note": "Worker node 10.66.133.111 (c7a.2xlarge, 8 vCPU) is CPU-saturated at 91% real usage 6 of 21 web pods landed on it (no topology spread on the… | Also durable fix is adding topologySpreadConstraints | to the web Deployment which i will look into", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1780046779856509", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1779945379858529?thread_ts=1779945379.858529&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-05-23_0700_2026-05-30_0700.html", "source_json": "production_reliability_dashboard_2026-05-23_0700_2026-05-30_0700.json" }, { "period_start": "2026-05-23T07:00:00+03:00", "period_end": "2026-05-30T07:00:00+03:00", "week_key": "2026-05-23T07:00__2026-05-30T07:00", "week_label": "W21", "source_type": "application", "source_label": "Application alert", "item_key": "TraefikServiceHighErrorRate", "item_label": "TraefikServiceHighErrorRate", "severity": "critical", "status": "Seen today", "count": 2, "last_seen": "2026-05-29 18:04", "last_seen_ts": 1780067083.244629, "services": "core-grafana-80 (2)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1780067083244629", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1780067083244629", "source_href": "reports/production_reliability_dashboard_2026-05-23_0700_2026-05-30_0700.html", "source_json": "production_reliability_dashboard_2026-05-23_0700_2026-05-30_0700.json" }, { "period_start": "2026-05-23T07:00:00+03:00", "period_end": "2026-05-30T07:00:00+03:00", "week_key": "2026-05-23T07:00__2026-05-30T07:00", "week_label": "W21", "source_type": "application", "source_label": "Application alert", "item_key": "TraefikServiceHighLatency", "item_label": "TraefikServiceHighLatency", "severity": "warning", "status": "Seen today", "count": 6, "last_seen": "2026-05-29 18:06", "last_seen_ts": 1780067168.183929, "services": "ai-api (6)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1780067168183929", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1780067168183929", "source_href": "reports/production_reliability_dashboard_2026-05-23_0700_2026-05-30_0700.html", "source_json": "production_reliability_dashboard_2026-05-23_0700_2026-05-30_0700.json" }, { "period_start": "2026-05-23T07:00:00+03:00", "period_end": "2026-05-30T07:00:00+03:00", "week_key": "2026-05-23T07:00__2026-05-30T07:00", "week_label": "W21", "source_type": "infrastructure", "source_label": "Infrastructure alarm", "item_key": "adservio-rds-mysql-master-memory-low", "item_label": "adservio-rds-mysql-master-memory-low", "severity": "warning", "status": "Flapping, latest OK", "count": 13, "last_seen": "2026-05-29 12:00", "last_seen_ts": 1780045223.0, "services": "", "signals": "ALARM 6, OK 7, Flips 12", "latest_note": "OK", "alert_url": "", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-05-23_0700_2026-05-30_0700.html", "source_json": "production_reliability_dashboard_2026-05-23_0700_2026-05-30_0700.json" }, { "period_start": "2026-05-23T07:00:00+03:00", "period_end": "2026-05-30T07:00:00+03:00", "week_key": "2026-05-23T07:00__2026-05-30T07:00", "week_label": "W21", "source_type": "infrastructure", "source_label": "Infrastructure alarm", "item_key": "adservio-rds-postgres-billing-cpu-high", "item_label": "adservio-rds-postgres-billing-cpu-high", "severity": "warning", "status": "Latest OK", "count": 4, "last_seen": "2026-05-29 10:23", "last_seen_ts": 1780039416.0, "services": "", "signals": "ALARM 2, OK 2, Flips 3", "latest_note": "OK", "alert_url": "", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-05-23_0700_2026-05-30_0700.html", "source_json": "production_reliability_dashboard_2026-05-23_0700_2026-05-30_0700.json" }, { "period_start": "2026-05-30T07:00:00+03:00", "period_end": "2026-06-06T07:00:00+03:00", "week_key": "2026-05-30T07:00__2026-06-06T07:00", "week_label": "W22", "source_type": "application", "source_label": "Application alert", "item_key": "AlertmanagerFailedToSendAlerts", "item_label": "AlertmanagerFailedToSendAlerts", "severity": "warning", "status": "Seen this week", "count": 1, "last_seen": "2026-06-03 14:10", "last_seen_ts": 1780485048.343579, "services": "grafana (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1780485048343579", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1780485048343579", "source_href": "reports/production_reliability_dashboard_2026-05-30_0700_2026-06-08_0218.html", "source_json": "production_reliability_dashboard_2026-05-30_0700_2026-06-08_0218.json" }, { "period_start": "2026-05-30T07:00:00+03:00", "period_end": "2026-06-06T07:00:00+03:00", "week_key": "2026-05-30T07:00__2026-06-06T07:00", "week_label": "W22", "source_type": "application", "source_label": "Application alert", "item_key": "KubeAggregatedAPIDown", "item_label": "KubeAggregatedAPIDown", "severity": "warning", "status": "Seen today", "count": 27, "last_seen": "2026-06-08 00:20", "last_seen_ts": 1780867249.341069, "services": "metrics-server (27)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1780867249341069", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1780867249341069", "source_href": "reports/production_reliability_dashboard_2026-05-30_0700_2026-06-08_0218.html", "source_json": "production_reliability_dashboard_2026-05-30_0700_2026-06-08_0218.json" }, { "period_start": "2026-05-30T07:00:00+03:00", "period_end": "2026-06-06T07:00:00+03:00", "week_key": "2026-05-30T07:00__2026-06-06T07:00", "week_label": "W22", "source_type": "application", "source_label": "Application alert", "item_key": "KubeHpaMaxedOut", "item_label": "KubeHpaMaxedOut", "severity": "warning", "status": "Seen this week", "count": 9, "last_seen": "2026-06-03 16:34", "last_seen_ts": 1780493662.476469, "services": "docgen2-api (6), grafana (2), notifications-event-manager (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1780493662476469", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1780493662476469", "source_href": "reports/production_reliability_dashboard_2026-05-30_0700_2026-06-08_0218.html", "source_json": "production_reliability_dashboard_2026-05-30_0700_2026-06-08_0218.json" }, { "period_start": "2026-05-30T07:00:00+03:00", "period_end": "2026-06-06T07:00:00+03:00", "week_key": "2026-05-30T07:00__2026-06-06T07:00", "week_label": "W22", "source_type": "application", "source_label": "Application alert", "item_key": "KubeJobFailed", "item_label": "KubeJobFailed", "severity": "warning", "status": "Seen this week", "count": 1, "last_seen": "2026-06-03 14:10", "last_seen_ts": 1780485048.348319, "services": "accommodations-sync-users (1), colecteaza-sms-note-abs (1), minicrm-sync (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1780485048348319", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1780485048348319", "source_href": "reports/production_reliability_dashboard_2026-05-30_0700_2026-06-08_0218.html", "source_json": "production_reliability_dashboard_2026-05-30_0700_2026-06-08_0218.json" }, { "period_start": "2026-05-30T07:00:00+03:00", "period_end": "2026-06-06T07:00:00+03:00", "week_key": "2026-05-30T07:00__2026-06-06T07:00", "week_label": "W22", "source_type": "application", "source_label": "Application alert", "item_key": "KubeletServerCertificateExpiration", "item_label": "KubeletServerCertificateExpiration", "severity": "warning", "status": "No recent signal", "count": 6, "last_seen": "2026-05-31 11:22", "last_seen_ts": 1780215753.737109, "services": "grafana (6)", "signals": "General investigation", "latest_note": "Cert auto-rotated over the weekend", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1780215753737109", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1780215753737109?thread_ts=1780215753.737109&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-05-30_0700_2026-06-08_0218.html", "source_json": "production_reliability_dashboard_2026-05-30_0700_2026-06-08_0218.json" }, { "period_start": "2026-05-30T07:00:00+03:00", "period_end": "2026-06-06T07:00:00+03:00", "week_key": "2026-05-30T07:00__2026-06-06T07:00", "week_label": "W22", "source_type": "application", "source_label": "Application alert", "item_key": "KubeMemoryOvercommit", "item_label": "KubeMemoryOvercommit", "severity": "warning", "status": "Seen this week", "count": 7, "last_seen": "2026-06-04 19:08", "last_seen_ts": 1780589337.971909, "services": "grafana (7)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1780589337971909", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1780589337971909", "source_href": "reports/production_reliability_dashboard_2026-05-30_0700_2026-06-08_0218.html", "source_json": "production_reliability_dashboard_2026-05-30_0700_2026-06-08_0218.json" }, { "period_start": "2026-05-30T07:00:00+03:00", "period_end": "2026-06-06T07:00:00+03:00", "week_key": "2026-05-30T07:00__2026-06-06T07:00", "week_label": "W22", "source_type": "application", "source_label": "Application alert", "item_key": "KubeNodeEviction", "item_label": "KubeNodeEviction", "severity": "warning", "status": "Seen this week", "count": 4, "last_seen": "2026-06-04 20:50", "last_seen_ts": 1780595443.530579, "services": "grafana (4)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1780595443530579", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1780595443530579", "source_href": "reports/production_reliability_dashboard_2026-05-30_0700_2026-06-08_0218.html", "source_json": "production_reliability_dashboard_2026-05-30_0700_2026-06-08_0218.json" }, { "period_start": "2026-05-30T07:00:00+03:00", "period_end": "2026-06-06T07:00:00+03:00", "week_key": "2026-05-30T07:00__2026-06-06T07:00", "week_label": "W22", "source_type": "application", "source_label": "Application alert", "item_key": "NodeSystemSaturation", "item_label": "NodeSystemSaturation", "severity": "warning", "status": "Seen this week", "count": 3, "last_seen": "2026-06-02 12:28", "last_seen_ts": 1780392529.733539, "services": "grafana (3)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1780392529733539", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1780392529733539", "source_href": "reports/production_reliability_dashboard_2026-05-30_0700_2026-06-08_0218.html", "source_json": "production_reliability_dashboard_2026-05-30_0700_2026-06-08_0218.json" }, { "period_start": "2026-05-30T07:00:00+03:00", "period_end": "2026-06-06T07:00:00+03:00", "week_key": "2026-05-30T07:00__2026-06-06T07:00", "week_label": "W22", "source_type": "application", "source_label": "Application alert", "item_key": "TargetDown", "item_label": "TargetDown", "severity": "warning", "status": "Seen this week", "count": 1, "last_seen": "2026-06-01 11:34", "last_seen_ts": 1780302897.768889, "services": "grafana (1)", "signals": "General investigation", "latest_note": "Brief node-exporter target gap caused by Cluster Autoscaler scaling down worker node 10.66.126.144 the node was cordoned, drained, and remo…", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1780302897768889", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1780302897768889?thread_ts=1780302897.768889&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-05-30_0700_2026-06-08_0218.html", "source_json": "production_reliability_dashboard_2026-05-30_0700_2026-06-08_0218.json" }, { "period_start": "2026-05-30T07:00:00+03:00", "period_end": "2026-06-06T07:00:00+03:00", "week_key": "2026-05-30T07:00__2026-06-06T07:00", "week_label": "W22", "source_type": "application", "source_label": "Application alert", "item_key": "TraefikServiceHighErrorRate", "item_label": "TraefikServiceHighErrorRate", "severity": "critical", "status": "Recent (72h)", "count": 7, "last_seen": "2026-06-05 09:40", "last_seen_ts": 1780641640.702279, "services": "core-grafana-80 (3), accommodations-api (2), web-80 (1), uni-api (1)", "signals": "General investigation", "latest_note": "@U09JYAWCGLB | uni-api on TUIASI is returning HTTP 500 from StudentServiceImpl.getCatalogCloseStatus and updateCatalogCloseStatus because the JPA query fo…", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1780641640702279", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1780641640702279?thread_ts=1780641640.702279&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-05-30_0700_2026-06-08_0218.html", "source_json": "production_reliability_dashboard_2026-05-30_0700_2026-06-08_0218.json" }, { "period_start": "2026-05-30T07:00:00+03:00", "period_end": "2026-06-06T07:00:00+03:00", "week_key": "2026-05-30T07:00__2026-06-06T07:00", "week_label": "W22", "source_type": "application", "source_label": "Application alert", "item_key": "TraefikServiceHighLatency", "item_label": "TraefikServiceHighLatency", "severity": "warning", "status": "Seen today", "count": 21, "last_seen": "2026-06-07 21:40", "last_seen_ts": 1780857608.184429, "services": "ai-api (11), social-api (8), web-80 (7), admission-api (6)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1780857608184429", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1780857608184429", "source_href": "reports/production_reliability_dashboard_2026-05-30_0700_2026-06-08_0218.html", "source_json": "production_reliability_dashboard_2026-05-30_0700_2026-06-08_0218.json" }, { "period_start": "2026-05-30T07:00:00+03:00", "period_end": "2026-06-06T07:00:00+03:00", "week_key": "2026-05-30T07:00__2026-06-06T07:00", "week_label": "W22", "source_type": "customer", "source_label": "Customer warning", "item_key": "Adservio Ro", "item_label": "Adservio Ro", "severity": "warning", "status": "Recovered recently", "count": 5, "last_seen": "2026-06-05 18:34", "last_seen_ts": 1780673697.0, "services": "unclassified", "signals": "Pingdom-only evidence so far", "latest_note": "Pingdom-only evidence so far", "alert_url": "http://www.adservio.ro/", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-05-30_0700_2026-06-08_0218.html", "source_json": "production_reliability_dashboard_2026-05-30_0700_2026-06-08_0218.json" }, { "period_start": "2026-05-30T07:00:00+03:00", "period_end": "2026-06-06T07:00:00+03:00", "week_key": "2026-05-30T07:00__2026-06-06T07:00", "week_label": "W22", "source_type": "infrastructure", "source_label": "Infrastructure alarm", "item_key": "adservio-rds-mysql-master-disk-queue-high", "item_label": "adservio-rds-mysql-master-disk-queue-high", "severity": "warning", "status": "Latest OK", "count": 2, "last_seen": "2026-06-06 04:23", "last_seen_ts": 1780709016.0, "services": "", "signals": "ALARM 1, OK 1, Flips 1", "latest_note": "OK", "alert_url": "", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-05-30_0700_2026-06-08_0218.html", "source_json": "production_reliability_dashboard_2026-05-30_0700_2026-06-08_0218.json" }, { "period_start": "2026-05-30T07:00:00+03:00", "period_end": "2026-06-06T07:00:00+03:00", "week_key": "2026-05-30T07:00__2026-06-06T07:00", "week_label": "W22", "source_type": "infrastructure", "source_label": "Infrastructure alarm", "item_key": "adservio-rds-postgres-billing-cpu-high", "item_label": "adservio-rds-postgres-billing-cpu-high", "severity": "warning", "status": "Latest OK", "count": 6, "last_seen": "2026-06-03 11:53", "last_seen_ts": 1780476814.0, "services": "", "signals": "ALARM 3, OK 3, Flips 5", "latest_note": "OK", "alert_url": "", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-05-30_0700_2026-06-08_0218.html", "source_json": "production_reliability_dashboard_2026-05-30_0700_2026-06-08_0218.json" }, { "period_start": "2026-05-30T07:00:00+03:00", "period_end": "2026-06-06T07:00:00+03:00", "week_key": "2026-05-30T07:00__2026-06-06T07:00", "week_label": "W22", "source_type": "infrastructure", "source_label": "Infrastructure alarm", "item_key": "adservio-rds-postgres-billing-disk-queue-high", "item_label": "adservio-rds-postgres-billing-disk-queue-high", "severity": "warning", "status": "Latest OK", "count": 2, "last_seen": "2026-06-01 10:13", "last_seen_ts": 1780298027.0, "services": "", "signals": "ALARM 1, OK 1, Flips 1", "latest_note": "OK", "alert_url": "", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-05-30_0700_2026-06-08_0218.html", "source_json": "production_reliability_dashboard_2026-05-30_0700_2026-06-08_0218.json" }, { "period_start": "2026-06-06T07:00:00+03:00", "period_end": "2026-06-13T07:00:00+03:00", "week_key": "2026-06-06T07:00__2026-06-13T07:00", "week_label": "W23", "source_type": "application", "source_label": "Application alert", "item_key": "KubeAggregatedAPIDown", "item_label": "KubeAggregatedAPIDown", "severity": "warning", "status": "Seen this week", "count": 12, "last_seen": "2026-06-08 04:25", "last_seen_ts": 1780881949.386399, "services": "metrics-server (12)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1780881949386399", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1780881949386399", "source_href": "reports/production_reliability_dashboard_2026-06-06_0700_2026-06-13_0700.html", "source_json": "production_reliability_dashboard_2026-06-06_0700_2026-06-13_0700.json" }, { "period_start": "2026-06-06T07:00:00+03:00", "period_end": "2026-06-13T07:00:00+03:00", "week_key": "2026-06-06T07:00__2026-06-13T07:00", "week_label": "W23", "source_type": "application", "source_label": "Application alert", "item_key": "KubeContainerMemoryHigh", "item_label": "KubeContainerMemoryHigh", "severity": "warning", "status": "Seen today", "count": 7, "last_seen": "2026-06-12 15:39", "last_seen_ts": 1781267948.443149, "services": "grafana (7)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781267948443149", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781267948443149", "source_href": "reports/production_reliability_dashboard_2026-06-06_0700_2026-06-13_0700.html", "source_json": "production_reliability_dashboard_2026-06-06_0700_2026-06-13_0700.json" }, { "period_start": "2026-06-06T07:00:00+03:00", "period_end": "2026-06-13T07:00:00+03:00", "week_key": "2026-06-06T07:00__2026-06-13T07:00", "week_label": "W23", "source_type": "application", "source_label": "Application alert", "item_key": "KubeDeploymentReplicasMismatch", "item_label": "KubeDeploymentReplicasMismatch", "severity": "warning", "status": "Seen today", "count": 2, "last_seen": "2026-06-12 10:16", "last_seen_ts": 1781248582.322309, "services": "web (2)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781248582322309", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781248582322309", "source_href": "reports/production_reliability_dashboard_2026-06-06_0700_2026-06-13_0700.html", "source_json": "production_reliability_dashboard_2026-06-06_0700_2026-06-13_0700.json" }, { "period_start": "2026-06-06T07:00:00+03:00", "period_end": "2026-06-13T07:00:00+03:00", "week_key": "2026-06-06T07:00__2026-06-13T07:00", "week_label": "W23", "source_type": "application", "source_label": "Application alert", "item_key": "KubeHpaMaxedOut", "item_label": "KubeHpaMaxedOut", "severity": "warning", "status": "Seen today", "count": 2, "last_seen": "2026-06-12 10:55", "last_seen_ts": 1781250922.343699, "services": "notifications-event-manager (2)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781250922343699", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781250922343699", "source_href": "reports/production_reliability_dashboard_2026-06-06_0700_2026-06-13_0700.html", "source_json": "production_reliability_dashboard_2026-06-06_0700_2026-06-13_0700.json" }, { "period_start": "2026-06-06T07:00:00+03:00", "period_end": "2026-06-13T07:00:00+03:00", "week_key": "2026-06-06T07:00__2026-06-13T07:00", "week_label": "W23", "source_type": "application", "source_label": "Application alert", "item_key": "NodeDiskIOSaturation", "item_label": "NodeDiskIOSaturation", "severity": "warning", "status": "Seen today", "count": 1, "last_seen": "2026-06-12 23:34", "last_seen_ts": 1781296459.733139, "services": "grafana (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781296459733139", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781296459733139", "source_href": "reports/production_reliability_dashboard_2026-06-06_0700_2026-06-13_0700.html", "source_json": "production_reliability_dashboard_2026-06-06_0700_2026-06-13_0700.json" }, { "period_start": "2026-06-06T07:00:00+03:00", "period_end": "2026-06-13T07:00:00+03:00", "week_key": "2026-06-06T07:00__2026-06-13T07:00", "week_label": "W23", "source_type": "application", "source_label": "Application alert", "item_key": "NodeSystemSaturation", "item_label": "NodeSystemSaturation", "severity": "warning", "status": "Seen today", "count": 3, "last_seen": "2026-06-12 23:18", "last_seen_ts": 1781295529.711699, "services": "grafana (3)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781295529711699", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781295529711699", "source_href": "reports/production_reliability_dashboard_2026-06-06_0700_2026-06-13_0700.html", "source_json": "production_reliability_dashboard_2026-06-06_0700_2026-06-13_0700.json" }, { "period_start": "2026-06-06T07:00:00+03:00", "period_end": "2026-06-13T07:00:00+03:00", "week_key": "2026-06-06T07:00__2026-06-13T07:00", "week_label": "W23", "source_type": "application", "source_label": "Application alert", "item_key": "PagerDutyDeploySmoketest", "item_label": "PagerDutyDeploySmoketest", "severity": "critical", "status": "Seen this week", "count": 1, "last_seen": "2026-06-09 08:56", "last_seen_ts": 1780984597.585739, "services": "grafana (1)", "signals": "Release / migration issue", "latest_note": "*Alert:* `PagerDutyDeploySmoketest` (critical) | *When:* 2026-06-09 08:56 Europe/Bucharest | | *Service:* `grafana` | *TL;DR:* `grafana` is…", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1780984597585739", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1780984597585739?thread_ts=1780984597.585739&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-06-06_0700_2026-06-13_0700.html", "source_json": "production_reliability_dashboard_2026-06-06_0700_2026-06-13_0700.json" }, { "period_start": "2026-06-06T07:00:00+03:00", "period_end": "2026-06-13T07:00:00+03:00", "week_key": "2026-06-06T07:00__2026-06-13T07:00", "week_label": "W23", "source_type": "application", "source_label": "Application alert", "item_key": "PDDebugTest", "item_label": "PDDebugTest", "severity": "critical", "status": "Likely noise / resolved", "count": 1, "last_seen": "2026-06-09 09:27", "last_seen_ts": 1780986460.434049, "services": "grafana (1)", "signals": "Alert tuning / noise", "latest_note": "*Alert:* `PDDebugTest` (critical) | *When:* 2026-06-09 09:27 Europe/Bucharest | | *Service:* `grafana` | *TL;DR:* `grafana` is the alert ho…", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1780986460434049", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1780986460434049?thread_ts=1780986460.434049&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-06-06_0700_2026-06-13_0700.html", "source_json": "production_reliability_dashboard_2026-06-06_0700_2026-06-13_0700.json" }, { "period_start": "2026-06-06T07:00:00+03:00", "period_end": "2026-06-13T07:00:00+03:00", "week_key": "2026-06-06T07:00__2026-06-13T07:00", "week_label": "W23", "source_type": "application", "source_label": "Application alert", "item_key": "PlatformLatencyP95Critical1s", "item_label": "PlatformLatencyP95Critical1s", "severity": "critical", "status": "Seen today", "count": 2, "last_seen": "2026-06-12 10:27", "last_seen_ts": 1781249233.233189, "services": "grafana (2)", "signals": "Release / migration issue", "latest_note": "*Alert:* `PlatformLatencyP95Critical1s` (critical) | *When:* 2026-06-12 09:22 Europe/Bucharest | | *Service:* `grafana` | *TL;DR:* `grafana…", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781249233233189", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781245333188519?thread_ts=1781245333.188519&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-06-06_0700_2026-06-13_0700.html", "source_json": "production_reliability_dashboard_2026-06-06_0700_2026-06-13_0700.json" }, { "period_start": "2026-06-06T07:00:00+03:00", "period_end": "2026-06-13T07:00:00+03:00", "week_key": "2026-06-06T07:00__2026-06-13T07:00", "week_label": "W23", "source_type": "application", "source_label": "Application alert", "item_key": "PlatformLatencyP95Warning400ms", "item_label": "PlatformLatencyP95Warning400ms", "severity": "warning", "status": "Seen today", "count": 1, "last_seen": "2026-06-12 09:22", "last_seen_ts": 1781245358.165039, "services": "grafana (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781245358165039", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781245358165039", "source_href": "reports/production_reliability_dashboard_2026-06-06_0700_2026-06-13_0700.html", "source_json": "production_reliability_dashboard_2026-06-06_0700_2026-06-13_0700.json" }, { "period_start": "2026-06-06T07:00:00+03:00", "period_end": "2026-06-13T07:00:00+03:00", "week_key": "2026-06-06T07:00__2026-06-13T07:00", "week_label": "W23", "source_type": "application", "source_label": "Application alert", "item_key": "PlatformLatencyP99Warning5s", "item_label": "PlatformLatencyP99Warning5s", "severity": "warning", "status": "Seen today", "count": 1, "last_seen": "2026-06-12 09:28", "last_seen_ts": 1781245688.190789, "services": "grafana (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781245688190789", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781245688190789", "source_href": "reports/production_reliability_dashboard_2026-06-06_0700_2026-06-13_0700.html", "source_json": "production_reliability_dashboard_2026-06-06_0700_2026-06-13_0700.json" }, { "period_start": "2026-06-06T07:00:00+03:00", "period_end": "2026-06-13T07:00:00+03:00", "week_key": "2026-06-06T07:00__2026-06-13T07:00", "week_label": "W23", "source_type": "application", "source_label": "Application alert", "item_key": "SlackHealthCheck20260610", "item_label": "SlackHealthCheck20260610", "severity": "warning", "status": "Recent (72h)", "count": 1, "last_seen": "2026-06-10 10:53", "last_seen_ts": 1781077990.138859, "services": "grafana (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781077990138859", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781077990138859", "source_href": "reports/production_reliability_dashboard_2026-06-06_0700_2026-06-13_0700.html", "source_json": "production_reliability_dashboard_2026-06-06_0700_2026-06-13_0700.json" }, { "period_start": "2026-06-06T07:00:00+03:00", "period_end": "2026-06-13T07:00:00+03:00", "week_key": "2026-06-06T07:00__2026-06-13T07:00", "week_label": "W23", "source_type": "application", "source_label": "Application alert", "item_key": "SlackPipelineTest", "item_label": "SlackPipelineTest", "severity": "warning", "status": "Seen this week", "count": 1, "last_seen": "2026-06-09 13:06", "last_seen_ts": 1780999596.511429, "services": "grafana (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1780999596511429", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1780999596511429", "source_href": "reports/production_reliability_dashboard_2026-06-06_0700_2026-06-13_0700.html", "source_json": "production_reliability_dashboard_2026-06-06_0700_2026-06-13_0700.json" }, { "period_start": "2026-06-06T07:00:00+03:00", "period_end": "2026-06-13T07:00:00+03:00", "week_key": "2026-06-06T07:00__2026-06-13T07:00", "week_label": "W23", "source_type": "application", "source_label": "Application alert", "item_key": "TargetDown", "item_label": "TargetDown", "severity": "warning", "status": "Seen today", "count": 2, "last_seen": "2026-06-12 10:15", "last_seen_ts": 1781248557.756399, "services": "grafana (2)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781248557756399", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781248557756399", "source_href": "reports/production_reliability_dashboard_2026-06-06_0700_2026-06-13_0700.html", "source_json": "production_reliability_dashboard_2026-06-06_0700_2026-06-13_0700.json" }, { "period_start": "2026-06-06T07:00:00+03:00", "period_end": "2026-06-13T07:00:00+03:00", "week_key": "2026-06-06T07:00__2026-06-13T07:00", "week_label": "W23", "source_type": "application", "source_label": "Application alert", "item_key": "TraefikServiceHighErrorRate", "item_label": "TraefikServiceHighErrorRate", "severity": "critical", "status": "Seen today", "count": 3, "last_seen": "2026-06-12 10:21", "last_seen_ts": 1781248903.173649, "services": "social-api (1), accommodations-api (1), docgen2-api (1)", "signals": "Observability storage", "latest_note": "*Alert:* `TraefikServiceHighErrorRate` (critical) | *When:* 2026-06-09 08:15 Europe/Bucharest | | *Service:* `accommodations-api` | *TL;DR:…", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781248903173649", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1780982140717209?thread_ts=1780982140.717209&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-06-06_0700_2026-06-13_0700.html", "source_json": "production_reliability_dashboard_2026-06-06_0700_2026-06-13_0700.json" }, { "period_start": "2026-06-06T07:00:00+03:00", "period_end": "2026-06-13T07:00:00+03:00", "week_key": "2026-06-06T07:00__2026-06-13T07:00", "week_label": "W23", "source_type": "application", "source_label": "Application alert", "item_key": "TraefikServiceHighLatency", "item_label": "TraefikServiceHighLatency", "severity": "warning", "status": "Seen today", "count": 15, "last_seen": "2026-06-12 19:17", "last_seen_ts": 1781281058.175359, "services": "ai-api (13), admission-api (2), rooms-api (1), social-api (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781281058175359", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781281058175359", "source_href": "reports/production_reliability_dashboard_2026-06-06_0700_2026-06-13_0700.html", "source_json": "production_reliability_dashboard_2026-06-06_0700_2026-06-13_0700.json" }, { "period_start": "2026-06-06T07:00:00+03:00", "period_end": "2026-06-13T07:00:00+03:00", "week_key": "2026-06-06T07:00__2026-06-13T07:00", "week_label": "W23", "source_type": "infrastructure", "source_label": "Infrastructure alarm", "item_key": "adservio-rds-mysql-catalog-memory-low", "item_label": "adservio-rds-mysql-catalog-memory-low", "severity": "critical", "status": "Still alarming", "count": 1, "last_seen": "2026-06-12 08:21", "last_seen_ts": 1781241698.0, "services": "", "signals": "ALARM 1, OK 0, Flips 0", "latest_note": "ALARM", "alert_url": "", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-06-06_0700_2026-06-13_0700.html", "source_json": "production_reliability_dashboard_2026-06-06_0700_2026-06-13_0700.json" }, { "period_start": "2026-06-06T07:00:00+03:00", "period_end": "2026-06-13T07:00:00+03:00", "week_key": "2026-06-06T07:00__2026-06-13T07:00", "week_label": "W23", "source_type": "infrastructure", "source_label": "Infrastructure alarm", "item_key": "adservio-root-account-usage", "item_label": "adservio-root-account-usage", "severity": "warning", "status": "Latest OK", "count": 6, "last_seen": "2026-06-09 14:51", "last_seen_ts": 1781005912.0, "services": "", "signals": "ALARM 3, OK 3, Flips 5", "latest_note": "OK", "alert_url": "", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-06-06_0700_2026-06-13_0700.html", "source_json": "production_reliability_dashboard_2026-06-06_0700_2026-06-13_0700.json" }, { "period_start": "2026-06-14T07:00:00+03:00", "period_end": "2026-06-21T07:00:00+03:00", "week_key": "2026-06-14T07:00__2026-06-21T07:00", "week_label": "W24", "source_type": "application", "source_label": "Application alert", "item_key": "CPUThrottlingHigh", "item_label": "CPUThrottlingHigh", "severity": "warning", "status": "Recent (72h)", "count": 13, "last_seen": "2026-06-19 09:05", "last_seen_ts": 1781849107.894829, "services": "docgen2-api (13)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781849107894829", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781849107894829", "source_href": "reports/production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.html", "source_json": "production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.json" }, { "period_start": "2026-06-14T07:00:00+03:00", "period_end": "2026-06-21T07:00:00+03:00", "week_key": "2026-06-14T07:00__2026-06-21T07:00", "week_label": "W24", "source_type": "application", "source_label": "Application alert", "item_key": "KubeAggregatedAPIDown", "item_label": "KubeAggregatedAPIDown", "severity": "warning", "status": "Seen this week", "count": 1, "last_seen": "2026-06-15 21:00", "last_seen_ts": 1781546406.803219, "services": "metrics-server (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781546406803219", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781546406803219", "source_href": "reports/production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.html", "source_json": "production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.json" }, { "period_start": "2026-06-14T07:00:00+03:00", "period_end": "2026-06-21T07:00:00+03:00", "week_key": "2026-06-14T07:00__2026-06-21T07:00", "week_label": "W24", "source_type": "application", "source_label": "Application alert", "item_key": "KubeContainerMemoryHigh", "item_label": "KubeContainerMemoryHigh", "severity": "warning", "status": "Recent (72h)", "count": 15, "last_seen": "2026-06-18 17:22", "last_seen_ts": 1781792528.196929, "services": "grafana (15)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781792528196929", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781792528196929", "source_href": "reports/production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.html", "source_json": "production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.json" }, { "period_start": "2026-06-14T07:00:00+03:00", "period_end": "2026-06-21T07:00:00+03:00", "week_key": "2026-06-14T07:00__2026-06-21T07:00", "week_label": "W24", "source_type": "application", "source_label": "Application alert", "item_key": "KubeDeploymentReplicasMismatch", "item_label": "KubeDeploymentReplicasMismatch", "severity": "warning", "status": "Seen this week", "count": 14, "last_seen": "2026-06-17 18:28", "last_seen_ts": 1781710101.009429, "services": "core-getresponse-events-worker (9), service-websocket (8), subscriptions-api (7), library-api (6)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781710101009429", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781710101009429", "source_href": "reports/production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.html", "source_json": "production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.json" }, { "period_start": "2026-06-14T07:00:00+03:00", "period_end": "2026-06-21T07:00:00+03:00", "week_key": "2026-06-14T07:00__2026-06-21T07:00", "week_label": "W24", "source_type": "application", "source_label": "Application alert", "item_key": "KubeHpaMaxedOut", "item_label": "KubeHpaMaxedOut", "severity": "warning", "status": "Recent (72h)", "count": 22, "last_seen": "2026-06-18 13:21", "last_seen_ts": 1781778112.368699, "services": "docgen2-api (11), grafana (6), notifications-event-manager (5)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781778112368699", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781778112368699", "source_href": "reports/production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.html", "source_json": "production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.json" }, { "period_start": "2026-06-14T07:00:00+03:00", "period_end": "2026-06-21T07:00:00+03:00", "week_key": "2026-06-14T07:00__2026-06-21T07:00", "week_label": "W24", "source_type": "application", "source_label": "Application alert", "item_key": "KubeJobFailed", "item_label": "KubeJobFailed", "severity": "warning", "status": "Seen today", "count": 27, "last_seen": "2026-06-21 05:43", "last_seen_ts": 1782009803.085349, "services": "download-album (20), colecteaza-sms-note-abs (11), rezumat (10), publish-results (9)", "signals": "General investigation", "latest_note": "These are failed jobs which can be cleaned up from cluster , removed these failed jobs .", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1782009803085349", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781598712407259?thread_ts=1781598712.407259&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.html", "source_json": "production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.json" }, { "period_start": "2026-06-14T07:00:00+03:00", "period_end": "2026-06-21T07:00:00+03:00", "week_key": "2026-06-14T07:00__2026-06-21T07:00", "week_label": "W24", "source_type": "application", "source_label": "Application alert", "item_key": "KubeJobNotCompleted", "item_label": "KubeJobNotCompleted", "severity": "warning", "status": "Seen this week", "count": 1, "last_seen": "2026-06-16 15:30", "last_seen_ts": 1781613052.350639, "services": "social-local-cache (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781613052350639", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781613052350639", "source_href": "reports/production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.html", "source_json": "production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.json" }, { "period_start": "2026-06-14T07:00:00+03:00", "period_end": "2026-06-21T07:00:00+03:00", "week_key": "2026-06-14T07:00__2026-06-21T07:00", "week_label": "W24", "source_type": "application", "source_label": "Application alert", "item_key": "KubePersistentVolumeFillingUp", "item_label": "KubePersistentVolumeFillingUp", "severity": "warning", "status": "Recent (72h)", "count": 6, "last_seen": "2026-06-19 14:18", "last_seen_ts": 1781867901.735419, "services": "grafana (6), loki (6)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781867901735419", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781867901735419", "source_href": "reports/production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.html", "source_json": "production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.json" }, { "period_start": "2026-06-14T07:00:00+03:00", "period_end": "2026-06-21T07:00:00+03:00", "week_key": "2026-06-14T07:00__2026-06-21T07:00", "week_label": "W24", "source_type": "application", "source_label": "Application alert", "item_key": "KubePodCrashLooping", "item_label": "KubePodCrashLooping", "severity": "warning", "status": "Seen this week", "count": 6, "last_seen": "2026-06-17 18:29", "last_seen_ts": 1781710160.920999, "services": "core-getresponse-events-worker (4), service-websocket (3), library-api (1), notifications-event-manager (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781710160920999", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781710160920999", "source_href": "reports/production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.html", "source_json": "production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.json" }, { "period_start": "2026-06-14T07:00:00+03:00", "period_end": "2026-06-21T07:00:00+03:00", "week_key": "2026-06-14T07:00__2026-06-21T07:00", "week_label": "W24", "source_type": "application", "source_label": "Application alert", "item_key": "NodeDiskIOSaturation", "item_label": "NodeDiskIOSaturation", "severity": "warning", "status": "Recent (72h)", "count": 1, "last_seen": "2026-06-19 01:40", "last_seen_ts": 1781822449.841239, "services": "grafana (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781822449841239", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781822449841239", "source_href": "reports/production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.html", "source_json": "production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.json" }, { "period_start": "2026-06-14T07:00:00+03:00", "period_end": "2026-06-21T07:00:00+03:00", "week_key": "2026-06-14T07:00__2026-06-21T07:00", "week_label": "W24", "source_type": "application", "source_label": "Application alert", "item_key": "NodeHighNumberConntrackEntriesUsed", "item_label": "NodeHighNumberConntrackEntriesUsed", "severity": "warning", "status": "Recent (72h)", "count": 7, "last_seen": "2026-06-19 09:32", "last_seen_ts": 1781850739.718419, "services": "grafana (7)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781850739718419", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781850739718419", "source_href": "reports/production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.html", "source_json": "production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.json" }, { "period_start": "2026-06-14T07:00:00+03:00", "period_end": "2026-06-21T07:00:00+03:00", "week_key": "2026-06-14T07:00__2026-06-21T07:00", "week_label": "W24", "source_type": "application", "source_label": "Application alert", "item_key": "NodeSystemSaturation", "item_label": "NodeSystemSaturation", "severity": "warning", "status": "Recent (72h)", "count": 3, "last_seen": "2026-06-19 01:24", "last_seen_ts": 1781821489.760059, "services": "grafana (3)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781821489760059", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781821489760059", "source_href": "reports/production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.html", "source_json": "production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.json" }, { "period_start": "2026-06-14T07:00:00+03:00", "period_end": "2026-06-21T07:00:00+03:00", "week_key": "2026-06-14T07:00__2026-06-21T07:00", "week_label": "W24", "source_type": "application", "source_label": "Application alert", "item_key": "Platform5xxLowVolumeCritical100", "item_label": "Platform5xxLowVolumeCritical100", "severity": "critical", "status": "Seen this week", "count": 4, "last_seen": "2026-06-16 05:48", "last_seen_ts": 1781578123.277289, "services": "grafana (4)", "signals": "Observability storage", "latest_note": "*Alert:* `Platform5xxLowVolumeCritical100` (critical) | *When:* 2026-06-16 05:48 Europe/Bucharest | | *Service:* `grafana` | *TL;DR:* `graf…", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781578123277289", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781578123277289?thread_ts=1781578123.277289&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.html", "source_json": "production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.json" }, { "period_start": "2026-06-14T07:00:00+03:00", "period_end": "2026-06-21T07:00:00+03:00", "week_key": "2026-06-14T07:00__2026-06-21T07:00", "week_label": "W24", "source_type": "application", "source_label": "Application alert", "item_key": "Platform5xxLowVolumeWarning20", "item_label": "Platform5xxLowVolumeWarning20", "severity": "warning", "status": "Seen this week", "count": 1, "last_seen": "2026-06-16 04:11", "last_seen_ts": 1781572298.270849, "services": "grafana (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781572298270849", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781572298270849", "source_href": "reports/production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.html", "source_json": "production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.json" }, { "period_start": "2026-06-14T07:00:00+03:00", "period_end": "2026-06-21T07:00:00+03:00", "week_key": "2026-06-14T07:00__2026-06-21T07:00", "week_label": "W24", "source_type": "application", "source_label": "Application alert", "item_key": "Platform5xxRatioCritical3pct", "item_label": "Platform5xxRatioCritical3pct", "severity": "critical", "status": "Recent (72h)", "count": 2, "last_seen": "2026-06-18 16:18", "last_seen_ts": 1781788723.185949, "services": "grafana (2)", "signals": "Observability storage", "latest_note": "*Alert:* `Platform5xxRatioCritical3pct` (critical) | *When:* 2026-06-16 06:53 Europe/Bucharest | | *Service:* `grafana` | *TL;DR:* `grafana…", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781788723185949", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781582023177819?thread_ts=1781582023.177819&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.html", "source_json": "production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.json" }, { "period_start": "2026-06-14T07:00:00+03:00", "period_end": "2026-06-21T07:00:00+03:00", "week_key": "2026-06-14T07:00__2026-06-21T07:00", "week_label": "W24", "source_type": "application", "source_label": "Application alert", "item_key": "Platform5xxRatioWarning1pct", "item_label": "Platform5xxRatioWarning1pct", "severity": "warning", "status": "Recent (72h)", "count": 2, "last_seen": "2026-06-18 16:18", "last_seen_ts": 1781788718.189639, "services": "grafana (2)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781788718189639", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781788718189639", "source_href": "reports/production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.html", "source_json": "production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.json" }, { "period_start": "2026-06-14T07:00:00+03:00", "period_end": "2026-06-21T07:00:00+03:00", "week_key": "2026-06-14T07:00__2026-06-21T07:00", "week_label": "W24", "source_type": "application", "source_label": "Application alert", "item_key": "PlatformLatencyP95Critical1s", "item_label": "PlatformLatencyP95Critical1s", "severity": "critical", "status": "Seen this week", "count": 7, "last_seen": "2026-06-15 14:01", "last_seen_ts": 1781521303.190719, "services": "grafana (7)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781521303190719", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781521303190719", "source_href": "reports/production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.html", "source_json": "production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.json" }, { "period_start": "2026-06-14T07:00:00+03:00", "period_end": "2026-06-21T07:00:00+03:00", "week_key": "2026-06-14T07:00__2026-06-21T07:00", "week_label": "W24", "source_type": "application", "source_label": "Application alert", "item_key": "PlatformLatencyP95Warning400ms", "item_label": "PlatformLatencyP95Warning400ms", "severity": "warning", "status": "Seen this week", "count": 6, "last_seen": "2026-06-15 14:01", "last_seen_ts": 1781521298.186529, "services": "grafana (6)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781521298186529", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781521298186529", "source_href": "reports/production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.html", "source_json": "production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.json" }, { "period_start": "2026-06-14T07:00:00+03:00", "period_end": "2026-06-21T07:00:00+03:00", "week_key": "2026-06-14T07:00__2026-06-21T07:00", "week_label": "W24", "source_type": "application", "source_label": "Application alert", "item_key": "PlatformLatencyP99Warning5s", "item_label": "PlatformLatencyP99Warning5s", "severity": "warning", "status": "Seen this week", "count": 5, "last_seen": "2026-06-15 14:07", "last_seen_ts": 1781521628.189509, "services": "grafana (5)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781521628189509", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781521628189509", "source_href": "reports/production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.html", "source_json": "production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.json" }, { "period_start": "2026-06-14T07:00:00+03:00", "period_end": "2026-06-21T07:00:00+03:00", "week_key": "2026-06-14T07:00__2026-06-21T07:00", "week_label": "W24", "source_type": "application", "source_label": "Application alert", "item_key": "TargetDown", "item_label": "TargetDown", "severity": "warning", "status": "Seen this week", "count": 2, "last_seen": "2026-06-15 09:31", "last_seen_ts": 1781505117.762349, "services": "grafana (2)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781505117762349", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781505117762349", "source_href": "reports/production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.html", "source_json": "production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.json" }, { "period_start": "2026-06-14T07:00:00+03:00", "period_end": "2026-06-21T07:00:00+03:00", "week_key": "2026-06-14T07:00__2026-06-21T07:00", "week_label": "W24", "source_type": "application", "source_label": "Application alert", "item_key": "TraefikServiceHighErrorRate", "item_label": "TraefikServiceHighErrorRate", "severity": "critical", "status": "Recent (72h)", "count": 25, "last_seen": "2026-06-19 10:43", "last_seen_ts": 1781854993.252559, "services": "web-80 (14), docgen2-api (4), uni-api (4), core-grafana-80 (3)", "signals": "Observability storage, Release / migration issue", "latest_note": "*Alert:* `TraefikServiceHighErrorRate` (critical) | *When:* 2026-06-16 16:38 Europe/Bucharest | | *Service:* `uni-api` | *TL;DR:* `uni-api`… | *Fix now:* Check the latest deployed revision for `/api/v2/uni/student-studies/students/1882/disciplines?semester=1` and rollback or hotfix…", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781854993252559", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781617123174359?thread_ts=1781617123.174359&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.html", "source_json": "production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.json" }, { "period_start": "2026-06-14T07:00:00+03:00", "period_end": "2026-06-21T07:00:00+03:00", "week_key": "2026-06-14T07:00__2026-06-21T07:00", "week_label": "W24", "source_type": "application", "source_label": "Application alert", "item_key": "TraefikServiceHighLatency", "item_label": "TraefikServiceHighLatency", "severity": "warning", "status": "Recent (72h)", "count": 26, "last_seen": "2026-06-19 17:15", "last_seen_ts": 1781878538.169589, "services": "social-api (17), web-80 (17), ai-api (17), subscriptions-api (13)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781878538169589", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1781878538169589", "source_href": "reports/production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.html", "source_json": "production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.json" }, { "period_start": "2026-06-14T07:00:00+03:00", "period_end": "2026-06-21T07:00:00+03:00", "week_key": "2026-06-14T07:00__2026-06-21T07:00", "week_label": "W24", "source_type": "infrastructure", "source_label": "Infrastructure alarm", "item_key": "adservio-rds-mysql-catalog2-memory-low", "item_label": "adservio-rds-mysql-catalog2-memory-low", "severity": "critical", "status": "Still alarming", "count": 32, "last_seen": "2026-06-21 03:59", "last_seen_ts": 1782003543.0, "services": "", "signals": "ALARM 17, OK 15, Flips 30", "latest_note": "ALARM", "alert_url": "", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.html", "source_json": "production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.json" }, { "period_start": "2026-06-14T07:00:00+03:00", "period_end": "2026-06-21T07:00:00+03:00", "week_key": "2026-06-14T07:00__2026-06-21T07:00", "week_label": "W24", "source_type": "infrastructure", "source_label": "Infrastructure alarm", "item_key": "adservio-rds-mysql-catalog2-storage-low", "item_label": "adservio-rds-mysql-catalog2-storage-low", "severity": "warning", "status": "Latest OK", "count": 4, "last_seen": "2026-06-16 06:00", "last_seen_ts": 1781578807.0, "services": "", "signals": "ALARM 2, OK 2, Flips 3", "latest_note": "OK", "alert_url": "", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.html", "source_json": "production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.json" }, { "period_start": "2026-06-14T07:00:00+03:00", "period_end": "2026-06-21T07:00:00+03:00", "week_key": "2026-06-14T07:00__2026-06-21T07:00", "week_label": "W24", "source_type": "infrastructure", "source_label": "Infrastructure alarm", "item_key": "adservio-rds-mysql-catalog2-swap-high", "item_label": "adservio-rds-mysql-catalog2-swap-high", "severity": "critical", "status": "Still alarming", "count": 3, "last_seen": "2026-06-17 06:42", "last_seen_ts": 1781667734.0, "services": "", "signals": "ALARM 2, OK 1, Flips 2", "latest_note": "ALARM", "alert_url": "", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.html", "source_json": "production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.json" }, { "period_start": "2026-06-14T07:00:00+03:00", "period_end": "2026-06-21T07:00:00+03:00", "week_key": "2026-06-14T07:00__2026-06-21T07:00", "week_label": "W24", "source_type": "infrastructure", "source_label": "Infrastructure alarm", "item_key": "adservio-rds-mysql-catalog3-connections-high", "item_label": "adservio-rds-mysql-catalog3-connections-high", "severity": "warning", "status": "Latest OK", "count": 1, "last_seen": "2026-06-15 20:04", "last_seen_ts": 1781543091.0, "services": "", "signals": "ALARM 0, OK 1, Flips 0", "latest_note": "OK", "alert_url": "", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.html", "source_json": "production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.json" }, { "period_start": "2026-06-14T07:00:00+03:00", "period_end": "2026-06-21T07:00:00+03:00", "week_key": "2026-06-14T07:00__2026-06-21T07:00", "week_label": "W24", "source_type": "infrastructure", "source_label": "Infrastructure alarm", "item_key": "adservio-rds-mysql-catalog3-cpu-high", "item_label": "adservio-rds-mysql-catalog3-cpu-high", "severity": "warning", "status": "Latest OK", "count": 1, "last_seen": "2026-06-15 20:05", "last_seen_ts": 1781543136.0, "services": "", "signals": "ALARM 0, OK 1, Flips 0", "latest_note": "OK", "alert_url": "", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.html", "source_json": "production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.json" }, { "period_start": "2026-06-14T07:00:00+03:00", "period_end": "2026-06-21T07:00:00+03:00", "week_key": "2026-06-14T07:00__2026-06-21T07:00", "week_label": "W24", "source_type": "infrastructure", "source_label": "Infrastructure alarm", "item_key": "adservio-rds-mysql-catalog3-disk-queue-high", "item_label": "adservio-rds-mysql-catalog3-disk-queue-high", "severity": "warning", "status": "Latest OK", "count": 1, "last_seen": "2026-06-15 20:04", "last_seen_ts": 1781543070.0, "services": "", "signals": "ALARM 0, OK 1, Flips 0", "latest_note": "OK", "alert_url": "", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.html", "source_json": "production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.json" }, { "period_start": "2026-06-14T07:00:00+03:00", "period_end": "2026-06-21T07:00:00+03:00", "week_key": "2026-06-14T07:00__2026-06-21T07:00", "week_label": "W24", "source_type": "infrastructure", "source_label": "Infrastructure alarm", "item_key": "adservio-rds-mysql-catalog3-memory-low", "item_label": "adservio-rds-mysql-catalog3-memory-low", "severity": "warning", "status": "Latest OK", "count": 1, "last_seen": "2026-06-15 20:05", "last_seen_ts": 1781543127.0, "services": "", "signals": "ALARM 0, OK 1, Flips 0", "latest_note": "OK", "alert_url": "", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.html", "source_json": "production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.json" }, { "period_start": "2026-06-14T07:00:00+03:00", "period_end": "2026-06-21T07:00:00+03:00", "week_key": "2026-06-14T07:00__2026-06-21T07:00", "week_label": "W24", "source_type": "infrastructure", "source_label": "Infrastructure alarm", "item_key": "adservio-rds-mysql-catalog3-read-latency-high", "item_label": "adservio-rds-mysql-catalog3-read-latency-high", "severity": "warning", "status": "Latest OK", "count": 1, "last_seen": "2026-06-15 20:05", "last_seen_ts": 1781543130.0, "services": "", "signals": "ALARM 0, OK 1, Flips 0", "latest_note": "OK", "alert_url": "", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.html", "source_json": "production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.json" }, { "period_start": "2026-06-14T07:00:00+03:00", "period_end": "2026-06-21T07:00:00+03:00", "week_key": "2026-06-14T07:00__2026-06-21T07:00", "week_label": "W24", "source_type": "infrastructure", "source_label": "Infrastructure alarm", "item_key": "adservio-rds-mysql-catalog3-storage-low", "item_label": "adservio-rds-mysql-catalog3-storage-low", "severity": "warning", "status": "Latest OK", "count": 1, "last_seen": "2026-06-15 20:05", "last_seen_ts": 1781543120.0, "services": "", "signals": "ALARM 0, OK 1, Flips 0", "latest_note": "OK", "alert_url": "", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.html", "source_json": "production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.json" }, { "period_start": "2026-06-14T07:00:00+03:00", "period_end": "2026-06-21T07:00:00+03:00", "week_key": "2026-06-14T07:00__2026-06-21T07:00", "week_label": "W24", "source_type": "infrastructure", "source_label": "Infrastructure alarm", "item_key": "adservio-rds-mysql-catalog3-swap-high", "item_label": "adservio-rds-mysql-catalog3-swap-high", "severity": "warning", "status": "Latest OK", "count": 1, "last_seen": "2026-06-15 20:04", "last_seen_ts": 1781543056.0, "services": "", "signals": "ALARM 0, OK 1, Flips 0", "latest_note": "OK", "alert_url": "", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.html", "source_json": "production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.json" }, { "period_start": "2026-06-14T07:00:00+03:00", "period_end": "2026-06-21T07:00:00+03:00", "week_key": "2026-06-14T07:00__2026-06-21T07:00", "week_label": "W24", "source_type": "infrastructure", "source_label": "Infrastructure alarm", "item_key": "adservio-rds-mysql-catalog3-write-latency-high", "item_label": "adservio-rds-mysql-catalog3-write-latency-high", "severity": "warning", "status": "Latest OK", "count": 1, "last_seen": "2026-06-15 20:05", "last_seen_ts": 1781543103.0, "services": "", "signals": "ALARM 0, OK 1, Flips 0", "latest_note": "OK", "alert_url": "", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.html", "source_json": "production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.json" }, { "period_start": "2026-06-14T07:00:00+03:00", "period_end": "2026-06-21T07:00:00+03:00", "week_key": "2026-06-14T07:00__2026-06-21T07:00", "week_label": "W24", "source_type": "infrastructure", "source_label": "Infrastructure alarm", "item_key": "adservio-root-account-usage", "item_label": "adservio-root-account-usage", "severity": "warning", "status": "Latest OK", "count": 4, "last_seen": "2026-06-16 15:07", "last_seen_ts": 1781611679.0, "services": "", "signals": "ALARM 2, OK 2, Flips 3", "latest_note": "OK", "alert_url": "", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.html", "source_json": "production_reliability_dashboard_2026-06-14_0700_2026-06-21_0700.json" }, { "period_start": "2026-06-21T07:00:00+03:00", "period_end": "2026-06-28T07:00:00+03:00", "week_key": "2026-06-21T07:00__2026-06-28T07:00", "week_label": "W25", "source_type": "application", "source_label": "Application alert", "item_key": "KubeContainerMemoryHigh", "item_label": "KubeContainerMemoryHigh", "severity": "warning", "status": "Seen this week", "count": 7, "last_seen": "2026-06-25 00:07", "last_seen_ts": 1782335258.196479, "services": "grafana (7), loki (2)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1782335258196479", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1782335258196479", "source_href": "reports/production_reliability_dashboard_2026-06-21_0700_2026-06-28_0700.html", "source_json": "production_reliability_dashboard_2026-06-21_0700_2026-06-28_0700.json" }, { "period_start": "2026-06-21T07:00:00+03:00", "period_end": "2026-06-28T07:00:00+03:00", "week_key": "2026-06-21T07:00__2026-06-28T07:00", "week_label": "W25", "source_type": "application", "source_label": "Application alert", "item_key": "KubeContainerWaiting", "item_label": "KubeContainerWaiting", "severity": "warning", "status": "Seen today", "count": 20, "last_seen": "2026-06-28 06:03", "last_seen_ts": 1782615802.786819, "services": "grafana (20)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1782615802786819", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1782615802786819", "source_href": "reports/production_reliability_dashboard_2026-06-21_0700_2026-06-28_0700.html", "source_json": "production_reliability_dashboard_2026-06-21_0700_2026-06-28_0700.json" }, { "period_start": "2026-06-21T07:00:00+03:00", "period_end": "2026-06-28T07:00:00+03:00", "week_key": "2026-06-21T07:00__2026-06-28T07:00", "week_label": "W25", "source_type": "application", "source_label": "Application alert", "item_key": "KubeDeploymentReplicasMismatch", "item_label": "KubeDeploymentReplicasMismatch", "severity": "warning", "status": "Recent (72h)", "count": 2, "last_seen": "2026-06-26 10:16", "last_seen_ts": 1782458182.340369, "services": "s3-migration (1), billing-retry-payment-worker (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1782458182340369", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1782458182340369", "source_href": "reports/production_reliability_dashboard_2026-06-21_0700_2026-06-28_0700.html", "source_json": "production_reliability_dashboard_2026-06-21_0700_2026-06-28_0700.json" }, { "period_start": "2026-06-21T07:00:00+03:00", "period_end": "2026-06-28T07:00:00+03:00", "week_key": "2026-06-21T07:00__2026-06-28T07:00", "week_label": "W25", "source_type": "application", "source_label": "Application alert", "item_key": "KubeHpaMaxedOut", "item_label": "KubeHpaMaxedOut", "severity": "warning", "status": "Recent (72h)", "count": 4, "last_seen": "2026-06-25 10:04", "last_seen_ts": 1782371092.339509, "services": "docgen2-api (4)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1782371092339509", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1782371092339509", "source_href": "reports/production_reliability_dashboard_2026-06-21_0700_2026-06-28_0700.html", "source_json": "production_reliability_dashboard_2026-06-21_0700_2026-06-28_0700.json" }, { "period_start": "2026-06-21T07:00:00+03:00", "period_end": "2026-06-28T07:00:00+03:00", "week_key": "2026-06-21T07:00__2026-06-28T07:00", "week_label": "W25", "source_type": "application", "source_label": "Application alert", "item_key": "KubeJobFailed", "item_label": "KubeJobFailed", "severity": "warning", "status": "Seen today", "count": 17, "last_seen": "2026-06-28 03:42", "last_seen_ts": 1782607372.613359, "services": "download-album (16), colecteaza-sms-note-abs (11), rezumat (9), admission-end-session (7)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1782607372613359", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1782607372613359", "source_href": "reports/production_reliability_dashboard_2026-06-21_0700_2026-06-28_0700.html", "source_json": "production_reliability_dashboard_2026-06-21_0700_2026-06-28_0700.json" }, { "period_start": "2026-06-21T07:00:00+03:00", "period_end": "2026-06-28T07:00:00+03:00", "week_key": "2026-06-21T07:00__2026-06-28T07:00", "week_label": "W25", "source_type": "application", "source_label": "Application alert", "item_key": "KubeJobNotCompleted", "item_label": "KubeJobNotCompleted", "severity": "warning", "status": "Seen today", "count": 14, "last_seen": "2026-06-28 04:47", "last_seen_ts": 1782611242.662569, "services": "social-migration (14)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1782611242662569", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1782611242662569", "source_href": "reports/production_reliability_dashboard_2026-06-21_0700_2026-06-28_0700.html", "source_json": "production_reliability_dashboard_2026-06-21_0700_2026-06-28_0700.json" }, { "period_start": "2026-06-21T07:00:00+03:00", "period_end": "2026-06-28T07:00:00+03:00", "week_key": "2026-06-21T07:00__2026-06-28T07:00", "week_label": "W25", "source_type": "application", "source_label": "Application alert", "item_key": "KubePdbNotEnoughHealthyPods", "item_label": "KubePdbNotEnoughHealthyPods", "severity": "warning", "status": "Seen this week", "count": 1, "last_seen": "2026-06-22 14:28", "last_seen_ts": 1782127732.341889, "services": "grafana (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1782127732341889", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1782127732341889", "source_href": "reports/production_reliability_dashboard_2026-06-21_0700_2026-06-28_0700.html", "source_json": "production_reliability_dashboard_2026-06-21_0700_2026-06-28_0700.json" }, { "period_start": "2026-06-21T07:00:00+03:00", "period_end": "2026-06-28T07:00:00+03:00", "week_key": "2026-06-21T07:00__2026-06-28T07:00", "week_label": "W25", "source_type": "application", "source_label": "Application alert", "item_key": "KubePersistentVolumeFillingUp", "item_label": "KubePersistentVolumeFillingUp", "severity": "warning", "status": "Seen this week", "count": 1, "last_seen": "2026-06-22 10:43", "last_seen_ts": 1782114231.685519, "services": "grafana (1), loki (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1782114231685519", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1782114231685519", "source_href": "reports/production_reliability_dashboard_2026-06-21_0700_2026-06-28_0700.html", "source_json": "production_reliability_dashboard_2026-06-21_0700_2026-06-28_0700.json" }, { "period_start": "2026-06-21T07:00:00+03:00", "period_end": "2026-06-28T07:00:00+03:00", "week_key": "2026-06-21T07:00__2026-06-28T07:00", "week_label": "W25", "source_type": "application", "source_label": "Application alert", "item_key": "KubePodCrashLooping", "item_label": "KubePodCrashLooping", "severity": "warning", "status": "Recent (72h)", "count": 4, "last_seen": "2026-06-26 23:06", "last_seen_ts": 1782504412.351399, "services": "tempo-0 (1), otel-collector (1), billing-cycle-worker (1), billing-retry-payment-worker (1)", "signals": "General investigation", "latest_note": "Same OOM killed in tempo we need to patch this once the the release freez is lifted", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1782504412351399", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1782206782339809?thread_ts=1782206782.339809&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-06-21_0700_2026-06-28_0700.html", "source_json": "production_reliability_dashboard_2026-06-21_0700_2026-06-28_0700.json" }, { "period_start": "2026-06-21T07:00:00+03:00", "period_end": "2026-06-28T07:00:00+03:00", "week_key": "2026-06-21T07:00__2026-06-28T07:00", "week_label": "W25", "source_type": "application", "source_label": "Application alert", "item_key": "Platform5xxLowVolumeCritical100", "item_label": "Platform5xxLowVolumeCritical100", "severity": "critical", "status": "Recent (72h)", "count": 1, "last_seen": "2026-06-26 22:56", "last_seen_ts": 1782503773.196499, "services": "grafana (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1782503773196499", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1782503773196499", "source_href": "reports/production_reliability_dashboard_2026-06-21_0700_2026-06-28_0700.html", "source_json": "production_reliability_dashboard_2026-06-21_0700_2026-06-28_0700.json" }, { "period_start": "2026-06-21T07:00:00+03:00", "period_end": "2026-06-28T07:00:00+03:00", "week_key": "2026-06-21T07:00__2026-06-28T07:00", "week_label": "W25", "source_type": "application", "source_label": "Application alert", "item_key": "Platform5xxLowVolumeWarning20", "item_label": "Platform5xxLowVolumeWarning20", "severity": "warning", "status": "Recent (72h)", "count": 2, "last_seen": "2026-06-26 22:56", "last_seen_ts": 1782503798.283349, "services": "grafana (2)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1782503798283349", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1782503798283349", "source_href": "reports/production_reliability_dashboard_2026-06-21_0700_2026-06-28_0700.html", "source_json": "production_reliability_dashboard_2026-06-21_0700_2026-06-28_0700.json" }, { "period_start": "2026-06-21T07:00:00+03:00", "period_end": "2026-06-28T07:00:00+03:00", "week_key": "2026-06-21T07:00__2026-06-28T07:00", "week_label": "W25", "source_type": "application", "source_label": "Application alert", "item_key": "TargetDown", "item_label": "TargetDown", "severity": "warning", "status": "Seen this week", "count": 1, "last_seen": "2026-06-23 17:18", "last_seen_ts": 1782224337.768899, "services": "grafana (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1782224337768899", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1782224337768899", "source_href": "reports/production_reliability_dashboard_2026-06-21_0700_2026-06-28_0700.html", "source_json": "production_reliability_dashboard_2026-06-21_0700_2026-06-28_0700.json" }, { "period_start": "2026-06-21T07:00:00+03:00", "period_end": "2026-06-28T07:00:00+03:00", "week_key": "2026-06-21T07:00__2026-06-28T07:00", "week_label": "W25", "source_type": "application", "source_label": "Application alert", "item_key": "TraefikServiceHighErrorRate", "item_label": "TraefikServiceHighErrorRate", "severity": "critical", "status": "Seen today", "count": 12, "last_seen": "2026-06-27 09:21", "last_seen_ts": 1782541273.269979, "services": "web-80 (4), accommodations-api (4), docgen2-api (3), notifications-api (1)", "signals": "Release / migration issue, Observability storage", "latest_note": "info{ | \"ClientHost\": \"140.248.36.239\", | \"DownstreamContentSize\": 11, | \"DownstreamStatus\": 502, | \"Duration\": 1436165, | \"RequestHost\": \"…", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1782541273269979", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1782303190693079?thread_ts=1782303190.693079&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-06-21_0700_2026-06-28_0700.html", "source_json": "production_reliability_dashboard_2026-06-21_0700_2026-06-28_0700.json" }, { "period_start": "2026-06-21T07:00:00+03:00", "period_end": "2026-06-28T07:00:00+03:00", "week_key": "2026-06-21T07:00__2026-06-28T07:00", "week_label": "W25", "source_type": "application", "source_label": "Application alert", "item_key": "TraefikServiceHighLatency", "item_label": "TraefikServiceHighLatency", "severity": "warning", "status": "Recent (72h)", "count": 5, "last_seen": "2026-06-26 17:31", "last_seen_ts": 1782484298.182399, "services": "ai-api (3), admission-api (2)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1782484298182399", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1782484298182399", "source_href": "reports/production_reliability_dashboard_2026-06-21_0700_2026-06-28_0700.html", "source_json": "production_reliability_dashboard_2026-06-21_0700_2026-06-28_0700.json" }, { "period_start": "2026-06-21T07:00:00+03:00", "period_end": "2026-06-28T07:00:00+03:00", "week_key": "2026-06-21T07:00__2026-06-28T07:00", "week_label": "W25", "source_type": "infrastructure", "source_label": "Infrastructure alarm", "item_key": "adservio-rds-mysql-catalog-memory-low", "item_label": "adservio-rds-mysql-catalog-memory-low", "severity": "warning", "status": "Flapping, latest OK", "count": 13, "last_seen": "2026-06-28 06:53", "last_seen_ts": 1782618837.0, "services": "", "signals": "ALARM 6, OK 7, Flips 12", "latest_note": "OK", "alert_url": "", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-06-21_0700_2026-06-28_0700.html", "source_json": "production_reliability_dashboard_2026-06-21_0700_2026-06-28_0700.json" }, { "period_start": "2026-06-21T07:00:00+03:00", "period_end": "2026-06-28T07:00:00+03:00", "week_key": "2026-06-21T07:00__2026-06-28T07:00", "week_label": "W25", "source_type": "infrastructure", "source_label": "Infrastructure alarm", "item_key": "adservio-rds-mysql-catalog-swap-high", "item_label": "adservio-rds-mysql-catalog-swap-high", "severity": "warning", "status": "Flapping, latest OK", "count": 7, "last_seen": "2026-06-28 05:22", "last_seen_ts": 1782613358.0, "services": "", "signals": "ALARM 3, OK 4, Flips 6", "latest_note": "OK", "alert_url": "", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-06-21_0700_2026-06-28_0700.html", "source_json": "production_reliability_dashboard_2026-06-21_0700_2026-06-28_0700.json" }, { "period_start": "2026-06-21T07:00:00+03:00", "period_end": "2026-06-28T07:00:00+03:00", "week_key": "2026-06-21T07:00__2026-06-28T07:00", "week_label": "W25", "source_type": "infrastructure", "source_label": "Infrastructure alarm", "item_key": "adservio-rds-mysql-catalog2-memory-low", "item_label": "adservio-rds-mysql-catalog2-memory-low", "severity": "warning", "status": "Flapping, latest OK", "count": 49, "last_seen": "2026-06-24 07:34", "last_seen_ts": 1782275643.0, "services": "", "signals": "ALARM 24, OK 25, Flips 48", "latest_note": "OK", "alert_url": "", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-06-21_0700_2026-06-28_0700.html", "source_json": "production_reliability_dashboard_2026-06-21_0700_2026-06-28_0700.json" }, { "period_start": "2026-06-28T07:00:00+03:00", "period_end": "2026-07-05T07:00:00+03:00", "week_key": "2026-06-28T07:00__2026-07-05T07:00", "week_label": "W26", "source_type": "application", "source_label": "Application alert", "item_key": "AlertmanagerFailedToSendAlerts", "item_label": "AlertmanagerFailedToSendAlerts", "severity": "warning", "status": "Seen this week", "count": 1, "last_seen": "2026-06-29 10:34", "last_seen_ts": 1782718480.387679, "services": "grafana (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1782718480387679", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1782718480387679", "source_href": "reports/production_reliability_dashboard_2026-06-28_0700_2026-07-05_0700.html", "source_json": "production_reliability_dashboard_2026-06-28_0700_2026-07-05_0700.json" }, { "period_start": "2026-06-28T07:00:00+03:00", "period_end": "2026-07-05T07:00:00+03:00", "week_key": "2026-06-28T07:00__2026-07-05T07:00", "week_label": "W26", "source_type": "application", "source_label": "Application alert", "item_key": "KubeContainerMemoryHigh", "item_label": "KubeContainerMemoryHigh", "severity": "warning", "status": "Seen this week", "count": 1, "last_seen": "2026-07-01 11:25", "last_seen_ts": 1782894308.205599, "services": "grafana (1), loki (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1782894308205599", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1782894308205599", "source_href": "reports/production_reliability_dashboard_2026-06-28_0700_2026-07-05_0700.html", "source_json": "production_reliability_dashboard_2026-06-28_0700_2026-07-05_0700.json" }, { "period_start": "2026-06-28T07:00:00+03:00", "period_end": "2026-07-05T07:00:00+03:00", "week_key": "2026-06-28T07:00__2026-07-05T07:00", "week_label": "W26", "source_type": "application", "source_label": "Application alert", "item_key": "KubeContainerWaiting", "item_label": "KubeContainerWaiting", "severity": "warning", "status": "Recent (72h)", "count": 15, "last_seen": "2026-07-02 18:26", "last_seen_ts": 1783005982.342399, "services": "grafana (15)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1783005982342399", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1783005982342399", "source_href": "reports/production_reliability_dashboard_2026-06-28_0700_2026-07-05_0700.html", "source_json": "production_reliability_dashboard_2026-06-28_0700_2026-07-05_0700.json" }, { "period_start": "2026-06-28T07:00:00+03:00", "period_end": "2026-07-05T07:00:00+03:00", "week_key": "2026-06-28T07:00__2026-07-05T07:00", "week_label": "W26", "source_type": "application", "source_label": "Application alert", "item_key": "KubeDeploymentReplicasMismatch", "item_label": "KubeDeploymentReplicasMismatch", "severity": "warning", "status": "Recent (72h)", "count": 4, "last_seen": "2026-07-03 00:55", "last_seen_ts": 1783029322.323329, "services": "core-getresponse-events-worker (3), superset-worker (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1783029322323329", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1783029322323329", "source_href": "reports/production_reliability_dashboard_2026-06-28_0700_2026-07-05_0700.html", "source_json": "production_reliability_dashboard_2026-06-28_0700_2026-07-05_0700.json" }, { "period_start": "2026-06-28T07:00:00+03:00", "period_end": "2026-07-05T07:00:00+03:00", "week_key": "2026-06-28T07:00__2026-07-05T07:00", "week_label": "W26", "source_type": "application", "source_label": "Application alert", "item_key": "KubeDeploymentRolloutStuck", "item_label": "KubeDeploymentRolloutStuck", "severity": "warning", "status": "Recent (72h)", "count": 2, "last_seen": "2026-07-02 17:51", "last_seen_ts": 1783003882.344379, "services": "core-grafana (1), superset-worker (1)", "signals": "General investigation", "latest_note": "i am working on it currently .", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1783003882344379", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1782970132337069?thread_ts=1782970132.337069&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-06-28_0700_2026-07-05_0700.html", "source_json": "production_reliability_dashboard_2026-06-28_0700_2026-07-05_0700.json" }, { "period_start": "2026-06-28T07:00:00+03:00", "period_end": "2026-07-05T07:00:00+03:00", "week_key": "2026-06-28T07:00__2026-07-05T07:00", "week_label": "W26", "source_type": "application", "source_label": "Application alert", "item_key": "KubeHpaMaxedOut", "item_label": "KubeHpaMaxedOut", "severity": "warning", "status": "Recent (72h)", "count": 3, "last_seen": "2026-07-03 13:43", "last_seen_ts": 1783075402.342399, "services": "grafana (2), notifications-event-manager (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1783075402342399", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1783075402342399", "source_href": "reports/production_reliability_dashboard_2026-06-28_0700_2026-07-05_0700.html", "source_json": "production_reliability_dashboard_2026-06-28_0700_2026-07-05_0700.json" }, { "period_start": "2026-06-28T07:00:00+03:00", "period_end": "2026-07-05T07:00:00+03:00", "week_key": "2026-06-28T07:00__2026-07-05T07:00", "week_label": "W26", "source_type": "application", "source_label": "Application alert", "item_key": "KubeJobFailed", "item_label": "KubeJobFailed", "severity": "warning", "status": "Seen today", "count": 54, "last_seen": "2026-07-05 03:19", "last_seen_ts": 1783210792.321659, "services": "colecteaza-sms-note-abs (29), download-album (27), rezumat (26), minicrm-sync (24)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1783210792321659", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1783210792321659", "source_href": "reports/production_reliability_dashboard_2026-06-28_0700_2026-07-05_0700.html", "source_json": "production_reliability_dashboard_2026-06-28_0700_2026-07-05_0700.json" }, { "period_start": "2026-06-28T07:00:00+03:00", "period_end": "2026-07-05T07:00:00+03:00", "week_key": "2026-06-28T07:00__2026-07-05T07:00", "week_label": "W26", "source_type": "application", "source_label": "Application alert", "item_key": "KubeJobNotCompleted", "item_label": "KubeJobNotCompleted", "severity": "warning", "status": "Seen this week", "count": 15, "last_seen": "2026-06-29 13:23", "last_seen_ts": 1782728601.728159, "services": "social-migration (15)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1782728601728159", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1782728601728159", "source_href": "reports/production_reliability_dashboard_2026-06-28_0700_2026-07-05_0700.html", "source_json": "production_reliability_dashboard_2026-06-28_0700_2026-07-05_0700.json" }, { "period_start": "2026-06-28T07:00:00+03:00", "period_end": "2026-07-05T07:00:00+03:00", "week_key": "2026-06-28T07:00__2026-07-05T07:00", "week_label": "W26", "source_type": "application", "source_label": "Application alert", "item_key": "KubePodContainerRestartingFrequently", "item_label": "KubePodContainerRestartingFrequently", "severity": "warning", "status": "Recent (72h)", "count": 11, "last_seen": "2026-07-03 09:22", "last_seen_ts": 1783059728.201549, "services": "core-getresponse-events-worker (8), colecteaza-sms-note-abs (7), download-album (7), rezumat (7)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1783059728201549", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1783059728201549", "source_href": "reports/production_reliability_dashboard_2026-06-28_0700_2026-07-05_0700.html", "source_json": "production_reliability_dashboard_2026-06-28_0700_2026-07-05_0700.json" }, { "period_start": "2026-06-28T07:00:00+03:00", "period_end": "2026-07-05T07:00:00+03:00", "week_key": "2026-06-28T07:00__2026-07-05T07:00", "week_label": "W26", "source_type": "application", "source_label": "Application alert", "item_key": "KubePodCrashLooping", "item_label": "KubePodCrashLooping", "severity": "warning", "status": "Recent (72h)", "count": 4, "last_seen": "2026-07-03 00:55", "last_seen_ts": 1783029322.409929, "services": "core-getresponse-events-worker (4), notifications-event-manager (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1783029322409929", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1783029322409929", "source_href": "reports/production_reliability_dashboard_2026-06-28_0700_2026-07-05_0700.html", "source_json": "production_reliability_dashboard_2026-06-28_0700_2026-07-05_0700.json" }, { "period_start": "2026-06-28T07:00:00+03:00", "period_end": "2026-07-05T07:00:00+03:00", "week_key": "2026-06-28T07:00__2026-07-05T07:00", "week_label": "W26", "source_type": "application", "source_label": "Application alert", "item_key": "KubePodNotReady", "item_label": "KubePodNotReady", "severity": "warning", "status": "Recent (72h)", "count": 2, "last_seen": "2026-07-02 17:41", "last_seen_ts": 1783003282.339159, "services": "core-grafana (1), superset-worker (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1783003282339159", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1783003282339159", "source_href": "reports/production_reliability_dashboard_2026-06-28_0700_2026-07-05_0700.html", "source_json": "production_reliability_dashboard_2026-06-28_0700_2026-07-05_0700.json" }, { "period_start": "2026-06-28T07:00:00+03:00", "period_end": "2026-07-05T07:00:00+03:00", "week_key": "2026-06-28T07:00__2026-07-05T07:00", "week_label": "W26", "source_type": "application", "source_label": "Application alert", "item_key": "Platform5xxLowVolumeCritical100", "item_label": "Platform5xxLowVolumeCritical100", "severity": "critical", "status": "Recent (72h)", "count": 3, "last_seen": "2026-07-03 00:47", "last_seen_ts": 1783028833.205409, "services": "grafana (3)", "signals": "Release / migration issue", "latest_note": "this was due to recent deployment on production , the 5xx errors . should be gone now , as the social migration", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1783028833205409", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1782717823201259?thread_ts=1782717823.201259&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-06-28_0700_2026-07-05_0700.html", "source_json": "production_reliability_dashboard_2026-06-28_0700_2026-07-05_0700.json" }, { "period_start": "2026-06-28T07:00:00+03:00", "period_end": "2026-07-05T07:00:00+03:00", "week_key": "2026-06-28T07:00__2026-07-05T07:00", "week_label": "W26", "source_type": "application", "source_label": "Application alert", "item_key": "Platform5xxLowVolumeWarning20", "item_label": "Platform5xxLowVolumeWarning20", "severity": "warning", "status": "Recent (72h)", "count": 5, "last_seen": "2026-07-03 00:45", "last_seen_ts": 1783028738.187739, "services": "grafana (5)", "signals": "Release / migration issue", "latest_note": "this was due to recent deployment on production , the 5xx errors . should be gone now , as the social migration", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1783028738187739", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1782717788186129?thread_ts=1782717788.186129&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-06-28_0700_2026-07-05_0700.html", "source_json": "production_reliability_dashboard_2026-06-28_0700_2026-07-05_0700.json" }, { "period_start": "2026-06-28T07:00:00+03:00", "period_end": "2026-07-05T07:00:00+03:00", "week_key": "2026-06-28T07:00__2026-07-05T07:00", "week_label": "W26", "source_type": "application", "source_label": "Application alert", "item_key": "PlatformLatencyP99Warning5s", "item_label": "PlatformLatencyP99Warning5s", "severity": "warning", "status": "Seen this week", "count": 1, "last_seen": "2026-06-29 22:56", "last_seen_ts": 1782762998.196329, "services": "grafana (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1782762998196329", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1782762998196329", "source_href": "reports/production_reliability_dashboard_2026-06-28_0700_2026-07-05_0700.html", "source_json": "production_reliability_dashboard_2026-06-28_0700_2026-07-05_0700.json" }, { "period_start": "2026-06-28T07:00:00+03:00", "period_end": "2026-07-05T07:00:00+03:00", "week_key": "2026-06-28T07:00__2026-07-05T07:00", "week_label": "W26", "source_type": "application", "source_label": "Application alert", "item_key": "TraefikServiceHighErrorRate", "item_label": "TraefikServiceHighErrorRate", "severity": "critical", "status": "Recent (72h)", "count": 8, "last_seen": "2026-07-03 00:55", "last_seen_ts": 1783029313.266719, "services": "web-80 (3), admission-api (2), library-api (2), social-api (1)", "signals": "Release / migration issue", "latest_note": "admission-api is single-replica (replicas=1). During the rev 271 deploy the one pod was replaced, so there was a brief window with no ready…", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1783029313266719", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1782718333173119?thread_ts=1782718333.173119&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-06-28_0700_2026-07-05_0700.html", "source_json": "production_reliability_dashboard_2026-06-28_0700_2026-07-05_0700.json" }, { "period_start": "2026-06-28T07:00:00+03:00", "period_end": "2026-07-05T07:00:00+03:00", "week_key": "2026-06-28T07:00__2026-07-05T07:00", "week_label": "W26", "source_type": "application", "source_label": "Application alert", "item_key": "TraefikServiceHighLatency", "item_label": "TraefikServiceHighLatency", "severity": "warning", "status": "Recent (72h)", "count": 4, "last_seen": "2026-07-03 00:42", "last_seen_ts": 1783028528.175269, "services": "web-80 (2), admission-api (2)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1783028528175269", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1783028528175269", "source_href": "reports/production_reliability_dashboard_2026-06-28_0700_2026-07-05_0700.html", "source_json": "production_reliability_dashboard_2026-06-28_0700_2026-07-05_0700.json" }, { "period_start": "2026-07-04T07:00:00+03:00", "period_end": "2026-07-11T07:00:00+03:00", "week_key": "2026-07-04T07:00__2026-07-11T07:00", "week_label": "W27", "source_type": "application", "source_label": "Application alert", "item_key": "[Composite] RDSMemorySwapCriticalTest", "item_label": "[Composite] RDSMemorySwapCriticalTest", "severity": "critical", "status": "Seen this week", "count": 1, "last_seen": "2026-07-07 10:20", "last_seen_ts": 1783408823.182899, "services": "grafana (1)", "signals": "General investigation", "latest_note": "Test", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1783408823182899", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1783408823182899?thread_ts=1783408823.182899&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-07-04_0700_2026-07-11_0700.html", "source_json": "production_reliability_dashboard_2026-07-04_0700_2026-07-11_0700.json" }, { "period_start": "2026-07-04T07:00:00+03:00", "period_end": "2026-07-11T07:00:00+03:00", "week_key": "2026-07-04T07:00__2026-07-11T07:00", "week_label": "W27", "source_type": "application", "source_label": "Application alert", "item_key": "KubeAPIErrorBudgetBurn", "item_label": "KubeAPIErrorBudgetBurn", "severity": "warning", "status": "Recent (72h)", "count": 1, "last_seen": "2026-07-08 12:09", "last_seen_ts": 1783501778.219019, "services": "grafana (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1783501778219019", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1783501778219019", "source_href": "reports/production_reliability_dashboard_2026-07-04_0700_2026-07-11_0700.html", "source_json": "production_reliability_dashboard_2026-07-04_0700_2026-07-11_0700.json" }, { "period_start": "2026-07-04T07:00:00+03:00", "period_end": "2026-07-11T07:00:00+03:00", "week_key": "2026-07-04T07:00__2026-07-11T07:00", "week_label": "W27", "source_type": "application", "source_label": "Application alert", "item_key": "KubeHpaMaxedOut", "item_label": "KubeHpaMaxedOut", "severity": "warning", "status": "Seen today", "count": 3, "last_seen": "2026-07-10 16:34", "last_seen_ts": 1783690492.349929, "services": "grafana (3)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1783690492349929", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1783690492349929", "source_href": "reports/production_reliability_dashboard_2026-07-04_0700_2026-07-11_0700.html", "source_json": "production_reliability_dashboard_2026-07-04_0700_2026-07-11_0700.json" }, { "period_start": "2026-07-04T07:00:00+03:00", "period_end": "2026-07-11T07:00:00+03:00", "week_key": "2026-07-04T07:00__2026-07-11T07:00", "week_label": "W27", "source_type": "application", "source_label": "Application alert", "item_key": "KubeJobFailed", "item_label": "KubeJobFailed", "severity": "warning", "status": "Seen this week", "count": 19, "last_seen": "2026-07-07 12:29", "last_seen_ts": 1783416592.656369, "services": "minicrm-sync (19)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1783416592656369", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1783416592656369", "source_href": "reports/production_reliability_dashboard_2026-07-04_0700_2026-07-11_0700.html", "source_json": "production_reliability_dashboard_2026-07-04_0700_2026-07-11_0700.json" }, { "period_start": "2026-07-04T07:00:00+03:00", "period_end": "2026-07-11T07:00:00+03:00", "week_key": "2026-07-04T07:00__2026-07-11T07:00", "week_label": "W27", "source_type": "application", "source_label": "Application alert", "item_key": "KubePodContainerRestartingFrequently", "item_label": "KubePodContainerRestartingFrequently", "severity": "warning", "status": "Seen today", "count": 2, "last_seen": "2026-07-10 16:38", "last_seen_ts": 1783690688.208159, "services": "service-websocket (1), library (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1783690688208159", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1783690688208159", "source_href": "reports/production_reliability_dashboard_2026-07-04_0700_2026-07-11_0700.html", "source_json": "production_reliability_dashboard_2026-07-04_0700_2026-07-11_0700.json" }, { "period_start": "2026-07-04T07:00:00+03:00", "period_end": "2026-07-11T07:00:00+03:00", "week_key": "2026-07-04T07:00__2026-07-11T07:00", "week_label": "W27", "source_type": "application", "source_label": "Application alert", "item_key": "RDSMemorySwapCriticalTest", "item_label": "RDSMemorySwapCriticalTest", "severity": "critical", "status": "Seen this week", "count": 1, "last_seen": "2026-07-07 09:37", "last_seen_ts": 1783406237.544529, "services": "grafana (1)", "signals": "General investigation", "latest_note": "Test !!! please ignoree", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1783406237544529", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1783406237544529?thread_ts=1783406237.544529&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-07-04_0700_2026-07-11_0700.html", "source_json": "production_reliability_dashboard_2026-07-04_0700_2026-07-11_0700.json" }, { "period_start": "2026-07-04T07:00:00+03:00", "period_end": "2026-07-11T07:00:00+03:00", "week_key": "2026-07-04T07:00__2026-07-11T07:00", "week_label": "W27", "source_type": "application", "source_label": "Application alert", "item_key": "TraefikServiceHighErrorRate", "item_label": "TraefikServiceHighErrorRate", "severity": "critical", "status": "Recent (72h)", "count": 4, "last_seen": "2026-07-09 15:15", "last_seen_ts": 1783599343.177329, "services": "docgen2-api (1), accommodations-api (1), uni-api (1), superset-http (1)", "signals": "Release / migration issue", "latest_note": "No app errors at all the 5xx are 19× 502s that occur only during its own CI deploys (single | replica, no graceful drain, traefik briefly r…", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1783599343177329", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1783430743173129?thread_ts=1783430743.173129&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-07-04_0700_2026-07-11_0700.html", "source_json": "production_reliability_dashboard_2026-07-04_0700_2026-07-11_0700.json" }, { "period_start": "2026-07-04T07:00:00+03:00", "period_end": "2026-07-11T07:00:00+03:00", "week_key": "2026-07-04T07:00__2026-07-11T07:00", "week_label": "W27", "source_type": "application", "source_label": "Application alert", "item_key": "TraefikServiceHighLatency", "item_label": "TraefikServiceHighLatency", "severity": "warning", "status": "Seen today", "count": 4, "last_seen": "2026-07-10 17:06", "last_seen_ts": 1783692368.176599, "services": "admission-api (4)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1783692368176599", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1783692368176599", "source_href": "reports/production_reliability_dashboard_2026-07-04_0700_2026-07-11_0700.html", "source_json": "production_reliability_dashboard_2026-07-04_0700_2026-07-11_0700.json" }, { "period_start": "2026-07-11T07:00:00+03:00", "period_end": "2026-07-18T07:00:00+03:00", "week_key": "2026-07-11T07:00__2026-07-18T07:00", "week_label": "W28", "source_type": "application", "source_label": "Application alert", "item_key": "AlertmanagerFailedToSendAlerts", "item_label": "AlertmanagerFailedToSendAlerts", "severity": "warning", "status": "Seen this week", "count": 1, "last_seen": "2026-07-13 07:42", "last_seen_ts": 1783917748.434879, "services": "grafana (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1783917748434879", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1783917748434879", "source_href": "reports/production_reliability_dashboard_2026-07-11_0700_2026-07-18_0700.html", "source_json": "production_reliability_dashboard_2026-07-11_0700_2026-07-18_0700.json" }, { "period_start": "2026-07-11T07:00:00+03:00", "period_end": "2026-07-18T07:00:00+03:00", "week_key": "2026-07-11T07:00__2026-07-18T07:00", "week_label": "W28", "source_type": "application", "source_label": "Application alert", "item_key": "etcdInsufficientMembers", "item_label": "etcdInsufficientMembers", "severity": "critical", "status": "Recent (72h)", "count": 1, "last_seen": "2026-07-15 10:40", "last_seen_ts": 1784101235.640569, "services": "etcd (1)", "signals": "Release / migration issue", "latest_note": "Looks good i did a helm deployed on tuiasi that rolled out alert manger pod and these alerts are triggered all good on tuiasi cluster . | false positive alerts.", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1784101235640569", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1784101235640569?thread_ts=1784101235.640569&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-07-11_0700_2026-07-18_0700.html", "source_json": "production_reliability_dashboard_2026-07-11_0700_2026-07-18_0700.json" }, { "period_start": "2026-07-11T07:00:00+03:00", "period_end": "2026-07-18T07:00:00+03:00", "week_key": "2026-07-11T07:00__2026-07-18T07:00", "week_label": "W28", "source_type": "application", "source_label": "Application alert", "item_key": "etcdMembersDown", "item_label": "etcdMembersDown", "severity": "warning", "status": "Recent (72h)", "count": 3, "last_seen": "2026-07-15 18:50", "last_seen_ts": 1784130605.624389, "services": "etcd (3)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1784130605624389", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1784130605624389", "source_href": "reports/production_reliability_dashboard_2026-07-11_0700_2026-07-18_0700.html", "source_json": "production_reliability_dashboard_2026-07-11_0700_2026-07-18_0700.json" }, { "period_start": "2026-07-11T07:00:00+03:00", "period_end": "2026-07-18T07:00:00+03:00", "week_key": "2026-07-11T07:00__2026-07-18T07:00", "week_label": "W28", "source_type": "application", "source_label": "Application alert", "item_key": "KubeAggregatedAPIDown", "item_label": "KubeAggregatedAPIDown", "severity": "warning", "status": "Recent (72h)", "count": 3, "last_seen": "2026-07-15 18:49", "last_seen_ts": 1784130587.167189, "services": "metrics-server (3)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1784130587167189", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1784130587167189", "source_href": "reports/production_reliability_dashboard_2026-07-11_0700_2026-07-18_0700.html", "source_json": "production_reliability_dashboard_2026-07-11_0700_2026-07-18_0700.json" }, { "period_start": "2026-07-11T07:00:00+03:00", "period_end": "2026-07-18T07:00:00+03:00", "week_key": "2026-07-11T07:00__2026-07-18T07:00", "week_label": "W28", "source_type": "application", "source_label": "Application alert", "item_key": "KubeAPIErrorBudgetBurn", "item_label": "KubeAPIErrorBudgetBurn", "severity": "warning", "status": "Seen this week", "count": 5, "last_seen": "2026-07-14 11:53", "last_seen_ts": 1784019218.246799, "services": "grafana (5)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1784019218246799", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1784019218246799", "source_href": "reports/production_reliability_dashboard_2026-07-11_0700_2026-07-18_0700.html", "source_json": "production_reliability_dashboard_2026-07-11_0700_2026-07-18_0700.json" }, { "period_start": "2026-07-11T07:00:00+03:00", "period_end": "2026-07-18T07:00:00+03:00", "week_key": "2026-07-11T07:00__2026-07-18T07:00", "week_label": "W28", "source_type": "application", "source_label": "Application alert", "item_key": "KubeDeploymentReplicasMismatch", "item_label": "KubeDeploymentReplicasMismatch", "severity": "warning", "status": "Seen this week", "count": 1, "last_seen": "2026-07-13 07:41", "last_seen_ts": 1783917685.756579, "services": "accommodations-api (1), library-api (1), rooms-api (1)", "signals": "General investigation", "latest_note": "Both pods are fine now accommodations-api , library-api and rooms api recovered on their own about 1.5–2h ago and have been stable since (a…", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1783917685756579", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1783917685756579?thread_ts=1783917685.756579&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-07-11_0700_2026-07-18_0700.html", "source_json": "production_reliability_dashboard_2026-07-11_0700_2026-07-18_0700.json" }, { "period_start": "2026-07-11T07:00:00+03:00", "period_end": "2026-07-18T07:00:00+03:00", "week_key": "2026-07-11T07:00__2026-07-18T07:00", "week_label": "W28", "source_type": "application", "source_label": "Application alert", "item_key": "KubeHpaMaxedOut", "item_label": "KubeHpaMaxedOut", "severity": "warning", "status": "Recent (72h)", "count": 2, "last_seen": "2026-07-16 01:12", "last_seen_ts": 1784153572.430159, "services": "grafana (2)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1784153572430159", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1784153572430159", "source_href": "reports/production_reliability_dashboard_2026-07-11_0700_2026-07-18_0700.html", "source_json": "production_reliability_dashboard_2026-07-11_0700_2026-07-18_0700.json" }, { "period_start": "2026-07-11T07:00:00+03:00", "period_end": "2026-07-18T07:00:00+03:00", "week_key": "2026-07-11T07:00__2026-07-18T07:00", "week_label": "W28", "source_type": "application", "source_label": "Application alert", "item_key": "KubeJobFailed", "item_label": "KubeJobFailed", "severity": "warning", "status": "Recent (72h)", "count": 2, "last_seen": "2026-07-16 05:25", "last_seen_ts": 1784168752.389219, "services": "send-codes (2)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1784168752389219", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1784168752389219", "source_href": "reports/production_reliability_dashboard_2026-07-11_0700_2026-07-18_0700.html", "source_json": "production_reliability_dashboard_2026-07-11_0700_2026-07-18_0700.json" }, { "period_start": "2026-07-11T07:00:00+03:00", "period_end": "2026-07-18T07:00:00+03:00", "week_key": "2026-07-11T07:00__2026-07-18T07:00", "week_label": "W28", "source_type": "application", "source_label": "Application alert", "item_key": "KubePodContainerRestartingFrequently", "item_label": "KubePodContainerRestartingFrequently", "severity": "warning", "status": "Seen today", "count": 4, "last_seen": "2026-07-17 16:23", "last_seen_ts": 1784294615.700009, "services": "accommodations-api (2), library-api (2), rooms-api (2), core-grafana (2)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1784294615700009", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1784294615700009", "source_href": "reports/production_reliability_dashboard_2026-07-11_0700_2026-07-18_0700.html", "source_json": "production_reliability_dashboard_2026-07-11_0700_2026-07-18_0700.json" }, { "period_start": "2026-07-11T07:00:00+03:00", "period_end": "2026-07-18T07:00:00+03:00", "week_key": "2026-07-11T07:00__2026-07-18T07:00", "week_label": "W28", "source_type": "application", "source_label": "Application alert", "item_key": "KubePodCrashLooping", "item_label": "KubePodCrashLooping", "severity": "warning", "status": "Recent (72h)", "count": 2, "last_seen": "2026-07-16 01:12", "last_seen_ts": 1784153572.430259, "services": "library-api (2), accommodations-api (1), rooms-api (1)", "signals": "General investigation", "latest_note": "Both pods are fine now accommodations-api , library-api and rooms api recovered on their own about 1.5–2h ago and have been stable since (a…", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1784153572430259", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1783917689938669?thread_ts=1783917689.938669&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-07-11_0700_2026-07-18_0700.html", "source_json": "production_reliability_dashboard_2026-07-11_0700_2026-07-18_0700.json" }, { "period_start": "2026-07-11T07:00:00+03:00", "period_end": "2026-07-18T07:00:00+03:00", "week_key": "2026-07-11T07:00__2026-07-18T07:00", "week_label": "W28", "source_type": "application", "source_label": "Application alert", "item_key": "PagerDutyTest", "item_label": "PagerDutyTest", "severity": "critical", "status": "Recent (72h)", "count": 1, "last_seen": "2026-07-15 10:41", "last_seen_ts": 1784101269.161109, "services": "grafana (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1784101269161109", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1784101269161109", "source_href": "reports/production_reliability_dashboard_2026-07-11_0700_2026-07-18_0700.html", "source_json": "production_reliability_dashboard_2026-07-11_0700_2026-07-18_0700.json" }, { "period_start": "2026-07-11T07:00:00+03:00", "period_end": "2026-07-18T07:00:00+03:00", "week_key": "2026-07-11T07:00__2026-07-18T07:00", "week_label": "W28", "source_type": "application", "source_label": "Application alert", "item_key": "PlatformLatencyP95Critical1s", "item_label": "PlatformLatencyP95Critical1s", "severity": "critical", "status": "Recent (72h)", "count": 1, "last_seen": "2026-07-16 01:05", "last_seen_ts": 1784153143.284889, "services": "grafana (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1784153143284889", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1784153143284889", "source_href": "reports/production_reliability_dashboard_2026-07-11_0700_2026-07-18_0700.html", "source_json": "production_reliability_dashboard_2026-07-11_0700_2026-07-18_0700.json" }, { "period_start": "2026-07-11T07:00:00+03:00", "period_end": "2026-07-18T07:00:00+03:00", "week_key": "2026-07-11T07:00__2026-07-18T07:00", "week_label": "W28", "source_type": "application", "source_label": "Application alert", "item_key": "PlatformLatencyP95Warning400ms", "item_label": "PlatformLatencyP95Warning400ms", "severity": "warning", "status": "Recent (72h)", "count": 1, "last_seen": "2026-07-16 01:06", "last_seen_ts": 1784153168.164319, "services": "grafana (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1784153168164319", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1784153168164319", "source_href": "reports/production_reliability_dashboard_2026-07-11_0700_2026-07-18_0700.html", "source_json": "production_reliability_dashboard_2026-07-11_0700_2026-07-18_0700.json" }, { "period_start": "2026-07-11T07:00:00+03:00", "period_end": "2026-07-18T07:00:00+03:00", "week_key": "2026-07-11T07:00__2026-07-18T07:00", "week_label": "W28", "source_type": "application", "source_label": "Application alert", "item_key": "RESOLVED - TraefikServiceHighLatency", "item_label": "RESOLVED - TraefikServiceHighLatency", "severity": "warning", "status": "Seen today", "count": 1, "last_seen": "2026-07-17 22:31", "last_seen_ts": 1784316668.182999, "services": "admission-api (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1784316668182999", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1784316668182999", "source_href": "reports/production_reliability_dashboard_2026-07-11_0700_2026-07-18_0700.html", "source_json": "production_reliability_dashboard_2026-07-11_0700_2026-07-18_0700.json" }, { "period_start": "2026-07-11T07:00:00+03:00", "period_end": "2026-07-18T07:00:00+03:00", "week_key": "2026-07-11T07:00__2026-07-18T07:00", "week_label": "W28", "source_type": "application", "source_label": "Application alert", "item_key": "TargetDown", "item_label": "TargetDown", "severity": "warning", "status": "Recent (72h)", "count": 3, "last_seen": "2026-07-15 18:50", "last_seen_ts": 1784130608.551579, "services": "etcd (3)", "signals": "General investigation", "latest_note": "tuiasi is having metrics server issues and these are all false alarms . will fix it with a new ticket", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1784130608551579", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1784115908511709?thread_ts=1784115908.511709&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-07-11_0700_2026-07-18_0700.html", "source_json": "production_reliability_dashboard_2026-07-11_0700_2026-07-18_0700.json" }, { "period_start": "2026-07-11T07:00:00+03:00", "period_end": "2026-07-18T07:00:00+03:00", "week_key": "2026-07-11T07:00__2026-07-18T07:00", "week_label": "W28", "source_type": "application", "source_label": "Application alert", "item_key": "TraefikServiceHighErrorRate", "item_label": "TraefikServiceHighErrorRate", "severity": "critical", "status": "Recent (72h)", "count": 12, "last_seen": "2026-07-16 12:21", "last_seen_ts": 1784193673.171289, "services": "accommodations-api (10), web-80 (1), core-grafana-80 (1)", "signals": "General investigation", "latest_note": "This should go away caused due codex agent running queries", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1784193673171289", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1784193673171289?thread_ts=1784193673.171289&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-07-11_0700_2026-07-18_0700.html", "source_json": "production_reliability_dashboard_2026-07-11_0700_2026-07-18_0700.json" }, { "period_start": "2026-07-11T07:00:00+03:00", "period_end": "2026-07-18T07:00:00+03:00", "week_key": "2026-07-11T07:00__2026-07-18T07:00", "week_label": "W28", "source_type": "application", "source_label": "Application alert", "item_key": "TraefikServiceHighLatency", "item_label": "TraefikServiceHighLatency", "severity": "warning", "status": "Seen today", "count": 19, "last_seen": "2026-07-17 22:26", "last_seen_ts": 1784316368.174779, "services": "admission-api (18), web-80 (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1784316368174779", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1784316368174779", "source_href": "reports/production_reliability_dashboard_2026-07-11_0700_2026-07-18_0700.html", "source_json": "production_reliability_dashboard_2026-07-11_0700_2026-07-18_0700.json" }, { "period_start": "2026-07-11T07:00:00+03:00", "period_end": "2026-07-18T07:00:00+03:00", "week_key": "2026-07-11T07:00__2026-07-18T07:00", "week_label": "W28", "source_type": "application", "source_label": "Application alert", "item_key": "Watchdog", "item_label": "Watchdog", "severity": "warning", "status": "Seen today", "count": 6, "last_seen": "2026-07-17 23:05", "last_seen_ts": 1784318739.124599, "services": "grafana (6)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1784318739124599", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1784318739124599", "source_href": "reports/production_reliability_dashboard_2026-07-11_0700_2026-07-18_0700.html", "source_json": "production_reliability_dashboard_2026-07-11_0700_2026-07-18_0700.json" }, { "period_start": "2026-07-11T07:00:00+03:00", "period_end": "2026-07-18T07:00:00+03:00", "week_key": "2026-07-11T07:00__2026-07-18T07:00", "week_label": "W28", "source_type": "customer", "source_label": "Customer warning", "item_key": "Adservio Ro", "item_label": "Adservio Ro", "severity": "warning", "status": "Recovered recently", "count": 1, "last_seen": "2026-07-17 18:05", "last_seen_ts": 1784300720.0, "services": "unclassified", "signals": "Pingdom-only evidence so far", "latest_note": "Pingdom-only evidence so far", "alert_url": "http://www.adservio.ro/ro", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-07-11_0700_2026-07-18_0700.html", "source_json": "production_reliability_dashboard_2026-07-11_0700_2026-07-18_0700.json" } ] }