Coverage for app\services\health_service.py: 12%
172 statements
« prev ^ index » next coverage.py v7.3.2, created at 2026-01-15 13:39 +0300
1import httpx
2import logging
3import os
4from datetime import datetime, timedelta, UTC
5from sqlalchemy.orm import Session
6from typing import List, Dict, Any
8from ..schemas.system import (
9 SystemHealthOverview,
10 SystemComponentStatus,
11 ComponentStatus,
12 KPICard
13)
14from ..db import School
16logger = logging.getLogger(__name__)
18# Service URLs (from environment or defaults)
19AI_TRACKER_URL = os.getenv("AI_TRACKER_URL", "http://ai-tracker:8000")
20GTO_BACKEND_URL = os.getenv("GTO_BACKEND_URL", "http://gto-backend:8000")
21SCHOOL_NODE_URL = os.getenv("SCHOOL_NODE_URL", "http://school-template:8100")
22TIMESCALE_DB_HOST = os.getenv("TIMESCALE_DB_HOST", "db")
23TIMESCALE_DB_PORT = int(os.getenv("TIMESCALE_DB_PORT", "5432"))
24TIMESCALE_DB_HOST_FALLBACK = os.getenv("TIMESCALE_DB_HOST_FALLBACK", "127.0.0.1")
25TIMESCALE_DB_PORT_FALLBACK = int(os.getenv("TIMESCALE_DB_PORT_FALLBACK", "5438"))
27class HealthService:
28 def __init__(self, db: Session):
29 self.db = db
31 async def get_system_overview(self) -> SystemHealthOverview:
32 components: List[SystemComponentStatus] = []
33 summary = {"ok": 0, "warning": 0, "error": 0, "offline": 0}
35 # 1. Check Admin-Core (Self)
36 components.append(SystemComponentStatus(
37 id="admin-core",
38 name="Admin Core",
39 type="service",
40 status=ComponentStatus.OK,
41 message="Operational"
42 ))
43 summary["ok"] += 1
45 # 2. Check GTO-Backend
46 try:
47 async with httpx.AsyncClient(timeout=2.0) as client:
48 resp = await client.get(f"{GTO_BACKEND_URL}/health")
49 if resp.status_code == 200:
50 components.append(SystemComponentStatus(
51 id="gto-backend",
52 name="GTO Backend",
53 type="service",
54 status=ComponentStatus.OK,
55 message="Operational"
56 ))
57 summary["ok"] += 1
58 else:
59 components.append(SystemComponentStatus(
60 id="gto-backend",
61 name="GTO Backend",
62 type="service",
63 status=ComponentStatus.ERROR,
64 message=f"Status code: {resp.status_code}"
65 ))
66 summary["error"] += 1
67 except Exception as e:
68 components.append(SystemComponentStatus(
69 id="gto-backend",
70 name="GTO Backend",
71 type="service",
72 status=ComponentStatus.ERROR,
73 message=str(e)
74 ))
75 summary["error"] += 1
77 # 3. Check AI-Tracker & Cameras
78 try:
79 async with httpx.AsyncClient(timeout=2.0) as client:
80 # Check service itself
81 resp = await client.get(f"{AI_TRACKER_URL}/metrics")
82 if resp.status_code == 200:
83 components.append(SystemComponentStatus(
84 id="ai-tracker",
85 name="AI Tracker",
86 type="service",
87 status=ComponentStatus.OK,
88 message="Operational"
89 ))
90 summary["ok"] += 1
91 else:
92 components.append(SystemComponentStatus(
93 id="ai-tracker",
94 name="AI Tracker",
95 type="service",
96 status=ComponentStatus.WARNING,
97 message=f"Metrics endpoint returned {resp.status_code}"
98 ))
99 summary["warning"] += 1
101 # Check cameras
102 resp_cam = await client.get(f"{AI_TRACKER_URL}/cameras")
103 if resp_cam.status_code == 200:
104 data = resp_cam.json()
105 cameras = data.get("cameras", [])
106 for cam in cameras:
107 # cam is likely {"id": "cam1", "source": "0", "status": "ok" or something}
108 # Based on ai-tracker code, it returns cam_manager.status() which returns a list of dicts
109 # We need to verify the structure, but let's assume it has id and source.
110 # Wait, ai-tracker code: return {"cameras": cam_manager.status()}
111 # cam_manager.status() returns list of dicts.
113 # Let's assume structure based on what we saw in ai-tracker code (it wasn't fully shown but hinted)
114 # We will treat them as components.
115 cam_id = cam.get("id", "unknown")
116 cam_status_str = cam.get("status", "ok")
118 status = ComponentStatus.OK
119 if cam_status_str != "ok":
120 status = ComponentStatus.ERROR
122 components.append(SystemComponentStatus(
123 id=f"camera-{cam_id}",
124 name=f"Camera {cam_id}",
125 type="camera",
126 status=status,
127 message=f"Source: {cam.get('source')}",
128 metadata=cam
129 ))
130 summary[status.value] += 1
131 except Exception as e:
132 components.append(SystemComponentStatus(
133 id="ai-tracker",
134 name="AI Tracker",
135 type="service",
136 status=ComponentStatus.ERROR,
137 message=f"Connection failed: {str(e)}"
138 ))
139 summary["error"] += 1
141 # 4. Check School Nodes (from DB Heartbeats)
142 try:
143 schools = self.db.query(School).all()
144 except Exception as e:
145 components.append(SystemComponentStatus(
146 id="school-nodes",
147 name="School Nodes",
148 type="node_group",
149 status=ComponentStatus.WARNING,
150 message=f"DB schema issue: {str(e)}",
151 metadata={}
152 ))
153 summary["warning"] += 1
154 schools = []
155 online_schools = 0
156 offline_schools = 0
158 for school in schools:
159 # Determine status based on last_heartbeat
160 is_online = False
161 if school.last_heartbeat:
162 # Consider online if heartbeat within last 5 minutes
163 if datetime.now(UTC) - school.last_heartbeat.replace(tzinfo=UTC) < timedelta(minutes=5):
164 is_online = True
166 status = ComponentStatus.OK if is_online else ComponentStatus.OFFLINE
167 if is_online:
168 online_schools += 1
169 else:
170 offline_schools += 1
172 # We don't add every school to components list to avoid clutter,
173 # but we could add a summary component or critical ones.
174 # For now let's add a summary component for schools.
176 components.append(SystemComponentStatus(
177 id="school-nodes",
178 name="School Nodes",
179 type="node_group",
180 status=ComponentStatus.OK if offline_schools == 0 else ComponentStatus.WARNING,
181 message=f"{online_schools} online / {offline_schools} offline",
182 metadata={"online": online_schools, "offline": offline_schools, "total": len(schools)}
183 ))
184 if offline_schools > 0:
185 summary["warning"] += 1 # Or error depending on policy
186 else:
187 summary["ok"] += 1
189 # 5. Check Local DB (TimescaleDB)
190 try:
191 import socket
192 s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
193 s.settimeout(1.5)
194 db_ok = False
195 try:
196 if s.connect_ex((TIMESCALE_DB_HOST, TIMESCALE_DB_PORT)) == 0:
197 db_ok = True
198 finally:
199 try:
200 s.close()
201 except Exception:
202 pass
203 # Fallback to localhost mapped port if needed
204 if not db_ok:
205 s2 = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
206 s2.settimeout(1.5)
207 try:
208 if s2.connect_ex((TIMESCALE_DB_HOST_FALLBACK, TIMESCALE_DB_PORT_FALLBACK)) == 0:
209 db_ok = True
210 finally:
211 try:
212 s2.close()
213 except Exception:
214 pass
215 components.append(SystemComponentStatus(
216 id="timescale-db",
217 name="Local DB (TimescaleDB)",
218 type="database",
219 status=ComponentStatus.OK if db_ok else ComponentStatus.ERROR,
220 message="Operational" if db_ok else "Connection failed"
221 ))
222 summary["ok" if db_ok else "error"] += 1
223 except Exception as e:
224 components.append(SystemComponentStatus(
225 id="timescale-db",
226 name="Local DB (TimescaleDB)",
227 type="database",
228 status=ComponentStatus.ERROR,
229 message=str(e)
230 ))
231 summary["error"] += 1
233 # 6. Check Hardware Controller (via School Node diagnostics)
234 try:
235 async with httpx.AsyncClient(timeout=2.0) as client:
236 resp = await client.get(f"{SCHOOL_NODE_URL}/v1/diagnostics/hardware")
237 if resp.status_code == 200:
238 components.append(SystemComponentStatus(
239 id="hardware-controller",
240 name="Hardware Controller",
241 type="service",
242 status=ComponentStatus.OK,
243 message="Operational",
244 metadata=resp.json()
245 ))
246 summary["ok"] += 1
247 else:
248 components.append(SystemComponentStatus(
249 id="hardware-controller",
250 name="Hardware Controller",
251 type="service",
252 status=ComponentStatus.WARNING,
253 message=f"Status code: {resp.status_code}"
254 ))
255 summary["warning"] += 1
256 except Exception as e:
257 # Try localhost fallback for dev
258 try:
259 async with httpx.AsyncClient(timeout=2.0) as client:
260 resp = await client.get("http://127.0.0.1:8100/v1/diagnostics/hardware")
261 if resp.status_code == 200:
262 components.append(SystemComponentStatus(
263 id="hardware-controller",
264 name="Hardware Controller",
265 type="service",
266 status=ComponentStatus.OK,
267 message="Operational",
268 metadata=resp.json()
269 ))
270 summary["ok"] += 1
271 else:
272 raise RuntimeError(f"Status code: {resp.status_code}")
273 except Exception:
274 components.append(SystemComponentStatus(
275 id="hardware-controller",
276 name="Hardware Controller",
277 type="service",
278 status=ComponentStatus.ERROR,
279 message=str(e)
280 ))
281 summary["error"] += 1
283 # Calculate overall status
284 overall = ComponentStatus.OK
285 if summary["error"] > 0:
286 overall = ComponentStatus.ERROR
287 elif summary["warning"] > 0:
288 overall = ComponentStatus.WARNING
290 return SystemHealthOverview(
291 overall_status=overall,
292 components=components,
293 summary=summary
294 )
296 def register_heartbeat(self, school_id: str):
297 school = self.db.query(School).filter(School.id == school_id).first()
298 if school:
299 school.last_heartbeat = datetime.now(UTC)
300 self.db.commit()
301 return True
302 return False
304 async def get_kpis(self) -> List[KPICard]:
305 kpis = []
306 try:
307 async with httpx.AsyncClient(timeout=3.0) as client:
308 # Call GTO-Backend /api/v1/analytics/stats
309 # Note: analytics.py aliases /stats to get_stats which returns DashboardData
310 url = f"{GTO_BACKEND_URL}/api/v1/analytics/stats"
311 resp = await client.get(url)
313 if resp.status_code == 200:
314 data = resp.json()
315 # data follows DashboardData schema: { kpi: List[KPIData], ... }
316 backend_kpis = data.get("kpi", [])
318 for bk in backend_kpis:
319 title = bk.get("title", "Unknown")
320 value = bk.get("value", "0")
321 change = bk.get("change", "")
323 # Determine ID based on title (heuristic)
324 kid = "kpi_" + title.lower().replace(" ", "_")
325 if "всего тренировок" in title.lower():
326 kid = "total_workouts"
327 elif "активных участников" in title.lower():
328 kid = "active_users"
329 elif "успешная сдача" in title.lower():
330 kid = "success_rate"
331 elif "часов активности" in title.lower():
332 kid = "activity_hours"
334 # Parse trend direction
335 trend_dir = "neutral"
336 if "+" in change or "up" in change:
337 trend_dir = "up"
338 elif "-" in change or "down" in change:
339 trend_dir = "down"
341 kpis.append(KPICard(
342 id=kid,
343 title=title,
344 value=value,
345 # unit is often embedded in value in GTO backend currently
346 status=ComponentStatus.OK,
347 trend=None, # GTO backend returns formatted string like "+5 (total)"
348 trend_direction=trend_dir
349 ))
350 else:
351 logger.warning(f"GTO Backend returned {resp.status_code} for KPIs")
352 kpis.append(KPICard(
353 id="gto_error",
354 title="Связь с GTO Backend",
355 value="Ошибка",
356 status=ComponentStatus.WARNING
357 ))
359 except Exception as e:
360 logger.error(f"Failed to fetch KPIs from GTO Backend: {e}")
361 kpis.append(KPICard(
362 id="gto_offline",
363 title="Связь с GTO Backend",
364 value="Нет связи",
365 status=ComponentStatus.OFFLINE
366 ))
368 return kpis