From 2e853aad93db59cd72f50a22c2a18d60d095575c Mon Sep 17 00:00:00 2001
From: Michel Spils <msp@informatik.uni-kiel.de>
Date: Mon, 25 Nov 2024 09:38:48 +0100
Subject: [PATCH] dashboard metrics
---
src/app.py | 270 +++++++++++++++++++++++++-----
src/dashboard/styles.py | 32 +++-
src/dashboard/tables_and_plots.py | 140 +++++-----------
3 files changed, 298 insertions(+), 144 deletions(-)
diff --git a/src/app.py b/src/app.py
index bb7532c..828c7d7 100644
--- a/src/app.py
+++ b/src/app.py
@@ -4,26 +4,22 @@ from datetime import datetime, timedelta
import dash_bootstrap_components as dbc
import oracledb
import pandas as pd
-#import plotly.express as px
-#import plotly.graph_objs as go
-import dash
-from dash import Dash, dash_table, dcc, html,ctx
+from dash import Dash, dash_table, dcc, html
from dash.dependencies import MATCH, Input, Output, State
from sqlalchemy import and_, create_engine, desc, func, select,distinct
from sqlalchemy.orm import Session
-from dashboard.styles import TABLE_STYLE
+from dashboard.styles import TABLE_STYLE, TAB_STYLE
#from dash_tools.layout_helper import create_collapsible_section
from dashboard.tables_and_plots import (create_historical_plot,
create_historical_table,
create_inp_forecast_status_table,
- create_input_forecasts_plot,
- create_log_table,
+ create_log_table, create_metrics_plots,
create_model_forecasts_plot,
create_status_summary_chart)
from dashboard.constants import NUM_RECENT_FORECASTS, NUM_RECENT_LOGS, BUFFER_TIME,LOOKBACK_EXTERNAL_FORECAST_EXISTENCE
-from utils.db_tools.orm_classes import (InputForecasts, Log, Modell,
+from utils.db_tools.orm_classes import (InputForecasts, Log, Metric, Modell,
ModellSensor, PegelForecasts,
SensorData)
@@ -78,23 +74,40 @@ class ForecastMonitor:
def setup_layout(self):
# Header
- header = dbc.Navbar(
- dbc.Container(
- dbc.Row([
- dbc.Col(
- html.H1("Pegel Dashboard", className="mb-0 text-white"),
- width="auto"
- )
- ]),
- fluid=True
- ),
- color="primary",
- dark=True,
- className="mb-4"
- )
+ header = html.Div([
+ # Main header bar
+ dbc.Navbar(
+ [
+ dbc.Container([
+ dbc.Row([
+ dbc.Col(html.H1("Pegel Dashboard",
+ className="mb-0 text-white d-flex align-items-center"#className="mb-0 text-white"
+ ),
+ width="auto"),
+ dbc.Col(dbc.Tabs([
+ dbc.Tab(label="Modell Monitoring",tab_id="model-monitoring",**TAB_STYLE),
+ dbc.Tab(label="System Logs",tab_id="system-logs",**TAB_STYLE)
+ ],
+ id="main-tabs",
+ active_tab="model-monitoring",
+ className="nav-tabs border-0",
+ ),
+ width="auto",
+ className="d-flex align-items-end"
+ )
+ ]),
+ ],
+ fluid=True,
+ className="px-4"),
+ ],
+ color="primary",
+ dark=True
+ )],
+ className="mb-4")
+
# Main content
- main_content = dbc.Container([
+ model_monitoring_tab = dbc.Container([
# Row for the entire dashboard content
dbc.Row([
# Left column - fixed width
@@ -125,10 +138,72 @@ class ForecastMonitor:
#self._make_section("Input Forecasts",'input-forecasts','inp-fcst-view'),
# Recent Logs Section
self._make_section("Logs",'logs','log-view'),
+ # Metrics Section
+ self._make_section("Metriken", 'metrics', 'metrics-view'),
], xs=12, md=7) # Responsive column
])
], fluid=True, className="py-4")
+ # System Logs Tab Content
+ system_logs_tab = dbc.Container([
+ dbc.Card([
+ dbc.CardBody([
+ html.H3("System Logs", className="mb-4"),
+ dcc.Interval( #TODO rausnehmen?
+ id='logs-update',
+ interval=30000 # Update every 30 seconds
+ ),
+ dash_table.DataTable(
+ id='system-log-table',
+ columns=[
+ {'name': 'Zeitstempel', 'id': 'timestamp'},
+ {'name': 'Level', 'id': 'level'},
+ {'name': 'Pegel', 'id': 'sensor'},
+ {'name': 'Nachricht', 'id': 'message'},
+ {'name': 'Modul', 'id': 'module'},
+ {'name': 'Funktion', 'id': 'function'},
+ {'name': 'Zeile', 'id': 'line'},
+ {'name': 'Exception', 'id': 'exception'}
+ ],
+ **TABLE_STYLE,
+ page_size=20,
+ page_action='native',
+ sort_action='native',
+ sort_mode='multi',
+ filter_action='native',
+ )
+ ])
+ ])
+ ], fluid=True, className="py-4")
+
+ # Add a container to hold the tab content
+ tab_content = html.Div(
+ id="tab-content",
+ children=[
+ dbc.Container(
+ id="page-content",
+ children=[],
+ fluid=True,
+ className="py-4"
+ )
+ ]
+ )
+ # Main content with tabs
+ main_content = dbc.Tabs([
+ dbc.Tab(model_monitoring_tab, label="Modell Monitoring", tab_id="model-monitoring"),
+ dbc.Tab(system_logs_tab, label="System Logs", tab_id="system-logs")
+ ], id="main-tabs", active_tab="model-monitoring")
+
+ # Combine all elements
+ self.app.layout = html.Div([
+ header,
+ html.Div(id="tab-content", children=[
+ model_monitoring_tab,
+ system_logs_tab
+ ], style={"display": "none"}), # Hidden container for both tabs
+ html.Div(id="visible-tab-content") # Container for the visible tab
+ ], className="min-vh-100 bg-light")
+
# Combine all elements
self.app.layout = html.Div([
header,
@@ -279,7 +354,7 @@ class ForecastMonitor:
# Get all sensors for this model
model_sensors = session.query(ModellSensor).filter(
ModellSensor.modell_id == model_id
- ).all()
+ ).all()
sensor_names = [ms.sensor_name for ms in model_sensors]
@@ -299,6 +374,32 @@ class ForecastMonitor:
except Exception as e:
print(f"Error getting historical data: {str(e)}")
return pd.DataFrame()
+
+ def get_model_metrics(self, model_id, metric_name='mae'):
+ """Get metrics for a specific model. Assumes that each model only has on set of metrics with the same name"""
+
+ try:
+
+ metric_names = [f"train_{metric_name}", f"test_{metric_name}", f"val_{metric_name}",
+ f"train_{metric_name}_flood", f"test_{metric_name}_flood", f"val_{metric_name}_flood"]
+ # Get metrics for the model
+ stmt = select(Metric).where(Metric.model_id == model_id,Metric.metric_name.in_(metric_names))
+
+ df = pd.read_sql(sql=stmt, con=self.engine)
+
+ train_start,train_end = df[df["metric_name"] == f"train_{metric_name}"][["start_time","end_time"]].iloc[0]
+ test_start,test_end = df[df["metric_name"] == f"test_{metric_name}"][["start_time","end_time"]].iloc[0]
+ times = [train_start,train_end,test_start,test_end]
+
+ # Group metrics by tag and timestep
+ df = df.pivot(index="timestep",columns="metric_name",values="value")
+
+ return df, times
+ except Exception as e:
+ print(f"Error getting model metrics: {str(e)}")
+ return None, None
+
+
def setup_callbacks(self):
@@ -361,7 +462,7 @@ class ForecastMonitor:
[Output('fcst-view', 'children'),
Output('historical-view', 'children'),
Output('log-view', 'children'),
- #Output('inp-fcst-view', 'children'),
+ Output('metrics-view', 'children'),
Output('current-sensor-names', 'data')], # Removed input-forecasts-view
[Input('status-table', 'selected_rows')],
[State('status-table', 'data')]
@@ -370,8 +471,8 @@ class ForecastMonitor:
if not selected_rows:
return (html.Div("Wähle ein Modell um Vorhersagen anzuzeigen."),
html.Div("Wähle ein Modell um Messwerte anzuzeigen."),
- #html.Div("Wähle ein Modell um Eingangsvorhersagen anzuzeigen."),
html.Div("Wähle ein Modell um Logs anzuzeigen."),
+ html.Div("Wähle ein Modell um Metriken anzuzeigen."),
None) # Removed input forecasts return
selected_row = table_data[selected_rows[0]]
@@ -409,21 +510,6 @@ class ForecastMonitor:
historical_view = html.Div("Keine Messdaten verfügbar")
- ## Get Input Forecasts
- #df_inp_fcst = self.get_input_forecasts(sensor_names)
- #if not df_inp_fcst.empty:
- # fig_inp_fcst = create_input_forecasts_plot(df_inp_fcst,df_historical)
- # inp_fcst_table = create_inp_forecast_status_table(df_inp_fcst)
- # inp_fcst_view = html.Div([
- # html.H4(f"Input Forecasts for {model_name}"),
- # dcc.Graph(figure=fig_inp_fcst),
- # html.H4("Input Forecast Status", style={'marginTop': '20px', 'marginBottom': '10px'}),
- # html.Div(inp_fcst_table, style={'width': '100%', 'padding': '10px'})
- # ])
- #else:
- # inp_fcst_view = html.Div("No input forecasts available")
-
-
# Get forecast data
df_inp_fcst,ext_forecast_names = self.get_input_forecasts(sensor_names)
inp_fcst_table = create_inp_forecast_status_table(df_inp_fcst,ext_forecast_names)
@@ -439,12 +525,33 @@ class ForecastMonitor:
else:
fcst_view = html.Div("No forecasts available")
+ # Get metrics
+ metrics_view = html.Div([
+ html.H4(f"Modell Metriken für {model_name}"),
+ dbc.Row([
+ dbc.Col([
+ html.Label("Metric:", className="fw-bold mb-2"),
+ dcc.Dropdown(
+ id={'type': 'metric-selector', 'section': 'metrics'},
+ options=[
+ {'label': metric.upper(), 'value': metric}
+ for metric in ['mae', 'mse', 'nse', 'kge', 'p10', 'p20', 'r2', 'rmse', 'wape']
+ ],
+ value='mae',
+ clearable=False,
+ className="mb-4"
+ )
+ ], width=4)
+ ]),
+ html.Div(id='metric-plots-container'),
+ html.Div(id='metric-info-container', className="mt-4")
+ ])
return (fcst_view,
historical_view,
log_view,
- #inp_fcst_view,
- sensor_names) # Removed input forecasts return
+ metrics_view,
+ sensor_names)
@self.app.callback(
@@ -478,6 +585,83 @@ class ForecastMonitor:
return df_filtered.to_dict('records')
+
+ # Add new callback for updating metric plots
+ @self.app.callback(
+ [Output('metric-plots-container', 'children'),
+ Output('metric-info-container', 'children')],
+ [Input({'type': 'metric-selector', 'section': 'metrics'}, 'value'),
+ Input('status-table', 'selected_rows')],
+ [State('status-table', 'data')]
+ )
+ def update_metric_plots(metric_name, selected_rows, table_data):
+ if not selected_rows or not metric_name:
+ return html.Div(), html.Div()
+
+ selected_row = table_data[selected_rows[0]]
+ model_id = selected_row['model_id']
+
+ metrics_data, edge_times = self.get_model_metrics(model_id, metric_name)
+ if metrics_data.empty:
+ return html.Div("Keine Metriken verfügbar"), html.Div()
+
+ fig = create_metrics_plots(metrics_data, metric_name)
+
+ plots = html.Div([dcc.Graph(figure=fig, className="mb-4")])
+
+ info = html.Div([
+ html.H5("Zeitraum der Metrikberechnung:", className="h6 mb-3"),
+ dbc.Row([
+ dbc.Col([
+ html.Strong("Start Trainingszeitraum: "),
+ html.Span(edge_times[0].strftime('%Y-%m-%d %H:%M:%S'))
+ ], width=6),
+ dbc.Col([
+ html.Strong("Start Validierungszeitraum: "),
+ html.Span(edge_times[1].strftime('%Y-%m-%d %H:%M:%S'))
+ ], width=6),
+ dbc.Col([
+ html.Strong("Start Testzeitraum: "),
+ html.Span(edge_times[2].strftime('%Y-%m-%d %H:%M:%S'))
+ ], width=6),
+ dbc.Col([
+ html.Strong("Ende Testzeitraum: "),
+ html.Span(edge_times[3].strftime('%Y-%m-%d %H:%M:%S'))
+ ], width=6)
+ ])
+ ], className="bg-light p-3 rounded")
+
+ return plots, info
+
+ @self.app.callback(
+ Output('system-log-table', 'data'),
+ Input('logs-update', 'n_intervals')
+ )
+ def update_system_logs(n):
+ """Get all logs from the database"""
+ try:
+ with Session(self.engine) as session:
+ # Get last 1000 logs
+ logs = session.query(Log).order_by(
+ desc(Log.created)
+ ).limit(1000).all()
+
+ return [{
+ 'timestamp': log.created.strftime('%Y-%m-%d %H:%M:%S'),
+ 'level': log.loglevelname,
+ 'sensor': log.gauge,
+ 'message': log.message,
+ 'module': log.module,
+ 'function': log.funcname,
+ 'line': log.lineno,
+ 'exception': log.exception or ''
+ } for log in logs]
+
+ except Exception as e:
+ print(f"Error getting system logs: {str(e)}")
+ return []
+
+
# Update the collapse callback
@self.app.callback(
[Output({'type': 'collapse-content', 'section': MATCH}, 'is_open'),
diff --git a/src/dashboard/styles.py b/src/dashboard/styles.py
index 3ac83db..93c40ba 100644
--- a/src/dashboard/styles.py
+++ b/src/dashboard/styles.py
@@ -7,12 +7,17 @@ COLORS = {
'danger': '#dc3545',
'light_gray': '#f4f4f4',
'border': 'rgba(0,0,0,.125)',
- 'background': '#f8f9fa'
+ 'background': '#f8f9fa',
+ 'warning_bg': '#fff3e0',
+ 'error_bg': '#ffebee',
+ 'warning' : '#ef6c00',
+ 'error': '#c62828',
}
+
COLOR_SET = px.colors.qualitative.Set3
-# Table Styles
+# Table Styles (This is messy, the conditional style data is not the same for each table)
TABLE_STYLE = {
'style_table': {
'overflowX': 'auto',
@@ -43,10 +48,33 @@ TABLE_STYLE = {
{
'if': {'filter_query': '{has_current_forecast} = "✗"', "column_id": "has_current_forecast"},
'color': COLORS['danger']
+ },
+ {
+ 'if': {'filter_query': '{level} = "ERROR"'},
+ 'backgroundColor': COLORS['error_bg'],
+ 'color': COLORS['error']
+ },
+ {
+ 'if': {'filter_query': '{level} = "WARNING"'},
+ 'backgroundColor': COLORS['warning_bg'],
+ 'color': COLORS['warning']
}
]
}
+TAB_STYLE = {
+ "active_label_class_name": 'fw-bold',
+ "label_style": {
+ 'color': 'rgba(255, 255, 255, 0.9)',
+ 'padding': '1rem 1.5rem'
+ },
+ "active_label_style": {
+ 'color': 'white',
+ 'background': 'rgba(255, 255, 255, 0.1)'
+ }
+ }
+
+
# Historical Table Style
HISTORICAL_TABLE_STYLE = {
'style_table': {
diff --git a/src/dashboard/tables_and_plots.py b/src/dashboard/tables_and_plots.py
index 300fad2..7348ee4 100644
--- a/src/dashboard/tables_and_plots.py
+++ b/src/dashboard/tables_and_plots.py
@@ -330,7 +330,7 @@ def create_inp_forecast_status_table(df_forecast,ext_forecast_names):
dash.html.Div: Div containing configured DataTable
"""
# Get unique sensor names
- sensor_names = sorted(df_forecast['sensor_name'].unique())
+ #sensor_names = sorted(df_forecast['sensor_name'].unique())
# Create index of last 48 hours at 3-hour intervals
last_required_hour = datetime.now().replace(minute=0, second=0, microsecond=0)
@@ -384,112 +384,54 @@ def create_inp_forecast_status_table(df_forecast,ext_forecast_names):
)
)
-def create_input_forecasts_plot(df_forecast, df_historical):
- """
- Creates a figure with subplots for each sensor, showing forecast lines and historical data
-
- Args:
- df_forecast: DataFrame with columns tstamp, sensor_name, member, h1-h48
- df_historical: DataFrame with sensor_name columns and timestamp index
-
- Returns:
- plotly.graph_objects.Figure: Figure with subplots
- """
- # Get unique sensors and members
- sensors = df_forecast['sensor_name'].unique()
-
- # Create a color sequence for different forecast start times
- colors = px.colors.qualitative.Set3
+
+def create_metrics_plots(df_metrics, metric_name):
+ """Create plots for model metrics"""
- # Create subplot figure
fig = make_subplots(
- rows=len(sensors),
+ rows=2,
cols=1,
- subplot_titles=[f'Sensor: {sensor}' for sensor in sensors],
+ subplot_titles=["Gesamt", "Flutbereich (95. Percentil)"],
vertical_spacing=0.1
)
- # For each sensor
- for sensor_idx, sensor in enumerate(sensors, 1):
- # Add historical data
- if sensor in df_historical.columns:
- fig.add_trace(
- go.Scatter(
- x=df_historical.index,
- y=df_historical[sensor],
- name=sensor,
- legendgroup=sensor,
- showlegend=True, # Show in legend for all sensors
- line=dict(color='black', width=2),
- hovertemplate='Time: %{x}<br>Value: %{y:.2f}<br>Historical<extra></extra>'
- ),
- row=sensor_idx,
- col=1
- )
-
- sensor_data = df_forecast[df_forecast['sensor_name'] == sensor]
- members = sensor_data['member'].unique()
+ for col in df_metrics:
+ row = 2 if 'flood' in col else 1
+ set_name = col.split("_")[0]
+ if set_name == 'train':
+ color = COLOR_SET[0]
+ elif set_name == 'val':
+ color = COLOR_SET[1]
+ elif set_name == 'test':
+ color = COLOR_SET[2]
+ else:
+ color = COLOR_SET[3]
-
- # Get unique forecast start times for color assignment
- start_times = sorted(sensor_data['tstamp'].unique())
- start_time_colors = {t: COLOR_SET[i % len(COLOR_SET)] for i, t in enumerate(start_times)}
-
- # For each member
- for member in members:
- member_data = sensor_data[sensor_data['member'] == member]
-
- # For each forecast (row)
- for _, row in member_data.iterrows():
- start_time = row['tstamp']
- legend_group = f'{sensor} {start_time.strftime("%Y%m%d%H%M")}'
- timestamps = [start_time + timedelta(hours=i) for i in range(1, 49)]
- values = [row[f'h{i}'] for i in range(1, 49)]
-
- fig.add_trace(
- go.Scatter(
- x=timestamps,
- y=values,
- name=f'{start_time.strftime("%Y-%m-%d %H:%M")}',
- legendgroup=legend_group,
- showlegend=(member == members[0]).item(),
- line=dict(
- color=start_time_colors[start_time],
- width=1,
- dash='solid' if member == members[0] else 'dot'
- ),
- hovertemplate=(
- 'Time: %{x}<br>'
- 'Value: %{y:.2f}<br>'
- f'Member: {member}<br>'
- f'Start: {start_time}<extra></extra>'
- )
- ),
- row=sensor_idx,
- col=1
- )
-
- # Update layout
+ fig.add_trace(
+ go.Scatter(
+ x=df_metrics.index,
+ y=df_metrics[col],
+ name=set_name,
+ legendgroup=set_name,
+ showlegend= row==1,
+ mode='lines+markers',
+ line={"color":color},
+ ),
+ row=row,
+ col=1
+ )
+
fig.update_layout(
- height=400 * len(sensors), # Adjust height based on number of sensors
- #title='Forecast Values by Sensor with Historical Data',
+ xaxis_title='Vorhersagehorizont',
+ yaxis_title=metric_name.upper(),
+ height=600,
showlegend=True,
- legend=dict(
- yanchor="top",
- y=0.99,
- xanchor="left",
- x=1.05,
- groupclick="togglegroup", # Allows clicking group title to toggle all traces
- itemsizing='constant', # Makes legend items constant size
- tracegroupgap=5 # Add small gap between groups
- ),
- margin=dict(r=150)
+ legend={
+ "yanchor": 'top',
+ "y": 0.99,
+ "xanchor": 'left',
+ "x": 1.05},
+ margin={"r":50}
)
-
- # Update all x and y axes labels
- for i in range(len(sensors)):
- fig.update_xaxes(title_text="Time", row=i+1, col=1)
- fig.update_yaxes(title_text="Value", row=i+1, col=1)
-
- return fig
\ No newline at end of file
+ return fig
--
GitLab