Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
K
KI-WaVo
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Package registry
Container registry
Model registry
Operate
Environments
Terraform modules
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
GitLab community forum
Contribute to GitLab
Provide feedback
Terms and privacy
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
Landesamt für Umwelt
KI-WaVo
Commits
3c3c00d6
Commit
3c3c00d6
authored
8 months ago
by
Michel Spils
Browse files
Options
Downloads
Patches
Plain Diff
forecast view
parent
1f5937c9
Branches
Branches containing commit
No related tags found
No related merge requests found
Changes
2
Show whitespace changes
Inline
Side-by-side
Showing
2 changed files
src/dash_tools/style_configs.py
+94
-0
94 additions, 0 deletions
src/dash_tools/style_configs.py
src/dashboard.py
+83
-17
83 additions, 17 deletions
src/dashboard.py
with
177 additions
and
17 deletions
src/dash_tools/style_configs.py
+
94
−
0
View file @
3c3c00d6
...
...
@@ -7,6 +7,100 @@ from datetime import timedelta, datetime
import
plotly.express
as
px
import
pandas
as
pd
def
create_model_forecasts_plot
(
df_forecasts
,
df_historical
):
"""
Create a plot showing recent model forecasts alongside historical data
"""
#TODO drop empty rows?
sensor_name
=
df_forecasts
[
"
sensor_name
"
][
0
]
colors
=
px
.
colors
.
qualitative
.
Set3
start_times
=
sorted
(
df_forecasts
[
"
tstamp
"
].
unique
())
start_time_colors
=
{
t
:
colors
[
i
%
len
(
colors
)]
for
i
,
t
in
enumerate
(
start_times
)}
fig
=
go
.
Figure
()
df_measured
=
df_historical
[
start_times
[
0
]
-
timedelta
(
days
=
3
):]
# Add historical data
fig
.
add_trace
(
go
.
Scatter
(
x
=
df_measured
.
index
,
y
=
df_measured
[
sensor_name
],
name
=
f
"
Historical -
{
sensor_name
}
"
,
line
=
dict
(
color
=
'
black
'
,
width
=
2
),
mode
=
'
lines
'
))
df_forecasts
=
df_forecasts
.
round
(
1
)
members
=
df_forecasts
[
'
member
'
].
unique
()
for
member
in
members
:
member_data
=
df_forecasts
[
df_forecasts
[
'
member
'
]
==
member
]
for
_
,
row
in
member_data
.
iterrows
():
start_time
=
row
[
'
tstamp
'
]
legend_group
=
f
'
{
start_time
.
strftime
(
"
%Y%m%d%H%M
"
)
}
'
legendgrouptitle_text
=
f
'
{
start_time
.
strftime
(
"
%Y-%m-%d %H
:
%
M
"
)
}
'
# Convert h1-h48 columns to forecast points
timestamps
=
[
start_time
+
timedelta
(
hours
=
i
)
for
i
in
range
(
1
,
49
)]
#values = row[5:]
values
=
[
row
[
f
'
h
{
i
}
'
]
for
i
in
range
(
1
,
49
)]
fig
.
add_trace
(
go
.
Scatter
(
x
=
timestamps
,
y
=
values
,
name
=
f
'
{
start_time
.
strftime
(
"
%Y-%m-%d %H
:
%
M
"
)
}
'
,
#name=f'M{member} {start_time.strftime("%Y-%m-%d %H:%M")}',
legendgroup
=
legend_group
,
showlegend
=
(
member
==
members
[
0
]).
item
(),
# and sensor_idx == 1, # Show all traces in legend
line
=
dict
(
color
=
start_time_colors
[
start_time
],
width
=
1
,
#dash='solid' if member == members[0] else 'dot'
),
hovertemplate
=
(
'
Time: %{x}<br>
'
'
Value: %{y:.2f}<br>
'
f
'
Member:
{
member
}
<br>
'
f
'
Start:
{
start_time
}
<extra></extra>
'
)
),
)
#fig.add_trace(go.Scatter(
# x=timestamps,
# y=row[5:],
# name=f"Forecast Member {member} ({row['tstamp']})",
# line=dict(color=colors[member % len(colors)], width=1, dash='dot'),
# opacity=1,
# showlegend=True
#))
####
#sensor_data = df_forecast[df_forecast['sensor_name'] == sensor]
#members = sensor_data['member'].unique()
# Add trace to the subplot
####
fig
.
update_layout
(
height
=
600
,
title
=
'
Model Forecasts
'
,
xaxis_title
=
'
Time
'
,
yaxis_title
=
'
Gauge [cm]
'
,
#hovermode='x unified',
legend
=
dict
(
yanchor
=
"
top
"
,
y
=
0.99
,
xanchor
=
"
left
"
,
x
=
1.05
,
#bgcolor='rgba(255, 255, 255, 0.8)'
)
)
return
fig
def
create_log_table
(
logs_data
):
"""
Creates a configured DataTable for logging display
...
...
This diff is collapsed.
Click to expand it.
src/dashboard.py
+
83
−
17
View file @
3c3c00d6
from
re
import
A
from
dash
import
html
,
dcc
,
dash_table
,
Dash
from
dash.dependencies
import
Input
,
Output
,
State
,
MATCH
import
plotly.express
as
px
...
...
@@ -6,15 +7,20 @@ import pandas as pd
from
datetime
import
datetime
,
timedelta
from
sqlalchemy
import
create_engine
,
select
,
and_
,
desc
from
sqlalchemy.orm
import
Session
from
utils.db_tools.orm_classes
import
Base
,
InputForecasts
,
Modell
,
PegelForecasts
,
Sensor
,
Log
,
ModellSensor
,
SensorData
from
utils.db_tools.orm_classes
import
InputForecasts
,
Modell
,
PegelForecasts
,
Log
,
ModellSensor
,
SensorData
import
oracledb
import
os
from
sqlalchemy
import
select
,
func
from
dash_tools.style_configs
import
create_log_table
,
create_historical_plot
,
create_historical_table
,
create_input_forecasts_plot
,
create_inp_forecast_status_table
from
dash_tools.style_configs
import
(
create_log_table
,
create_historical_plot
,
create_historical_table
,
create_input_forecasts_plot
,
create_inp_forecast_status_table
,
create_model_forecasts_plot
)
from
dash_tools.layout_helper
import
create_collapsible_section
NUM_RECENT_INPUT_FORECASTS
=
12
NUM_RECENT_PEGEL_FORECASTS
=
5
class
ForecastMonitor
:
...
...
@@ -27,7 +33,7 @@ class ForecastMonitor:
self
.
con
=
oracledb
.
connect
(
**
self
.
db_params
)
self
.
engine
=
create_engine
(
"
oracle+oracledb://
"
,
creator
=
lambda
:
self
.
con
)
self
.
app
=
Dash
(
__name__
)
self
.
app
=
Dash
(
__name__
,
suppress_callback_exceptions
=
True
)
self
.
setup_layout
()
self
.
setup_callbacks
()
...
...
@@ -86,6 +92,38 @@ class ForecastMonitor:
print
(
f
"
Error getting model status:
{
str
(
e
)
}
"
)
return
None
def
get_recent_forecasts
(
self
,
actual_model_name
,
limit
=
NUM_RECENT_PEGEL_FORECASTS
):
"""
Get recent forecasts for a specific model and sensor
"""
try
:
subq
=
(
select
(
PegelForecasts
,
func
.
row_number
()
.
over
(
partition_by
=
[
PegelForecasts
.
member
],
order_by
=
PegelForecasts
.
tstamp
.
desc
()
).
label
(
'
rn
'
))
.
where
(
PegelForecasts
.
model
==
actual_model_name
)
.
subquery
()
)
stmt
=
(
select
(
subq
)
.
where
(
subq
.
c
.
rn
<=
NUM_RECENT_PEGEL_FORECASTS
)
.
order_by
(
subq
.
c
.
member
,
subq
.
c
.
tstamp
.
desc
()
)
)
df
=
pd
.
read_sql
(
sql
=
stmt
,
con
=
self
.
engine
)
df
.
drop
(
columns
=
[
'
rn
'
],
inplace
=
True
)
return
df
except
Exception
as
e
:
print
(
f
"
Error getting recent forecasts:
{
str
(
e
)
}
"
)
return
pd
.
DataFrame
()
def
get_input_forecasts
(
self
,
sensor_names
):
"""
Get 3 most recent input forecasts for the given sensor names
"""
try
:
...
...
@@ -120,7 +158,7 @@ class ForecastMonitor:
return
df
except
Exception
as
e
:
raise
RuntimeError
(
f
"
Error getting input forecasts:
{
str
(
e
)
}
"
)
raise
RuntimeError
(
f
"
Error getting input forecasts:
{
str
(
e
)
}
"
)
from
e
def
get_recent_logs
(
self
,
sensor_name
):
...
...
@@ -164,7 +202,7 @@ class ForecastMonitor:
# Get last 144 hours of sensor data
time_threshold
=
datetime
.
now
()
-
timedelta
(
hours
=
144
)
#
time_threshold= pd.to_datetime("2024-09-13 14:00:00.000") - timedelta(hours=144) #TODO rausnehmen
time_threshold
=
pd
.
to_datetime
(
"
2024-09-13 14:00:00.000
"
)
-
timedelta
(
hours
=
144
)
#TODO rausnehmen
stmt
=
select
(
SensorData
).
where
(
SensorData
.
tstamp
>=
time_threshold
,
...
...
@@ -177,7 +215,7 @@ class ForecastMonitor:
except
Exception
as
e
:
print
(
f
"
Error getting historical data:
{
str
(
e
)
}
"
)
return
[],
[]
return
pd
.
DataFrame
()
def
setup_layout
(
self
):
self
.
app
.
layout
=
html
.
Div
([
...
...
@@ -223,7 +261,11 @@ class ForecastMonitor:
# Right column
html
.
Div
([
dcc
.
Store
(
id
=
'
current-sensor-names
'
),
# Store current sensor names
create_collapsible_section
(
"
Model Forecasts
"
,
html
.
Div
(
id
=
'
fcst-view
'
),
is_open
=
True
),
create_collapsible_section
(
"
Recent Logs
"
,
html
.
Div
(
id
=
'
log-view
'
),
...
...
@@ -329,7 +371,10 @@ class ForecastMonitor:
{
'
name
'
:
'
Last Valid
'
,
'
id
'
:
'
last_forecast_time
'
},
{
'
name
'
:
'
Created
'
,
'
id
'
:
'
forecast_created
'
},
{
'
name
'
:
'
Target Sensor
'
,
'
id
'
:
'
sensor_name
'
},
{
'
name
'
:
'
model_id
'
,
'
id
'
:
'
model_id
'
,
'
hideable
'
:
True
}
{
'
name
'
:
'
model_id
'
,
'
id
'
:
'
model_id
'
,
'
hideable
'
:
True
},
#{'name': 'actual_model_name', 'id': 'actual_model_name', 'hidden': True}
#{'name': 'actual_model_name', 'id': 'actual_model_name', 'visible': False}
{
'
name
'
:
'
actual_model_name
'
,
'
id
'
:
'
actual_model_name
'
,
'
hideable
'
:
True
}
],
data
=
[{
'
model_name
'
:
row
[
'
model_name
'
],
...
...
@@ -337,8 +382,10 @@ class ForecastMonitor:
'
last_forecast_time
'
:
row
[
'
last_forecast_time
'
].
strftime
(
'
%Y-%m-%d %H:%M:%S
'
)
if
row
[
'
last_forecast_time
'
]
else
'
No valid forecast
'
,
'
forecast_created
'
:
row
[
'
forecast_created
'
].
strftime
(
'
%Y-%m-%d %H:%M:%S
'
)
if
row
[
'
forecast_created
'
]
else
'
N/A
'
,
'
sensor_name
'
:
row
[
'
sensor_name
'
],
'
model_id
'
:
row
[
'
model_id
'
]
'
model_id
'
:
row
[
'
model_id
'
],
'
actual_model_name
'
:
row
[
'
actual_model_name
'
]
}
for
row
in
status
[
'
model_status
'
]],
hidden_columns
=
[
'
model_id
'
,
'
actual_model_name
'
],
# Specify hidden columns here
style_data_conditional
=
[
{
'
if
'
:
{
'
filter_query
'
:
'
{has_current_forecast} =
"
✓
"'
,
"
column_id
"
:
"
has_current_forecast
"
},
...
...
@@ -380,7 +427,8 @@ class ForecastMonitor:
@self.app.callback
(
[
Output
(
'
log-view
'
,
'
children
'
),
[
Output
(
'
fcst-view
'
,
'
children
'
),
Output
(
'
log-view
'
,
'
children
'
),
Output
(
'
historical-view
'
,
'
children
'
),
Output
(
'
inp-fcst-view
'
,
'
children
'
),
Output
(
'
current-sensor-names
'
,
'
data
'
)],
# Removed input-forecasts-view
...
...
@@ -389,7 +437,8 @@ class ForecastMonitor:
)
def
update_right_column
(
selected_rows
,
table_data
):
if
not
selected_rows
:
return
(
html
.
Div
(
"
Select a model to view logs
"
),
return
(
html
.
Div
(
"
Select a model to view Forecasts
"
),
html
.
Div
(
"
Select a model to view logs
"
),
html
.
Div
(
"
Select a model to view Input Forecasts
"
),
html
.
Div
(
"
Select a model to view historical data
"
),
None
)
# Removed input forecasts return
...
...
@@ -398,6 +447,7 @@ class ForecastMonitor:
sensor_name
=
selected_row
[
'
sensor_name
'
]
model_id
=
selected_row
[
'
model_id
'
]
model_name
=
selected_row
[
'
model_name
'
]
actual_model_name
=
selected_row
[
'
actual_model_name
'
]
# Get logs
logs
=
self
.
get_recent_logs
(
sensor_name
)
...
...
@@ -407,6 +457,8 @@ class ForecastMonitor:
log_table
])
# Get historical data
df_historical
=
self
.
get_historical_data
(
model_id
)
sensor_names
=
list
(
df_historical
.
columns
)
...
...
@@ -423,21 +475,35 @@ class ForecastMonitor:
else
:
historical_view
=
html
.
Div
(
"
No historical data available
"
)
# Get forecast data
df_forecasts
=
self
.
get_recent_forecasts
(
actual_model_name
)
if
not
df_forecasts
.
empty
:
fig_fcst
=
create_model_forecasts_plot
(
df_forecasts
,
df_historical
)
fcst_view
=
html
.
Div
([
html
.
H4
(
f
"
Gauge Forecasts for
{
model_name
}
"
),
dcc
.
Graph
(
figure
=
fig_fcst
),
#html.H4("Input Forecast Status", style={'marginTop': '20px', 'marginBottom': '10px'}),
#html.Div(inp_fcst_table, style={'width': '100%', 'padding': '10px'})
])
else
:
fcst_view
=
html
.
Div
(
"
No forecasts available
"
)
# Get Input Forecasts
df_inp_fcst
=
self
.
get_input_forecasts
(
sensor_names
)
if
not
df_inp_fcst
.
empty
:
fig_fcst
=
create_input_forecasts_plot
(
df_inp_fcst
,
df_historical
)
fig_
inp_
fcst
=
create_input_forecasts_plot
(
df_inp_fcst
,
df_historical
)
inp_fcst_table
=
create_inp_forecast_status_table
(
df_inp_fcst
)
inp_fcst_view
=
html
.
Div
([
html
.
H4
(
f
"
Input Forecasts for
{
model_name
}
"
),
dcc
.
Graph
(
figure
=
fig_fcst
),
dcc
.
Graph
(
figure
=
fig_
inp_
fcst
),
html
.
H4
(
"
Input Forecast Status
"
,
style
=
{
'
marginTop
'
:
'
20px
'
,
'
marginBottom
'
:
'
10px
'
}),
html
.
Div
(
inp_fcst_table
,
style
=
{
'
width
'
:
'
100%
'
,
'
padding
'
:
'
10px
'
})
])
else
:
inp_fcst_view
=
html
.
Div
(
"
No input forecasts available
"
)
return
(
log_view
,
return
(
fcst_view
,
log_view
,
historical_view
,
inp_fcst_view
,
sensor_names
)
# Removed input forecasts return
...
...
@@ -470,5 +536,5 @@ if __name__ == '__main__':
dsn
=
"
localhost/XE
"
)
#monitor.run(host="172.17.0.1", port=8050, debug=True)
monitor
.
run
(
host
=
"
134.245.232.166
"
,
port
=
8050
,
debug
=
True
)
\ No newline at end of file
#monitor.run(host="134.245.232.166", port=8050, debug=True)
monitor
.
run
()
\ No newline at end of file
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment