diff --git a/.gitignore b/.gitignore index ce97fe0..5830bc8 100644 --- a/.gitignore +++ b/.gitignore @@ -37,25 +37,11 @@ logs/*.jsonl .states/ # SQLite database (will contain local data) -*.db -*.sqlite +#*.db +#*.sqlite # Snowflake result cache data/cache/ -# Uploaded data files -data/uploads/ - -# Exported analysis results -data/exports/ - -# Analysis output files -output/*.html -output/*.csv -*.html - # VS Code workspace settings .vscode/ - -# User uploaded files -uploaded_files/ diff --git a/01_nhs_classic.html b/01_nhs_classic.html new file mode 100644 index 0000000..6362d0a --- /dev/null +++ b/01_nhs_classic.html @@ -0,0 +1,551 @@ + + + + + +HCD Analysis — NHS Classic + + + + + + +
+
+ +
+
HCD Analysis
+
+
+
+ Dashboard › Pathway Analysis +
+
+ 656,247 records + Last updated: 2h ago +
+
+ + + + + +
+ +
+
+
Unique Patients
+
37,842
+
across all trusts
+
+
+
Drug Types
+
89
+
high-cost drugs tracked
+
+
+
Total Cost
+
£145.2M
+
current period spend
+
+
+
Indication Match
+
93%
+
GP diagnosis confirmed
+
+
+ + +
+
+ View +
+ + +
+
+
+
+ Initiated + +
+
+ Last seen + +
+
+
+ Drugs + +
+
+ Directorates + +
+
+ + +
+
+
+
Patient Pathway Visualization
+
Trust → Directorate → Drug → Patient Pathway
+
+
+
+ + + +
+ + + +
+
+ + + + + + + diff --git a/RALPH_PROMPT.md b/RALPH_PROMPT.md index 13b9c7d..26eb409 100644 --- a/RALPH_PROMPT.md +++ b/RALPH_PROMPT.md @@ -2,7 +2,7 @@ You are operating inside an automated loop improving Plotly charts in an NHS patient pathway analysis Dash application. Each iteration you receive fresh context — you have NO memory of previous iterations. Your only memory is the filesystem. -**Current Focus**: Fix chart bugs, improve visual polish, add new analytics charts. See IMPLEMENTATION_PLAN.md for the full task list organized into Phases A–D. +**Current Focus**: Phase E — Redesign temporal trends as a standalone 3rd view with directorate overview + drug drill-down, fix chart height, rename cost labels. See IMPLEMENTATION_PLAN.md for the full task list organized into Phases A–E. ## First Actions Every Iteration @@ -27,8 +27,16 @@ Then run `git log --oneline -5` to see recent commits. - `dash_app/data/queries.py` — Thin wrappers. Add wrapper for each new query. - `dash_app/components/chart_card.py` — TAB_DEFINITIONS for Patient Pathways tabs. +**When working on the Trends view** (Phase E), also read: +- `dash_app/components/trends.py` — Trends landing + detail components (create if doesn't exist) +- `dash_app/callbacks/trends.py` — Trends view callbacks (create if doesn't exist) +- `dash_app/components/sidebar.py` — Sidebar navigation (3 items: Patient Pathways, Trust Comparison, Trends) +- `dash_app/callbacks/navigation.py` — View switching (3-way) +- `dash_app/callbacks/filters.py` — `update_app_state()` handles nav clicks +- `dash_app/app.py` — Layout with 3 view containers + app-state initial data + **When modifying UI components**, read: -- `dash_app/components/trust_comparison.py` — TC landing + dashboard layout. +- `dash_app/components/trust_comparison.py` — TC landing + dashboard layout (reference for Trends landing/detail pattern). - `dash_app/assets/nhs.css` — All CSS styles. ## Narration @@ -72,6 +80,10 @@ Work on ONE task per iteration. Build incrementally and verify as you go. - **SQLite**: `import sqlite3` — read-only access to `data/pathways.db` - **CSS**: All in `dash_app/assets/nhs.css` — auto-served by Dash +### Plotly Skill + +**IMPORTANT**: When creating or modifying chart functions in `plotly_generator.py`, invoke the `/plotly` skill first. This loads Plotly reference documentation (chart types, graph objects, layouts, interactivity) that helps produce better chart code. Use it before writing any Plotly figure code. + ### plotly_generator.py Patterns All chart functions follow the same pattern: @@ -235,6 +247,7 @@ DO NOT output it if any task is still `[ ]` or `[B]` or `[~]`. - **New query functions** go in `src/data_processing/pathway_queries.py` with thin wrappers in `dash_app/data/queries.py` - **dcc.Store for state** — no server-side globals - **Lazy tab rendering** — only compute the active tab's chart +- **3-view architecture** — Patient Pathways, Trust Comparison, Trends (Phase E). View switching via `active_view` in app-state. - Keep commits atomic and well-described - If stuck for 2+ attempts, document in progress.txt and move on - `python run_dash.py` must work after every task diff --git a/dash_app/assets/nhs.css b/dash_app/assets/nhs.css index 76873ed..ae90035 100644 --- a/dash_app/assets/nhs.css +++ b/dash_app/assets/nhs.css @@ -187,12 +187,12 @@ body { margin-left: var(--sidebar-w); margin-top: var(--header-total-h); padding: 24px; - height: calc(100vh - var(--header-total-h)); + min-height: calc(100vh - var(--header-total-h)); display: flex; flex-direction: column; gap: 20px; overflow-y: auto; } -/* View containers — flex chain for chart to fill height */ +/* View containers */ #view-container { flex: 1; display: flex; flex-direction: column; min-height: 0; } @@ -268,12 +268,6 @@ body { #pathway-chart { flex: 1; min-height: 0; } -/* Propagate flex height into Plotly-rendered divs when no explicit figure height is set */ -#pathway-chart .js-plotly-plot, -#pathway-chart .plot-container, -#pathway-chart .svg-container { - height: 100% !important; -} .chart-card > .dash-loading-callback, .chart-card > .dash-loading-callback > div { flex: 1; display: flex; flex-direction: column; min-height: 0; diff --git a/dash_app/components/chart_card.py b/dash_app/components/chart_card.py index ae67b00..c949184 100644 --- a/dash_app/components/chart_card.py +++ b/dash_app/components/chart_card.py @@ -106,8 +106,6 @@ def make_chart_card(): children=[ dcc.Graph( id="pathway-chart", - style={"flex": "1", "minHeight": "0"}, - responsive=True, config={ "displayModeBar": True, "displaylogo": False, diff --git a/dash_app/components/trends.py b/dash_app/components/trends.py index 3fe5992..2713ca6 100644 --- a/dash_app/components/trends.py +++ b/dash_app/components/trends.py @@ -46,7 +46,8 @@ def make_trends_landing(): dcc.Graph( id="trends-overview-chart", config={"displayModeBar": False, "displaylogo": False}, - style={"height": "500px"}, + style={"height": "calc(100vh - 220px)", "minHeight": "400px"}, + responsive=True, ), ]), ], @@ -105,7 +106,8 @@ def make_trends_detail(): dcc.Graph( id="trends-detail-chart", config={"displayModeBar": False, "displaylogo": False}, - style={"height": "500px"}, + style={"height": "calc(100vh - 220px)", "minHeight": "400px"}, + responsive=True, ), ]), ], diff --git a/guardrails.md b/guardrails.md index 9acffc5..1c615c8 100644 --- a/guardrails.md +++ b/guardrails.md @@ -21,11 +21,16 @@ If you discover a new failure pattern during your work, add it to this file. - **Rule**: Chart figure functions go in `src/visualization/plotly_generator.py`. Query functions go in `src/data_processing/pathway_queries.py`. Dash callbacks should CALL INTO `src/`, not duplicate the code. - **Why**: Duplicating SQL queries and figure logic creates copies that drift apart. -### Do NOT modify pathways.db schema or data +### Do NOT modify pathways.db schema or data from Dash callbacks - **When**: Querying the database from Dash callbacks -- **Rule**: Read-only access. Use `sqlite3.connect(db_path)` with SELECT queries only. Never INSERT, UPDATE, DELETE, or ALTER. -- **Exception**: Phase D tasks (D.1 trends) may add new tables — this requires explicit planning. -- **Why**: pathways.db is populated by `python -m cli.refresh_pathways`. The Dash app is a read-only consumer. +- **Rule**: Read-only access from Dash. Use `sqlite3.connect(db_path)` with SELECT queries only. Never INSERT, UPDATE, DELETE, or ALTER from the Dash app. +- **Exception**: The standalone `cli/compute_trends.py` script may CREATE and INSERT into the `pathway_trends` table. This is a separate CLI command, not part of the Dash app or the main refresh pipeline. +- **Why**: pathways.db is populated by CLI commands. The Dash app is a read-only consumer. + +### Trend computation uses existing pipeline functions as-is +- **When**: Building `cli/compute_trends.py` +- **Rule**: Import and call `fetch_and_transform_data()` and `process_pathway_for_date_filter()` from `pathway_pipeline.py`. Do NOT modify these functions. Do NOT modify `schema.py`, `reference_data.py`, or `refresh_pathways.py`. The new script creates its own table via `CREATE TABLE IF NOT EXISTS`. +- **Why**: The historical snapshot approach works by calling existing functions with different `max_date` values. No pipeline changes needed. --- @@ -133,12 +138,17 @@ If you discover a new failure pattern during your work, add it to this file. - **Rule**: Always re-read `src/visualization/plotly_generator.py` at the start of the iteration. Line numbers in IMPLEMENTATION_PLAN.md are approximate and shift as edits accumulate. Search for function names, not line numbers. - **Why**: Previous iterations may have changed the file, shifting all line numbers. - +### Trends view state in app-state +- **When**: Working on the Trends view (E.2–E.4) +- **Rule**: `selected_trends_directorate` must be initialized as `None` in the `app-state` dcc.Store initial data in `app.py`. The Trends view uses landing/detail toggle based on this value (same pattern as Trust Comparison's `selected_comparison_directorate`). +- **Why**: Missing initial state causes KeyError on first page load. + +### Removing callback Outputs/Inputs requires updating ALL return paths +- **When**: Removing Outputs or Inputs from an existing callback (e.g., E.1 removing trends toggle from update_chart) +- **Rule**: When removing an Output from a callback, you MUST update EVERY `return` statement in that callback to match the new Output count. Count the number of return statements before editing and verify the same count after. The `update_chart()` callback currently has 4+ return paths. +- **Why**: Mismatched return tuple length causes `InvalidCallbackReturnValue` at runtime. diff --git a/ralph.ps1 b/ralph.ps1 index bcede68..b281984 100644 --- a/ralph.ps1 +++ b/ralph.ps1 @@ -262,6 +262,31 @@ while ($true) { Write-Host " [ERROR] API overloaded after $maxRetries retries, giving up." -ForegroundColor Red } } + # Check for usage limit with cooldown (e.g. "Usage limit reached. Reset at 3 pm") + elseif ($outputString -match "(?i)usage limit reached.*reset at (\d{1,2})(?::(\d{2}))?\s*(am|pm)") { + $resetHour = [int]$Matches[1] + $resetMinute = if ($Matches[2]) { [int]$Matches[2] } else { 0 } + $resetAmPm = $Matches[3] + + if ($resetAmPm -ieq "pm" -and $resetHour -ne 12) { $resetHour += 12 } + elseif ($resetAmPm -ieq "am" -and $resetHour -eq 12) { $resetHour = 0 } + + $now = Get-Date + $resetTime = Get-Date -Hour $resetHour -Minute $resetMinute -Second 0 + if ($resetTime -le $now) { $resetTime = $resetTime.AddDays(1) } + $resetTime = $resetTime.AddMinutes(2) + + $waitSeconds = [Math]::Ceiling(($resetTime - $now).TotalSeconds) + $waitMinutes = [Math]::Ceiling($waitSeconds / 60) + + Write-Host "" + Write-Host " [USAGE LIMIT] Reset at $($Matches[1]) $resetAmPm. Cooling down ~$waitMinutes minutes (until $($resetTime.ToString('HH:mm')))..." -ForegroundColor Yellow + Start-Sleep -Seconds $waitSeconds + Write-Host " [USAGE LIMIT] Cooldown complete. Retrying iteration..." -ForegroundColor Green + + $apiOverloaded = $true + # Don't increment retryCount — deterministic wait, not a flaky error + } } while ($apiOverloaded -and $retryCount -lt $maxRetries) $outputString | Set-Content -Path $logFile -Encoding UTF8 diff --git a/src/visualization/plotly_generator.py b/src/visualization/plotly_generator.py index a87189c..0be84c1 100644 --- a/src/visualization/plotly_generator.py +++ b/src/visualization/plotly_generator.py @@ -135,6 +135,8 @@ def _base_layout(title: str, **overrides) -> dict: plot_bgcolor="rgba(0,0,0,0)", autosize=True, font=dict(family=CHART_FONT_FAMILY), + xaxis=dict(automargin=True), + yaxis=dict(automargin=True), ) layout.update(overrides) return layout @@ -337,6 +339,7 @@ def create_icicle_from_nodes(nodes: list[dict], title: str = "") -> go.Figure: layout = _base_layout( display_title, margin=dict(t=40, l=8, r=8, b=24), + height=700, hoverlabel=dict( bgcolor="#FFFFFF", bordercolor="#CBD5E1", @@ -444,7 +447,7 @@ def create_market_share_figure(data: list[dict], title: str = "") -> go.Figure: yaxis=dict(title="", automargin=True), legend=_smart_legend(n_drugs, legend_title="Drug"), margin=dict(t=50, l=8, **legend_margins), - height=max(400, len(seen_dirs) * 60 + 200), + height=max(600, len(seen_dirs) * 60 + 200), ) fig.update_layout(**layout) @@ -607,8 +610,8 @@ def create_cost_effectiveness_figure( automargin=True, tickfont=dict(size=11), ), - margin=dict(t=50, l=8, r=24, b=40), - height=max(450, len(filtered) * 28 + 150), + margin=dict(t=50, l=8, r=80, b=40), + height=max(600, len(filtered) * 28 + 150), ) fig.update_layout(**layout) @@ -719,8 +722,10 @@ def create_cost_waterfall_figure( gridcolor=GRID_COLOR, zeroline=True, zerolinecolor="#CBD5E1", + automargin=True, ), - margin=dict(t=60, l=8, r=24, b=40), + margin=dict(t=60, l=8, r=24, b=80), + height=max(600, len(data) * 50 + 200), showlegend=False, bargap=0.25, ) @@ -833,7 +838,7 @@ def create_sankey_figure( layout.update( font=dict(family=CHART_FONT_FAMILY, size=12), margin=dict(t=60, l=30, r=30, b=30), - height=max(500, len(unique_bases) * 35 + 200), + height=max(600, len(unique_bases) * 35 + 200), ) fig.update_layout(**layout) @@ -889,7 +894,7 @@ def create_dosing_figure( ), yaxis=dict(automargin=True, tickfont=dict(size=11)), margin=dict(t=60, l=20, **legend_margins), - height=max(450, n_rows * 40 + 150), + height=max(600, n_rows * 40 + 150), bargap=0.15, bargroupgap=0.05, showlegend=True, @@ -1280,7 +1285,7 @@ def create_heatmap_figure( chart_title = f"{chart_title} — {title}" n_dirs = len(directories) - fig_height = max(400, 80 + n_dirs * 40) + fig_height = max(600, 80 + n_dirs * 40) layout = _base_layout(chart_title) layout.update( @@ -1289,6 +1294,7 @@ def create_heatmap_figure( tickfont=dict(size=11, color="#425563"), tickangle=-45, side="bottom", + automargin=True, ), yaxis=dict( title="", @@ -1426,7 +1432,7 @@ def create_duration_figure( chart_title += f"
{title}" n_bars = len(data) - fig_height = max(400, 40 + n_bars * 28) + fig_height = max(600, 40 + n_bars * 28) layout = _base_layout(chart_title) layout.update( @@ -1444,7 +1450,7 @@ def create_duration_figure( automargin=True, autorange="reversed", ), - margin=dict(t=60, l=8, r=80, b=50), + margin=dict(t=60, l=8, r=100, b=50), height=fig_height, showlegend=False, ) @@ -1652,10 +1658,10 @@ def create_trust_heatmap_figure( layout = _base_layout(chart_title) layout.update( - xaxis=dict(title="", tickfont=dict(size=11, color="#425563"), tickangle=-45, side="bottom"), + xaxis=dict(title="", tickfont=dict(size=11, color="#425563"), tickangle=-45, side="bottom", automargin=True), yaxis=dict(title="", tickfont=dict(size=12, color="#425563"), autorange="reversed", automargin=True), margin=dict(t=60, l=8, r=80, b=120), - height=max(300, 80 + n_trusts * 50), + height=max(400, 80 + n_trusts * 50), ) fig.update_layout(**layout) @@ -1814,7 +1820,7 @@ def create_retention_funnel_figure( layout.update( margin=dict(t=60, l=8, r=8, b=40), yaxis=dict(automargin=True), - height=max(300, len(data) * 80 + 120), + height=max(600, len(data) * 80 + 120), ) fig.update_layout(**layout) @@ -1884,7 +1890,7 @@ def create_pathway_depth_figure( title="Patients", gridcolor=GRID_COLOR, ), - height=max(300, len(data) * 70 + 120), + height=max(600, len(data) * 70 + 120), bargap=0.3, ) fig.update_layout(**layout) @@ -1979,6 +1985,7 @@ def create_duration_cost_scatter_figure( layout = _base_layout(display_title) layout.update( margin=dict(t=60, l=8, **legend_margins), + height=600, xaxis=dict( title="Average Treatment Duration (days)", gridcolor=GRID_COLOR, @@ -2076,6 +2083,7 @@ def create_drug_network_figure(data: dict, title: str = "") -> go.Figure: layout = _base_layout(display_title) layout.update( margin=dict(t=60, l=24, r=24, b=24), + height=600, xaxis=dict(visible=False, scaleanchor="y", scaleratio=1), yaxis=dict(visible=False), ) @@ -2175,7 +2183,7 @@ def create_drug_timeline_figure(data: list[dict], title: str = "") -> go.Figure: # Layout n_bars = len(data) bar_height = 28 - dynamic_height = max(400, n_bars * bar_height + 120) + dynamic_height = max(600, n_bars * bar_height + 120) n_dirs = len(directories) legend_margins = _smart_legend_margin(n_dirs) @@ -2268,7 +2276,7 @@ def create_dosing_distribution_figure( n_bars = len(sorted_data) bar_height = 24 - dynamic_height = max(400, n_bars * bar_height + 120) + dynamic_height = max(600, n_bars * bar_height + 120) n_dirs = len(directories) legend_margins = _smart_legend_margin(n_dirs) @@ -2322,25 +2330,30 @@ def create_trend_figure( display_title = title or "Temporal Trends" - # Group data by name (drug or directory) + # Group data by name (drug or directory), sorting periods chronologically from collections import defaultdict - series = defaultdict(lambda: {"periods": [], "values": []}) + series = defaultdict(list) for row in data: name = row.get("name", "") - series[name]["periods"].append(row["period_end"]) - series[name]["values"].append(row.get("value", 0)) + series[name].append((row["period_end"], row.get("value", 0))) + + # Sort each series by period + for name in series: + series[name].sort(key=lambda x: x[0]) n_series = len(series) fig = go.Figure() - for i, (name, s) in enumerate(sorted(series.items())): + for i, (name, points) in enumerate(sorted(series.items())): + periods = [p[0] for p in points] + values = [p[1] for p in points] colour = DRUG_PALETTE[i % len(DRUG_PALETTE)] fig.add_trace(go.Scatter( - x=s["periods"], - y=s["values"], + x=periods, + y=values, mode="lines+markers", name=name, - customdata=[name] * len(s["periods"]), + customdata=[name] * len(periods), line=dict(color=colour, width=2), marker=dict(color=colour, size=6), hovertemplate=( @@ -2365,7 +2378,9 @@ def create_trend_figure( xaxis=dict( title="Period", gridcolor=GRID_COLOR, - type="category", + type="date", + dtick="M6", + tickformat="%b %Y", ), yaxis=dict( title=y_label,