Good. I've read all the required files. Let me understand the current state:

**Phase 1 is complete** (Tasks 1.1, 1.2, 1.3 all done). The next task is **Task 2.1: Create `assign_drug_indications()` function** ÔÇö this is the core drug-aware matching logic.

**Why Task 2.1**:
- All Phase 1 dependencies are complete (Snowflake query returns ALL matches with code_frequency, drug mapping loaded from CSV, asthma merged)
- Task 2.1 is the foundation for Phase 2 ÔÇö Task 2.2 (tiebreaker logic) is naturally part of 2.1
- This can be built and tested with mock data (no Snowflake needed)
- Phase 3 (pipeline integration) depends on this function existing

Let me read the current refresh_pathways.py to understand the integration point and how gp_matches_df is currently used.


     1ÔåÆ"""
     2ÔåÆCLI command for refreshing pathway data from Snowflake.
     3ÔåÆ
     4ÔåÆThis command fetches activity data from Snowflake, processes it through the
     5ÔåÆpathway pipeline for all 6 date filter combinations, and stores the results
     6ÔåÆin the SQLite pathway_nodes table. Supports two chart types:
     7ÔåÆ- "directory": Trust ÔåÆ Directory ÔåÆ Drug ÔåÆ Pathway (default)
     8ÔåÆ- "indication": Trust ÔåÆ Search_Term ÔåÆ Drug ÔåÆ Pathway (requires GP diagnosis lookup)
     9ÔåÆ
    10ÔåÆUsage:
    11ÔåÆ    python -m cli.refresh_pathways
    12ÔåÆ    python -m cli.refresh_pathways --minimum-patients 10
    13ÔåÆ    python -m cli.refresh_pathways --provider-codes RGT,RM1
    14ÔåÆ    python -m cli.refresh_pathways --chart-type all
    15ÔåÆ    python -m cli.refresh_pathways --chart-type directory
    16ÔåÆ    python -m cli.refresh_pathways --dry-run
    17ÔåÆ
    18ÔåÆRun `python -m cli.refresh_pathways --help` for full options.
    19ÔåÆ"""
    20ÔåÆ
    21ÔåÆimport argparse
    22ÔåÆimport json
    23ÔåÆimport sqlite3
    24ÔåÆimport sys
    25ÔåÆimport time
    26ÔåÆimport uuid
    27ÔåÆfrom datetime import datetime
    28ÔåÆfrom pathlib import Path
    29ÔåÆfrom typing import Optional
    30ÔåÆ
    31ÔåÆfrom core import PathConfig, default_paths
    32ÔåÆfrom core.logging_config import get_logger, setup_logging
    33ÔåÆfrom data_processing.database import DatabaseManager, DatabaseConfig
    34ÔåÆfrom data_processing.schema import (
    35ÔåÆ    clear_pathway_nodes,
    36ÔåÆ    get_pathway_table_counts,
    37ÔåÆ    verify_pathway_tables_exist,
    38ÔåÆ    create_pathway_tables,
    39ÔåÆ)
    40ÔåÆfrom data_processing.pathway_pipeline import (
    41ÔåÆ    ChartType,
    42ÔåÆ    DATE_FILTER_CONFIGS,
    43ÔåÆ    fetch_and_transform_data,
    44ÔåÆ    process_all_date_filters,
    45ÔåÆ    process_pathway_for_date_filter,
    46ÔåÆ    process_indication_pathway_for_date_filter,
    47ÔåÆ    extract_denormalized_fields,
    48ÔåÆ    extract_indication_fields,
    49ÔåÆ    convert_to_records,
    50ÔåÆ)
    51ÔåÆfrom data_processing.diagnosis_lookup import get_patient_indication_groups
    52ÔåÆ
    53ÔåÆlogger = get_logger(__name__)
    54ÔåÆ
    55ÔåÆ
    56ÔåÆdef get_default_filters(paths: PathConfig) -> tuple[list[str], list[str], list[str]]:
    57ÔåÆ    """
    58ÔåÆ    Load default filter values from reference files.
    59ÔåÆ
    60ÔåÆ    Returns:
    61ÔåÆ        Tuple of (trust_filter, drug_filter, directory_filter)
    62ÔåÆ    """
    63ÔåÆ    import pandas as pd
    64ÔåÆ
    65ÔåÆ    # Load default trusts
    66ÔåÆ    trust_filter = []
    67ÔåÆ    if paths.default_trusts_csv.exists():
    68ÔåÆ        try:
    69ÔåÆ            trusts_df = pd.read_csv(paths.default_trusts_csv)
    70ÔåÆ            # Use the "Name" column which contains trust names
    71ÔåÆ            if 'Name' in trusts_df.columns:
    72ÔåÆ                trust_filter = trusts_df['Name'].dropna().tolist()
    73ÔåÆ            else:
    74ÔåÆ                # Fallback to first column if no Name column
    75ÔåÆ                trust_filter = trusts_df.iloc[:, 0].dropna().tolist()
    76ÔåÆ            logger.info(f"Loaded {len(trust_filter)} default trusts")
    77ÔåÆ        except Exception as e:
    78ÔåÆ            logger.warning(f"Could not load default trusts: {e}")
    79ÔåÆ
    80ÔåÆ    # Load default drugs (Include=1 in include.csv)
    81ÔåÆ    drug_filter = []
    82ÔåÆ    if paths.include_csv.exists():
    83ÔåÆ        try:
    84ÔåÆ            drugs_df = pd.read_csv(paths.include_csv)
    85ÔåÆ            if 'Include' in drugs_df.columns:
    86ÔåÆ                drug_filter = drugs_df[drugs_df['Include'] == 1].iloc[:, 0].dropna().tolist()
    87ÔåÆ            else:
    88ÔåÆ                # Assume first column contains drug names if no Include column
    89ÔåÆ                drug_filter = drugs_df.iloc[:, 0].dropna().tolist()
    90ÔåÆ            logger.info(f"Loaded {len(drug_filter)} default drugs")
    91ÔåÆ        except Exception as e:
    92ÔåÆ            logger.warning(f"Could not load default drugs: {e}")
    93ÔåÆ
    94ÔåÆ    # Load default directories
    95ÔåÆ    directory_filter = []
    96ÔåÆ    if paths.directory_list_csv.exists():
    97ÔåÆ        try:
    98ÔåÆ            dirs_df = pd.read_csv(paths.directory_list_csv)
    99ÔåÆ            # Assume first column contains directory names
   100ÔåÆ            directory_filter = dirs_df.iloc[:, 0].dropna().tolist()
   101ÔåÆ            logger.info(f"Loaded {len(directory_filter)} default directories")
   102ÔåÆ        except Exception as e:
   103ÔåÆ            logger.warning(f"Could not load default directories: {e}")
   104ÔåÆ
   105ÔåÆ    return trust_filter, drug_filter, directory_filter
   106ÔåÆ
   107ÔåÆ
   108ÔåÆdef insert_pathway_records(
   109ÔåÆ    conn: sqlite3.Connection,
   110ÔåÆ    records: list[dict],
   111ÔåÆ) -> int:
   112ÔåÆ    """
   113ÔåÆ    Insert pathway records into pathway_nodes table.
   114ÔåÆ
   115ÔåÆ    Uses INSERT OR REPLACE to handle updates to existing records.
   116ÔåÆ
   117ÔåÆ    Args:
   118ÔåÆ        conn: SQLite connection
   119ÔåÆ        records: List of record dicts from convert_to_records()
   120ÔåÆ
   121ÔåÆ    Returns:
   122ÔåÆ        Number of records inserted
   123ÔåÆ    """
   124ÔåÆ    if not records:
   125ÔåÆ        return 0
   126ÔåÆ
   127ÔåÆ    # Column order matching pathway_nodes schema (includes chart_type)
   128ÔåÆ    columns = [
   129ÔåÆ        'date_filter_id', 'chart_type', 'parents', 'ids', 'labels', 'level',
   130ÔåÆ        'value', 'cost', 'costpp', 'cost_pp_pa', 'colour',
   131ÔåÆ        'first_seen', 'last_seen', 'first_seen_parent', 'last_seen_parent',
   132ÔåÆ        'average_spacing', 'average_administered', 'avg_days',
   133ÔåÆ        'trust_name', 'directory', 'drug_sequence', 'data_refresh_id'
   134ÔåÆ    ]
   135ÔåÆ
   136ÔåÆ    placeholders = ', '.join(['?' for _ in columns])
   137ÔåÆ    column_names = ', '.join(columns)
   138ÔåÆ
   139ÔåÆ    insert_sql = f"""
   140ÔåÆ        INSERT OR REPLACE INTO pathway_nodes ({column_names})
   141ÔåÆ        VALUES ({placeholders})
   142ÔåÆ    """
   143ÔåÆ
   144ÔåÆ    # Convert records to tuples in column order
   145ÔåÆ    rows = []
   146ÔåÆ    for record in records:
   147ÔåÆ        row = tuple(record.get(col) for col in columns)
   148ÔåÆ        rows.append(row)
   149ÔåÆ
   150ÔåÆ    cursor = conn.executemany(insert_sql, rows)
   151ÔåÆ    return cursor.rowcount
   152ÔåÆ
   153ÔåÆ
   154ÔåÆdef log_refresh_start(
   155ÔåÆ    conn: sqlite3.Connection,
   156ÔåÆ    refresh_id: str,
   157ÔåÆ    date_from: Optional[str] = None,
   158ÔåÆ    date_to: Optional[str] = None,
   159ÔåÆ) -> None:
   160ÔåÆ    """Log the start of a refresh operation."""
   161ÔåÆ    conn.execute("""
   162ÔåÆ        INSERT INTO pathway_refresh_log
   163ÔåÆ        (refresh_id, started_at, status, snowflake_query_date_from, snowflake_query_date_to)
   164ÔåÆ        VALUES (?, ?, 'running', ?, ?)
   165ÔåÆ    """, (refresh_id, datetime.now().isoformat(), date_from, date_to))
   166ÔåÆ    conn.commit()
   167ÔåÆ
   168ÔåÆ
   169ÔåÆdef log_refresh_complete(
   170ÔåÆ    conn: sqlite3.Connection,
   171ÔåÆ    refresh_id: str,
   172ÔåÆ    record_count: int,
   173ÔåÆ    date_filter_counts: dict[str, int],
   174ÔåÆ    duration_seconds: float,
   175ÔåÆ) -> None:
   176ÔåÆ    """Log the successful completion of a refresh operation."""
   177ÔåÆ    conn.execute("""
   178ÔåÆ        UPDATE pathway_refresh_log
   179ÔåÆ        SET completed_at = ?,
   180ÔåÆ            status = 'completed',
   181ÔåÆ            record_count = ?,
   182ÔåÆ            date_filter_counts = ?,
   183ÔåÆ            processing_duration_seconds = ?
   184ÔåÆ        WHERE refresh_id = ?
   185ÔåÆ    """, (
   186ÔåÆ        datetime.now().isoformat(),
   187ÔåÆ        record_count,
   188ÔåÆ        json.dumps(date_filter_counts),
   189ÔåÆ        duration_seconds,
   190ÔåÆ        refresh_id,
   191ÔåÆ    ))
   192ÔåÆ    conn.commit()
   193ÔåÆ
   194ÔåÆ
   195ÔåÆdef log_refresh_failed(
   196ÔåÆ    conn: sqlite3.Connection,
   197ÔåÆ    refresh_id: str,
   198ÔåÆ    error_message: str,
   199ÔåÆ    duration_seconds: float,
   200ÔåÆ) -> None:
   201ÔåÆ    """Log a failed refresh operation."""
   202ÔåÆ    conn.execute("""
   203ÔåÆ        UPDATE pathway_refresh_log
   204ÔåÆ        SET completed_at = ?,
   205ÔåÆ            status = 'failed',
   206ÔåÆ            error_message = ?,
   207ÔåÆ            processing_duration_seconds = ?
   208ÔåÆ        WHERE refresh_id = ?
   209ÔåÆ    """, (
   210ÔåÆ        datetime.now().isoformat(),
   211ÔåÆ        error_message,
   212ÔåÆ        duration_seconds,
   213ÔåÆ        refresh_id,
   214ÔåÆ    ))
   215ÔåÆ    conn.commit()
   216ÔåÆ
   217ÔåÆ
   218ÔåÆdef refresh_pathways(
   219ÔåÆ    minimum_patients: int = 5,
   220ÔåÆ    provider_codes: Optional[list[str]] = None,
   221ÔåÆ    trust_filter: Optional[list[str]] = None,
   222ÔåÆ    drug_filter: Optional[list[str]] = None,
   223ÔåÆ    directory_filter: Optional[list[str]] = None,
   224ÔåÆ    db_path: Optional[Path] = None,
   225ÔåÆ    paths: Optional[PathConfig] = None,
   226ÔåÆ    dry_run: bool = False,
   227ÔåÆ    chart_type: str = "directory",
   228ÔåÆ) -> tuple[bool, str, dict]:
   229ÔåÆ    """
   230ÔåÆ    Main refresh function that orchestrates the full pipeline.
   231ÔåÆ
   232ÔåÆ    Args:
   233ÔåÆ        minimum_patients: Minimum patients to include a pathway
   234ÔåÆ        provider_codes: List of provider codes to filter Snowflake query
   235ÔåÆ        trust_filter: List of trust names to include in pathways
   236ÔåÆ        drug_filter: List of drug names to include in pathways
   237ÔåÆ        directory_filter: List of directories to include in pathways
   238ÔåÆ        db_path: Path to SQLite database (uses default if None)
   239ÔåÆ        paths: PathConfig for file paths
   240ÔåÆ        dry_run: If True, don't actually insert records
   241ÔåÆ        chart_type: Which chart type to process: "directory", "indication", or "all"
   242ÔåÆ
   243ÔåÆ    Returns:
   244ÔåÆ        Tuple of (success: bool, message: str, stats: dict)
   245ÔåÆ    """
   246ÔåÆ    if paths is None:
   247ÔåÆ        paths = default_paths
   248ÔåÆ
   249ÔåÆ    # Set up database connection
   250ÔåÆ    if db_path:
   251ÔåÆ        db_config = DatabaseConfig(db_path=db_path)
   252ÔåÆ    else:
   253ÔåÆ        db_config = DatabaseConfig(data_dir=paths.data_dir)
   254ÔåÆ
   255ÔåÆ    db_manager = DatabaseManager(db_config)
   256ÔåÆ
   257ÔåÆ    # Load default filters if not provided
   258ÔåÆ    default_trusts, default_drugs, default_dirs = get_default_filters(paths)
   259ÔåÆ
   260ÔåÆ    if trust_filter is None:
   261ÔåÆ        trust_filter = default_trusts
   262ÔåÆ    if drug_filter is None:
   263ÔåÆ        drug_filter = default_drugs
   264ÔåÆ    if directory_filter is None:
   265ÔåÆ        directory_filter = default_dirs
   266ÔåÆ
   267ÔåÆ    # Ensure we have some filters
   268ÔåÆ    if not drug_filter:
   269ÔåÆ        return False, "No drugs specified and could not load defaults", {}
   270ÔåÆ
   271ÔåÆ    # Determine which chart types to process
   272ÔåÆ    if chart_type == "all":
   273ÔåÆ        chart_types_to_process: list[ChartType] = ["directory", "indication"]
   274ÔåÆ    else:
   275ÔåÆ        chart_types_to_process = [chart_type]  # type: ignore
   276ÔåÆ
   277ÔåÆ    logger.info("=" * 60)
   278ÔåÆ    logger.info("Pathway Data Refresh Starting")
   279ÔåÆ    logger.info("=" * 60)
   280ÔåÆ    logger.info(f"Minimum patients: {minimum_patients}")
   281ÔåÆ    logger.info(f"Trust filter: {len(trust_filter)} trusts")
   282ÔåÆ    logger.info(f"Drug filter: {len(drug_filter)} drugs")
   283ÔåÆ    logger.info(f"Directory filter: {len(directory_filter)} directories")
   284ÔåÆ    logger.info(f"Provider codes: {provider_codes or 'All'}")
   285ÔåÆ    logger.info(f"Chart type(s): {', '.join(chart_types_to_process)}")
   286ÔåÆ    logger.info(f"Database: {db_manager.db_path}")
   287ÔåÆ    logger.info(f"Dry run: {dry_run}")
   288ÔåÆ    logger.info("=" * 60)
   289ÔåÆ
   290ÔåÆ    start_time = time.time()
   291ÔåÆ    refresh_id = str(uuid.uuid4())[:8]
   292ÔåÆ    stats = {
   293ÔåÆ        "refresh_id": refresh_id,
   294ÔåÆ        "date_filter_counts": {},
   295ÔåÆ        "total_records": 0,
   296ÔåÆ        "snowflake_rows": 0,
   297ÔåÆ    }
   298ÔåÆ
   299ÔåÆ    try:
   300ÔåÆ        # Verify database and tables
   301ÔåÆ        with db_manager.get_connection() as conn:
   302ÔåÆ            missing_tables = verify_pathway_tables_exist(conn)
   303ÔåÆ            if missing_tables:
   304ÔåÆ                logger.info(f"Creating missing tables: {missing_tables}")
   305ÔåÆ                create_pathway_tables(conn)
   306ÔåÆ
   307ÔåÆ            # Log refresh start
   308ÔåÆ            if not dry_run:
   309ÔåÆ                log_refresh_start(conn, refresh_id)
   310ÔåÆ
   311ÔåÆ        # Step 1: Fetch data from Snowflake
   312ÔåÆ        logger.info("")
   313ÔåÆ        logger.info("Step 1/4: Fetching data from Snowflake...")
   314ÔåÆ        df = fetch_and_transform_data(
   315ÔåÆ            provider_codes=provider_codes,
   316ÔåÆ            paths=paths,
   317ÔåÆ        )
   318ÔåÆ
   319ÔåÆ        if df.empty:
   320ÔåÆ            msg = "No data returned from Snowflake"
   321ÔåÆ            logger.error(msg)
   322ÔåÆ            with db_manager.get_connection() as conn:
   323ÔåÆ                log_refresh_failed(conn, refresh_id, msg, time.time() - start_time)
   324ÔåÆ            return False, msg, stats
   325ÔåÆ
   326ÔåÆ        stats["snowflake_rows"] = len(df)
   327ÔåÆ        logger.info(f"Fetched {len(df)} records from Snowflake")
   328ÔåÆ
   329ÔåÆ        # Step 2: Process all date filters for each chart type
   330ÔåÆ        num_date_filters = len(DATE_FILTER_CONFIGS)
   331ÔåÆ        num_chart_types = len(chart_types_to_process)
   332ÔåÆ        total_datasets = num_date_filters * num_chart_types
   333ÔåÆ
   334ÔåÆ        logger.info("")
   335ÔåÆ        logger.info(f"Step 2/4: Processing pathway data for {total_datasets} datasets "
   336ÔåÆ                   f"({num_date_filters} date filters x {num_chart_types} chart types)...")
   337ÔåÆ
   338ÔåÆ        # Store results keyed by "date_filter_id:chart_type"
   339ÔåÆ        results: dict[str, list[dict]] = {}
   340ÔåÆ
   341ÔåÆ        for current_chart_type in chart_types_to_process:
   342ÔåÆ            logger.info("")
   343ÔåÆ            logger.info(f"Processing chart type: {current_chart_type}")
   344ÔåÆ
   345ÔåÆ            if current_chart_type == "directory":
   346ÔåÆ                # Use existing process_all_date_filters for directory charts
   347ÔåÆ                dir_results = process_all_date_filters(
   348ÔåÆ                    df=df,
   349ÔåÆ                    trust_filter=trust_filter,
   350ÔåÆ                    drug_filter=drug_filter,
   351ÔåÆ                    directory_filter=directory_filter,
   352ÔåÆ                    minimum_patients=minimum_patients,
   353ÔåÆ                    refresh_id=refresh_id,
   354ÔåÆ                    paths=paths,
   355ÔåÆ                )
   356ÔåÆ                # Add results with chart_type suffix
   357ÔåÆ                for filter_id, records in dir_results.items():
   358ÔåÆ                    # Records already have chart_type set by convert_to_records
   359ÔåÆ                    results[f"{filter_id}:directory"] = records
   360ÔåÆ
   361ÔåÆ            elif current_chart_type == "indication":
   362ÔåÆ                # For indication charts, we need to look up GP diagnoses for all patients
   363ÔåÆ                # using the new Snowflake-direct approach via get_patient_indication_groups()
   364ÔåÆ                logger.info("Building indication groups from GP diagnosis lookups (Snowflake-direct)...")
   365ÔåÆ
   366ÔåÆ                # Check Snowflake availability
   367ÔåÆ                from data_processing.snowflake_connector import get_connector, is_snowflake_available
   368ÔåÆ
   369ÔåÆ                if not is_snowflake_available():
   370ÔåÆ                    logger.warning("Snowflake not available - cannot process indication charts")
   371ÔåÆ                    for config in DATE_FILTER_CONFIGS:
   372ÔåÆ                        results[f"{config.id}:indication"] = []
   373ÔåÆ                    continue
   374ÔåÆ
   375ÔåÆ                try:
   376ÔåÆ                    import pandas as pd
   377ÔåÆ                    connector = get_connector()
   378ÔåÆ
   379ÔåÆ                    # Step 1: Extract unique PseudoNHSNoLinked values from df
   380ÔåÆ                    # This is the patient identifier that matches PatientPseudonym in GP records
   381ÔåÆ                    if 'PseudoNHSNoLinked' not in df.columns:
   382ÔåÆ                        logger.error("DataFrame missing 'PseudoNHSNoLinked' column - cannot lookup GP records")
   383ÔåÆ                        for config in DATE_FILTER_CONFIGS:
   384ÔåÆ                            results[f"{config.id}:indication"] = []
   385ÔåÆ                        continue
   386ÔåÆ
   387ÔåÆ                    # Get unique patient pseudonyms for GP lookup (avoid redundant queries)
   388ÔåÆ                    patient_pseudonyms = df['PseudoNHSNoLinked'].dropna().unique().tolist()
   389ÔåÆ
   390ÔåÆ                    logger.info(f"Looking up GP diagnoses for {len(patient_pseudonyms)} unique patients...")
   391ÔåÆ
   392ÔåÆ                    # Step 2: Call the new Snowflake-direct indication lookup
   393ÔåÆ                    gp_matches_df = get_patient_indication_groups(
   394ÔåÆ                        patient_pseudonyms=patient_pseudonyms,
   395ÔåÆ                        connector=connector,
   396ÔåÆ                        batch_size=500,
   397ÔåÆ                    )
   398ÔåÆ
   399ÔåÆ                    # Step 3: Build indication_df mapping UPID -> Indication_Group
   400ÔåÆ                    # For matched patients: Indication_Group = Search_Term
   401ÔåÆ                    # For unmatched patients: Indication_Group = Directory + " (no GP dx)"
   402ÔåÆ                    #
   403ÔåÆ                    # IMPORTANT: We need ALL unique UPIDs, not just unique PseudoNHSNoLinked.
   404ÔåÆ                    # A patient can have multiple UPIDs if they visited multiple providers.
   405ÔåÆ
   406ÔåÆ                    # Get all unique UPID records with their PseudoNHSNoLinked and Directory
   407ÔåÆ                    upid_lookup = df[['UPID', 'PseudoNHSNoLinked', 'Directory']].drop_duplicates(
   408ÔåÆ                        subset=['UPID']
   409ÔåÆ                    ).copy()
   410ÔåÆ
   411ÔåÆ                    if gp_matches_df.empty:
   412ÔåÆ                        logger.warning("No GP matches found - all patients will use fallback directory")
   413ÔåÆ                        # All patients use fallback
   414ÔåÆ                        indication_records = []
   415ÔåÆ                        for _, row in upid_lookup.iterrows():
   416ÔåÆ                            directory = row['Directory']
   417ÔåÆ                            indication_records.append({
   418ÔåÆ                                'UPID': row['UPID'],
   419ÔåÆ                                'Indication_Group': str(directory) + " (no GP dx)" if pd.notna(directory) else "UNKNOWN (no GP dx)",
   420ÔåÆ                                'Source': 'FALLBACK',
   421ÔåÆ                            })
   422ÔåÆ                        indication_df = pd.DataFrame(indication_records)
   423ÔåÆ                    else:
   424ÔåÆ                        # Create lookup dict: PseudoNHSNoLinked -> Search_Term
   425ÔåÆ                        match_lookup = dict(zip(
   426ÔåÆ                            gp_matches_df['PatientPseudonym'],
   427ÔåÆ                            gp_matches_df['Search_Term']
   428ÔåÆ                        ))
   429ÔåÆ
   430ÔåÆ                        # Build indication records for each unique UPID
   431ÔåÆ                        indication_records = []
   432ÔåÆ                        for _, row in upid_lookup.iterrows():
   433ÔåÆ                            pseudo = row['PseudoNHSNoLinked']
   434ÔåÆ                            upid = row['UPID']
   435ÔåÆ                            directory = row['Directory']
   436ÔåÆ
   437ÔåÆ                            if pd.notna(pseudo) and pseudo in match_lookup:
   438ÔåÆ                                indication_records.append({
   439ÔåÆ                                    'UPID': upid,
   440ÔåÆ                                    'Indication_Group': match_lookup[pseudo],
   441ÔåÆ                                    'Source': 'DIAGNOSIS',
   442ÔåÆ                                })
   443ÔåÆ                            else:
   444ÔåÆ                                # Use fallback: Directory + " (no GP dx)"
   445ÔåÆ                                fallback_label = str(directory) + " (no GP dx)" if pd.notna(directory) else "UNKNOWN (no GP dx)"
   446ÔåÆ                                indication_records.append({
   447ÔåÆ                                    'UPID': upid,
   448ÔåÆ                                    'Indication_Group': fallback_label,
   449ÔåÆ                                    'Source': 'FALLBACK',
   450ÔåÆ                                })
   451ÔåÆ
   452ÔåÆ                        indication_df = pd.DataFrame(indication_records)
   453ÔåÆ
   454ÔåÆ                    # Log coverage statistics
   455ÔåÆ                    if not indication_df.empty:
   456ÔåÆ                        diagnosis_count = (indication_df['Source'] == 'DIAGNOSIS').sum()
   457ÔåÆ                        fallback_count = (indication_df['Source'] == 'FALLBACK').sum()
   458ÔåÆ                        total = len(indication_df)
   459ÔåÆ                        stats["diagnosis_coverage"] = {
   460ÔåÆ                            "diagnosis": int(diagnosis_count),
   461ÔåÆ                            "fallback": int(fallback_count),
   462ÔåÆ                            "total": total,
   463ÔåÆ                            "diagnosis_pct": round(100 * diagnosis_count / total, 1) if total > 0 else 0,
   464ÔåÆ                        }
   465ÔåÆ                        logger.info(f"Indication coverage: {diagnosis_count}/{total} ({stats['diagnosis_coverage']['diagnosis_pct']}%) diagnosis-matched")
   466ÔåÆ
   467ÔåÆ                        # Log top indication groups
   468ÔåÆ                        top_indications = indication_df[indication_df['Source'] == 'DIAGNOSIS']['Indication_Group'].value_counts().head(5)
   469ÔåÆ                        if len(top_indications) > 0:
   470ÔåÆ                            logger.info(f"Top 5 indications: {dict(top_indications)}")
   471ÔåÆ
   472ÔåÆ                        # Rename column for compatibility with generate_icicle_chart_indication
   473ÔåÆ                        # It expects indication_df to have 'Directory' column (mapped from Indication_Group)
   474ÔåÆ                        indication_df_for_chart = indication_df[['UPID', 'Indication_Group']].copy()
   475ÔåÆ                        indication_df_for_chart = indication_df_for_chart.rename(columns={'Indication_Group': 'Directory'})
   476ÔåÆ                        # Ensure unique UPID index (build_hierarchy requires uniquely valued Index)
   477ÔåÆ                        # Keep first occurrence - DIAGNOSIS entries should come before FALLBACK
   478ÔåÆ                        indication_df_for_chart = indication_df_for_chart.drop_duplicates(subset=['UPID'], keep='first')
   479ÔåÆ                        indication_df_for_chart = indication_df_for_chart.set_index('UPID')
   480ÔåÆ
   481ÔåÆ                        # Process each date filter with indication grouping
   482ÔåÆ                        for config in DATE_FILTER_CONFIGS:
   483ÔåÆ                            logger.info(f"Processing indication pathway for {config.id}")
   484ÔåÆ
   485ÔåÆ                            ice_df = process_indication_pathway_for_date_filter(
   486ÔåÆ                                df=df,
   487ÔåÆ                                indication_df=indication_df_for_chart,
   488ÔåÆ                                config=config,
   489ÔåÆ                                trust_filter=trust_filter,
   490ÔåÆ                                drug_filter=drug_filter,
   491ÔåÆ                                directory_filter=directory_filter,
   492ÔåÆ                                minimum_patients=minimum_patients,
   493ÔåÆ                                paths=paths,
   494ÔåÆ                            )
   495ÔåÆ
   496ÔåÆ                            if ice_df is None:
   497ÔåÆ                                logger.warning(f"No indication pathway data for {config.id}")
   498ÔåÆ                                results[f"{config.id}:indication"] = []
   499ÔåÆ                                continue
   500ÔåÆ
   501ÔåÆ                            # Extract denormalized fields (using indication variant)
   502ÔåÆ                            ice_df = extract_indication_fields(ice_df)
   503ÔåÆ
   504ÔåÆ                            # Convert to records with chart_type="indication"
   505ÔåÆ                            records = convert_to_records(ice_df, config.id, refresh_id, chart_type="indication")
   506ÔåÆ                            results[f"{config.id}:indication"] = records
   507ÔåÆ
   508ÔåÆ                            logger.info(f"Completed {config.id}:indication: {len(records)} nodes")
   509ÔåÆ                    else:
   510ÔåÆ                        logger.warning("Empty indication_df - skipping indication charts")
   511ÔåÆ                        for config in DATE_FILTER_CONFIGS:
   512ÔåÆ                            results[f"{config.id}:indication"] = []
   513ÔåÆ
   514ÔåÆ                except Exception as e:
   515ÔåÆ                    logger.error(f"Error processing indication charts: {e}")
   516ÔåÆ                    logger.exception(e)
   517ÔåÆ                    for config in DATE_FILTER_CONFIGS:
   518ÔåÆ                        results[f"{config.id}:indication"] = []
   519ÔåÆ
   520ÔåÆ        # Count records per filter and chart type
   521ÔåÆ        stats["chart_type_counts"] = {}
   522ÔåÆ        for key, records in results.items():
   523ÔåÆ            stats["date_filter_counts"][key] = len(records)
   524ÔåÆ            stats["total_records"] += len(records)
   525ÔåÆ            # Also track by chart type
   526ÔåÆ            _, ct = key.split(":")
   527ÔåÆ            stats["chart_type_counts"][ct] = stats["chart_type_counts"].get(ct, 0) + len(records)
   528ÔåÆ
   529ÔåÆ        logger.info("")
   530ÔåÆ        logger.info(f"Processed {stats['total_records']} total pathway nodes")
   531ÔåÆ        for chart_type_name, count in stats.get("chart_type_counts", {}).items():
   532ÔåÆ            logger.info(f"  {chart_type_name}: {count} nodes total")
   533ÔåÆ        for key, count in sorted(stats["date_filter_counts"].items()):
   534ÔåÆ            if count > 0:
   535ÔåÆ                logger.info(f"    {key}: {count} nodes")
   536ÔåÆ
   537ÔåÆ        if dry_run:
   538ÔåÆ            logger.info("")
   539ÔåÆ            logger.info("DRY RUN - Skipping database insertion")
   540ÔåÆ            elapsed = time.time() - start_time
   541ÔåÆ            return True, f"Dry run complete: {stats['total_records']} records would be inserted", stats
   542ÔåÆ
   543ÔåÆ        # Step 3: Clear existing data and insert new records
   544ÔåÆ        logger.info("")
   545ÔåÆ        logger.info("Step 3/4: Clearing existing pathway data and inserting new records...")
   546ÔåÆ
   547ÔåÆ        with db_manager.get_transaction() as conn:
   548ÔåÆ            # Clear all existing pathway nodes
   549ÔåÆ            deleted = clear_pathway_nodes(conn)
   550ÔåÆ            logger.info(f"Cleared {deleted} existing pathway nodes")
   551ÔåÆ
   552ÔåÆ            # Insert new records for each date filter + chart type combination
   553ÔåÆ            total_inserted = 0
   554ÔåÆ            for key, records in results.items():
   555ÔåÆ                if records:
   556ÔåÆ                    inserted = insert_pathway_records(conn, records)
   557ÔåÆ                    total_inserted += len(records)
   558ÔåÆ                    logger.info(f"  Inserted {len(records)} records for {key}")
   559ÔåÆ
   560ÔåÆ        # Step 4: Log completion
   561ÔåÆ        logger.info("")
   562ÔåÆ        logger.info("Step 4/4: Logging refresh completion...")
   563ÔåÆ
   564ÔåÆ        elapsed = time.time() - start_time
   565ÔåÆ
   566ÔåÆ        with db_manager.get_connection() as conn:
   567ÔåÆ            log_refresh_complete(
   568ÔåÆ                conn=conn,
   569ÔåÆ                refresh_id=refresh_id,
   570ÔåÆ                record_count=stats["total_records"],
   571ÔåÆ                date_filter_counts=stats["date_filter_counts"],
   572ÔåÆ                duration_seconds=elapsed,
   573ÔåÆ            )
   574ÔåÆ
   575ÔåÆ            # Verify final counts
   576ÔåÆ            counts = get_pathway_table_counts(conn)
   577ÔåÆ            logger.info(f"Final table counts: {counts}")
   578ÔåÆ
   579ÔåÆ        logger.info("")
   580ÔåÆ        logger.info("=" * 60)
   581ÔåÆ        logger.info(f"Refresh completed successfully in {elapsed:.1f} seconds")
   582ÔåÆ        logger.info(f"Total records: {stats['total_records']}")
   583ÔåÆ        logger.info(f"Refresh ID: {refresh_id}")
   584ÔåÆ        logger.info("=" * 60)
   585ÔåÆ
   586ÔåÆ        return True, f"Refresh complete: {stats['total_records']} records in {elapsed:.1f}s", stats
   587ÔåÆ
   588ÔåÆ    except Exception as e:
   589ÔåÆ        elapsed = time.time() - start_time
   590ÔåÆ        error_msg = f"Refresh failed: {e}"
   591ÔåÆ        logger.error(error_msg, exc_info=True)
   592ÔåÆ
   593ÔåÆ        try:
   594ÔåÆ            with db_manager.get_connection() as conn:
   595ÔåÆ                log_refresh_failed(conn, refresh_id, str(e), elapsed)
   596ÔåÆ        except Exception:
   597ÔåÆ            pass  # Don't fail the error handling
   598ÔåÆ
   599ÔåÆ        return False, error_msg, stats
   600ÔåÆ
   601ÔåÆ
   602ÔåÆdef main() -> int:
   603ÔåÆ    """CLI entry point."""
   604ÔåÆ    parser = argparse.ArgumentParser(
   605ÔåÆ        description="Refresh pathway data from Snowflake",
   606ÔåÆ        formatter_class=argparse.RawDescriptionHelpFormatter,
   607ÔåÆ        epilog="""
   608ÔåÆExamples:
   609ÔåÆ    # Basic refresh with defaults (directory chart only)
   610ÔåÆ    python -m cli.refresh_pathways
   611ÔåÆ
   612ÔåÆ    # Refresh both chart types (directory and indication)
   613ÔåÆ    python -m cli.refresh_pathways --chart-type all
   614ÔåÆ
   615ÔåÆ    # Refresh only indication-based charts
   616ÔåÆ    python -m cli.refresh_pathways --chart-type indication
   617ÔåÆ
   618ÔåÆ    # Refresh with custom minimum patients
   619ÔåÆ    python -m cli.refresh_pathways --minimum-patients 10
   620ÔåÆ
   621ÔåÆ    # Refresh specific providers only
   622ÔåÆ    python -m cli.refresh_pathways --provider-codes RGT,RM1
   623ÔåÆ
   624ÔåÆ    # Dry run to see what would be processed
   625ÔåÆ    python -m cli.refresh_pathways --dry-run
   626ÔåÆ
   627ÔåÆ    # Verbose output
   628ÔåÆ    python -m cli.refresh_pathways --verbose
   629ÔåÆ        """
   630ÔåÆ    )
   631ÔåÆ
   632ÔåÆ    parser.add_argument(
   633ÔåÆ        "--minimum-patients",
   634ÔåÆ        type=int,
   635ÔåÆ        default=5,
   636ÔåÆ        help="Minimum patients to include a pathway (default: 5)"
   637ÔåÆ    )
   638ÔåÆ
   639ÔåÆ    parser.add_argument(
   640ÔåÆ        "--provider-codes",
   641ÔåÆ        type=str,
   642ÔåÆ        default=None,
   643ÔåÆ        help="Comma-separated list of provider codes to filter (default: all)"
   644ÔåÆ    )
   645ÔåÆ
   646ÔåÆ    parser.add_argument(
   647ÔåÆ        "--db-path",
   648ÔåÆ        type=str,
   649ÔåÆ        default=None,
   650ÔåÆ        help="Path to SQLite database (default: data/pathways.db)"
   651ÔåÆ    )
   652ÔåÆ
   653ÔåÆ    parser.add_argument(
   654ÔåÆ        "--dry-run",
   655ÔåÆ        action="store_true",
   656ÔåÆ        help="Process data but don't insert into database"
   657ÔåÆ    )
   658ÔåÆ
   659ÔåÆ    parser.add_argument(
   660ÔåÆ        "--chart-type",
   661ÔåÆ        type=str,
   662ÔåÆ        choices=["directory", "indication", "all"],
   663ÔåÆ        default="directory",
   664ÔåÆ        help="Chart type to process: 'directory' (default), 'indication', or 'all'"
   665ÔåÆ    )
   666ÔåÆ
   667ÔåÆ    parser.add_argument(
   668ÔåÆ        "--verbose", "-v",
   669ÔåÆ        action="store_true",
   670ÔåÆ        help="Enable verbose logging"
   671ÔåÆ    )
   672ÔåÆ
   673ÔåÆ    args = parser.parse_args()
   674ÔåÆ
   675ÔåÆ    # Configure logging
   676ÔåÆ    import logging
   677ÔåÆ    log_level = logging.DEBUG if args.verbose else logging.INFO
   678ÔåÆ    setup_logging(level=log_level)
   679ÔåÆ
   680ÔåÆ    # Parse provider codes
   681ÔåÆ    provider_codes = None
   682ÔåÆ    if args.provider_codes:
   683ÔåÆ        provider_codes = [code.strip() for code in args.provider_codes.split(",")]
   684ÔåÆ
   685ÔåÆ    # Parse db path
   686ÔåÆ    db_path = Path(args.db_path) if args.db_path else None
   687ÔåÆ
   688ÔåÆ    # Run the refresh
   689ÔåÆ    success, message, stats = refresh_pathways(
   690ÔåÆ        minimum_patients=args.minimum_patients,
   691ÔåÆ        provider_codes=provider_codes,
   692ÔåÆ        db_path=db_path,
   693ÔåÆ        dry_run=args.dry_run,
   694ÔåÆ        chart_type=args.chart_type,
   695ÔåÆ    )
   696ÔåÆ
   697ÔåÆ    if success:
   698ÔåÆ        print(f"\n[OK] {message}")
   699ÔåÆ        return 0
   700ÔåÆ    else:
   701ÔåÆ        print(f"\n[FAILED] {message}", file=sys.stderr)
   702ÔåÆ        return 1
   703ÔåÆ
   704ÔåÆ
   705ÔåÆif __name__ == "__main__":
   706ÔåÆ    sys.exit(main())
   707ÔåÆ

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>

"""
CLI command for refreshing pathway data from Snowflake.

This command fetches activity data from Snowflake, processes it through the
pathway pipeline for all 6 date filter combinations, and stores the results
in the SQLite pathway_nodes table. Supports two chart types:
- "directory": Trust ÔåÆ Directory ÔåÆ Drug ÔåÆ Pathway (default)
- "indication": Trust ÔåÆ Search_Term ÔåÆ Drug ÔåÆ Pathway (requires GP diagnosis lookup)

Usage:
    python -m cli.refresh_pathways
    python -m cli.refresh_pathways --minimum-patients 10
    python -m cli.refresh_pathways --provider-codes RGT,RM1
    python -m cli.refresh_pathways --chart-type all
    python -m cli.refresh_pathways --chart-type directory
    python -m cli.refresh_pathways --dry-run

Run `python -m cli.refresh_pathways --help` for full options.
"""

import argparse
import json
import sqlite3
import sys
import time
import uuid
from datetime import datetime
from pathlib import Path
from typing import Optional

from core import PathConfig, default_paths
from core.logging_config import get_logger, setup_logging
from data_processing.database import DatabaseManager, DatabaseConfig
from data_processing.schema import (
    clear_pathway_nodes,
    get_pathway_table_counts,
    verify_pathway_tables_exist,
    create_pathway_tables,
)
from data_processing.pathway_pipeline import (
    ChartType,
    DATE_FILTER_CONFIGS,
    fetch_and_transform_data,
    process_all_date_filters,
    process_pathway_for_date_filter,
    process_indication_pathway_for_date_filter,
    extract_denormalized_fields,
    extract_indication_fields,
    convert_to_records,
)
from data_processing.diagnosis_lookup import get_patient_indication_groups

logger = get_logger(__name__)

def get_default_filters(paths: PathConfig) -> tuple[list[str], list[str], list[str]]:
    """
    Load default filter values from reference files.

    Returns:
        Tuple of (trust_filter, drug_filter, directory_filter)
    """
    import pandas as pd

    # Load default trusts
    trust_filter = []
    if paths.default_trusts_csv.exists():
        try:
            trusts_df = pd.read_csv(paths.default_trusts_csv)
            # Use the "Name" column which contains trust names
            if 'Name' in trusts_df.columns:
                trust_filter = trusts_df['Name'].dropna().tolist()
            else:
                # Fallback to first column if no Name column
                trust_filter = trusts_df.iloc[:, 0].dropna().tolist()
            logger.info(f"Loaded {len(trust_filter)} default trusts")
        except Exception as e:
            logger.warning(f"Could not load default trusts: {e}")

    # Load default drugs (Include=1 in include.csv)
    drug_filter = []
    if paths.include_csv.exists():
        try:
            drugs_df = pd.read_csv(paths.include_csv)
            if 'Include' in drugs_df.columns:
                drug_filter = drugs_df[drugs_df['Include'] == 1].iloc[:, 0].dropna().tolist()
            else:
                # Assume first column contains drug names if no Include column
                drug_filter = drugs_df.iloc[:, 0].dropna().tolist()
            logger.info(f"Loaded {len(drug_filter)} default drugs")
        except Exception as e:
            logger.warning(f"Could not load default drugs: {e}")

    # Load default directories
    directory_filter = []
    if paths.directory_list_csv.exists():
        try:
            dirs_df = pd.read_csv(paths.directory_list_csv)
            # Assume first column contains directory names
            directory_filter = dirs_df.iloc[:, 0].dropna().tolist()
            logger.info(f"Loaded {len(directory_filter)} default directories")
        except Exception as e:
            logger.warning(f"Could not load default directories: {e}")

    return trust_filter, drug_filter, directory_filter

def insert_pathway_records(
    conn: sqlite3.Connection,
    records: list[dict],
) -> int:
    """
    Insert pathway records into pathway_nodes table.

    Uses INSERT OR REPLACE to handle updates to existing records.

    Args:
        conn: SQLite connection
        records: List of record dicts from convert_to_records()

    Returns:
        Number of records inserted
    """
    if not records:
        return 0

    # Column order matching pathway_nodes schema (includes chart_type)
    columns = [
        'date_filter_id', 'chart_type', 'parents', 'ids', 'labels', 'level',
        'value', 'cost', 'costpp', 'cost_pp_pa', 'colour',
        'first_seen', 'last_seen', 'first_seen_parent', 'last_seen_parent',
        'average_spacing', 'average_administered', 'avg_days',
        'trust_name', 'directory', 'drug_sequence', 'data_refresh_id'
    ]

    placeholders = ', '.join(['?' for _ in columns])
    column_names = ', '.join(columns)

    insert_sql = f"""
        INSERT OR REPLACE INTO pathway_nodes ({column_names})
        VALUES ({placeholders})
    """

    # Convert records to tuples in column order
    rows = []
    for record in records:
        row = tuple(record.get(col) for col in columns)
        rows.append(row)

    cursor = conn.executemany(insert_sql, rows)
    return cursor.rowcount

def log_refresh_start(
    conn: sqlite3.Connection,
    refresh_id: str,
    date_from: Optional[str] = None,
    date_to: Optional[str] = None,
) -> None:
    """Log the start of a refresh operation."""
    conn.execute("""
        INSERT INTO pathway_refresh_log
        (refresh_id, started_at, status, snowflake_query_date_from, snowflake_query_date_to)
        VALUES (?, ?, 'running', ?, ?)
    """, (refresh_id, datetime.now().isoformat(), date_from, date_to))
    conn.commit()

def log_refresh_complete(
    conn: sqlite3.Connection,
    refresh_id: str,
    record_count: int,
    date_filter_counts: dict[str, int],
    duration_seconds: float,
) -> None:
    """Log the successful completion of a refresh operation."""
    conn.execute("""
        UPDATE pathway_refresh_log
        SET completed_at = ?,
            status = 'completed',
            record_count = ?,
            date_filter_counts = ?,
            processing_duration_seconds = ?
        WHERE refresh_id = ?
    """, (
        datetime.now().isoformat(),
        record_count,
        json.dumps(date_filter_counts),
        duration_seconds,
        refresh_id,
    ))
    conn.commit()

def log_refresh_failed(
    conn: sqlite3.Connection,
    refresh_id: str,
    error_message: str,
    duration_seconds: float,
) -> None:
    """Log a failed refresh operation."""
    conn.execute("""
        UPDATE pathway_refresh_log
        SET completed_at = ?,
            status = 'failed',
            error_message = ?,
            processing_duration_seconds = ?
        WHERE refresh_id = ?
    """, (
        datetime.now().isoformat(),
        error_message,
        duration_seconds,
        refresh_id,
    ))
    conn.commit()

def refresh_pathways(
    minimum_patients: int = 5,
    provider_codes: Optional[list[str]] = None,
    trust_filter: Optional[list[str]] = None,
    drug_filter: Optional[list[str]] = None,
    directory_filter: Optional[list[str]] = None,
    db_path: Optional[Path] = None,
    paths: Optional[PathConfig] = None,
    dry_run: bool = False,
    chart_type: str = "directory",
) -> tuple[bool, str, dict]:
    """
    Main refresh function that orchestrates the full pipeline.

    Args:
        minimum_patients: Minimum patients to include a pathway
        provider_codes: List of provider codes to filter Snowflake query
        trust_filter: List of trust names to include in pathways
        drug_filter: List of drug names to include in pathways
        directory_filter: List of directories to include in pathways
        db_path: Path to SQLite database (uses default if None)
        paths: PathConfig for file paths
        dry_run: If True, don't actually insert records
        chart_type: Which chart type to process: "directory", "indication", or "all"

    Returns:
        Tuple of (success: bool, message: str, stats: dict)
    """
    if paths is None:
        paths = default_paths

    # Set up database connection
    if db_path:
        db_config = DatabaseConfig(db_path=db_path)
    else:
        db_config = DatabaseConfig(data_dir=paths.data_dir)

    db_manager = DatabaseManager(db_config)

    # Load default filters if not provided
    default_trusts, default_drugs, default_dirs = get_default_filters(paths)

    if trust_filter is None:
        trust_filter = default_trusts
    if drug_filter is None:
        drug_filter = default_drugs
    if directory_filter is None:
        directory_filter = default_dirs

    # Ensure we have some filters
    if not drug_filter:
        return False, "No drugs specified and could not load defaults", {}

    # Determine which chart types to process
    if chart_type == "all":
        chart_types_to_process: list[ChartType] = ["directory", "indication"]
    else:
        chart_types_to_process = [chart_type]  # type: ignore

    logger.info("=" * 60)
    logger.info("Pathway Data Refresh Starting")
    logger.info("=" * 60)
    logger.info(f"Minimum patients: {minimum_patients}")
    logger.info(f"Trust filter: {len(trust_filter)} trusts")
    logger.info(f"Drug filter: {len(drug_filter)} drugs")
    logger.info(f"Directory filter: {len(directory_filter)} directories")
    logger.info(f"Provider codes: {provider_codes or 'All'}")
    logger.info(f"Chart type(s): {', '.join(chart_types_to_process)}")
    logger.info(f"Database: {db_manager.db_path}")
    logger.info(f"Dry run: {dry_run}")
    logger.info("=" * 60)

    start_time = time.time()
    refresh_id = str(uuid.uuid4())[:8]
    stats = {
        "refresh_id": refresh_id,
        "date_filter_counts": {},
        "total_records": 0,
        "snowflake_rows": 0,
    }

    try:
        # Verify database and tables
        with db_manager.get_connection() as conn:
            missing_tables = verify_pathway_tables_exist(conn)
            if missing_tables:
                logger.info(f"Creating missing tables: {missing_tables}")
                create_pathway_tables(conn)

            # Log refresh start
            if not dry_run:
                log_refresh_start(conn, refresh_id)

        # Step 1: Fetch data from Snowflake
        logger.info("")
        logger.info("Step 1/4: Fetching data from Snowflake...")
        df = fetch_and_transform_data(
            provider_codes=provider_codes,
            paths=paths,
        )

        if df.empty:
            msg = "No data returned from Snowflake"
            logger.error(msg)
            with db_manager.get_connection() as conn:
                log_refresh_failed(conn, refresh_id, msg, time.time() - start_time)
            return False, msg, stats

        stats["snowflake_rows"] = len(df)
        logger.info(f"Fetched {len(df)} records from Snowflake")

        # Step 2: Process all date filters for each chart type
        num_date_filters = len(DATE_FILTER_CONFIGS)
        num_chart_types = len(chart_types_to_process)
        total_datasets = num_date_filters * num_chart_types

        logger.info("")
        logger.info(f"Step 2/4: Processing pathway data for {total_datasets} datasets "
                   f"({num_date_filters} date filters x {num_chart_types} chart types)...")

        # Store results keyed by "date_filter_id:chart_type"
        results: dict[str, list[dict]] = {}

        for current_chart_type in chart_types_to_process:
            logger.info("")
            logger.info(f"Processing chart type: {current_chart_type}")

            if current_chart_type == "directory":
                # Use existing process_all_date_filters for directory charts
                dir_results = process_all_date_filters(
                    df=df,
                    trust_filter=trust_filter,
                    drug_filter=drug_filter,
                    directory_filter=directory_filter,
                    minimum_patients=minimum_patients,
                    refresh_id=refresh_id,
                    paths=paths,
                )
                # Add results with chart_type suffix
                for filter_id, records in dir_results.items():
                    # Records already have chart_type set by convert_to_records
                    results[f"{filter_id}:directory"] = records

            elif current_chart_type == "indication":
                # For indication charts, we need to look up GP diagnoses for all patients
                # using the new Snowflake-direct approach via get_patient_indication_groups()
                logger.info("Building indication groups from GP diagnosis lookups (Snowflake-direct)...")

                # Check Snowflake availability
                from data_processing.snowflake_connector import get_connector, is_snowflake_available

                if not is_snowflake_available():
                    logger.warning("Snowflake not available - cannot process indication charts")
                    for config in DATE_FILTER_CONFIGS:
                        results[f"{config.id}:indication"] = []
                    continue

                try:
                    import pandas as pd
                    connector = get_connector()

                    # Step 1: Extract unique PseudoNHSNoLinked values from df
                    # This is the patient identifier that matches PatientPseudonym in GP records
                    if 'PseudoNHSNoLinked' not in df.columns:
                        logger.error("DataFrame missing 'PseudoNHSNoLinked' column - cannot lookup GP records")
                        for config in DATE_FILTER_CONFIGS:
                            results[f"{config.id}:indication"] = []
                        continue

                    # Get unique patient pseudonyms for GP lookup (avoid redundant queries)
                    patient_pseudonyms = df['PseudoNHSNoLinked'].dropna().unique().tolist()

                    logger.info(f"Looking up GP diagnoses for {len(patient_pseudonyms)} unique patients...")

                    # Step 2: Call the new Snowflake-direct indication lookup
                    gp_matches_df = get_patient_indication_groups(
                        patient_pseudonyms=patient_pseudonyms,
                        connector=connector,
                        batch_size=500,
                    )

                    # Step 3: Build indication_df mapping UPID -> Indication_Group
                    # For matched patients: Indication_Group = Search_Term
                    # For unmatched patients: Indication_Group = Directory + " (no GP dx)"
                    #
                    # IMPORTANT: We need ALL unique UPIDs, not just unique PseudoNHSNoLinked.
                    # A patient can have multiple UPIDs if they visited multiple providers.

                    # Get all unique UPID records with their PseudoNHSNoLinked and Directory
                    upid_lookup = df[['UPID', 'PseudoNHSNoLinked', 'Directory']].drop_duplicates(
                        subset=['UPID']
                    ).copy()

                    if gp_matches_df.empty:
                        logger.warning("No GP matches found - all patients will use fallback directory")
                        # All patients use fallback
                        indication_records = []
                        for _, row in upid_lookup.iterrows():
                            directory = row['Directory']
                            indication_records.append({
                                'UPID': row['UPID'],
                                'Indication_Group': str(directory) + " (no GP dx)" if pd.notna(directory) else "UNKNOWN (no GP dx)",
                                'Source': 'FALLBACK',
                            })
                        indication_df = pd.DataFrame(indication_records)
                    else:
                        # Create lookup dict: PseudoNHSNoLinked -> Search_Term
                        match_lookup = dict(zip(
                            gp_matches_df['PatientPseudonym'],
                            gp_matches_df['Search_Term']
                        ))

                        # Build indication records for each unique UPID
                        indication_records = []
                        for _, row in upid_lookup.iterrows():
                            pseudo = row['PseudoNHSNoLinked']
                            upid = row['UPID']
                            directory = row['Directory']

                            if pd.notna(pseudo) and pseudo in match_lookup:
                                indication_records.append({
                                    'UPID': upid,
                                    'Indication_Group': match_lookup[pseudo],
                                    'Source': 'DIAGNOSIS',
                                })
                            else:
                                # Use fallback: Directory + " (no GP dx)"
                                fallback_label = str(directory) + " (no GP dx)" if pd.notna(directory) else "UNKNOWN (no GP dx)"
                                indication_records.append({
                                    'UPID': upid,
                                    'Indication_Group': fallback_label,
                                    'Source': 'FALLBACK',
                                })

                        indication_df = pd.DataFrame(indication_records)

                    # Log coverage statistics
                    if not indication_df.empty:
                        diagnosis_count = (indication_df['Source'] == 'DIAGNOSIS').sum()
                        fallback_count = (indication_df['Source'] == 'FALLBACK').sum()
                        total = len(indication_df)
                        stats["diagnosis_coverage"] = {
                            "diagnosis": int(diagnosis_count),
                            "fallback": int(fallback_count),
                            "total": total,
                            "diagnosis_pct": round(100 * diagnosis_count / total, 1) if total > 0 else 0,
                        }
                        logger.info(f"Indication coverage: {diagnosis_count}/{total} ({stats['diagnosis_coverage']['diagnosis_pct']}%) diagnosis-matched")

                        # Log top indication groups
                        top_indications = indication_df[indication_df['Source'] == 'DIAGNOSIS']['Indication_Group'].value_counts().head(5)
                        if len(top_indications) > 0:
                            logger.info(f"Top 5 indications: {dict(top_indications)}")

                        # Rename column for compatibility with generate_icicle_chart_indication
                        # It expects indication_df to have 'Directory' column (mapped from Indication_Group)
                        indication_df_for_chart = indication_df[['UPID', 'Indication_Group']].copy()
                        indication_df_for_chart = indication_df_for_chart.rename(columns={'Indication_Group': 'Directory'})
                        # Ensure unique UPID index (build_hierarchy requires uniquely valued Index)
                        # Keep first occurrence - DIAGNOSIS entries should come before FALLBACK
                        indication_df_for_chart = indication_df_for_chart.drop_duplicates(subset=['UPID'], keep='first')
                        indication_df_for_chart = indication_df_for_chart.set_index('UPID')

                        # Process each date filter with indication grouping
                        for config in DATE_FILTER_CONFIGS:
                            logger.info(f"Processing indication pathway for {config.id}")

                            ice_df = process_indication_pathway_for_date_filter(
                                df=df,
                                indication_df=indication_df_for_chart,
                                config=config,
                                trust_filter=trust_filter,
                                drug_filter=drug_filter,
                                directory_filter=directory_filter,
                                minimum_patients=minimum_patients,
                                paths=paths,
                            )

                            if ice_df is None:
                                logger.warning(f"No indication pathway data for {config.id}")
                                results[f"{config.id}:indication"] = []
                                continue

                            # Extract denormalized fields (using indication variant)
                            ice_df = extract_indication_fields(ice_df)

                            # Convert to records with chart_type="indication"
                            records = convert_to_records(ice_df, config.id, refresh_id, chart_type="indication")
                            results[f"{config.id}:indication"] = records

                            logger.info(f"Completed {config.id}:indication: {len(records)} nodes")
                    else:
                        logger.warning("Empty indication_df - skipping indication charts")
                        for config in DATE_FILTER_CONFIGS:
                            results[f"{config.id}:indication"] = []

                except Exception as e:
                    logger.error(f"Error processing indication charts: {e}")
                    logger.exception(e)
                    for config in DATE_FILTER_CONFIGS:
                        results[f"{config.id}:indication"] = []

        # Count records per filter and chart type
        stats["chart_type_counts"] = {}
        for key, records in results.items():
            stats["date_filter_counts"][key] = len(records)
            stats["total_records"] += len(records)
            # Also track by chart type
            _, ct = key.split(":")
            stats["chart_type_counts"][ct] = stats["chart_type_counts"].get(ct, 0) + len(records)

        logger.info("")
        logger.info(f"Processed {stats['total_records']} total pathway nodes")
        for chart_type_name, count in stats.get("chart_type_counts", {}).items():
            logger.info(f"  {chart_type_name}: {count} nodes total")
        for key, count in sorted(stats["date_filter_counts"].items()):
            if count > 0:
                logger.info(f"    {key}: {count} nodes")

        if dry_run:
            logger.info("")
            logger.info("DRY RUN - Skipping database insertion")
            elapsed = time.time() - start_time
            return True, f"Dry run complete: {stats['total_records']} records would be inserted", stats

        # Step 3: Clear existing data and insert new records
        logger.info("")
        logger.info("Step 3/4: Clearing existing pathway data and inserting new records...")

        with db_manager.get_transaction() as conn:
            # Clear all existing pathway nodes
            deleted = clear_pathway_nodes(conn)
            logger.info(f"Cleared {deleted} existing pathway nodes")

            # Insert new records for each date filter + chart type combination
            total_inserted = 0
            for key, records in results.items():
                if records:
                    inserted = insert_pathway_records(conn, records)
                    total_inserted += len(records)
                    logger.info(f"  Inserted {len(records)} records for {key}")

        # Step 4: Log completion
        logger.info("")
        logger.info("Step 4/4: Logging refresh completion...")

        elapsed = time.time() - start_time

        with db_manager.get_connection() as conn:
            log_refresh_complete(
                conn=conn,
                refresh_id=refresh_id,
                record_count=stats["total_records"],
                date_filter_counts=stats["date_filter_counts"],
                duration_seconds=elapsed,
            )

            # Verify final counts
            counts = get_pathway_table_counts(conn)
            logger.info(f"Final table counts: {counts}")

        logger.info("")
        logger.info("=" * 60)
        logger.info(f"Refresh completed successfully in {elapsed:.1f} seconds")
        logger.info(f"Total records: {stats['total_records']}")
        logger.info(f"Refresh ID: {refresh_id}")
        logger.info("=" * 60)

        return True, f"Refresh complete: {stats['total_records']} records in {elapsed:.1f}s", stats

    except Exception as e:
        elapsed = time.time() - start_time
        error_msg = f"Refresh failed: {e}"
        logger.error(error_msg, exc_info=True)

        try:
            with db_manager.get_connection() as conn:
                log_refresh_failed(conn, refresh_id, str(e), elapsed)
        except Exception:
            pass  # Don't fail the error handling

        return False, error_msg, stats

def main() -> int:
    """CLI entry point."""
    parser = argparse.ArgumentParser(
        description="Refresh pathway data from Snowflake",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog="""
Examples:
    # Basic refresh with defaults (directory chart only)
    python -m cli.refresh_pathways

    # Refresh both chart types (directory and indication)
    python -m cli.refresh_pathways --chart-type all

    # Refresh only indication-based charts
    python -m cli.refresh_pathways --chart-type indication

    # Refresh with custom minimum patients
    python -m cli.refresh_pathways --minimum-patients 10

    # Refresh specific providers only
    python -m cli.refresh_pathways --provider-codes RGT,RM1

    # Dry run to see what would be processed
    python -m cli.refresh_pathways --dry-run

    # Verbose output
    python -m cli.refresh_pathways --verbose
        """
    )

    parser.add_argument(
        "--minimum-patients",
        type=int,
        default=5,
        help="Minimum patients to include a pathway (default: 5)"
    )

    parser.add_argument(
        "--provider-codes",
        type=str,
        default=None,
        help="Comma-separated list of provider codes to filter (default: all)"
    )

    parser.add_argument(
        "--db-path",
        type=str,
        default=None,
        help="Path to SQLite database (default: data/pathways.db)"
    )

    parser.add_argument(
        "--dry-run",
        action="store_true",
        help="Process data but don't insert into database"
    )

    parser.add_argument(
        "--chart-type",
        type=str,
        choices=["directory", "indication", "all"],
        default="directory",
        help="Chart type to process: 'directory' (default), 'indication', or 'all'"
    )

    parser.add_argument(
        "--verbose", "-v",
        action="store_true",
        help="Enable verbose logging"
    )

    args = parser.parse_args()

    # Configure logging
    import logging
    log_level = logging.DEBUG if args.verbose else logging.INFO
    setup_logging(level=log_level)

    # Parse provider codes
    provider_codes = None
    if args.provider_codes:
        provider_codes = [code.strip() for code in args.provider_codes.split(",")]

    # Parse db path
    db_path = Path(args.db_path) if args.db_path else None

    # Run the refresh
    success, message, stats = refresh_pathways(
        minimum_patients=args.minimum_patients,
        provider_codes=provider_codes,
        db_path=db_path,
        dry_run=args.dry_run,
        chart_type=args.chart_type,
    )

    if success:
        print(f"\n[OK] {message}")
        return 0
    else:
        print(f"\n[FAILED] {message}", file=sys.stderr)
        return 1

if __name__ == "__main__":
    sys.exit(main())


Now I have the full picture. I can see the critical issue in `refresh_pathways.py` lines 424-428 ÔÇö `dict(zip(...))` which only keeps the LAST match per patient (since the new query returns multiple rows per patient). This will be fixed in Task 3.1.

Now I'm ready to build Task 2.1: `assign_drug_indications()`. Let me mark it in-progress and start implementing.


TodoWrite

Read project files (progress.txt, IMPLEMENTATION_PLAN.md, guardrails.md)

Build assign_drug_indications() function (Task 2.1 + 2.2)

Validate code (py_compile, import check, mock data test)

Commit and update progress.txt


Todos have been modified successfully. Ensure that you continue to use the todo list to track your progress. Please proceed with the current tasks if applicable

Read project files (progress.txt, IMPLEMENTATION_PLAN.md, guardrails.md)

Build assign_drug_indications() function (Task 2.1 + 2.2)

Validate code (py_compile, import check, mock data test)

Commit and update progress.txt


Now implementing the `assign_drug_indications()` function. The key logic:

1. For each UPID + Drug Name pair in the HCD data:
   - Look up patient's GP-matched Search_Terms (from gp_matches_df via PseudoNHSNoLinked)
   - Look up which Search_Terms list this drug (from drug mapping)
   - Intersect ÔåÆ valid indications
   - Pick highest code_frequency as tiebreaker (or alphabetical if tied)
   - Modify UPID: `{UPID}|{search_term}` or fallback `{UPID}|{Directory} (no GP dx)`

2. Build indication_df mapping modified UPID ÔåÆ Search_Term

I'll add this function to `diagnosis_lookup.py` since that's where all the related functions live.


  1090ÔåÆ# === Drug-to-indication mapping from DimSearchTerm.csv ===
  1091ÔåÆ
  1092ÔåÆ
  1093ÔåÆ# Merge related Search_Terms into canonical names.
  1094ÔåÆ# Asthma variants are clinically the same condition at different severity levels.
  1095ÔåÆ# Urticaria is a separate condition ÔÇö do NOT merge with asthma.
  1096ÔåÆSEARCH_TERM_MERGE_MAP: dict[str, str] = {
  1097ÔåÆ    "allergic asthma": "asthma",
  1098ÔåÆ    "severe persistent allergic asthma": "asthma",
  1099ÔåÆ}
  1100ÔåÆ
  1101ÔåÆ
  1102ÔåÆdef load_drug_indication_mapping(
  1103ÔåÆ    csv_path: Optional[str] = None,
  1104ÔåÆ) -> tuple[dict[str, list[str]], dict[str, list[str]]]:
  1105ÔåÆ    """
  1106ÔåÆ    Load the drug-to-Search_Term mapping from DimSearchTerm.csv.
  1107ÔåÆ
  1108ÔåÆ    Builds two lookup dicts:
  1109ÔåÆ    - fragment_to_search_terms: drug fragment (UPPERCASE) -> list of Search_Terms containing it
  1110ÔåÆ    - search_term_to_fragments: search_term -> list of drug fragments (UPPERCASE)
  1111ÔåÆ
  1112ÔåÆ    DimSearchTerm.csv has columns: Search_Term, CleanedDrugName, PrimaryDirectorate
  1113ÔåÆ    CleanedDrugName is pipe-separated (e.g., "ADALIMUMAB|GOLIMUMAB|IXEKIZUMAB").
  1114ÔåÆ
  1115ÔåÆ    Note: A Search_Term can appear multiple times with different PrimaryDirectorates
  1116ÔåÆ    (e.g., "diabetes" appears under both DIABETIC MEDICINE and OPHTHALMOLOGY).
  1117ÔåÆ    Drug fragments from all rows for the same Search_Term are combined.
  1118ÔåÆ
  1119ÔåÆ    Asthma-related Search_Terms ("allergic asthma", "severe persistent allergic asthma")
  1120ÔåÆ    are merged into "asthma" to match the CLUSTER_MAPPING_SQL normalization.
  1121ÔåÆ    "urticaria" stays separate.
  1122ÔåÆ
  1123ÔåÆ    Args:
  1124ÔåÆ        csv_path: Path to DimSearchTerm.csv. Defaults to data/DimSearchTerm.csv.
  1125ÔåÆ
  1126ÔåÆ    Returns:
  1127ÔåÆ        Tuple of (fragment_to_search_terms, search_term_to_fragments)
  1128ÔåÆ    """
  1129ÔåÆ    if csv_path is None:
  1130ÔåÆ        csv_path = str(Path(__file__).parent.parent / "data" / "DimSearchTerm.csv")
  1131ÔåÆ
  1132ÔåÆ    fragment_to_search_terms: dict[str, list[str]] = {}
  1133ÔåÆ    search_term_to_fragments: dict[str, list[str]] = {}
  1134ÔåÆ
  1135ÔåÆ    try:
  1136ÔåÆ        with open(csv_path, "r", encoding="utf-8") as f:
  1137ÔåÆ            reader = csv.DictReader(f)
  1138ÔåÆ            for row in reader:
  1139ÔåÆ                search_term = row.get("Search_Term", "").strip()

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>

# === Drug-to-indication mapping from DimSearchTerm.csv ===

# Merge related Search_Terms into canonical names.
# Asthma variants are clinically the same condition at different severity levels.
# Urticaria is a separate condition ÔÇö do NOT merge with asthma.
SEARCH_TERM_MERGE_MAP: dict[str, str] = {
    "allergic asthma": "asthma",
    "severe persistent allergic asthma": "asthma",
}

def load_drug_indication_mapping(
    csv_path: Optional[str] = None,
) -> tuple[dict[str, list[str]], dict[str, list[str]]]:
    """
    Load the drug-to-Search_Term mapping from DimSearchTerm.csv.

    Builds two lookup dicts:
    - fragment_to_search_terms: drug fragment (UPPERCASE) -> list of Search_Terms containing it
    - search_term_to_fragments: search_term -> list of drug fragments (UPPERCASE)

    DimSearchTerm.csv has columns: Search_Term, CleanedDrugName, PrimaryDirectorate
    CleanedDrugName is pipe-separated (e.g., "ADALIMUMAB|GOLIMUMAB|IXEKIZUMAB").

    Note: A Search_Term can appear multiple times with different PrimaryDirectorates
    (e.g., "diabetes" appears under both DIABETIC MEDICINE and OPHTHALMOLOGY).
    Drug fragments from all rows for the same Search_Term are combined.

    Asthma-related Search_Terms ("allergic asthma", "severe persistent allergic asthma")
    are merged into "asthma" to match the CLUSTER_MAPPING_SQL normalization.
    "urticaria" stays separate.

    Args:
        csv_path: Path to DimSearchTerm.csv. Defaults to data/DimSearchTerm.csv.

    Returns:
        Tuple of (fragment_to_search_terms, search_term_to_fragments)
    """
    if csv_path is None:
        csv_path = str(Path(__file__).parent.parent / "data" / "DimSearchTerm.csv")

    fragment_to_search_terms: dict[str, list[str]] = {}
    search_term_to_fragments: dict[str, list[str]] = {}

    try:
        with open(csv_path, "r", encoding="utf-8") as f:
            reader = csv.DictReader(f)
            for row in reader:
                search_term = row.get("Search_Term", "").strip()


Now I'll add the `assign_drug_indications()` function. I'll insert it after `get_search_terms_for_drug()` (line 1204) and before the `CLUSTER_MAPPING_SQL` section (line 1207).


Edit

    return matched_terms

# === NEW APPROACH: Query Snowflake directly using cluster CTE ===

    return matched_terms

def assign_drug_indications(
    df: "pd.DataFrame",
    gp_matches_df: "pd.DataFrame",
    search_term_to_fragments: dict[str, list[str]],
) -> "tuple[pd.DataFrame, pd.DataFrame]":
    """
    Assign drug-aware indications by cross-referencing GP diagnoses with drug mappings.

    For each UPID + Drug Name pair in the HCD data:
    1. Look up patient's GP-matched Search_Terms (via PseudoNHSNoLinked ÔåÆ gp_matches_df)
    2. Look up which Search_Terms list this drug (via search_term_to_fragments)
    3. Intersect = valid indications for this drug-patient pair
    4. If 1 match: use it
    5. If multiple: pick highest code_frequency (most GP coding = most likely indication)
    6. If tied frequency: alphabetical Search_Term for determinism
    7. If 0 matches: fallback to "{Directory} (no GP dx)"

    Modifies UPID to include indication: "{UPID}|{search_term}" or "{UPID}|{Directory} (no GP dx)".
    This makes drugs under different indications appear as separate pathways.

    Args:
        df: HCD DataFrame with columns: UPID, Drug Name, PseudoNHSNoLinked, Directory
        gp_matches_df: GP matches from get_patient_indication_groups() with columns:
            PatientPseudonym, Search_Term, code_frequency
            (multiple rows per patient, one per matched Search_Term)
        search_term_to_fragments: From load_drug_indication_mapping()[1].
            Maps search_term -> list of drug fragments (UPPERCASE).

    Returns:
        Tuple of (modified_df, indication_df):
        - modified_df: Copy of df with UPID replaced by "{UPID}|{indication}"
        - indication_df: DataFrame indexed by modified UPID with 'Directory' column
            containing the Search_Term (or fallback label) for pathway hierarchy
    """
    import pandas as pd

    modified_df = df.copy()

    # Build GP match lookup: PseudoNHSNoLinked -> {Search_Term: code_frequency}
    gp_lookup: dict[str, dict[str, int]] = {}
    if not gp_matches_df.empty:
        for _, row in gp_matches_df.iterrows():
            pseudo = row['PatientPseudonym']
            term = row['Search_Term']
            freq = int(row.get('code_frequency', 0))
            if pseudo not in gp_lookup:
                gp_lookup[pseudo] = {}
            gp_lookup[pseudo][term] = freq

    logger.info(f"GP lookup built: {len(gp_lookup)} patients with GP matches")

    # Cache drug -> Search_Terms lookups to avoid recomputing per row
    drug_search_terms_cache: dict[str, list[str]] = {}

    def get_drug_terms(drug_name: str) -> list[str]:
        if drug_name not in drug_search_terms_cache:
            drug_search_terms_cache[drug_name] = get_search_terms_for_drug(
                drug_name, search_term_to_fragments
            )
        return drug_search_terms_cache[drug_name]

    # Process each row: determine indication for each UPID + Drug Name
    # We work at the (UPID, Drug Name) pair level, then apply to all rows
    # Key: (UPID, Drug Name) -> (matched_search_term_or_fallback, is_diagnosis)
    pair_indication_cache: dict[tuple[str, str], tuple[str, bool]] = {}

    # Get unique (UPID, Drug Name, PseudoNHSNoLinked, Directory) combos
    required_cols = ['UPID', 'Drug Name', 'PseudoNHSNoLinked', 'Directory']
    unique_pairs = modified_df[required_cols].drop_duplicates(
        subset=['UPID', 'Drug Name']
    )

    match_count = 0
    fallback_count = 0
    tiebreak_count = 0

    for _, pair_row in unique_pairs.iterrows():
        upid = pair_row['UPID']
        drug_name = pair_row['Drug Name']
        pseudo = pair_row['PseudoNHSNoLinked']
        directory = pair_row['Directory']

        # Get Search_Terms this drug maps to
        drug_terms = get_drug_terms(drug_name)

        # Get patient's GP-matched Search_Terms
        patient_gp_terms = gp_lookup.get(pseudo, {}) if pd.notna(pseudo) else {}

        # Intersect: Search_Terms that list this drug AND patient has GP dx for
        valid_indications = {
            term: patient_gp_terms[term]
            for term in drug_terms
            if term in patient_gp_terms
        }

        if len(valid_indications) == 1:
            matched_term = next(iter(valid_indications))
            pair_indication_cache[(upid, drug_name)] = (matched_term, True)
            match_count += 1

        elif len(valid_indications) > 1:
            # Tiebreaker: highest code_frequency, then alphabetical
            sorted_terms = sorted(
                valid_indications.items(),
                key=lambda x: (-x[1], x[0]),
            )
            matched_term = sorted_terms[0][0]
            pair_indication_cache[(upid, drug_name)] = (matched_term, True)
            match_count += 1
            tiebreak_count += 1

        else:
            # No intersection: fallback to directory
            if pd.notna(directory):
                fallback_label = f"{directory} (no GP dx)"
            else:
                fallback_label = "UNKNOWN (no GP dx)"
            pair_indication_cache[(upid, drug_name)] = (fallback_label, False)
            fallback_count += 1

    total_pairs = len(unique_pairs)
    logger.info(f"Drug-indication matching complete:")
    logger.info(f"  Total UPID-Drug pairs: {total_pairs}")
    logger.info(f"  Matched (GP dx + drug mapping): {match_count} ({100*match_count/total_pairs:.1f}%)" if total_pairs > 0 else f"  Matched: 0")
    logger.info(f"  Tiebreaker used: {tiebreak_count}")
    logger.info(f"  Fallback (no match): {fallback_count} ({100*fallback_count/total_pairs:.1f}%)" if total_pairs > 0 else f"  Fallback: 0")

    # Apply modified UPIDs to all rows
    # Build vectorized lookup: original UPID + Drug Name -> modified UPID
    def build_modified_upid(row: "pd.Series") -> str:
        upid = row['UPID']
        drug_name = row['Drug Name']
        key = (upid, drug_name)
        if key in pair_indication_cache:
            indication, _ = pair_indication_cache[key]
            return f"{upid}|{indication}"
        # Shouldn't happen, but fallback
        return f"{upid}|UNKNOWN (no GP dx)"

    modified_df['UPID'] = modified_df.apply(build_modified_upid, axis=1)

    # Build indication_df: modified UPID -> Search_Term/fallback label
    # This maps each unique modified UPID to its indication for the pathway hierarchy
    indication_records: list[dict[str, str]] = []
    seen_upids: set[str] = set()

    for (original_upid, drug_name), (indication, is_diagnosis) in pair_indication_cache.items():
        modified_upid = f"{original_upid}|{indication}"
        if modified_upid not in seen_upids:
            indication_records.append({
                'UPID': modified_upid,
                'Directory': indication,
            })
            seen_upids.add(modified_upid)

    indication_df = pd.DataFrame(indication_records)
    if not indication_df.empty:
        indication_df = indication_df.set_index('UPID')

    logger.info(f"  Unique modified UPIDs: {len(indication_df)}")

    # Log top indications
    if not indication_df.empty:
        top_terms = indication_df['Directory'].value_counts().head(5)
        logger.info(f"  Top 5 indications: {dict(top_terms)}")

    return modified_df, indication_df

# === NEW APPROACH: Query Snowflake directly using cluster CTE ===