     1ÔåÆ"""
     2ÔåÆDiagnosis lookup module for NHS Patient Pathway Analysis.
     3ÔåÆ
     4ÔåÆProvides functions to validate patient indications by checking GP diagnosis records
     5ÔåÆagainst SNOMED cluster codes. Uses the drug-to-cluster mapping from
     6ÔåÆdrug_indication_clusters.csv and queries Snowflake for SNOMED codes and GP records.
     7ÔåÆ
     8ÔåÆKey workflow:
     9ÔåÆ1. Get drug's valid indication clusters from local mapping
    10ÔåÆ2. Get all SNOMED codes for those clusters from Snowflake
    11ÔåÆ3. Check if patient has any of those SNOMED codes in GP records
    12ÔåÆ4. Report indication validation status
    13ÔåÆ
    14ÔåÆIMPORTANT: HCD activity data indication codes are UNRELIABLE. This module uses
    15ÔåÆGP/Primary Care data (PrimaryCareClinicalCoding) as the authoritative source.
    16ÔåÆ"""
    17ÔåÆ
    18ÔåÆfrom dataclasses import dataclass, field
    19ÔåÆfrom datetime import date, datetime
    20ÔåÆfrom pathlib import Path
    21ÔåÆfrom typing import Optional, Callable, Any, cast, TYPE_CHECKING
    22ÔåÆimport csv
    23ÔåÆ
    24ÔåÆif TYPE_CHECKING:
    25ÔåÆ    import pandas as pd
    26ÔåÆ
    27ÔåÆfrom core.logging_config import get_logger
    28ÔåÆfrom data_processing.database import DatabaseManager, default_db_manager
    29ÔåÆfrom data_processing.snowflake_connector import (
    30ÔåÆ    SnowflakeConnector,
    31ÔåÆ    get_connector,
    32ÔåÆ    is_snowflake_available,
    33ÔåÆ    is_snowflake_configured,
    34ÔåÆ    SNOWFLAKE_AVAILABLE,
    35ÔåÆ)
    36ÔåÆfrom data_processing.cache import get_cache, is_cache_enabled
    37ÔåÆ
    38ÔåÆlogger = get_logger(__name__)
    39ÔåÆ
    40ÔåÆ
    41ÔåÆ@dataclass
    42ÔåÆclass ClusterSnomedCodes:
    43ÔåÆ    """SNOMED codes for a clinical coding cluster."""
    44ÔåÆ    cluster_id: str
    45ÔåÆ    cluster_description: str
    46ÔåÆ    snomed_codes: list[str] = field(default_factory=list)
    47ÔåÆ    snomed_descriptions: dict[str, str] = field(default_factory=dict)
    48ÔåÆ
    49ÔåÆ    @property
    50ÔåÆ    def code_count(self) -> int:
    51ÔåÆ        return len(self.snomed_codes)
    52ÔåÆ
    53ÔåÆ
    54ÔåÆ@dataclass
    55ÔåÆclass IndicationValidationResult:
    56ÔåÆ    """Result of validating a patient's indication for a drug."""
    57ÔåÆ    patient_pseudonym: str
    58ÔåÆ    drug_name: str
    59ÔåÆ    has_valid_indication: bool
    60ÔåÆ    matched_cluster_id: Optional[str] = None
    61ÔåÆ    matched_snomed_code: Optional[str] = None
    62ÔåÆ    matched_snomed_description: Optional[str] = None
    63ÔåÆ    checked_clusters: list[str] = field(default_factory=list)
    64ÔåÆ    total_codes_checked: int = 0
    65ÔåÆ    source: str = "GP_SNOMED"  # GP_SNOMED | NONE
    66ÔåÆ    error_message: Optional[str] = None
    67ÔåÆ
    68ÔåÆ
    69ÔåÆ@dataclass
    70ÔåÆclass DrugIndicationMatchRate:
    71ÔåÆ    """Match rate statistics for a drug's indication validation."""
    72ÔåÆ    drug_name: str
    73ÔåÆ    total_patients: int
    74ÔåÆ    patients_with_indication: int
    75ÔåÆ    patients_without_indication: int
    76ÔåÆ    match_rate: float  # 0.0 to 1.0
    77ÔåÆ    clusters_checked: list[str] = field(default_factory=list)
    78ÔåÆ    sample_unmatched: list[str] = field(default_factory=list)  # Sample patient IDs
    79ÔåÆ
    80ÔåÆ
    81ÔåÆ@dataclass
    82ÔåÆclass DrugSnomedMapping:
    83ÔåÆ    """SNOMED code mapping for a drug from ref_drug_snomed_mapping."""
    84ÔåÆ    snomed_code: str
    85ÔåÆ    snomed_description: str
    86ÔåÆ    search_term: str
    87ÔåÆ    primary_directorate: str
    88ÔåÆ    indication: str = ""
    89ÔåÆ    ta_id: str = ""
    90ÔåÆ
    91ÔåÆ
    92ÔåÆ@dataclass
    93ÔåÆclass DirectSnomedMatchResult:
    94ÔåÆ    """Result of direct SNOMED code lookup in GP records."""
    95ÔåÆ    patient_pseudonym: str
    96ÔåÆ    matched: bool
    97ÔåÆ    snomed_code: Optional[str] = None
    98ÔåÆ    snomed_description: Optional[str] = None
    99ÔåÆ    search_term: Optional[str] = None
   100ÔåÆ    primary_directorate: Optional[str] = None
   101ÔåÆ    event_date: Optional[datetime] = None
   102ÔåÆ    source: str = "DIRECT_SNOMED"  # DIRECT_SNOMED | NONE
   103ÔåÆ
   104ÔåÆ
   105ÔåÆ@dataclass
   106ÔåÆclass DirectorateAssignment:
   107ÔåÆ    """Result of directorate assignment for a patient-drug combination."""
   108ÔåÆ    upid: str
   109ÔåÆ    drug_name: str
   110ÔåÆ    directorate: Optional[str]
   111ÔåÆ    search_term: Optional[str] = None
   112ÔåÆ    source: str = "FALLBACK"  # DIAGNOSIS | FALLBACK
   113ÔåÆ    snomed_code: Optional[str] = None
   114ÔåÆ    event_date: Optional[datetime] = None
   115ÔåÆ
   116ÔåÆ
   117ÔåÆdef get_drug_clusters(
   118ÔåÆ    drug_name: str,
   119ÔåÆ    db_manager: Optional[DatabaseManager] = None
   120ÔåÆ) -> list[dict]:
   121ÔåÆ    """
   122ÔåÆ    Get all SNOMED cluster mappings for a drug from local SQLite.
   123ÔåÆ
   124ÔåÆ    Args:
   125ÔåÆ        drug_name: Drug name to look up (case-insensitive)
   126ÔåÆ        db_manager: Optional DatabaseManager (defaults to default_db_manager)
   127ÔåÆ
   128ÔåÆ    Returns:
   129ÔåÆ        List of dicts with keys: drug_name, indication, cluster_id,
   130ÔåÆ        cluster_description, nice_ta_reference
   131ÔåÆ    """
   132ÔåÆ    if db_manager is None:
   133ÔåÆ        db_manager = default_db_manager
   134ÔåÆ
   135ÔåÆ    query = """
   136ÔåÆ        SELECT drug_name, indication, cluster_id, cluster_description, nice_ta_reference
   137ÔåÆ        FROM ref_drug_indication_clusters
   138ÔåÆ        WHERE UPPER(drug_name) = UPPER(?)
   139ÔåÆ        ORDER BY indication, cluster_id
   140ÔåÆ    """
   141ÔåÆ
   142ÔåÆ    try:
   143ÔåÆ        with db_manager.get_connection() as conn:
   144ÔåÆ            cursor = conn.execute(query, (drug_name,))
   145ÔåÆ            rows = cursor.fetchall()
   146ÔåÆ
   147ÔåÆ            results = []
   148ÔåÆ            for row in rows:
   149ÔåÆ                results.append({
   150ÔåÆ                    "drug_name": row["drug_name"],
   151ÔåÆ                    "indication": row["indication"],
   152ÔåÆ                    "cluster_id": row["cluster_id"],
   153ÔåÆ                    "cluster_description": row["cluster_description"],
   154ÔåÆ                    "nice_ta_reference": row["nice_ta_reference"],
   155ÔåÆ                })
   156ÔåÆ
   157ÔåÆ            logger.debug(f"Found {len(results)} cluster mappings for drug '{drug_name}'")
   158ÔåÆ            return results
   159ÔåÆ
   160ÔåÆ    except Exception as e:
   161ÔåÆ        logger.error(f"Error getting clusters for drug '{drug_name}': {e}")
   162ÔåÆ        return []
   163ÔåÆ
   164ÔåÆ
   165ÔåÆdef get_drug_cluster_ids(
   166ÔåÆ    drug_name: str,
   167ÔåÆ    db_manager: Optional[DatabaseManager] = None
   168ÔåÆ) -> list[str]:
   169ÔåÆ    """
   170ÔåÆ    Get unique cluster IDs for a drug.
   171ÔåÆ
   172ÔåÆ    Args:
   173ÔåÆ        drug_name: Drug name to look up
   174ÔåÆ        db_manager: Optional DatabaseManager
   175ÔåÆ
   176ÔåÆ    Returns:
   177ÔåÆ        List of unique cluster IDs
   178ÔåÆ    """
   179ÔåÆ    clusters = get_drug_clusters(drug_name, db_manager)
   180ÔåÆ    return list(set(c["cluster_id"] for c in clusters))
   181ÔåÆ
   182ÔåÆ
   183ÔåÆdef get_drug_snomed_codes(
   184ÔåÆ    drug_name: str,
   185ÔåÆ    db_manager: Optional[DatabaseManager] = None
   186ÔåÆ) -> list[DrugSnomedMapping]:
   187ÔåÆ    """
   188ÔåÆ    Get all SNOMED codes for a drug from local ref_drug_snomed_mapping table.
   189ÔåÆ
   190ÔåÆ    This uses the enriched mapping CSV data loaded into SQLite, which provides
   191ÔåÆ    direct SNOMED-to-drug mappings with Search_Term and PrimaryDirectorate.
   192ÔåÆ
   193ÔåÆ    Args:
   194ÔåÆ        drug_name: Drug name to look up (case-insensitive, matches cleaned_drug_name)
   195ÔåÆ        db_manager: Optional DatabaseManager (defaults to default_db_manager)
   196ÔåÆ
   197ÔåÆ    Returns:
   198ÔåÆ        List of DrugSnomedMapping with snomed_code, snomed_description,
   199ÔåÆ        search_term, primary_directorate, indication, ta_id
   200ÔåÆ    """
   201ÔåÆ    if db_manager is None:
   202ÔåÆ        db_manager = default_db_manager
   203ÔåÆ
   204ÔåÆ    query = """
   205ÔåÆ        SELECT DISTINCT
   206ÔåÆ            snomed_code,
   207ÔåÆ            snomed_description,
   208ÔåÆ            search_term,
   209ÔåÆ            primary_directorate,
   210ÔåÆ            indication,
   211ÔåÆ            ta_id
   212ÔåÆ        FROM ref_drug_snomed_mapping
   213ÔåÆ        WHERE UPPER(cleaned_drug_name) = UPPER(?)
   214ÔåÆ           OR UPPER(drug_name) = UPPER(?)
   215ÔåÆ        ORDER BY search_term, snomed_code
   216ÔåÆ    """
   217ÔåÆ
   218ÔåÆ    try:
   219ÔåÆ        with db_manager.get_connection() as conn:
   220ÔåÆ            cursor = conn.execute(query, (drug_name, drug_name))
   221ÔåÆ            rows = cursor.fetchall()
   222ÔåÆ
   223ÔåÆ            results = []
   224ÔåÆ            for row in rows:
   225ÔåÆ                results.append(DrugSnomedMapping(
   226ÔåÆ                    snomed_code=row["snomed_code"],
   227ÔåÆ                    snomed_description=row["snomed_description"] or "",
   228ÔåÆ                    search_term=row["search_term"] or "",
   229ÔåÆ                    primary_directorate=row["primary_directorate"] or "",
   230ÔåÆ                    indication=row["indication"] or "",
   231ÔåÆ                    ta_id=row["ta_id"] or "",
   232ÔåÆ                ))
   233ÔåÆ
   234ÔåÆ            logger.debug(f"Found {len(results)} SNOMED mappings for drug '{drug_name}'")
   235ÔåÆ            return results
   236ÔåÆ
   237ÔåÆ    except Exception as e:
   238ÔåÆ        logger.error(f"Error getting SNOMED codes for drug '{drug_name}': {e}")
   239ÔåÆ        return []
   240ÔåÆ
   241ÔåÆ
   242ÔåÆdef patient_has_indication_direct(
   243ÔåÆ    patient_pseudonym: str,
   244ÔåÆ    drug_snomed_mappings: list[DrugSnomedMapping],
   245ÔåÆ    connector: Optional[SnowflakeConnector] = None,
   246ÔåÆ    before_date: Optional[date] = None,
   247ÔåÆ) -> DirectSnomedMatchResult:
   248ÔåÆ    """
   249ÔåÆ    Check if patient has any of the SNOMED codes in their GP records.
   250ÔåÆ
   251ÔåÆ    This is the direct SNOMED lookup - it queries PrimaryCareClinicalCoding
   252ÔåÆ    for exact SNOMED code matches (not via cluster). Returns the most recent
   253ÔåÆ    match by EventDateTime if multiple matches exist.
   254ÔåÆ
   255ÔåÆ    Args:
   256ÔåÆ        patient_pseudonym: Patient's pseudonymised NHS number
   257ÔåÆ        drug_snomed_mappings: List of DrugSnomedMapping from get_drug_snomed_codes()
   258ÔåÆ        connector: Optional SnowflakeConnector (defaults to singleton)
   259ÔåÆ        before_date: Optional date - only check diagnoses before this date
   260ÔåÆ
   261ÔåÆ    Returns:
   262ÔåÆ        DirectSnomedMatchResult with match details (most recent by EventDateTime)
   263ÔåÆ    """
   264ÔåÆ    result = DirectSnomedMatchResult(
   265ÔåÆ        patient_pseudonym=patient_pseudonym,
   266ÔåÆ        matched=False,
   267ÔåÆ        source="NONE",
   268ÔåÆ    )
   269ÔåÆ
   270ÔåÆ    if not drug_snomed_mappings:
   271ÔåÆ        return result
   272ÔåÆ
   273ÔåÆ    if not SNOWFLAKE_AVAILABLE:
   274ÔåÆ        logger.warning("Snowflake connector not available")
   275ÔåÆ        return result
   276ÔåÆ
   277ÔåÆ    if not is_snowflake_configured():
   278ÔåÆ        logger.warning("Snowflake not configured - cannot check GP records")
   279ÔåÆ        return result
   280ÔåÆ
   281ÔåÆ    if connector is None:
   282ÔåÆ        connector = get_connector()
   283ÔåÆ
   284ÔåÆ    # Build lookup dict for mapping snomed_code -> (search_term, primary_directorate, snomed_description)
   285ÔåÆ    snomed_lookup = {
   286ÔåÆ        m.snomed_code: (m.search_term, m.primary_directorate, m.snomed_description)
   287ÔåÆ        for m in drug_snomed_mappings
   288ÔåÆ    }
   289ÔåÆ
   290ÔåÆ    # Get unique SNOMED codes
   291ÔåÆ    snomed_codes = list(snomed_lookup.keys())
   292ÔåÆ
   293ÔåÆ    # Build placeholders for SNOMED codes
   294ÔåÆ    placeholders = ", ".join(["%s"] * len(snomed_codes))
   295ÔåÆ
   296ÔåÆ    # Query to find most recent matching SNOMED code in GP records
   297ÔåÆ    query = f'''
   298ÔåÆ        SELECT
   299ÔåÆ            "SNOMEDCode",
   300ÔåÆ            "EventDateTime"
   301ÔåÆ        FROM DATA_HUB.PHM."PrimaryCareClinicalCoding"
   302ÔåÆ        WHERE "PatientPseudonym" = %s
   303ÔåÆ            AND "SNOMEDCode" IN ({placeholders})
   304ÔåÆ    '''
   305ÔåÆ
   306ÔåÆ    params: list = [patient_pseudonym] + snomed_codes
   307ÔåÆ
   308ÔåÆ    if before_date:
   309ÔåÆ        query += ' AND "EventDateTime" < %s'
   310ÔåÆ        params.append(before_date.isoformat())
   311ÔåÆ
   312ÔåÆ    query += ' ORDER BY "EventDateTime" DESC LIMIT 1'
   313ÔåÆ
   314ÔåÆ    try:
   315ÔåÆ        results = connector.execute_dict(query, tuple(params))
   316ÔåÆ
   317ÔåÆ        if results:
   318ÔåÆ            row = results[0]
   319ÔåÆ            matched_code = row.get("SNOMEDCode")
   320ÔåÆ            event_dt = row.get("EventDateTime")
   321ÔåÆ
   322ÔåÆ            if matched_code and matched_code in snomed_lookup:
   323ÔåÆ                search_term, primary_dir, snomed_desc = snomed_lookup[matched_code]
   324ÔåÆ
   325ÔåÆ                return DirectSnomedMatchResult(
   326ÔåÆ                    patient_pseudonym=patient_pseudonym,
   327ÔåÆ                    matched=True,
   328ÔåÆ                    snomed_code=matched_code,
   329ÔåÆ                    snomed_description=snomed_desc,
   330ÔåÆ                    search_term=search_term,
   331ÔåÆ                    primary_directorate=primary_dir,
   332ÔåÆ                    event_date=event_dt,
   333ÔåÆ                    source="DIRECT_SNOMED",
   334ÔåÆ                )
   335ÔåÆ
   336ÔåÆ        return result
   337ÔåÆ
   338ÔåÆ    except Exception as e:
   339ÔåÆ        logger.error(f"Error checking direct SNOMED for patient '{patient_pseudonym}': {e}")
   340ÔåÆ        return result
   341ÔåÆ
   342ÔåÆ
   343ÔåÆdef get_directorate_from_diagnosis(
   344ÔåÆ    upid: str,
   345ÔåÆ    drug_name: str,
   346ÔåÆ    connector: Optional[SnowflakeConnector] = None,
   347ÔåÆ    db_manager: Optional[DatabaseManager] = None,
   348ÔåÆ    before_date: Optional[date] = None,
   349ÔåÆ) -> DirectorateAssignment:
   350ÔåÆ    """
   351ÔåÆ    Get directorate assignment for a patient-drug combination using diagnosis-based lookup.
   352ÔåÆ
   353ÔåÆ    This function attempts to assign a directorate based on the patient's GP records
   354ÔåÆ    (direct SNOMED code matching). If no match is found, it returns a FALLBACK result
   355ÔåÆ    indicating that the caller should use alternative assignment methods (e.g.,
   356ÔåÆ    department_identification() from tools/data.py).
   357ÔåÆ
   358ÔåÆ    Workflow:
   359ÔåÆ    1. Get all SNOMED codes for the drug from ref_drug_snomed_mapping
   360ÔåÆ    2. Query patient's GP records for matching SNOMED codes
   361ÔåÆ    3. If match found ÔåÆ return diagnosis-based directorate and search_term
   362ÔåÆ    4. If no match ÔåÆ return FALLBACK result (caller handles fallback logic)
   363ÔåÆ
   364ÔåÆ    Args:
   365ÔåÆ        upid: Patient's unique patient ID (Provider Code[:3] + PersonKey)
   366ÔåÆ        drug_name: Drug name to look up
   367ÔåÆ        connector: Optional SnowflakeConnector (defaults to singleton)
   368ÔåÆ        db_manager: Optional DatabaseManager (defaults to default_db_manager)
   369ÔåÆ        before_date: Optional date - only check diagnoses before this date
   370ÔåÆ
   371ÔåÆ    Returns:
   372ÔåÆ        DirectorateAssignment with directorate, search_term, and source
   373ÔåÆ    """
   374ÔåÆ    result = DirectorateAssignment(
   375ÔåÆ        upid=upid,
   376ÔåÆ        drug_name=drug_name,
   377ÔåÆ        directorate=None,
   378ÔåÆ        source="FALLBACK",
   379ÔåÆ    )
   380ÔåÆ
   381ÔåÆ    # Step 1: Get SNOMED codes for the drug
   382ÔåÆ    drug_snomed_mappings = get_drug_snomed_codes(drug_name, db_manager)
   383ÔåÆ
   384ÔåÆ    if not drug_snomed_mappings:
   385ÔåÆ        logger.debug(f"No SNOMED mappings found for drug '{drug_name}' - using fallback")
   386ÔåÆ        return result
   387ÔåÆ
   388ÔåÆ    # Step 2: Check Snowflake availability
   389ÔåÆ    if not SNOWFLAKE_AVAILABLE:
   390ÔåÆ        logger.debug("Snowflake not available - using fallback")
   391ÔåÆ        return result
   392ÔåÆ
   393ÔåÆ    if not is_snowflake_configured():
   394ÔåÆ        logger.debug("Snowflake not configured - using fallback")
   395ÔåÆ        return result
   396ÔåÆ
   397ÔåÆ    # Step 3: Get patient pseudonym from UPID
   398ÔåÆ    # UPID format is Provider Code (3 chars) + PersonKey
   399ÔåÆ    # We need to query Snowflake to get the PatientPseudonym for this PersonKey
   400ÔåÆ    # However, patient_has_indication_direct expects PatientPseudonym, not UPID
   401ÔåÆ    # For now, we'll use UPID as the identifier - the actual integration
   402ÔåÆ    # will need to happen at the DataFrame level where we have PersonKey
   403ÔåÆ    #
   404ÔåÆ    # NOTE: This function will be called from the pipeline where we have
   405ÔåÆ    # access to PatientPseudonym. The UPID is passed for logging/tracking.
   406ÔåÆ
   407ÔåÆ    # Actually, looking at the pipeline, we need PatientPseudonym, not UPID.
   408ÔåÆ    # The caller should pass the PatientPseudonym or we need to look it up.
   409ÔåÆ    # For now, let's assume the caller will use this in a batch context
   410ÔåÆ    # where they can map UPID -> PatientPseudonym.
   411ÔåÆ
   412ÔåÆ    # Let me reconsider: the function signature takes UPID but we need
   413ÔåÆ    # PatientPseudonym for Snowflake. In the pipeline context (fetch_and_transform_data),
   414ÔåÆ    # we'll have the PersonKey column which IS the PatientPseudonym.
   415ÔåÆ    # So UPID = ProviderCode[:3] + PersonKey, and PersonKey = PatientPseudonym.
   416ÔåÆ    #
   417ÔåÆ    # We can extract PatientPseudonym from UPID by removing the first 3 chars.
   418ÔåÆ    patient_pseudonym = upid[3:] if len(upid) > 3 else upid
   419ÔåÆ
   420ÔåÆ    # Step 4: Check patient's GP records for matching SNOMED codes
   421ÔåÆ    match_result = patient_has_indication_direct(
   422ÔåÆ        patient_pseudonym=patient_pseudonym,
   423ÔåÆ        drug_snomed_mappings=drug_snomed_mappings,
   424ÔåÆ        connector=connector,
   425ÔåÆ        before_date=before_date,
   426ÔåÆ    )
   427ÔåÆ
   428ÔåÆ    if match_result.matched and match_result.primary_directorate:
   429ÔåÆ        return DirectorateAssignment(
   430ÔåÆ            upid=upid,
   431ÔåÆ            drug_name=drug_name,
   432ÔåÆ            directorate=match_result.primary_directorate,
   433ÔåÆ            search_term=match_result.search_term,
   434ÔåÆ            source="DIAGNOSIS",
   435ÔåÆ            snomed_code=match_result.snomed_code,
   436ÔåÆ            event_date=match_result.event_date,
   437ÔåÆ        )
   438ÔåÆ
   439ÔåÆ    # No match found - return fallback result
   440ÔåÆ    return result
   441ÔåÆ
   442ÔåÆ
   443ÔåÆdef get_cluster_snomed_codes(
   444ÔåÆ    cluster_id: str,
   445ÔåÆ    connector: Optional[SnowflakeConnector] = None,
   446ÔåÆ    use_cache: bool = True,
   447ÔåÆ) -> ClusterSnomedCodes:
   448ÔåÆ    """
   449ÔåÆ    Get all SNOMED codes for a cluster from Snowflake.
   450ÔåÆ
   451ÔåÆ    Queries the ClinicalCodingClusterSnomedCodes table to get all SNOMED codes
   452ÔåÆ    that belong to the specified cluster.
   453ÔåÆ
   454ÔåÆ    Args:
   455ÔåÆ        cluster_id: Cluster ID to look up (e.g., 'RARTH_COD', 'PSORIASIS_COD')
   456ÔåÆ        connector: Optional SnowflakeConnector (defaults to singleton)
   457ÔåÆ        use_cache: Whether to use cached results (default True)
   458ÔåÆ
   459ÔåÆ    Returns:
   460ÔåÆ        ClusterSnomedCodes with list of SNOMED codes and descriptions
   461ÔåÆ    """
   462ÔåÆ    if not SNOWFLAKE_AVAILABLE:
   463ÔåÆ        logger.warning("Snowflake connector not available")
   464ÔåÆ        return ClusterSnomedCodes(cluster_id=cluster_id, cluster_description="")
   465ÔåÆ
   466ÔåÆ    if not is_snowflake_configured():
   467ÔåÆ        logger.warning("Snowflake not configured - cannot get cluster codes")
   468ÔåÆ        return ClusterSnomedCodes(cluster_id=cluster_id, cluster_description="")
   469ÔåÆ
   470ÔåÆ    # Check cache first
   471ÔåÆ    cache_key = f"cluster_snomed_{cluster_id}"
   472ÔåÆ    if use_cache and is_cache_enabled():
   473ÔåÆ        cache = get_cache()
   474ÔåÆ        cached = cache.get(cache_key)
   475ÔåÆ        if cached is not None and len(cached) > 0:
   476ÔåÆ            logger.debug(f"Using cached SNOMED codes for cluster '{cluster_id}'")
   477ÔåÆ            cached_dict = cached[0]  # First element is our data dict
   478ÔåÆ            return ClusterSnomedCodes(
   479ÔåÆ                cluster_id=cluster_id,
   480ÔåÆ                cluster_description=str(cached_dict.get("description", "")),
   481ÔåÆ                snomed_codes=list(cached_dict.get("codes", [])),
   482ÔåÆ                snomed_descriptions=dict(cached_dict.get("descriptions", {})),
   483ÔåÆ            )
   484ÔåÆ
   485ÔåÆ    if connector is None:
   486ÔåÆ        connector = get_connector()
   487ÔåÆ
   488ÔåÆ    query = '''
   489ÔåÆ        SELECT DISTINCT
   490ÔåÆ            "Cluster_ID",
   491ÔåÆ            "Cluster_Description",
   492ÔåÆ            "SNOMEDCode",
   493ÔåÆ            "SNOMEDDescription"
   494ÔåÆ        FROM DATA_HUB.PHM."ClinicalCodingClusterSnomedCodes"
   495ÔåÆ        WHERE "Cluster_ID" = %s
   496ÔåÆ        ORDER BY "SNOMEDCode"
   497ÔåÆ    '''
   498ÔåÆ
   499ÔåÆ    try:
   500ÔåÆ        results = connector.execute_dict(query, (cluster_id,))
   501ÔåÆ
   502ÔåÆ        if not results:
   503ÔåÆ            logger.warning(f"No SNOMED codes found for cluster '{cluster_id}'")
   504ÔåÆ            return ClusterSnomedCodes(cluster_id=cluster_id, cluster_description="")
   505ÔåÆ
   506ÔåÆ        codes = []
   507ÔåÆ        descriptions = {}
   508ÔåÆ        description = results[0].get("Cluster_Description", "") if results else ""
   509ÔåÆ
   510ÔåÆ        for row in results:
   511ÔåÆ            code = row.get("SNOMEDCode")
   512ÔåÆ            if code:
   513ÔåÆ                codes.append(code)
   514ÔåÆ                descriptions[code] = row.get("SNOMEDDescription", "")
   515ÔåÆ
   516ÔåÆ        logger.info(f"Found {len(codes)} SNOMED codes for cluster '{cluster_id}'")
   517ÔåÆ
   518ÔåÆ        # Cache the results (using query-based cache with fake params)
   519ÔåÆ        if use_cache and is_cache_enabled():
   520ÔåÆ            cache = get_cache()
   521ÔåÆ            cache_data = [{
   522ÔåÆ                "description": description,
   523ÔåÆ                "codes": codes,
   524ÔåÆ                "descriptions": descriptions,
   525ÔåÆ            }]
   526ÔåÆ            cache.set(cache_key, None, cache_data)  # type: ignore[arg-type]
   527ÔåÆ
   528ÔåÆ        return ClusterSnomedCodes(
   529ÔåÆ            cluster_id=cluster_id,
   530ÔåÆ            cluster_description=description,
   531ÔåÆ            snomed_codes=codes,
   532ÔåÆ            snomed_descriptions=descriptions,
   533ÔåÆ        )
   534ÔåÆ
   535ÔåÆ    except Exception as e:
   536ÔåÆ        logger.error(f"Error getting SNOMED codes for cluster '{cluster_id}': {e}")
   537ÔåÆ        return ClusterSnomedCodes(cluster_id=cluster_id, cluster_description="")
   538ÔåÆ
   539ÔåÆ
   540ÔåÆdef patient_has_indication(
   541ÔåÆ    patient_pseudonym: str,
   542ÔåÆ    cluster_ids: list[str],
   543ÔåÆ    connector: Optional[SnowflakeConnector] = None,
   544ÔåÆ    before_date: Optional[date] = None,
   545ÔåÆ) -> tuple[bool, Optional[str], Optional[str], Optional[str]]:
   546ÔåÆ    """
   547ÔåÆ    Check if a patient has any SNOMED codes from the specified clusters in GP records.
   548ÔåÆ
   549ÔåÆ    Args:
   550ÔåÆ        patient_pseudonym: Patient's pseudonymised NHS number
   551ÔåÆ        cluster_ids: List of cluster IDs to check against
   552ÔåÆ        connector: Optional SnowflakeConnector
   553ÔåÆ        before_date: Optional date - only check diagnoses before this date
   554ÔåÆ
   555ÔåÆ    Returns:
   556ÔåÆ        Tuple of (has_indication, matched_cluster_id, matched_snomed_code, matched_description)
   557ÔåÆ    """
   558ÔåÆ    if not SNOWFLAKE_AVAILABLE or not is_snowflake_configured():
   559ÔåÆ        return False, None, None, None
   560ÔåÆ
   561ÔåÆ    if not cluster_ids:
   562ÔåÆ        return False, None, None, None
   563ÔåÆ
   564ÔåÆ    if connector is None:
   565ÔåÆ        connector = get_connector()
   566ÔåÆ
   567ÔåÆ    # Build placeholders for cluster IDs
   568ÔåÆ    placeholders = ", ".join(["%s"] * len(cluster_ids))
   569ÔåÆ
   570ÔåÆ    # Query to check if patient has any matching SNOMED code
   571ÔåÆ    query = f'''
   572ÔåÆ        SELECT
   573ÔåÆ            pc."SNOMEDCode",
   574ÔåÆ            cc."Cluster_ID",
   575ÔåÆ            cc."SNOMEDDescription"
   576ÔåÆ        FROM DATA_HUB.PHM."PrimaryCareClinicalCoding" pc
   577ÔåÆ        INNER JOIN DATA_HUB.PHM."ClinicalCodingClusterSnomedCodes" cc
   578ÔåÆ            ON pc."SNOMEDCode" = cc."SNOMEDCode"
   579ÔåÆ        WHERE pc."PatientPseudonym" = %s
   580ÔåÆ            AND cc."Cluster_ID" IN ({placeholders})
   581ÔåÆ    '''
   582ÔåÆ
   583ÔåÆ    params = [patient_pseudonym] + cluster_ids
   584ÔåÆ
   585ÔåÆ    if before_date:
   586ÔåÆ        query += ' AND pc."EventDateTime" < %s'
   587ÔåÆ        params.append(before_date.isoformat())
   588ÔåÆ
   589ÔåÆ    query += ' LIMIT 1'
   590ÔåÆ
   591ÔåÆ    try:
   592ÔåÆ        results = connector.execute_dict(query, tuple(params))
   593ÔåÆ
   594ÔåÆ        if results:
   595ÔåÆ            row = results[0]
   596ÔåÆ            return (
   597ÔåÆ                True,
   598ÔåÆ                row.get("Cluster_ID"),
   599ÔåÆ                row.get("SNOMEDCode"),
   600ÔåÆ                row.get("SNOMEDDescription"),
   601ÔåÆ            )
   602ÔåÆ
   603ÔåÆ        return False, None, None, None
   604ÔåÆ
   605ÔåÆ    except Exception as e:
   606ÔåÆ        logger.error(f"Error checking indication for patient '{patient_pseudonym}': {e}")
   607ÔåÆ        return False, None, None, None
   608ÔåÆ
   609ÔåÆ
   610ÔåÆdef validate_indication(
   611ÔåÆ    patient_pseudonym: str,
   612ÔåÆ    drug_name: str,
   613ÔåÆ    connector: Optional[SnowflakeConnector] = None,
   614ÔåÆ    db_manager: Optional[DatabaseManager] = None,
   615ÔåÆ    before_date: Optional[date] = None,
   616ÔåÆ) -> IndicationValidationResult:
   617ÔåÆ    """
   618ÔåÆ    Validate that a patient has an appropriate indication for a drug.
   619ÔåÆ
   620ÔåÆ    Full validation workflow:
   621ÔåÆ    1. Get drug's valid indication clusters from local mapping
   622ÔåÆ    2. Check if patient has any matching SNOMED codes in GP records
   623ÔåÆ    3. Return detailed validation result
   624ÔåÆ
   625ÔåÆ    Args:
   626ÔåÆ        patient_pseudonym: Patient's pseudonymised NHS number
   627ÔåÆ        drug_name: Drug name to validate indication for
   628ÔåÆ        connector: Optional SnowflakeConnector
   629ÔåÆ        db_manager: Optional DatabaseManager
   630ÔåÆ        before_date: Optional date - only check diagnoses before this date
   631ÔåÆ
   632ÔåÆ    Returns:
   633ÔåÆ        IndicationValidationResult with validation details
   634ÔåÆ    """
   635ÔåÆ    result = IndicationValidationResult(
   636ÔåÆ        patient_pseudonym=patient_pseudonym,
   637ÔåÆ        drug_name=drug_name,
   638ÔåÆ        has_valid_indication=False,
   639ÔåÆ    )
   640ÔåÆ
   641ÔåÆ    # Step 1: Get drug's cluster mappings
   642ÔåÆ    cluster_ids = get_drug_cluster_ids(drug_name, db_manager)
   643ÔåÆ
   644ÔåÆ    if not cluster_ids:
   645ÔåÆ        result.error_message = f"No cluster mappings found for drug '{drug_name}'"
   646ÔåÆ        result.source = "NONE"
   647ÔåÆ        return result
   648ÔåÆ
   649ÔåÆ    result.checked_clusters = cluster_ids
   650ÔåÆ
   651ÔåÆ    # Step 2: Check Snowflake availability
   652ÔåÆ    if not SNOWFLAKE_AVAILABLE:
   653ÔåÆ        result.error_message = "Snowflake connector not installed"
   654ÔåÆ        result.source = "NONE"
   655ÔåÆ        return result
   656ÔåÆ
   657ÔåÆ    if not is_snowflake_configured():
   658ÔåÆ        result.error_message = "Snowflake not configured"
   659ÔåÆ        result.source = "NONE"
   660ÔåÆ        return result
   661ÔåÆ
   662ÔåÆ    # Step 3: Check patient GP records
   663ÔåÆ    has_indication, matched_cluster, matched_code, matched_desc = patient_has_indication(
   664ÔåÆ        patient_pseudonym=patient_pseudonym,
   665ÔåÆ        cluster_ids=cluster_ids,
   666ÔåÆ        connector=connector,
   667ÔåÆ        before_date=before_date,
   668ÔåÆ    )
   669ÔåÆ
   670ÔåÆ    result.has_valid_indication = has_indication
   671ÔåÆ    result.matched_cluster_id = matched_cluster
   672ÔåÆ    result.matched_snomed_code = matched_code
   673ÔåÆ    result.matched_snomed_description = matched_desc
   674ÔåÆ    result.source = "GP_SNOMED" if has_indication else "NONE"
   675ÔåÆ
   676ÔåÆ    return result
   677ÔåÆ
   678ÔåÆ
   679ÔåÆdef get_indication_match_rate(
   680ÔåÆ    drug_name: str,
   681ÔåÆ    patient_pseudonyms: list[str],
   682ÔåÆ    connector: Optional[SnowflakeConnector] = None,
   683ÔåÆ    db_manager: Optional[DatabaseManager] = None,
   684ÔåÆ    sample_unmatched_count: int = 10,
   685ÔåÆ) -> DrugIndicationMatchRate:
   686ÔåÆ    """
   687ÔåÆ    Calculate indication match rate for a drug across a list of patients.
   688ÔåÆ
   689ÔåÆ    Args:
   690ÔåÆ        drug_name: Drug name to check
   691ÔåÆ        patient_pseudonyms: List of patient pseudonymised NHS numbers
   692ÔåÆ        connector: Optional SnowflakeConnector
   693ÔåÆ        db_manager: Optional DatabaseManager
   694ÔåÆ        sample_unmatched_count: Number of unmatched patient IDs to include in sample
   695ÔåÆ
   696ÔåÆ    Returns:
   697ÔåÆ        DrugIndicationMatchRate with match statistics
   698ÔåÆ    """
   699ÔåÆ    if connector is None and SNOWFLAKE_AVAILABLE and is_snowflake_configured():
   700ÔåÆ        connector = get_connector()
   701ÔåÆ
   702ÔåÆ    cluster_ids = get_drug_cluster_ids(drug_name, db_manager)
   703ÔåÆ
   704ÔåÆ    total = len(patient_pseudonyms)
   705ÔåÆ    matched = 0
   706ÔåÆ    unmatched = 0
   707ÔåÆ    sample_unmatched: list[str] = []
   708ÔåÆ
   709ÔåÆ    if not cluster_ids:
   710ÔåÆ        logger.warning(f"No cluster mappings for drug '{drug_name}' - all patients will be unmatched")
   711ÔåÆ        return DrugIndicationMatchRate(
   712ÔåÆ            drug_name=drug_name,
   713ÔåÆ            total_patients=total,
   714ÔåÆ            patients_with_indication=0,
   715ÔåÆ            patients_without_indication=total,
   716ÔåÆ            match_rate=0.0,
   717ÔåÆ            clusters_checked=[],
   718ÔåÆ            sample_unmatched=patient_pseudonyms[:sample_unmatched_count],
   719ÔåÆ        )
   720ÔåÆ
   721ÔåÆ    for i, pseudonym in enumerate(patient_pseudonyms):
   722ÔåÆ        if i > 0 and i % 100 == 0:
   723ÔåÆ            logger.info(f"Validating indications: {i}/{total} ({100*i/total:.1f}%)")
   724ÔåÆ
   725ÔåÆ        has_indication, _, _, _ = patient_has_indication(
   726ÔåÆ            patient_pseudonym=pseudonym,
   727ÔåÆ            cluster_ids=cluster_ids,
   728ÔåÆ            connector=connector,
   729ÔåÆ        )
   730ÔåÆ
   731ÔåÆ        if has_indication:
   732ÔåÆ            matched += 1
   733ÔåÆ        else:
   734ÔåÆ            unmatched += 1
   735ÔåÆ            if len(sample_unmatched) < sample_unmatched_count:
   736ÔåÆ                sample_unmatched.append(pseudonym)
   737ÔåÆ
   738ÔåÆ    match_rate = matched / total if total > 0 else 0.0
   739ÔåÆ
   740ÔåÆ    logger.info(f"Indication match rate for '{drug_name}': {100*match_rate:.1f}% ({matched}/{total})")
   741ÔåÆ
   742ÔåÆ    return DrugIndicationMatchRate(
   743ÔåÆ        drug_name=drug_name,
   744ÔåÆ        total_patients=total,
   745ÔåÆ        patients_with_indication=matched,
   746ÔåÆ        patients_without_indication=unmatched,
   747ÔåÆ        match_rate=match_rate,
   748ÔåÆ        clusters_checked=cluster_ids,
   749ÔåÆ        sample_unmatched=sample_unmatched,
   750ÔåÆ    )
   751ÔåÆ
   752ÔåÆ
   753ÔåÆdef batch_validate_indications(
   754ÔåÆ    patient_drug_pairs: list[tuple[str, str]],
   755ÔåÆ    connector: Optional[SnowflakeConnector] = None,
   756ÔåÆ    db_manager: Optional[DatabaseManager] = None,
   757ÔåÆ    progress_callback: Optional[Callable[[int, int], None]] = None,
   758ÔåÆ) -> list[IndicationValidationResult]:
   759ÔåÆ    """
   760ÔåÆ    Validate indications for multiple patient-drug pairs efficiently.
   761ÔåÆ
   762ÔåÆ    Args:
   763ÔåÆ        patient_drug_pairs: List of (patient_pseudonym, drug_name) tuples
   764ÔåÆ        connector: Optional SnowflakeConnector
   765ÔåÆ        db_manager: Optional DatabaseManager
   766ÔåÆ        progress_callback: Optional callback(current, total) for progress updates
   767ÔåÆ
   768ÔåÆ    Returns:
   769ÔåÆ        List of IndicationValidationResult for each pair
   770ÔåÆ    """
   771ÔåÆ    results = []
   772ÔåÆ    total = len(patient_drug_pairs)
   773ÔåÆ
   774ÔåÆ    # Cache cluster lookups by drug
   775ÔåÆ    drug_clusters_cache = {}
   776ÔåÆ
   777ÔåÆ    for i, (pseudonym, drug_name) in enumerate(patient_drug_pairs):
   778ÔåÆ        if progress_callback:
   779ÔåÆ            progress_callback(i + 1, total)
   780ÔåÆ
   781ÔåÆ        # Get clusters from cache or lookup
   782ÔåÆ        drug_upper = drug_name.upper()
   783ÔåÆ        if drug_upper not in drug_clusters_cache:
   784ÔåÆ            drug_clusters_cache[drug_upper] = get_drug_cluster_ids(drug_name, db_manager)
   785ÔåÆ
   786ÔåÆ        cluster_ids = drug_clusters_cache[drug_upper]
   787ÔåÆ
   788ÔåÆ        if not cluster_ids:
   789ÔåÆ            results.append(IndicationValidationResult(
   790ÔåÆ                patient_pseudonym=pseudonym,
   791ÔåÆ                drug_name=drug_name,
   792ÔåÆ                has_valid_indication=False,
   793ÔåÆ                source="NONE",
   794ÔåÆ                error_message=f"No cluster mappings for drug '{drug_name}'",
   795ÔåÆ            ))
   796ÔåÆ            continue
   797ÔåÆ
   798ÔåÆ        # Check patient indication
   799ÔåÆ        has_indication, matched_cluster, matched_code, matched_desc = patient_has_indication(
   800ÔåÆ            patient_pseudonym=pseudonym,
   801ÔåÆ            cluster_ids=cluster_ids,
   802ÔåÆ            connector=connector,
   803ÔåÆ        )
   804ÔåÆ
   805ÔåÆ        results.append(IndicationValidationResult(
   806ÔåÆ            patient_pseudonym=pseudonym,
   807ÔåÆ            drug_name=drug_name,
   808ÔåÆ            has_valid_indication=has_indication,
   809ÔåÆ            matched_cluster_id=matched_cluster,
   810ÔåÆ            matched_snomed_code=matched_code,
   811ÔåÆ            matched_snomed_description=matched_desc,
   812ÔåÆ            checked_clusters=cluster_ids,
   813ÔåÆ            source="GP_SNOMED" if has_indication else "NONE",
   814ÔåÆ        ))
   815ÔåÆ
   816ÔåÆ    matched_count = sum(1 for r in results if r.has_valid_indication)
   817ÔåÆ    logger.info(f"Batch validation complete: {matched_count}/{total} ({100*matched_count/total:.1f}%) with valid indications")
   818ÔåÆ
   819ÔåÆ    return results
   820ÔåÆ
   821ÔåÆ
   822ÔåÆdef get_available_clusters(
   823ÔåÆ    connector: Optional[SnowflakeConnector] = None,
   824ÔåÆ) -> list[dict]:
   825ÔåÆ    """
   826ÔåÆ    Get list of all available SNOMED clusters from Snowflake.
   827ÔåÆ
   828ÔåÆ    Returns:
   829ÔåÆ        List of dicts with cluster_id, cluster_description, code_count
   830ÔåÆ    """
   831ÔåÆ    if not SNOWFLAKE_AVAILABLE or not is_snowflake_configured():
   832ÔåÆ        logger.warning("Snowflake not available - cannot list clusters")
   833ÔåÆ        return []
   834ÔåÆ
   835ÔåÆ    if connector is None:
   836ÔåÆ        connector = get_connector()
   837ÔåÆ
   838ÔåÆ    query = '''
   839ÔåÆ        SELECT
   840ÔåÆ            "Cluster_ID",
   841ÔåÆ            "Cluster_Description",
   842ÔåÆ            COUNT(DISTINCT "SNOMEDCode") as code_count
   843ÔåÆ        FROM DATA_HUB.PHM."ClinicalCodingClusterSnomedCodes"
   844ÔåÆ        GROUP BY "Cluster_ID", "Cluster_Description"
   845ÔåÆ        ORDER BY "Cluster_ID"
   846ÔåÆ    '''
   847ÔåÆ
   848ÔåÆ    try:
   849ÔåÆ        results = connector.execute_dict(query)
   850ÔåÆ
   851ÔåÆ        clusters = []
   852ÔåÆ        for row in results:
   853ÔåÆ            clusters.append({
   854ÔåÆ                "cluster_id": row.get("Cluster_ID"),
   855ÔåÆ                "cluster_description": row.get("Cluster_Description"),
   856ÔåÆ                "code_count": row.get("code_count", 0),
   857ÔåÆ            })
   858ÔåÆ
   859ÔåÆ        logger.info(f"Found {len(clusters)} available SNOMED clusters")
   860ÔåÆ        return clusters
   861ÔåÆ
   862ÔåÆ    except Exception as e:
   863ÔåÆ        logger.error(f"Error getting available clusters: {e}")
   864ÔåÆ        return []
   865ÔåÆ
   866ÔåÆ
   867ÔåÆdef batch_lookup_indication_groups(
   868ÔåÆ    df: "pd.DataFrame",
   869ÔåÆ    connector: Optional[SnowflakeConnector] = None,
   870ÔåÆ    db_manager: Optional[DatabaseManager] = None,
   871ÔåÆ    batch_size: int = 500,
   872ÔåÆ) -> "pd.DataFrame":
   873ÔåÆ    """
   874ÔåÆ    Batch lookup GP diagnosis-based indication groups for a DataFrame of patients.
   875ÔåÆ
   876ÔåÆ    This is the efficient batch version of get_directorate_from_diagnosis().
   877ÔåÆ    Instead of querying Snowflake per patient, it batches the lookups for performance.
   878ÔåÆ
   879ÔåÆ    Strategy:
   880ÔåÆ    1. Get all unique (PersonKey, Drug Name) pairs from DataFrame
   881ÔåÆ    2. For each unique drug, get all SNOMED codes from local SQLite
   882ÔåÆ    3. Build batched Snowflake queries to check GP records
   883ÔåÆ    4. Return indication_df mapping UPID ÔåÆ Indication_Group
   884ÔåÆ
   885ÔåÆ    For unmatched patients, Indication_Group will be their Directory (with suffix).
   886ÔåÆ
   887ÔåÆ    Args:
   888ÔåÆ        df: DataFrame with columns: UPID, Drug Name, Directory, PersonKey
   889ÔåÆ        connector: Optional SnowflakeConnector (defaults to singleton)
   890ÔåÆ        db_manager: Optional DatabaseManager (defaults to default_db_manager)
   891ÔåÆ        batch_size: Number of patients per Snowflake query batch
   892ÔåÆ
   893ÔåÆ    Returns:
   894ÔåÆ        DataFrame with columns: UPID, Indication_Group, Source
   895ÔåÆ        - Indication_Group: Search_Term (if matched) or "Directory (no GP dx)" (if not)
   896ÔåÆ        - Source: "DIAGNOSIS" or "FALLBACK"
   897ÔåÆ    """
   898ÔåÆ    import pandas as pd
   899ÔåÆ
   900ÔåÆ    if db_manager is None:
   901ÔåÆ        db_manager = default_db_manager
   902ÔåÆ
   903ÔåÆ    logger.info(f"Starting batch indication lookup for {len(df)} records...")
   904ÔåÆ
   905ÔåÆ    # Step 1: Get unique (UPID, Drug Name, PseudoNHSNoLinked, Directory) combinations
   906ÔåÆ    # We need PseudoNHSNoLinked to query Snowflake - this matches PatientPseudonym in GP records
   907ÔåÆ    # Note: PersonKey is LocalPatientID which is provider-specific and does NOT match GP records
   908ÔåÆ    if 'PseudoNHSNoLinked' not in df.columns:
   909ÔåÆ        logger.error("DataFrame missing 'PseudoNHSNoLinked' column - cannot lookup GP records")
   910ÔåÆ        # Return fallback for all patients
   911ÔåÆ        result_df = df[['UPID', 'Directory']].drop_duplicates().copy()
   912ÔåÆ        result_df['Indication_Group'] = result_df['Directory'] + " (no GP dx)"
   913ÔåÆ        result_df['Source'] = "FALLBACK"
   914ÔåÆ        return result_df[['UPID', 'Indication_Group', 'Source']]
   915ÔåÆ
   916ÔåÆ    # Get unique patient-drug combinations (we need one lookup per patient-drug pair)
   917ÔåÆ    unique_pairs = df[['UPID', 'Drug Name', 'PseudoNHSNoLinked', 'Directory']].drop_duplicates()
   918ÔåÆ    logger.info(f"Found {len(unique_pairs)} unique patient-drug combinations")
   919ÔåÆ
   920ÔåÆ    # Step 2: Get all unique drugs and their SNOMED codes
   921ÔåÆ    unique_drugs = unique_pairs['Drug Name'].unique()
   922ÔåÆ    logger.info(f"Building SNOMED lookup for {len(unique_drugs)} unique drugs...")
   923ÔåÆ
   924ÔåÆ    # Build drug -> list of DrugSnomedMapping dict
   925ÔåÆ    drug_snomed_map: dict[str, list[DrugSnomedMapping]] = {}
   926ÔåÆ    all_snomed_codes: set[str] = set()
   927ÔåÆ    snomed_to_drug_searchterm: dict[str, list[tuple[str, str, str]]] = {}  # snomed -> [(drug, search_term, primary_dir), ...]
   928ÔåÆ
   929ÔåÆ    for drug_name in unique_drugs:
   930ÔåÆ        mappings = get_drug_snomed_codes(drug_name, db_manager)
   931ÔåÆ        drug_snomed_map[drug_name] = mappings
   932ÔåÆ
   933ÔåÆ        for m in mappings:
   934ÔåÆ            all_snomed_codes.add(m.snomed_code)
   935ÔåÆ            if m.snomed_code not in snomed_to_drug_searchterm:
   936ÔåÆ                snomed_to_drug_searchterm[m.snomed_code] = []
   937ÔåÆ            snomed_to_drug_searchterm[m.snomed_code].append(
   938ÔåÆ                (drug_name, m.search_term, m.primary_directorate)
   939ÔåÆ            )
   940ÔåÆ
   941ÔåÆ    logger.info(f"Total SNOMED codes to check: {len(all_snomed_codes)}")
   942ÔåÆ
   943ÔåÆ    # Step 3: Check Snowflake availability
   944ÔåÆ    if not SNOWFLAKE_AVAILABLE or not is_snowflake_configured():
   945ÔåÆ        logger.warning("Snowflake not available - returning fallback for all patients")
   946ÔåÆ        result_df = unique_pairs[['UPID', 'Directory']].copy()
   947ÔåÆ        result_df['Indication_Group'] = result_df['Directory'] + " (no GP dx)"
   948ÔåÆ        result_df['Source'] = "FALLBACK"
   949ÔåÆ        return result_df[['UPID', 'Indication_Group', 'Source']].drop_duplicates(subset=['UPID'])
   950ÔåÆ
   951ÔåÆ    if connector is None:
   952ÔåÆ        connector = get_connector()
   953ÔåÆ
   954ÔåÆ    # Step 4: Query GP records for all patients in batches
   955ÔåÆ    # The query finds the most recent matching SNOMED code for each patient
   956ÔåÆ
   957ÔåÆ    # Get unique PseudoNHSNoLinked values (each = one patient in GP records)
   958ÔåÆ    unique_patients = unique_pairs[['PseudoNHSNoLinked', 'UPID', 'Directory']].drop_duplicates(subset=['PseudoNHSNoLinked'])
   959ÔåÆ    patient_pseudonyms = unique_patients['PseudoNHSNoLinked'].tolist()
   960ÔåÆ
   961ÔåÆ    logger.info(f"Querying GP records for {len(patient_pseudonyms)} unique patients in batches of {batch_size}...")
   962ÔåÆ
   963ÔåÆ    # Results dict: PersonKey -> (snomed_code, event_date)
   964ÔåÆ    gp_matches: dict[str, tuple[str, Any]] = {}
   965ÔåÆ
   966ÔåÆ    # Convert SNOMED codes to list for query
   967ÔåÆ    snomed_list = list(all_snomed_codes)
   968ÔåÆ
   969ÔåÆ    if not snomed_list:
   970ÔåÆ        logger.warning("No SNOMED codes to check - returning fallback for all patients")
   971ÔåÆ        result_df = unique_pairs[['UPID', 'Directory']].copy()
   972ÔåÆ        result_df['Indication_Group'] = result_df['Directory'] + " (no GP dx)"
   973ÔåÆ        result_df['Source'] = "FALLBACK"
   974ÔåÆ        return result_df[['UPID', 'Indication_Group', 'Source']].drop_duplicates(subset=['UPID'])
   975ÔåÆ
   976ÔåÆ    # Build SNOMED IN clause (reused across batches)
   977ÔåÆ    snomed_placeholders = ", ".join(["%s"] * len(snomed_list))
   978ÔåÆ
   979ÔåÆ    # Process patients in batches
   980ÔåÆ    for batch_start in range(0, len(patient_pseudonyms), batch_size):
   981ÔåÆ        batch_end = min(batch_start + batch_size, len(patient_pseudonyms))
   982ÔåÆ        batch_pseudonyms = patient_pseudonyms[batch_start:batch_end]
   983ÔåÆ
   984ÔåÆ        logger.info(f"Batch {batch_start//batch_size + 1}: patients {batch_start} to {batch_end}")
   985ÔåÆ
   986ÔåÆ        # Build patient IN clause
   987ÔåÆ        patient_placeholders = ", ".join(["%s"] * len(batch_pseudonyms))
   988ÔåÆ
   989ÔåÆ        # Query to find all matching SNOMED codes for these patients
   990ÔåÆ        # We'll get all matches and pick the most recent per patient in Python
   991ÔåÆ        query = f'''
   992ÔåÆ            SELECT
   993ÔåÆ                "PatientPseudonym",
   994ÔåÆ                "SNOMEDCode",
   995ÔåÆ                "EventDateTime"
   996ÔåÆ            FROM DATA_HUB.PHM."PrimaryCareClinicalCoding"
   997ÔåÆ            WHERE "PatientPseudonym" IN ({patient_placeholders})
   998ÔåÆ              AND "SNOMEDCode" IN ({snomed_placeholders})
   999ÔåÆ            ORDER BY "PatientPseudonym", "EventDateTime" DESC
  1000ÔåÆ        '''
  1001ÔåÆ
  1002ÔåÆ        params = tuple(batch_pseudonyms) + tuple(snomed_list)
  1003ÔåÆ
  1004ÔåÆ        try:
  1005ÔåÆ            results = connector.execute_dict(query, params)
  1006ÔåÆ
  1007ÔåÆ            # Process results - pick most recent per patient
  1008ÔåÆ            for row in results:
  1009ÔåÆ                person_key = row.get("PatientPseudonym")
  1010ÔåÆ                snomed_code = row.get("SNOMEDCode")
  1011ÔåÆ                event_date = row.get("EventDateTime")
  1012ÔåÆ
  1013ÔåÆ                if person_key and snomed_code:
  1014ÔåÆ                    # Keep only if we haven't seen this patient yet (first = most recent due to ORDER BY)
  1015ÔåÆ                    if person_key not in gp_matches:
  1016ÔåÆ                        gp_matches[person_key] = (snomed_code, event_date)
  1017ÔåÆ
  1018ÔåÆ        except Exception as e:
  1019ÔåÆ            logger.error(f"Error querying GP records for batch: {e}")
  1020ÔåÆ            # Continue with other batches
  1021ÔåÆ
  1022ÔåÆ    logger.info(f"Found GP matches for {len(gp_matches)} patients")
  1023ÔåÆ
  1024ÔåÆ    # Step 5: Build result DataFrame
  1025ÔåÆ    # For each unique_pair, determine Indication_Group based on match status
  1026ÔåÆ    results_list = []
  1027ÔåÆ
  1028ÔåÆ    # We need to dedupe by UPID - a patient might be on multiple drugs
  1029ÔåÆ    # Strategy: For each UPID, use the most recent match (if any)
  1030ÔåÆ    upid_to_match: dict[str, tuple[str, str]] = {}  # UPID -> (Indication_Group, Source)
  1031ÔåÆ
  1032ÔåÆ    for _, row in unique_pairs.iterrows():
  1033ÔåÆ        upid = row['UPID']
  1034ÔåÆ        drug_name = row['Drug Name']
  1035ÔåÆ        patient_pseudonym = row['PseudoNHSNoLinked']
  1036ÔåÆ        directory = row['Directory']
  1037ÔåÆ
  1038ÔåÆ        # Check if patient has GP match (using PseudoNHSNoLinked which matches PatientPseudonym in GP)
  1039ÔåÆ        if patient_pseudonym in gp_matches:
  1040ÔåÆ            matched_snomed, event_date = gp_matches[patient_pseudonym]
  1041ÔåÆ
  1042ÔåÆ            # Find the search_term for this SNOMED code and drug
  1043ÔåÆ            # (A SNOMED code might map to multiple drugs with different search_terms)
  1044ÔåÆ            if matched_snomed in snomed_to_drug_searchterm:
  1045ÔåÆ                # Look for match with current drug first
  1046ÔåÆ                search_term = None
  1047ÔåÆ                for drug, st, pd in snomed_to_drug_searchterm[matched_snomed]:
  1048ÔåÆ                    if drug.upper() == drug_name.upper():
  1049ÔåÆ                        search_term = st
  1050ÔåÆ                        break
  1051ÔåÆ                # If no drug-specific match, use any match
  1052ÔåÆ                if search_term is None:
  1053ÔåÆ                    search_term = snomed_to_drug_searchterm[matched_snomed][0][1]
  1054ÔåÆ
  1055ÔåÆ                # Only update if we don't have a match for this UPID yet
  1056ÔåÆ                if upid not in upid_to_match:
  1057ÔåÆ                    upid_to_match[upid] = (search_term, "DIAGNOSIS")
  1058ÔåÆ            else:
  1059ÔåÆ                # Shouldn't happen but fallback just in case
  1060ÔåÆ                if upid not in upid_to_match:
  1061ÔåÆ                    upid_to_match[upid] = (directory + " (no GP dx)", "FALLBACK")
  1062ÔåÆ        else:
  1063ÔåÆ            # No GP match - use fallback
  1064ÔåÆ            if upid not in upid_to_match:
  1065ÔåÆ                upid_to_match[upid] = (directory + " (no GP dx)", "FALLBACK")
  1066ÔåÆ
  1067ÔåÆ    # Build result DataFrame
  1068ÔåÆ    for upid, (indication_group, source) in upid_to_match.items():
  1069ÔåÆ        results_list.append({
  1070ÔåÆ            'UPID': upid,
  1071ÔåÆ            'Indication_Group': indication_group,
  1072ÔåÆ            'Source': source,
  1073ÔåÆ        })
  1074ÔåÆ
  1075ÔåÆ    result_df = pd.DataFrame(results_list)
  1076ÔåÆ
  1077ÔåÆ    # Log statistics
  1078ÔåÆ    diagnosis_count = len([s for s in result_df['Source'] if s == "DIAGNOSIS"])
  1079ÔåÆ    fallback_count = len([s for s in result_df['Source'] if s == "FALLBACK"])
  1080ÔåÆ    total = len(result_df)
  1081ÔåÆ
  1082ÔåÆ    logger.info(f"Indication lookup complete:")
  1083ÔåÆ    logger.info(f"  Total unique patients: {total}")
  1084ÔåÆ    logger.info(f"  DIAGNOSIS matches: {diagnosis_count} ({100*diagnosis_count/total:.1f}%)")
  1085ÔåÆ    logger.info(f"  FALLBACK (no GP match): {fallback_count} ({100*fallback_count/total:.1f}%)")
  1086ÔåÆ
  1087ÔåÆ    return result_df
  1088ÔåÆ
  1089ÔåÆ
  1090ÔåÆ# === Drug-to-indication mapping from DimSearchTerm.csv ===
  1091ÔåÆ
  1092ÔåÆ
  1093ÔåÆ# Merge related Search_Terms into canonical names.
  1094ÔåÆ# Asthma variants are clinically the same condition at different severity levels.
  1095ÔåÆ# Urticaria is a separate condition ÔÇö do NOT merge with asthma.
  1096ÔåÆSEARCH_TERM_MERGE_MAP: dict[str, str] = {
  1097ÔåÆ    "allergic asthma": "asthma",
  1098ÔåÆ    "severe persistent allergic asthma": "asthma",
  1099ÔåÆ}
  1100ÔåÆ
  1101ÔåÆ
  1102ÔåÆdef load_drug_indication_mapping(
  1103ÔåÆ    csv_path: Optional[str] = None,
  1104ÔåÆ) -> tuple[dict[str, list[str]], dict[str, list[str]]]:
  1105ÔåÆ    """
  1106ÔåÆ    Load the drug-to-Search_Term mapping from DimSearchTerm.csv.
  1107ÔåÆ
  1108ÔåÆ    Builds two lookup dicts:
  1109ÔåÆ    - fragment_to_search_terms: drug fragment (UPPERCASE) -> list of Search_Terms containing it
  1110ÔåÆ    - search_term_to_fragments: search_term -> list of drug fragments (UPPERCASE)
  1111ÔåÆ
  1112ÔåÆ    DimSearchTerm.csv has columns: Search_Term, CleanedDrugName, PrimaryDirectorate
  1113ÔåÆ    CleanedDrugName is pipe-separated (e.g., "ADALIMUMAB|GOLIMUMAB|IXEKIZUMAB").
  1114ÔåÆ
  1115ÔåÆ    Note: A Search_Term can appear multiple times with different PrimaryDirectorates
  1116ÔåÆ    (e.g., "diabetes" appears under both DIABETIC MEDICINE and OPHTHALMOLOGY).
  1117ÔåÆ    Drug fragments from all rows for the same Search_Term are combined.
  1118ÔåÆ
  1119ÔåÆ    Asthma-related Search_Terms ("allergic asthma", "severe persistent allergic asthma")
  1120ÔåÆ    are merged into "asthma" to match the CLUSTER_MAPPING_SQL normalization.
  1121ÔåÆ    "urticaria" stays separate.
  1122ÔåÆ
  1123ÔåÆ    Args:
  1124ÔåÆ        csv_path: Path to DimSearchTerm.csv. Defaults to data/DimSearchTerm.csv.
  1125ÔåÆ
  1126ÔåÆ    Returns:
  1127ÔåÆ        Tuple of (fragment_to_search_terms, search_term_to_fragments)
  1128ÔåÆ    """
  1129ÔåÆ    if csv_path is None:
  1130ÔåÆ        csv_path = str(Path(__file__).parent.parent / "data" / "DimSearchTerm.csv")
  1131ÔåÆ
  1132ÔåÆ    fragment_to_search_terms: dict[str, list[str]] = {}
  1133ÔåÆ    search_term_to_fragments: dict[str, list[str]] = {}
  1134ÔåÆ
  1135ÔåÆ    try:
  1136ÔåÆ        with open(csv_path, "r", encoding="utf-8") as f:
  1137ÔåÆ            reader = csv.DictReader(f)
  1138ÔåÆ            for row in reader:
  1139ÔåÆ                search_term = row.get("Search_Term", "").strip()
  1140ÔåÆ                drug_names_raw = row.get("CleanedDrugName", "").strip()
  1141ÔåÆ
  1142ÔåÆ                # Normalize asthma variants to canonical "asthma"
  1143ÔåÆ                search_term = SEARCH_TERM_MERGE_MAP.get(search_term, search_term)
  1144ÔåÆ
  1145ÔåÆ                if not search_term or not drug_names_raw:
  1146ÔåÆ                    continue
  1147ÔåÆ
  1148ÔåÆ                fragments = [frag.strip().upper() for frag in drug_names_raw.split("|") if frag.strip()]
  1149ÔåÆ
  1150ÔåÆ                # Build search_term -> fragments (accumulate for duplicate Search_Terms)
  1151ÔåÆ                if search_term not in search_term_to_fragments:
  1152ÔåÆ                    search_term_to_fragments[search_term] = []
  1153ÔåÆ                for frag in fragments:
  1154ÔåÆ                    if frag not in search_term_to_fragments[search_term]:
  1155ÔåÆ                        search_term_to_fragments[search_term].append(frag)
  1156ÔåÆ
  1157ÔåÆ                # Build fragment -> search_terms
  1158ÔåÆ                for frag in fragments:
  1159ÔåÆ                    if frag not in fragment_to_search_terms:
  1160ÔåÆ                        fragment_to_search_terms[frag] = []
  1161ÔåÆ                    if search_term not in fragment_to_search_terms[frag]:
  1162ÔåÆ                        fragment_to_search_terms[frag].append(search_term)
  1163ÔåÆ
  1164ÔåÆ        logger.info(
  1165ÔåÆ            f"Loaded drug-indication mapping: {len(search_term_to_fragments)} Search_Terms, "
  1166ÔåÆ            f"{len(fragment_to_search_terms)} drug fragments"
  1167ÔåÆ        )
  1168ÔåÆ
  1169ÔåÆ    except FileNotFoundError:
  1170ÔåÆ        logger.error(f"DimSearchTerm.csv not found at {csv_path}")
  1171ÔåÆ    except Exception as e:
  1172ÔåÆ        logger.error(f"Error loading DimSearchTerm.csv: {e}")
  1173ÔåÆ
  1174ÔåÆ    return fragment_to_search_terms, search_term_to_fragments
  1175ÔåÆ
  1176ÔåÆ
  1177ÔåÆdef get_search_terms_for_drug(
  1178ÔåÆ    drug_name: str,
  1179ÔåÆ    search_term_to_fragments: dict[str, list[str]],
  1180ÔåÆ) -> list[str]:
  1181ÔåÆ    """
  1182ÔåÆ    Get all Search_Terms that list a given drug using substring matching.
  1183ÔåÆ
  1184ÔåÆ    Checks if any drug fragment from DimSearchTerm is a SUBSTRING of the given
  1185ÔåÆ    drug name (case-insensitive). This handles both exact matches (ADALIMUMAB)
  1186ÔåÆ    and partial fragments (PEGYLATED, INHALED).
  1187ÔåÆ
  1188ÔåÆ    Args:
  1189ÔåÆ        drug_name: HCD drug name (e.g., "ADALIMUMAB 40MG", "PEGYLATED LIPOSOMAL DOXORUBICIN")
  1190ÔåÆ        search_term_to_fragments: Mapping of search_term -> list of drug fragments
  1191ÔåÆ
  1192ÔåÆ    Returns:
  1193ÔåÆ        List of Search_Terms whose drug fragments match the drug name
  1194ÔåÆ    """
  1195ÔåÆ    drug_name_upper = drug_name.upper()
  1196ÔåÆ    matched_terms: list[str] = []
  1197ÔåÆ
  1198ÔåÆ    for search_term, fragments in search_term_to_fragments.items():
  1199ÔåÆ        for frag in fragments:
  1200ÔåÆ            if frag in drug_name_upper:
  1201ÔåÆ                matched_terms.append(search_term)
  1202ÔåÆ                break  # One matching fragment is enough for this Search_Term
  1203ÔåÆ
  1204ÔåÆ    return matched_terms
  1205ÔåÆ
  1206ÔåÆ
  1207ÔåÆ# === NEW APPROACH: Query Snowflake directly using cluster CTE ===
  1208ÔåÆ
  1209ÔåÆ# The cluster query mapping (embedded from snomed_indication_mapping_query.sql)
  1210ÔåÆ# This maps Search_Term -> Cluster_ID for ~148 clinical conditions
  1211ÔåÆCLUSTER_MAPPING_SQL = """
  1212ÔåÆWITH SearchTermClusters AS (
  1213ÔåÆ    SELECT Search_Term, Cluster_ID FROM (VALUES
  1214ÔåÆ        ('acute lymphoblastic leukaemia', 'HAEMCANMORPH_COD'),
  1215ÔåÆ        ('acute myeloid leukaemia', 'C19HAEMCAN_COD'),
  1216ÔåÆ        ('acute promyelocytic leukaemia', 'HAEMCANMORPH_COD'),
  1217ÔåÆ        ('asthma', 'AST_COD'),
  1218ÔåÆ        ('allergic rhinitis', 'MILDINTAST_COD'),
  1219ÔåÆ        ('alzheimer''s disease', 'DEMALZ_COD'),
  1220ÔåÆ        ('amyloidosis', 'AMYLOID_COD'),
  1221ÔåÆ        ('anaemia', 'eFI2_AnaemiaTimeSensitive'),
  1222ÔåÆ        ('anaplastic large cell lymphoma', 'C19HAEMCAN_COD'),
  1223ÔåÆ        ('apixaban', 'DOACCON_COD'),
  1224ÔåÆ        ('aplastic anaemia', 'eFI2_AnaemiaEver'),
  1225ÔåÆ        ('arthritis', 'eFI2_InflammatoryArthritis'),
  1226ÔåÆ        ('asthma', 'eFI2_Asthma'),
  1227ÔåÆ        ('atopic dermatitis', 'ATOPDERM_COD'),
  1228ÔåÆ        ('atrial fibrillation', 'eFI2_AtrialFibrillation'),
  1229ÔåÆ        ('attention deficit hyperactivity disorder', 'ADHD_COD'),
  1230ÔåÆ        ('bipolar disorder', 'MH_COD'),
  1231ÔåÆ        ('bladder', 'eFI2_UrinaryIncontinence'),
  1232ÔåÆ        ('breast cancer', 'BRCANSCR_COD'),
  1233ÔåÆ        ('cardiomyopathy', 'eFI2_HarmfulDrinking'),
  1234ÔåÆ        ('cardiovascular disease', 'CVDRISKASS_COD'),
  1235ÔåÆ        ('cervical cancer', 'CSDEC_COD'),
  1236ÔåÆ        ('cholangiocarcinoma', 'eFI2_Cancer'),
  1237ÔåÆ        ('chronic kidney disease', 'CKD_COD'),
  1238ÔåÆ        ('chronic liver disease', 'eFI2_LiverProblems'),
  1239ÔåÆ        ('chronic lymphocytic leukaemia', 'EPPHAEMCAN_COD'),
  1240ÔåÆ        ('chronic myeloid leukaemia', 'EPPHAEMCAN_COD'),
  1241ÔåÆ        ('chronic obstructive pulmonary disease', 'eFI2_COPD'),
  1242ÔåÆ        ('colon cancer', 'eFI2_Cancer'),
  1243ÔåÆ        ('colorectal cancer', 'GICANREF_COD'),
  1244ÔåÆ        ('constipation', 'CHRONCONSTIP_COD'),
  1245ÔåÆ        ('covid-19', 'POSSPOSTCOVID_COD'),
  1246ÔåÆ        ('crohn''s disease', 'eFI2_InflammatoryBowelDisease'),
  1247ÔåÆ        ('cutaneous t-cell lymphoma', 'C19HAEMCAN_COD'),
  1248ÔåÆ        ('cystic fibrosis', 'CUST_ICB_CYSTIC_FIBROSIS'),
  1249ÔåÆ        ('deep vein thrombosis', 'VTE_COD'),
  1250ÔåÆ        ('depression', 'eFI2_Depression'),
  1251ÔåÆ        ('diabetes', 'eFI2_DiabetesEver'),
  1252ÔåÆ        ('diabetic retinopathy', 'DRSELIGIBILITY_COD'),
  1253ÔåÆ        ('diffuse large b-cell lymphoma', 'C19HAEMCAN_COD'),
  1254ÔåÆ        ('dravet syndrome', 'EPIL_COD'),
  1255ÔåÆ        ('drug misuse', 'ILLSUBINT_COD'),
  1256ÔåÆ        ('dyspepsia', 'eFI2_AbdominalPain'),
  1257ÔåÆ        ('epilepsy', 'eFI2_Seizures'),
  1258ÔåÆ        ('fallopian tube', 'STERIL_COD'),
  1259ÔåÆ        ('follicular lymphoma', 'C19HAEMCAN_COD'),
  1260ÔåÆ        ('gastric cancer', 'eFI2_Cancer'),
  1261ÔåÆ        ('giant cell arteritis', 'GCA_COD'),
  1262ÔåÆ        ('glioma', 'NHAEMCANMORPH_COD'),
  1263ÔåÆ        ('gout', 'eFI2_InflammatoryArthritis'),
  1264ÔåÆ        ('graft versus host disease', 'GVHD_COD'),
  1265ÔåÆ        ('granulomatosis with polyangiitis', 'WEGENERVASC_COD'),
  1266ÔåÆ        ('growth hormone deficiency', 'HYPOPITUITARY_COD'),
  1267ÔåÆ        ('hand eczema', 'ECZEMA_COD'),
  1268ÔåÆ        ('heart failure', 'eFI2_HeartFailure'),
  1269ÔåÆ        ('hepatitis b', 'HEPBCVAC_COD'),
  1270ÔåÆ        ('hepatocellular carcinoma', 'eFI2_Cancer'),
  1271ÔåÆ        ('hiv', 'PREFLANG_COD'),
  1272ÔåÆ        ('hodgkin lymphoma', 'HAEMCANMORPH_COD'),
  1273ÔåÆ        ('hormone receptor', 'eFI2_ThyroidProblems'),
  1274ÔåÆ        ('hypercholesterolaemia', 'CLASSFH_COD'),
  1275ÔåÆ        ('immune thrombocytopenia', 'ITP_COD'),
  1276ÔåÆ        ('influenza', 'FLUINVITE_COD'),
  1277ÔåÆ        ('insomnia', 'eFI2_SleepProblems'),
  1278ÔåÆ        ('irritable bowel syndrome', 'IBS_COD'),
  1279ÔåÆ        ('ischaemic stroke', 'OSTR_COD'),
  1280ÔåÆ        ('juvenile idiopathic arthritis', 'RARTHAD_COD'),
  1281ÔåÆ        ('kidney transplant', 'RENALTRANSP_COD'),
  1282ÔåÆ        ('leukaemia', 'eFI2_Cancer'),
  1283ÔåÆ        ('lung cancer', 'FTCANREF_COD'),
  1284ÔåÆ        ('lymphoma', 'C19HAEMCAN_COD'),
  1285ÔåÆ        ('macular degeneration', 'CUST_ICB_VISUAL_IMPAIRMENT'),
  1286ÔåÆ        ('macular oedema', 'CUST_ICB_VISUAL_IMPAIRMENT'),
  1287ÔåÆ        ('major depressive episodes', 'eFI2_Depression'),
  1288ÔåÆ        ('malignant melanoma', 'eFI2_Cancer'),
  1289ÔåÆ        ('malignant pleural mesothelioma', 'LUNGCAN_COD'),
  1290ÔåÆ        ('manic episode', 'MH_COD'),
  1291ÔåÆ        ('mantle cell lymphoma', 'HAEMCANMORPH_COD'),
  1292ÔåÆ        ('melanoma', 'eFI2_Cancer'),
  1293ÔåÆ        ('merkel cell carcinoma', 'C19CAN_COD'),
  1294ÔåÆ        ('migraine', 'eFI2_Headache'),
  1295ÔåÆ        ('motor neurone disease', 'MND_COD'),
  1296ÔåÆ        ('multiple myeloma', 'C19HAEMCAN_COD'),
  1297ÔåÆ        ('multiple sclerosis', 'MS_COD'),
  1298ÔåÆ        ('myelodysplastic', 'eFI2_AnaemiaEver'),
  1299ÔåÆ        ('myelofibrosis', 'MDS_COD'),
  1300ÔåÆ        ('myocardial infarction', 'eFI2_IschaemicHeartDisease'),
  1301ÔåÆ        ('myotonia', 'CNDATRISK2_COD'),
  1302ÔåÆ        ('narcolepsy', 'LD_COD'),
  1303ÔåÆ        ('neuroendocrine tumour', 'LUNGCAN_COD'),
  1304ÔåÆ        ('non-small cell lung cancer', 'LUNGCAN_COD'),
  1305ÔåÆ        ('non-small-cell lung cancer', 'FTCANREF_COD'),
  1306ÔåÆ        ('obesity', 'BMI30_COD'),
  1307ÔåÆ        ('osteoarthritis', 'CUST_ICB_OSTEOARTHRITIS'),
  1308ÔåÆ        ('osteoporosis', 'eFI2_Osteoporosis'),
  1309ÔåÆ        ('osteosarcoma', 'NHAEMCANMORPH_COD'),
  1310ÔåÆ        ('ovarian cancer', 'C19CAN_COD'),
  1311ÔåÆ        ('peripheral arterial disease', 'PADEXC_COD'),
  1312ÔåÆ        ('plaque psoriasis', 'PSORIASIS_COD'),
  1313ÔåÆ        ('polycystic kidney disease', 'EPPCONGMALF_COD'),
  1314ÔåÆ        ('polycythaemia vera', 'C19HAEMCAN_COD'),
  1315ÔåÆ        ('pregnancy', 'C19PREG_COD'),
  1316ÔåÆ        ('primary biliary cholangitis', 'eFI2_LiverProblems'),
  1317ÔåÆ        ('primary hypercholesterolaemia', 'FNFHYP_COD'),
  1318ÔåÆ        ('prostate cancer', 'EPPSOLIDCAN_COD'),
  1319ÔåÆ        ('psoriasis', 'PSORIASIS_COD'),
  1320ÔåÆ        ('psoriatic arthritis', 'RARTHAD_COD'),
  1321ÔåÆ        ('pulmonary embolism', 'eFI2_RespiratoryDiseaseTimeSensitive'),
  1322ÔåÆ        ('pulmonary fibrosis', 'ILD_COD'),
  1323ÔåÆ        ('relapsing multiple sclerosis', 'MS_COD'),
  1324ÔåÆ        ('renal cell carcinoma', 'C19CAN_COD'),
  1325ÔåÆ        ('renal transplantation', 'RENALTRANSP_COD'),
  1326ÔåÆ        ('retinal vein occlusion', 'CUST_ICB_VISUAL_IMPAIRMENT'),
  1327ÔåÆ        ('rheumatoid arthritis', 'eFI2_InflammatoryArthritis'),
  1328ÔåÆ        ('rivaroxaban', 'DOACCON_COD'),
  1329ÔåÆ        ('schizophrenia', 'MH_COD'),
  1330ÔåÆ        ('seizures', 'LSZFREQ_COD'),
  1331ÔåÆ        ('sepsis', 'C19ACTIVITY_COD'),
  1332ÔåÆ        ('asthma', 'SEVAST_COD'),
  1333ÔåÆ        ('sickle cell disease', 'SICKLE_COD'),
  1334ÔåÆ        ('sleep apnoea', 'CUST_ICB_NON_SEVERE_LDA'),
  1335ÔåÆ        ('smoking cessation', 'SMOKINGINT_COD'),
  1336ÔåÆ        ('soft tissue sarcoma', 'NHAEMCANMORPH_COD'),
  1337ÔåÆ        ('spinal muscular atrophy', 'MND_COD'),
  1338ÔåÆ        ('squamous cell', 'C19CAN_COD'),
  1339ÔåÆ        ('squamous cell carcinoma', 'C19CAN_COD'),
  1340ÔåÆ        ('stem cell transplant', 'ALLOTRANSP_COD'),
  1341ÔåÆ        ('stroke', 'eFI2_Stroke'),
  1342ÔåÆ        ('systemic lupus erythematosus', 'SLUPUS_COD'),
  1343ÔåÆ        ('systemic mastocytosis', 'HAEMCANMORPH_COD'),
  1344ÔåÆ        ('thrombocytopenic purpura', 'TTP_COD'),
  1345ÔåÆ        ('thrombotic thrombocytopenic purpura', 'TTP_COD'),
  1346ÔåÆ        ('thyroid cancer', 'C19CAN_COD'),
  1347ÔåÆ        ('tophaceous gout', 'CUST_ICB_OSTEOARTHRITIS'),
  1348ÔåÆ        ('transitional cell carcinoma', 'C19CAN_COD'),
  1349ÔåÆ        ('type 1 diabetes', 'DMTYPE1_COD'),
  1350ÔåÆ        ('type 2 diabetes', 'DMTYPE2_COD'),
  1351ÔåÆ        ('ulcerative colitis', 'eFI2_InflammatoryBowelDisease'),
  1352ÔåÆ        ('urothelial carcinoma', 'NHAEMCANMORPH_COD'),
  1353ÔåÆ        ('urticaria', 'XSAL_COD'),
  1354ÔåÆ        ('uveitis', 'CUST_ICB_VISUAL_IMPAIRMENT'),
  1355ÔåÆ        ('vascular disease', 'CVDINVITE_COD'),
  1356ÔåÆ        ('vasculitis', 'CRYOGLOBVASC_COD')
  1357ÔåÆ    ) AS t(Search_Term, Cluster_ID)
  1358ÔåÆ),
  1359ÔåÆ
  1360ÔåÆClusterCodes AS (
  1361ÔåÆ    SELECT
  1362ÔåÆ        stc.Search_Term,
  1363ÔåÆ        c."SNOMEDCode",
  1364ÔåÆ        c."SNOMEDDescription"
  1365ÔåÆ    FROM SearchTermClusters stc
  1366ÔåÆ    JOIN DATA_HUB.PHM."ClinicalCodingClusterSnomedCodes" c
  1367ÔåÆ        ON stc.Cluster_ID = c."Cluster_ID"
  1368ÔåÆ    WHERE c."SNOMEDCode" IS NOT NULL
  1369ÔåÆ),
  1370ÔåÆ
  1371ÔåÆExplicitCodes AS (
  1372ÔåÆ    SELECT Search_Term, SNOMEDCode, SNOMEDDescription FROM (VALUES
  1373ÔåÆ        ('acute coronary syndrome', '837091000000100', 'Manual mapping'),
  1374ÔåÆ        ('ankylosing spondylitis', '162930007', 'Manual mapping'),
  1375ÔåÆ        ('ankylosing spondylitis', '239805001', 'Manual mapping'),
  1376ÔåÆ        ('ankylosing spondylitis', '239810002', 'Manual mapping'),
  1377ÔåÆ        ('ankylosing spondylitis', '239811003', 'Manual mapping'),
  1378ÔåÆ        ('ankylosing spondylitis', '394990003', 'Manual mapping'),
  1379ÔåÆ        ('ankylosing spondylitis', '429712009', 'Manual mapping'),
  1380ÔåÆ        ('ankylosing spondylitis', '441562009', 'Manual mapping'),
  1381ÔåÆ        ('ankylosing spondylitis', '441680005', 'Manual mapping'),
  1382ÔåÆ        ('ankylosing spondylitis', '441930001', 'Manual mapping'),
  1383ÔåÆ        ('axial spondyloarthritis', '723116002', 'Manual mapping'),
  1384ÔåÆ        ('choroidal neovascularisation', '380621000000102', 'Manual mapping'),
  1385ÔåÆ        ('choroidal neovascularisation', '733124000', 'Manual mapping')
  1386ÔåÆ    ) AS t(Search_Term, SNOMEDCode, SNOMEDDescription)
  1387ÔåÆ),
  1388ÔåÆ
  1389ÔåÆAllIndicationCodes AS (
  1390ÔåÆ    SELECT Search_Term, "SNOMEDCode" AS SNOMEDCode, "SNOMEDDescription" AS SNOMEDDescription
  1391ÔåÆ    FROM ClusterCodes
  1392ÔåÆ    UNION ALL
  1393ÔåÆ    SELECT Search_Term, SNOMEDCode, SNOMEDDescription
  1394ÔåÆ    FROM ExplicitCodes
  1395ÔåÆ)
  1396ÔåÆ"""
  1397ÔåÆ
  1398ÔåÆ
  1399ÔåÆdef get_patient_indication_groups(
  1400ÔåÆ    patient_pseudonyms: list[str],
  1401ÔåÆ    connector: Optional[SnowflakeConnector] = None,
  1402ÔåÆ    batch_size: int = 500,
  1403ÔåÆ) -> "pd.DataFrame":
  1404ÔåÆ    """
  1405ÔåÆ    Batch lookup GP diagnosis-based indication groups using Snowflake cluster query.
  1406ÔåÆ
  1407ÔåÆ    This function queries Snowflake directly using the embedded cluster CTE
  1408ÔåÆ    (from snomed_indication_mapping_query.sql) to find patients with matching
  1409ÔåÆ    GP diagnoses. This is the NEW approach replacing the old SQLite-based lookup.
  1410ÔåÆ
  1411ÔåÆ    The query:
  1412ÔåÆ    1. Uses the cluster mapping CTE to get all Search_Term -> SNOMED code mappings
  1413ÔåÆ    2. Joins with PrimaryCareClinicalCoding to find patients with matching codes
  1414ÔåÆ    3. Returns the most recent match per patient (by EventDateTime)
  1415ÔåÆ
  1416ÔåÆ    Args:
  1417ÔåÆ        patient_pseudonyms: List of PseudoNHSNoLinked values (matches PatientPseudonym in GP records)
  1418ÔåÆ        connector: Optional SnowflakeConnector (defaults to singleton)
  1419ÔåÆ        batch_size: Number of patients per Snowflake query batch (default 500)
  1420ÔåÆ
  1421ÔåÆ    Returns:
  1422ÔåÆ        DataFrame with columns:
  1423ÔåÆ        - PatientPseudonym: The patient identifier (PseudoNHSNoLinked value)
  1424ÔåÆ        - Search_Term: The matched indication (e.g., "rheumatoid arthritis")
  1425ÔåÆ        - EventDateTime: Date of the GP diagnosis record
  1426ÔåÆ
  1427ÔåÆ        Patients not found in results have no matching GP diagnosis.
  1428ÔåÆ    """
  1429ÔåÆ    import pandas as pd
  1430ÔåÆ
  1431ÔåÆ    logger.info(f"Starting Snowflake-direct indication lookup for {len(patient_pseudonyms)} patients...")
  1432ÔåÆ
  1433ÔåÆ    # Handle edge case: empty patient list
  1434ÔåÆ    if not patient_pseudonyms:
  1435ÔåÆ        logger.warning("Empty patient list provided")
  1436ÔåÆ        return pd.DataFrame(columns=['PatientPseudonym', 'Search_Term', 'EventDateTime'])
  1437ÔåÆ
  1438ÔåÆ    # Check Snowflake availability
  1439ÔåÆ    if not SNOWFLAKE_AVAILABLE:
  1440ÔåÆ        logger.error("Snowflake connector not available - cannot lookup GP records")
  1441ÔåÆ        return pd.DataFrame(columns=['PatientPseudonym', 'Search_Term', 'EventDateTime'])
  1442ÔåÆ
  1443ÔåÆ    if not is_snowflake_configured():
  1444ÔåÆ        logger.error("Snowflake not configured - cannot lookup GP records")
  1445ÔåÆ        return pd.DataFrame(columns=['PatientPseudonym', 'Search_Term', 'EventDateTime'])
  1446ÔåÆ
  1447ÔåÆ    if connector is None:
  1448ÔåÆ        connector = get_connector()
  1449ÔåÆ
  1450ÔåÆ    # Results list to collect all matches
  1451ÔåÆ    all_results: list[dict] = []
  1452ÔåÆ
  1453ÔåÆ    # Process patients in batches
  1454ÔåÆ    total_patients = len(patient_pseudonyms)
  1455ÔåÆ    for batch_start in range(0, total_patients, batch_size):
  1456ÔåÆ        batch_end = min(batch_start + batch_size, total_patients)
  1457ÔåÆ        batch_pseudonyms = patient_pseudonyms[batch_start:batch_end]
  1458ÔåÆ        batch_num = batch_start // batch_size + 1
  1459ÔåÆ        total_batches = (total_patients + batch_size - 1) // batch_size
  1460ÔåÆ
  1461ÔåÆ        logger.info(f"Batch {batch_num}/{total_batches}: patients {batch_start + 1} to {batch_end}")
  1462ÔåÆ
  1463ÔåÆ        # Build patient IN clause placeholders
  1464ÔåÆ        patient_placeholders = ", ".join(["%s"] * len(batch_pseudonyms))
  1465ÔåÆ
  1466ÔåÆ        # Build the full query with cluster CTE
  1467ÔåÆ        # This finds the most recent matching diagnosis for each patient
  1468ÔåÆ        # Note: Column names must be aliased to ensure consistent casing in results
  1469ÔåÆ        query = f"""
  1470ÔåÆ{CLUSTER_MAPPING_SQL}
  1471ÔåÆSELECT
  1472ÔåÆ    pc."PatientPseudonym" AS "PatientPseudonym",
  1473ÔåÆ    aic.Search_Term AS "Search_Term",
  1474ÔåÆ    pc."EventDateTime" AS "EventDateTime"
  1475ÔåÆFROM DATA_HUB.PHM."PrimaryCareClinicalCoding" pc
  1476ÔåÆINNER JOIN AllIndicationCodes aic
  1477ÔåÆ    ON pc."SNOMEDCode" = aic.SNOMEDCode
  1478ÔåÆWHERE pc."PatientPseudonym" IN ({patient_placeholders})
  1479ÔåÆQUALIFY ROW_NUMBER() OVER (
  1480ÔåÆ    PARTITION BY pc."PatientPseudonym"
  1481ÔåÆ    ORDER BY pc."EventDateTime" DESC
  1482ÔåÆ) = 1
  1483ÔåÆ"""
  1484ÔåÆ
  1485ÔåÆ        try:
  1486ÔåÆ            results = connector.execute_dict(query, tuple(batch_pseudonyms))
  1487ÔåÆ
  1488ÔåÆ            for row in results:
  1489ÔåÆ                all_results.append({
  1490ÔåÆ                    'PatientPseudonym': row.get('PatientPseudonym'),
  1491ÔåÆ                    'Search_Term': row.get('Search_Term'),
  1492ÔåÆ                    'EventDateTime': row.get('EventDateTime'),
  1493ÔåÆ                })
  1494ÔåÆ
  1495ÔåÆ            logger.debug(f"Batch {batch_num}: found {len(results)} matches")
  1496ÔåÆ
  1497ÔåÆ        except Exception as e:
  1498ÔåÆ            logger.error(f"Error querying GP records for batch {batch_num}: {e}")
  1499ÔåÆ            # Continue with other batches - partial results are better than none
  1500ÔåÆ
  1501ÔåÆ    # Build result DataFrame
  1502ÔåÆ    result_df = pd.DataFrame(all_results)
  1503ÔåÆ
  1504ÔåÆ    # Log summary statistics
  1505ÔåÆ    if len(result_df) > 0:
  1506ÔåÆ        matched_count = len(result_df)
  1507ÔåÆ        match_rate = 100 * matched_count / total_patients
  1508ÔåÆ        unique_terms = result_df['Search_Term'].nunique()
  1509ÔåÆ        logger.info(f"Indication lookup complete:")
  1510ÔåÆ        logger.info(f"  Total patients queried: {total_patients}")
  1511ÔåÆ        logger.info(f"  Patients with GP match: {matched_count} ({match_rate:.1f}%)")
  1512ÔåÆ        logger.info(f"  Unique Search_Terms found: {unique_terms}")
  1513ÔåÆ
  1514ÔåÆ        # Log top Search_Terms
  1515ÔåÆ        top_terms = result_df['Search_Term'].value_counts().head(5)
  1516ÔåÆ        logger.info(f"  Top 5 indications: {dict(top_terms)}")
  1517ÔåÆ    else:
  1518ÔåÆ        logger.info(f"Indication lookup complete: 0 matches from {total_patients} patients")
  1519ÔåÆ
  1520ÔåÆ    return result_df
  1521ÔåÆ
  1522ÔåÆ
  1523ÔåÆ# Export public API
  1524ÔåÆ__all__ = [
  1525ÔåÆ    # Dataclasses
  1526ÔåÆ    "ClusterSnomedCodes",
  1527ÔåÆ    "IndicationValidationResult",
  1528ÔåÆ    "DrugIndicationMatchRate",
  1529ÔåÆ    "DrugSnomedMapping",
  1530ÔåÆ    "DirectSnomedMatchResult",
  1531ÔåÆ    "DirectorateAssignment",
  1532ÔåÆ    # Cluster-based lookup functions (existing)
  1533ÔåÆ    "get_drug_clusters",
  1534ÔåÆ    "get_drug_cluster_ids",
  1535ÔåÆ    "get_cluster_snomed_codes",
  1536ÔåÆ    "patient_has_indication",
  1537ÔåÆ    "validate_indication",
  1538ÔåÆ    "get_indication_match_rate",
  1539ÔåÆ    "batch_validate_indications",
  1540ÔåÆ    "get_available_clusters",
  1541ÔåÆ    # Direct SNOMED lookup functions (new)
  1542ÔåÆ    "get_drug_snomed_codes",
  1543ÔåÆ    "patient_has_indication_direct",
  1544ÔåÆ    # Diagnosis-based directorate assignment
  1545ÔåÆ    "get_directorate_from_diagnosis",
  1546ÔåÆ    # Batch lookup for indication groups
  1547ÔåÆ    "batch_lookup_indication_groups",
  1548ÔåÆ    # Drug-indication mapping from DimSearchTerm.csv
  1549ÔåÆ    "SEARCH_TERM_MERGE_MAP",
  1550ÔåÆ    "load_drug_indication_mapping",
  1551ÔåÆ    "get_search_terms_for_drug",
  1552ÔåÆ    # Snowflake-direct indication lookup (new approach)
  1553ÔåÆ    "get_patient_indication_groups",
  1554ÔåÆ    "CLUSTER_MAPPING_SQL",
  1555ÔåÆ]
  1556ÔåÆ

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>

"""
Diagnosis lookup module for NHS Patient Pathway Analysis.

Provides functions to validate patient indications by checking GP diagnosis records
against SNOMED cluster codes. Uses the drug-to-cluster mapping from
drug_indication_clusters.csv and queries Snowflake for SNOMED codes and GP records.

Key workflow:
1. Get drug's valid indication clusters from local mapping
2. Get all SNOMED codes for those clusters from Snowflake
3. Check if patient has any of those SNOMED codes in GP records
4. Report indication validation status

IMPORTANT: HCD activity data indication codes are UNRELIABLE. This module uses
GP/Primary Care data (PrimaryCareClinicalCoding) as the authoritative source.
"""

from dataclasses import dataclass, field
from datetime import date, datetime
from pathlib import Path
from typing import Optional, Callable, Any, cast, TYPE_CHECKING
import csv

if TYPE_CHECKING:
    import pandas as pd

from core.logging_config import get_logger
from data_processing.database import DatabaseManager, default_db_manager
from data_processing.snowflake_connector import (
    SnowflakeConnector,
    get_connector,
    is_snowflake_available,
    is_snowflake_configured,
    SNOWFLAKE_AVAILABLE,
)
from data_processing.cache import get_cache, is_cache_enabled

logger = get_logger(__name__)

@dataclass
class ClusterSnomedCodes:
    """SNOMED codes for a clinical coding cluster."""
    cluster_id: str
    cluster_description: str
    snomed_codes: list[str] = field(default_factory=list)
    snomed_descriptions: dict[str, str] = field(default_factory=dict)

    @property
    def code_count(self) -> int:
        return len(self.snomed_codes)

@dataclass
class IndicationValidationResult:
    """Result of validating a patient's indication for a drug."""
    patient_pseudonym: str
    drug_name: str
    has_valid_indication: bool
    matched_cluster_id: Optional[str] = None
    matched_snomed_code: Optional[str] = None
    matched_snomed_description: Optional[str] = None
    checked_clusters: list[str] = field(default_factory=list)
    total_codes_checked: int = 0
    source: str = "GP_SNOMED"  # GP_SNOMED | NONE
    error_message: Optional[str] = None

@dataclass
class DrugIndicationMatchRate:
    """Match rate statistics for a drug's indication validation."""
    drug_name: str
    total_patients: int
    patients_with_indication: int
    patients_without_indication: int
    match_rate: float  # 0.0 to 1.0
    clusters_checked: list[str] = field(default_factory=list)
    sample_unmatched: list[str] = field(default_factory=list)  # Sample patient IDs

@dataclass
class DrugSnomedMapping:
    """SNOMED code mapping for a drug from ref_drug_snomed_mapping."""
    snomed_code: str
    snomed_description: str
    search_term: str
    primary_directorate: str
    indication: str = ""
    ta_id: str = ""

@dataclass
class DirectSnomedMatchResult:
    """Result of direct SNOMED code lookup in GP records."""
    patient_pseudonym: str
    matched: bool
    snomed_code: Optional[str] = None
    snomed_description: Optional[str] = None
    search_term: Optional[str] = None
    primary_directorate: Optional[str] = None
    event_date: Optional[datetime] = None
    source: str = "DIRECT_SNOMED"  # DIRECT_SNOMED | NONE

@dataclass
class DirectorateAssignment:
    """Result of directorate assignment for a patient-drug combination."""
    upid: str
    drug_name: str
    directorate: Optional[str]
    search_term: Optional[str] = None
    source: str = "FALLBACK"  # DIAGNOSIS | FALLBACK
    snomed_code: Optional[str] = None
    event_date: Optional[datetime] = None

def get_drug_clusters(
    drug_name: str,
    db_manager: Optional[DatabaseManager] = None
) -> list[dict]:
    """
    Get all SNOMED cluster mappings for a drug from local SQLite.

    Args:
        drug_name: Drug name to look up (case-insensitive)
        db_manager: Optional DatabaseManager (defaults to default_db_manager)

    Returns:
        List of dicts with keys: drug_name, indication, cluster_id,
        cluster_description, nice_ta_reference
    """
    if db_manager is None:
        db_manager = default_db_manager

    query = """
        SELECT drug_name, indication, cluster_id, cluster_description, nice_ta_reference
        FROM ref_drug_indication_clusters
        WHERE UPPER(drug_name) = UPPER(?)
        ORDER BY indication, cluster_id
    """

    try:
        with db_manager.get_connection() as conn:
            cursor = conn.execute(query, (drug_name,))
            rows = cursor.fetchall()

            results = []
            for row in rows:
                results.append({
                    "drug_name": row["drug_name"],
                    "indication": row["indication"],
                    "cluster_id": row["cluster_id"],
                    "cluster_description": row["cluster_description"],
                    "nice_ta_reference": row["nice_ta_reference"],
                })

            logger.debug(f"Found {len(results)} cluster mappings for drug '{drug_name}'")
            return results

    except Exception as e:
        logger.error(f"Error getting clusters for drug '{drug_name}': {e}")
        return []

def get_drug_cluster_ids(
    drug_name: str,
    db_manager: Optional[DatabaseManager] = None
) -> list[str]:
    """
    Get unique cluster IDs for a drug.

    Args:
        drug_name: Drug name to look up
        db_manager: Optional DatabaseManager

    Returns:
        List of unique cluster IDs
    """
    clusters = get_drug_clusters(drug_name, db_manager)
    return list(set(c["cluster_id"] for c in clusters))

def get_drug_snomed_codes(
    drug_name: str,
    db_manager: Optional[DatabaseManager] = None
) -> list[DrugSnomedMapping]:
    """
    Get all SNOMED codes for a drug from local ref_drug_snomed_mapping table.

    This uses the enriched mapping CSV data loaded into SQLite, which provides
    direct SNOMED-to-drug mappings with Search_Term and PrimaryDirectorate.

    Args:
        drug_name: Drug name to look up (case-insensitive, matches cleaned_drug_name)
        db_manager: Optional DatabaseManager (defaults to default_db_manager)

    Returns:
        List of DrugSnomedMapping with snomed_code, snomed_description,
        search_term, primary_directorate, indication, ta_id
    """
    if db_manager is None:
        db_manager = default_db_manager

    query = """
        SELECT DISTINCT
            snomed_code,
            snomed_description,
            search_term,
            primary_directorate,
            indication,
            ta_id
        FROM ref_drug_snomed_mapping
        WHERE UPPER(cleaned_drug_name) = UPPER(?)
           OR UPPER(drug_name) = UPPER(?)
        ORDER BY search_term, snomed_code
    """

    try:
        with db_manager.get_connection() as conn:
            cursor = conn.execute(query, (drug_name, drug_name))
            rows = cursor.fetchall()

            results = []
            for row in rows:
                results.append(DrugSnomedMapping(
                    snomed_code=row["snomed_code"],
                    snomed_description=row["snomed_description"] or "",
                    search_term=row["search_term"] or "",
                    primary_directorate=row["primary_directorate"] or "",
                    indication=row["indication"] or "",
                    ta_id=row["ta_id"] or "",
                ))

            logger.debug(f"Found {len(results)} SNOMED mappings for drug '{drug_name}'")
            return results

    except Exception as e:
        logger.error(f"Error getting SNOMED codes for drug '{drug_name}': {e}")
        return []

def patient_has_indication_direct(
    patient_pseudonym: str,
    drug_snomed_mappings: list[DrugSnomedMapping],
    connector: Optional[SnowflakeConnector] = None,
    before_date: Optional[date] = None,
) -> DirectSnomedMatchResult:
    """
    Check if patient has any of the SNOMED codes in their GP records.

    This is the direct SNOMED lookup - it queries PrimaryCareClinicalCoding
    for exact SNOMED code matches (not via cluster). Returns the most recent
    match by EventDateTime if multiple matches exist.

    Args:
        patient_pseudonym: Patient's pseudonymised NHS number
        drug_snomed_mappings: List of DrugSnomedMapping from get_drug_snomed_codes()
        connector: Optional SnowflakeConnector (defaults to singleton)
        before_date: Optional date - only check diagnoses before this date

    Returns:
        DirectSnomedMatchResult with match details (most recent by EventDateTime)
    """
    result = DirectSnomedMatchResult(
        patient_pseudonym=patient_pseudonym,
        matched=False,
        source="NONE",
    )

    if not drug_snomed_mappings:
        return result

    if not SNOWFLAKE_AVAILABLE:
        logger.warning("Snowflake connector not available")
        return result

    if not is_snowflake_configured():
        logger.warning("Snowflake not configured - cannot check GP records")
        return result

    if connector is None:
        connector = get_connector()

    # Build lookup dict for mapping snomed_code -> (search_term, primary_directorate, snomed_description)
    snomed_lookup = {
        m.snomed_code: (m.search_term, m.primary_directorate, m.snomed_description)
        for m in drug_snomed_mappings
    }

    # Get unique SNOMED codes
    snomed_codes = list(snomed_lookup.keys())

    # Build placeholders for SNOMED codes
    placeholders = ", ".join(["%s"] * len(snomed_codes))

    # Query to find most recent matching SNOMED code in GP records
    query = f'''
        SELECT
            "SNOMEDCode",
            "EventDateTime"
        FROM DATA_HUB.PHM."PrimaryCareClinicalCoding"
        WHERE "PatientPseudonym" = %s
            AND "SNOMEDCode" IN ({placeholders})
    '''

    params: list = [patient_pseudonym] + snomed_codes

    if before_date:
        query += ' AND "EventDateTime" < %s'
        params.append(before_date.isoformat())

    query += ' ORDER BY "EventDateTime" DESC LIMIT 1'

    try:
        results = connector.execute_dict(query, tuple(params))

        if results:
            row = results[0]
            matched_code = row.get("SNOMEDCode")
            event_dt = row.get("EventDateTime")

            if matched_code and matched_code in snomed_lookup:
                search_term, primary_dir, snomed_desc = snomed_lookup[matched_code]

                return DirectSnomedMatchResult(
                    patient_pseudonym=patient_pseudonym,
                    matched=True,
                    snomed_code=matched_code,
                    snomed_description=snomed_desc,
                    search_term=search_term,
                    primary_directorate=primary_dir,
                    event_date=event_dt,
                    source="DIRECT_SNOMED",
                )

        return result

    except Exception as e:
        logger.error(f"Error checking direct SNOMED for patient '{patient_pseudonym}': {e}")
        return result

def get_directorate_from_diagnosis(
    upid: str,
    drug_name: str,
    connector: Optional[SnowflakeConnector] = None,
    db_manager: Optional[DatabaseManager] = None,
    before_date: Optional[date] = None,
) -> DirectorateAssignment:
    """
    Get directorate assignment for a patient-drug combination using diagnosis-based lookup.

    This function attempts to assign a directorate based on the patient's GP records
    (direct SNOMED code matching). If no match is found, it returns a FALLBACK result
    indicating that the caller should use alternative assignment methods (e.g.,
    department_identification() from tools/data.py).

    Workflow:
    1. Get all SNOMED codes for the drug from ref_drug_snomed_mapping
    2. Query patient's GP records for matching SNOMED codes
    3. If match found ÔåÆ return diagnosis-based directorate and search_term
    4. If no match ÔåÆ return FALLBACK result (caller handles fallback logic)

    Args:
        upid: Patient's unique patient ID (Provider Code[:3] + PersonKey)
        drug_name: Drug name to look up
        connector: Optional SnowflakeConnector (defaults to singleton)
        db_manager: Optional DatabaseManager (defaults to default_db_manager)
        before_date: Optional date - only check diagnoses before this date

    Returns:
        DirectorateAssignment with directorate, search_term, and source
    """
    result = DirectorateAssignment(
        upid=upid,
        drug_name=drug_name,
        directorate=None,
        source="FALLBACK",
    )

    # Step 1: Get SNOMED codes for the drug
    drug_snomed_mappings = get_drug_snomed_codes(drug_name, db_manager)

    if not drug_snomed_mappings:
        logger.debug(f"No SNOMED mappings found for drug '{drug_name}' - using fallback")
        return result

    # Step 2: Check Snowflake availability
    if not SNOWFLAKE_AVAILABLE:
        logger.debug("Snowflake not available - using fallback")
        return result

    if not is_snowflake_configured():
        logger.debug("Snowflake not configured - using fallback")
        return result

    # Step 3: Get patient pseudonym from UPID
    # UPID format is Provider Code (3 chars) + PersonKey
    # We need to query Snowflake to get the PatientPseudonym for this PersonKey
    # However, patient_has_indication_direct expects PatientPseudonym, not UPID
    # For now, we'll use UPID as the identifier - the actual integration
    # will need to happen at the DataFrame level where we have PersonKey
    #
    # NOTE: This function will be called from the pipeline where we have
    # access to PatientPseudonym. The UPID is passed for logging/tracking.

    # Actually, looking at the pipeline, we need PatientPseudonym, not UPID.
    # The caller should pass the PatientPseudonym or we need to look it up.
    # For now, let's assume the caller will use this in a batch context
    # where they can map UPID -> PatientPseudonym.

    # Let me reconsider: the function signature takes UPID but we need
    # PatientPseudonym for Snowflake. In the pipeline context (fetch_and_transform_data),
    # we'll have the PersonKey column which IS the PatientPseudonym.
    # So UPID = ProviderCode[:3] + PersonKey, and PersonKey = PatientPseudonym.
    #
    # We can extract PatientPseudonym from UPID by removing the first 3 chars.
    patient_pseudonym = upid[3:] if len(upid) > 3 else upid

    # Step 4: Check patient's GP records for matching SNOMED codes
    match_result = patient_has_indication_direct(
        patient_pseudonym=patient_pseudonym,
        drug_snomed_mappings=drug_snomed_mappings,
        connector=connector,
        before_date=before_date,
    )

    if match_result.matched and match_result.primary_directorate:
        return DirectorateAssignment(
            upid=upid,
            drug_name=drug_name,
            directorate=match_result.primary_directorate,
            search_term=match_result.search_term,
            source="DIAGNOSIS",
            snomed_code=match_result.snomed_code,
            event_date=match_result.event_date,
        )

    # No match found - return fallback result
    return result

def get_cluster_snomed_codes(
    cluster_id: str,
    connector: Optional[SnowflakeConnector] = None,
    use_cache: bool = True,
) -> ClusterSnomedCodes:
    """
    Get all SNOMED codes for a cluster from Snowflake.

    Queries the ClinicalCodingClusterSnomedCodes table to get all SNOMED codes
    that belong to the specified cluster.

    Args:
        cluster_id: Cluster ID to look up (e.g., 'RARTH_COD', 'PSORIASIS_COD')
        connector: Optional SnowflakeConnector (defaults to singleton)
        use_cache: Whether to use cached results (default True)

    Returns:
        ClusterSnomedCodes with list of SNOMED codes and descriptions
    """
    if not SNOWFLAKE_AVAILABLE:
        logger.warning("Snowflake connector not available")
        return ClusterSnomedCodes(cluster_id=cluster_id, cluster_description="")

    if not is_snowflake_configured():
        logger.warning("Snowflake not configured - cannot get cluster codes")
        return ClusterSnomedCodes(cluster_id=cluster_id, cluster_description="")

    # Check cache first
    cache_key = f"cluster_snomed_{cluster_id}"
    if use_cache and is_cache_enabled():
        cache = get_cache()
        cached = cache.get(cache_key)
        if cached is not None and len(cached) > 0:
            logger.debug(f"Using cached SNOMED codes for cluster '{cluster_id}'")
            cached_dict = cached[0]  # First element is our data dict
            return ClusterSnomedCodes(
                cluster_id=cluster_id,
                cluster_description=str(cached_dict.get("description", "")),
                snomed_codes=list(cached_dict.get("codes", [])),
                snomed_descriptions=dict(cached_dict.get("descriptions", {})),
            )

    if connector is None:
        connector = get_connector()

    query = '''
        SELECT DISTINCT
            "Cluster_ID",
            "Cluster_Description",
            "SNOMEDCode",
            "SNOMEDDescription"
        FROM DATA_HUB.PHM."ClinicalCodingClusterSnomedCodes"
        WHERE "Cluster_ID" = %s
        ORDER BY "SNOMEDCode"
    '''

    try:
        results = connector.execute_dict(query, (cluster_id,))

        if not results:
            logger.warning(f"No SNOMED codes found for cluster '{cluster_id}'")
            return ClusterSnomedCodes(cluster_id=cluster_id, cluster_description="")

        codes = []
        descriptions = {}
        description = results[0].get("Cluster_Description", "") if results else ""

        for row in results:
            code = row.get("SNOMEDCode")
            if code:
                codes.append(code)
                descriptions[code] = row.get("SNOMEDDescription", "")

        logger.info(f"Found {len(codes)} SNOMED codes for cluster '{cluster_id}'")

        # Cache the results (using query-based cache with fake params)
        if use_cache and is_cache_enabled():
            cache = get_cache()
            cache_data = [{
                "description": description,
                "codes": codes,
                "descriptions": descriptions,
            }]
            cache.set(cache_key, None, cache_data)  # type: ignore[arg-type]

        return ClusterSnomedCodes(
            cluster_id=cluster_id,
            cluster_description=description,
            snomed_codes=codes,
            snomed_descriptions=descriptions,
        )

    except Exception as e:
        logger.error(f"Error getting SNOMED codes for cluster '{cluster_id}': {e}")
        return ClusterSnomedCodes(cluster_id=cluster_id, cluster_description="")

def patient_has_indication(
    patient_pseudonym: str,
    cluster_ids: list[str],
    connector: Optional[SnowflakeConnector] = None,
    before_date: Optional[date] = None,
) -> tuple[bool, Optional[str], Optional[str], Optional[str]]:
    """
    Check if a patient has any SNOMED codes from the specified clusters in GP records.

    Args:
        patient_pseudonym: Patient's pseudonymised NHS number
        cluster_ids: List of cluster IDs to check against
        connector: Optional SnowflakeConnector
        before_date: Optional date - only check diagnoses before this date

    Returns:
        Tuple of (has_indication, matched_cluster_id, matched_snomed_code, matched_description)
    """
    if not SNOWFLAKE_AVAILABLE or not is_snowflake_configured():
        return False, None, None, None

    if not cluster_ids:
        return False, None, None, None

    if connector is None:
        connector = get_connector()

    # Build placeholders for cluster IDs
    placeholders = ", ".join(["%s"] * len(cluster_ids))

    # Query to check if patient has any matching SNOMED code
    query = f'''
        SELECT
            pc."SNOMEDCode",
            cc."Cluster_ID",
            cc."SNOMEDDescription"
        FROM DATA_HUB.PHM."PrimaryCareClinicalCoding" pc
        INNER JOIN DATA_HUB.PHM."ClinicalCodingClusterSnomedCodes" cc
            ON pc."SNOMEDCode" = cc."SNOMEDCode"
        WHERE pc."PatientPseudonym" = %s
            AND cc."Cluster_ID" IN ({placeholders})
    '''

    params = [patient_pseudonym] + cluster_ids

    if before_date:
        query += ' AND pc."EventDateTime" < %s'
        params.append(before_date.isoformat())

    query += ' LIMIT 1'

    try:
        results = connector.execute_dict(query, tuple(params))

        if results:
            row = results[0]
            return (
                True,
                row.get("Cluster_ID"),
                row.get("SNOMEDCode"),
                row.get("SNOMEDDescription"),
            )

        return False, None, None, None

    except Exception as e:
        logger.error(f"Error checking indication for patient '{patient_pseudonym}': {e}")
        return False, None, None, None

def validate_indication(
    patient_pseudonym: str,
    drug_name: str,
    connector: Optional[SnowflakeConnector] = None,
    db_manager: Optional[DatabaseManager] = None,
    before_date: Optional[date] = None,
) -> IndicationValidationResult:
    """
    Validate that a patient has an appropriate indication for a drug.

    Full validation workflow:
    1. Get drug's valid indication clusters from local mapping
    2. Check if patient has any matching SNOMED codes in GP records
    3. Return detailed validation result

    Args:
        patient_pseudonym: Patient's pseudonymised NHS number
        drug_name: Drug name to validate indication for
        connector: Optional SnowflakeConnector
        db_manager: Optional DatabaseManager
        before_date: Optional date - only check diagnoses before this date

    Returns:
        IndicationValidationResult with validation details
    """
    result = IndicationValidationResult(
        patient_pseudonym=patient_pseudonym,
        drug_name=drug_name,
        has_valid_indication=False,
    )

    # Step 1: Get drug's cluster mappings
    cluster_ids = get_drug_cluster_ids(drug_name, db_manager)

    if not cluster_ids:
        result.error_message = f"No cluster mappings found for drug '{drug_name}'"
        result.source = "NONE"
        return result

    result.checked_clusters = cluster_ids

    # Step 2: Check Snowflake availability
    if not SNOWFLAKE_AVAILABLE:
        result.error_message = "Snowflake connector not installed"
        result.source = "NONE"
        return result

    if not is_snowflake_configured():
        result.error_message = "Snowflake not configured"
        result.source = "NONE"
        return result

    # Step 3: Check patient GP records
    has_indication, matched_cluster, matched_code, matched_desc = patient_has_indication(
        patient_pseudonym=patient_pseudonym,
        cluster_ids=cluster_ids,
        connector=connector,
        before_date=before_date,
    )

    result.has_valid_indication = has_indication
    result.matched_cluster_id = matched_cluster
    result.matched_snomed_code = matched_code
    result.matched_snomed_description = matched_desc
    result.source = "GP_SNOMED" if has_indication else "NONE"

    return result

def get_indication_match_rate(
    drug_name: str,
    patient_pseudonyms: list[str],
    connector: Optional[SnowflakeConnector] = None,
    db_manager: Optional[DatabaseManager] = None,
    sample_unmatched_count: int = 10,
) -> DrugIndicationMatchRate:
    """
    Calculate indication match rate for a drug across a list of patients.

    Args:
        drug_name: Drug name to check
        patient_pseudonyms: List of patient pseudonymised NHS numbers
        connector: Optional SnowflakeConnector
        db_manager: Optional DatabaseManager
        sample_unmatched_count: Number of unmatched patient IDs to include in sample

    Returns:
        DrugIndicationMatchRate with match statistics
    """
    if connector is None and SNOWFLAKE_AVAILABLE and is_snowflake_configured():
        connector = get_connector()

    cluster_ids = get_drug_cluster_ids(drug_name, db_manager)

    total = len(patient_pseudonyms)
    matched = 0
    unmatched = 0
    sample_unmatched: list[str] = []

    if not cluster_ids:
        logger.warning(f"No cluster mappings for drug '{drug_name}' - all patients will be unmatched")
        return DrugIndicationMatchRate(
            drug_name=drug_name,
            total_patients=total,
            patients_with_indication=0,
            patients_without_indication=total,
            match_rate=0.0,
            clusters_checked=[],
            sample_unmatched=patient_pseudonyms[:sample_unmatched_count],
        )

    for i, pseudonym in enumerate(patient_pseudonyms):
        if i > 0 and i % 100 == 0:
            logger.info(f"Validating indications: {i}/{total} ({100*i/total:.1f}%)")

        has_indication, _, _, _ = patient_has_indication(
            patient_pseudonym=pseudonym,
            cluster_ids=cluster_ids,
            connector=connector,
        )

        if has_indication:
            matched += 1
        else:
            unmatched += 1
            if len(sample_unmatched) < sample_unmatched_count:
                sample_unmatched.append(pseudonym)

    match_rate = matched / total if total > 0 else 0.0

    logger.info(f"Indication match rate for '{drug_name}': {100*match_rate:.1f}% ({matched}/{total})")

    return DrugIndicationMatchRate(
        drug_name=drug_name,
        total_patients=total,
        patients_with_indication=matched,
        patients_without_indication=unmatched,
        match_rate=match_rate,
        clusters_checked=cluster_ids,
        sample_unmatched=sample_unmatched,
    )

def batch_validate_indications(
    patient_drug_pairs: list[tuple[str, str]],
    connector: Optional[SnowflakeConnector] = None,
    db_manager: Optional[DatabaseManager] = None,
    progress_callback: Optional[Callable[[int, int], None]] = None,
) -> list[IndicationValidationResult]:
    """
    Validate indications for multiple patient-drug pairs efficiently.

    Args:
        patient_drug_pairs: List of (patient_pseudonym, drug_name) tuples
        connector: Optional SnowflakeConnector
        db_manager: Optional DatabaseManager
        progress_callback: Optional callback(current, total) for progress updates

    Returns:
        List of IndicationValidationResult for each pair
    """
    results = []
    total = len(patient_drug_pairs)

    # Cache cluster lookups by drug
    drug_clusters_cache = {}

    for i, (pseudonym, drug_name) in enumerate(patient_drug_pairs):
        if progress_callback:
            progress_callback(i + 1, total)

        # Get clusters from cache or lookup
        drug_upper = drug_name.upper()
        if drug_upper not in drug_clusters_cache:
            drug_clusters_cache[drug_upper] = get_drug_cluster_ids(drug_name, db_manager)

        cluster_ids = drug_clusters_cache[drug_upper]

        if not cluster_ids:
            results.append(IndicationValidationResult(
                patient_pseudonym=pseudonym,
                drug_name=drug_name,
                has_valid_indication=False,
                source="NONE",
                error_message=f"No cluster mappings for drug '{drug_name}'",
            ))
            continue

        # Check patient indication
        has_indication, matched_cluster, matched_code, matched_desc = patient_has_indication(
            patient_pseudonym=pseudonym,
            cluster_ids=cluster_ids,
            connector=connector,
        )

        results.append(IndicationValidationResult(
            patient_pseudonym=pseudonym,
            drug_name=drug_name,
            has_valid_indication=has_indication,
            matched_cluster_id=matched_cluster,
            matched_snomed_code=matched_code,
            matched_snomed_description=matched_desc,
            checked_clusters=cluster_ids,
            source="GP_SNOMED" if has_indication else "NONE",
        ))

    matched_count = sum(1 for r in results if r.has_valid_indication)
    logger.info(f"Batch validation complete: {matched_count}/{total} ({100*matched_count/total:.1f}%) with valid indications")

    return results

def get_available_clusters(
    connector: Optional[SnowflakeConnector] = None,
) -> list[dict]:
    """
    Get list of all available SNOMED clusters from Snowflake.

    Returns:
        List of dicts with cluster_id, cluster_description, code_count
    """
    if not SNOWFLAKE_AVAILABLE or not is_snowflake_configured():
        logger.warning("Snowflake not available - cannot list clusters")
        return []

    if connector is None:
        connector = get_connector()

    query = '''
        SELECT
            "Cluster_ID",
            "Cluster_Description",
            COUNT(DISTINCT "SNOMEDCode") as code_count
        FROM DATA_HUB.PHM."ClinicalCodingClusterSnomedCodes"
        GROUP BY "Cluster_ID", "Cluster_Description"
        ORDER BY "Cluster_ID"
    '''

    try:
        results = connector.execute_dict(query)

        clusters = []
        for row in results:
            clusters.append({
                "cluster_id": row.get("Cluster_ID"),
                "cluster_description": row.get("Cluster_Description"),
                "code_count": row.get("code_count", 0),
            })

        logger.info(f"Found {len(clusters)} available SNOMED clusters")
        return clusters

    except Exception as e:
        logger.error(f"Error getting available clusters: {e}")
        return []

def batch_lookup_indication_groups(
    df: "pd.DataFrame",
    connector: Optional[SnowflakeConnector] = None,
    db_manager: Optional[DatabaseManager] = None,
    batch_size: int = 500,
) -> "pd.DataFrame":
    """
    Batch lookup GP diagnosis-based indication groups for a DataFrame of patients.

    This is the efficient batch version of get_directorate_from_diagnosis().
    Instead of querying Snowflake per patient, it batches the lookups for performance.

    Strategy:
    1. Get all unique (PersonKey, Drug Name) pairs from DataFrame
    2. For each unique drug, get all SNOMED codes from local SQLite
    3. Build batched Snowflake queries to check GP records
    4. Return indication_df mapping UPID ÔåÆ Indication_Group

    For unmatched patients, Indication_Group will be their Directory (with suffix).

    Args:
        df: DataFrame with columns: UPID, Drug Name, Directory, PersonKey
        connector: Optional SnowflakeConnector (defaults to singleton)
        db_manager: Optional DatabaseManager (defaults to default_db_manager)
        batch_size: Number of patients per Snowflake query batch

    Returns:
        DataFrame with columns: UPID, Indication_Group, Source
        - Indication_Group: Search_Term (if matched) or "Directory (no GP dx)" (if not)
        - Source: "DIAGNOSIS" or "FALLBACK"
    """
    import pandas as pd

    if db_manager is None:
        db_manager = default_db_manager

    logger.info(f"Starting batch indication lookup for {len(df)} records...")

    # Step 1: Get unique (UPID, Drug Name, PseudoNHSNoLinked, Directory) combinations
    # We need PseudoNHSNoLinked to query Snowflake - this matches PatientPseudonym in GP records
    # Note: PersonKey is LocalPatientID which is provider-specific and does NOT match GP records
    if 'PseudoNHSNoLinked' not in df.columns:
        logger.error("DataFrame missing 'PseudoNHSNoLinked' column - cannot lookup GP records")
        # Return fallback for all patients
        result_df = df[['UPID', 'Directory']].drop_duplicates().copy()
        result_df['Indication_Group'] = result_df['Directory'] + " (no GP dx)"
        result_df['Source'] = "FALLBACK"
        return result_df[['UPID', 'Indication_Group', 'Source']]

    # Get unique patient-drug combinations (we need one lookup per patient-drug pair)
    unique_pairs = df[['UPID', 'Drug Name', 'PseudoNHSNoLinked', 'Directory']].drop_duplicates()
    logger.info(f"Found {len(unique_pairs)} unique patient-drug combinations")

    # Step 2: Get all unique drugs and their SNOMED codes
    unique_drugs = unique_pairs['Drug Name'].unique()
    logger.info(f"Building SNOMED lookup for {len(unique_drugs)} unique drugs...")

    # Build drug -> list of DrugSnomedMapping dict
    drug_snomed_map: dict[str, list[DrugSnomedMapping]] = {}
    all_snomed_codes: set[str] = set()
    snomed_to_drug_searchterm: dict[str, list[tuple[str, str, str]]] = {}  # snomed -> [(drug, search_term, primary_dir), ...]

    for drug_name in unique_drugs:
        mappings = get_drug_snomed_codes(drug_name, db_manager)
        drug_snomed_map[drug_name] = mappings

        for m in mappings:
            all_snomed_codes.add(m.snomed_code)
            if m.snomed_code not in snomed_to_drug_searchterm:
                snomed_to_drug_searchterm[m.snomed_code] = []
            snomed_to_drug_searchterm[m.snomed_code].append(
                (drug_name, m.search_term, m.primary_directorate)
            )

    logger.info(f"Total SNOMED codes to check: {len(all_snomed_codes)}")

    # Step 3: Check Snowflake availability
    if not SNOWFLAKE_AVAILABLE or not is_snowflake_configured():
        logger.warning("Snowflake not available - returning fallback for all patients")
        result_df = unique_pairs[['UPID', 'Directory']].copy()
        result_df['Indication_Group'] = result_df['Directory'] + " (no GP dx)"
        result_df['Source'] = "FALLBACK"
        return result_df[['UPID', 'Indication_Group', 'Source']].drop_duplicates(subset=['UPID'])

    if connector is None:
        connector = get_connector()

    # Step 4: Query GP records for all patients in batches
    # The query finds the most recent matching SNOMED code for each patient

    # Get unique PseudoNHSNoLinked values (each = one patient in GP records)
    unique_patients = unique_pairs[['PseudoNHSNoLinked', 'UPID', 'Directory']].drop_duplicates(subset=['PseudoNHSNoLinked'])
    patient_pseudonyms = unique_patients['PseudoNHSNoLinked'].tolist()

    logger.info(f"Querying GP records for {len(patient_pseudonyms)} unique patients in batches of {batch_size}...")

    # Results dict: PersonKey -> (snomed_code, event_date)
    gp_matches: dict[str, tuple[str, Any]] = {}

    # Convert SNOMED codes to list for query
    snomed_list = list(all_snomed_codes)

    if not snomed_list:
        logger.warning("No SNOMED codes to check - returning fallback for all patients")
        result_df = unique_pairs[['UPID', 'Directory']].copy()
        result_df['Indication_Group'] = result_df['Directory'] + " (no GP dx)"
        result_df['Source'] = "FALLBACK"
        return result_df[['UPID', 'Indication_Group', 'Source']].drop_duplicates(subset=['UPID'])

    # Build SNOMED IN clause (reused across batches)
    snomed_placeholders = ", ".join(["%s"] * len(snomed_list))

    # Process patients in batches
    for batch_start in range(0, len(patient_pseudonyms), batch_size):
        batch_end = min(batch_start + batch_size, len(patient_pseudonyms))
        batch_pseudonyms = patient_pseudonyms[batch_start:batch_end]

        logger.info(f"Batch {batch_start//batch_size + 1}: patients {batch_start} to {batch_end}")

        # Build patient IN clause
        patient_placeholders = ", ".join(["%s"] * len(batch_pseudonyms))

        # Query to find all matching SNOMED codes for these patients
        # We'll get all matches and pick the most recent per patient in Python
        query = f'''
            SELECT
                "PatientPseudonym",
                "SNOMEDCode",
                "EventDateTime"
            FROM DATA_HUB.PHM."PrimaryCareClinicalCoding"
            WHERE "PatientPseudonym" IN ({patient_placeholders})
              AND "SNOMEDCode" IN ({snomed_placeholders})
            ORDER BY "PatientPseudonym", "EventDateTime" DESC
        '''

        params = tuple(batch_pseudonyms) + tuple(snomed_list)

        try:
            results = connector.execute_dict(query, params)

            # Process results - pick most recent per patient
            for row in results:
                person_key = row.get("PatientPseudonym")
                snomed_code = row.get("SNOMEDCode")
                event_date = row.get("EventDateTime")

                if person_key and snomed_code:
                    # Keep only if we haven't seen this patient yet (first = most recent due to ORDER BY)
                    if person_key not in gp_matches:
                        gp_matches[person_key] = (snomed_code, event_date)

        except Exception as e:
            logger.error(f"Error querying GP records for batch: {e}")
            # Continue with other batches

    logger.info(f"Found GP matches for {len(gp_matches)} patients")

    # Step 5: Build result DataFrame
    # For each unique_pair, determine Indication_Group based on match status
    results_list = []

    # We need to dedupe by UPID - a patient might be on multiple drugs
    # Strategy: For each UPID, use the most recent match (if any)
    upid_to_match: dict[str, tuple[str, str]] = {}  # UPID -> (Indication_Group, Source)

    for _, row in unique_pairs.iterrows():
        upid = row['UPID']
        drug_name = row['Drug Name']
        patient_pseudonym = row['PseudoNHSNoLinked']
        directory = row['Directory']

        # Check if patient has GP match (using PseudoNHSNoLinked which matches PatientPseudonym in GP)
        if patient_pseudonym in gp_matches:
            matched_snomed, event_date = gp_matches[patient_pseudonym]

            # Find the search_term for this SNOMED code and drug
            # (A SNOMED code might map to multiple drugs with different search_terms)
            if matched_snomed in snomed_to_drug_searchterm:
                # Look for match with current drug first
                search_term = None
                for drug, st, pd in snomed_to_drug_searchterm[matched_snomed]:
                    if drug.upper() == drug_name.upper():
                        search_term = st
                        break
                # If no drug-specific match, use any match
                if search_term is None:
                    search_term = snomed_to_drug_searchterm[matched_snomed][0][1]

                # Only update if we don't have a match for this UPID yet
                if upid not in upid_to_match:
                    upid_to_match[upid] = (search_term, "DIAGNOSIS")
            else:
                # Shouldn't happen but fallback just in case
                if upid not in upid_to_match:
                    upid_to_match[upid] = (directory + " (no GP dx)", "FALLBACK")
        else:
            # No GP match - use fallback
            if upid not in upid_to_match:
                upid_to_match[upid] = (directory + " (no GP dx)", "FALLBACK")

    # Build result DataFrame
    for upid, (indication_group, source) in upid_to_match.items():
        results_list.append({
            'UPID': upid,
            'Indication_Group': indication_group,
            'Source': source,
        })

    result_df = pd.DataFrame(results_list)

    # Log statistics
    diagnosis_count = len([s for s in result_df['Source'] if s == "DIAGNOSIS"])
    fallback_count = len([s for s in result_df['Source'] if s == "FALLBACK"])
    total = len(result_df)

    logger.info(f"Indication lookup complete:")
    logger.info(f"  Total unique patients: {total}")
    logger.info(f"  DIAGNOSIS matches: {diagnosis_count} ({100*diagnosis_count/total:.1f}%)")
    logger.info(f"  FALLBACK (no GP match): {fallback_count} ({100*fallback_count/total:.1f}%)")

    return result_df

# === Drug-to-indication mapping from DimSearchTerm.csv ===

# Merge related Search_Terms into canonical names.
# Asthma variants are clinically the same condition at different severity levels.
# Urticaria is a separate condition ÔÇö do NOT merge with asthma.
SEARCH_TERM_MERGE_MAP: dict[str, str] = {
    "allergic asthma": "asthma",
    "severe persistent allergic asthma": "asthma",
}

def load_drug_indication_mapping(
    csv_path: Optional[str] = None,
) -> tuple[dict[str, list[str]], dict[str, list[str]]]:
    """
    Load the drug-to-Search_Term mapping from DimSearchTerm.csv.

    Builds two lookup dicts:
    - fragment_to_search_terms: drug fragment (UPPERCASE) -> list of Search_Terms containing it
    - search_term_to_fragments: search_term -> list of drug fragments (UPPERCASE)

    DimSearchTerm.csv has columns: Search_Term, CleanedDrugName, PrimaryDirectorate
    CleanedDrugName is pipe-separated (e.g., "ADALIMUMAB|GOLIMUMAB|IXEKIZUMAB").

    Note: A Search_Term can appear multiple times with different PrimaryDirectorates
    (e.g., "diabetes" appears under both DIABETIC MEDICINE and OPHTHALMOLOGY).
    Drug fragments from all rows for the same Search_Term are combined.

    Asthma-related Search_Terms ("allergic asthma", "severe persistent allergic asthma")
    are merged into "asthma" to match the CLUSTER_MAPPING_SQL normalization.
    "urticaria" stays separate.

    Args:
        csv_path: Path to DimSearchTerm.csv. Defaults to data/DimSearchTerm.csv.

    Returns:
        Tuple of (fragment_to_search_terms, search_term_to_fragments)
    """
    if csv_path is None:
        csv_path = str(Path(__file__).parent.parent / "data" / "DimSearchTerm.csv")

    fragment_to_search_terms: dict[str, list[str]] = {}
    search_term_to_fragments: dict[str, list[str]] = {}

    try:
        with open(csv_path, "r", encoding="utf-8") as f:
            reader = csv.DictReader(f)
            for row in reader:
                search_term = row.get("Search_Term", "").strip()
                drug_names_raw = row.get("CleanedDrugName", "").strip()

                # Normalize asthma variants to canonical "asthma"
                search_term = SEARCH_TERM_MERGE_MAP.get(search_term, search_term)

                if not search_term or not drug_names_raw:
                    continue

                fragments = [frag.strip().upper() for frag in drug_names_raw.split("|") if frag.strip()]

                # Build search_term -> fragments (accumulate for duplicate Search_Terms)
                if search_term not in search_term_to_fragments:
                    search_term_to_fragments[search_term] = []
                for frag in fragments:
                    if frag not in search_term_to_fragments[search_term]:
                        search_term_to_fragments[search_term].append(frag)

                # Build fragment -> search_terms
                for frag in fragments:
                    if frag not in fragment_to_search_terms:
                        fragment_to_search_terms[frag] = []
                    if search_term not in fragment_to_search_terms[frag]:
                        fragment_to_search_terms[frag].append(search_term)

        logger.info(
            f"Loaded drug-indication mapping: {len(search_term_to_fragments)} Search_Terms, "
            f"{len(fragment_to_search_terms)} drug fragments"
        )

    except FileNotFoundError:
        logger.error(f"DimSearchTerm.csv not found at {csv_path}")
    except Exception as e:
        logger.error(f"Error loading DimSearchTerm.csv: {e}")

    return fragment_to_search_terms, search_term_to_fragments

def get_search_terms_for_drug(
    drug_name: str,
    search_term_to_fragments: dict[str, list[str]],
) -> list[str]:
    """
    Get all Search_Terms that list a given drug using substring matching.

    Checks if any drug fragment from DimSearchTerm is a SUBSTRING of the given
    drug name (case-insensitive). This handles both exact matches (ADALIMUMAB)
    and partial fragments (PEGYLATED, INHALED).

    Args:
        drug_name: HCD drug name (e.g., "ADALIMUMAB 40MG", "PEGYLATED LIPOSOMAL DOXORUBICIN")
        search_term_to_fragments: Mapping of search_term -> list of drug fragments

    Returns:
        List of Search_Terms whose drug fragments match the drug name
    """
    drug_name_upper = drug_name.upper()
    matched_terms: list[str] = []

    for search_term, fragments in search_term_to_fragments.items():
        for frag in fragments:
            if frag in drug_name_upper:
                matched_terms.append(search_term)
                break  # One matching fragment is enough for this Search_Term

    return matched_terms

# === NEW APPROACH: Query Snowflake directly using cluster CTE ===

# The cluster query mapping (embedded from snomed_indication_mapping_query.sql)
# This maps Search_Term -> Cluster_ID for ~148 clinical conditions
CLUSTER_MAPPING_SQL = """
WITH SearchTermClusters AS (
    SELECT Search_Term, Cluster_ID FROM (VALUES
        ('acute lymphoblastic leukaemia', 'HAEMCANMORPH_COD'),
        ('acute myeloid leukaemia', 'C19HAEMCAN_COD'),
        ('acute promyelocytic leukaemia', 'HAEMCANMORPH_COD'),
        ('asthma', 'AST_COD'),
        ('allergic rhinitis', 'MILDINTAST_COD'),
        ('alzheimer''s disease', 'DEMALZ_COD'),
        ('amyloidosis', 'AMYLOID_COD'),
        ('anaemia', 'eFI2_AnaemiaTimeSensitive'),
        ('anaplastic large cell lymphoma', 'C19HAEMCAN_COD'),
        ('apixaban', 'DOACCON_COD'),
        ('aplastic anaemia', 'eFI2_AnaemiaEver'),
        ('arthritis', 'eFI2_InflammatoryArthritis'),
        ('asthma', 'eFI2_Asthma'),
        ('atopic dermatitis', 'ATOPDERM_COD'),
        ('atrial fibrillation', 'eFI2_AtrialFibrillation'),
        ('attention deficit hyperactivity disorder', 'ADHD_COD'),
        ('bipolar disorder', 'MH_COD'),
        ('bladder', 'eFI2_UrinaryIncontinence'),
        ('breast cancer', 'BRCANSCR_COD'),
        ('cardiomyopathy', 'eFI2_HarmfulDrinking'),
        ('cardiovascular disease', 'CVDRISKASS_COD'),
        ('cervical cancer', 'CSDEC_COD'),
        ('cholangiocarcinoma', 'eFI2_Cancer'),
        ('chronic kidney disease', 'CKD_COD'),
        ('chronic liver disease', 'eFI2_LiverProblems'),
        ('chronic lymphocytic leukaemia', 'EPPHAEMCAN_COD'),
        ('chronic myeloid leukaemia', 'EPPHAEMCAN_COD'),
        ('chronic obstructive pulmonary disease', 'eFI2_COPD'),
        ('colon cancer', 'eFI2_Cancer'),
        ('colorectal cancer', 'GICANREF_COD'),
        ('constipation', 'CHRONCONSTIP_COD'),
        ('covid-19', 'POSSPOSTCOVID_COD'),
        ('crohn''s disease', 'eFI2_InflammatoryBowelDisease'),
        ('cutaneous t-cell lymphoma', 'C19HAEMCAN_COD'),
        ('cystic fibrosis', 'CUST_ICB_CYSTIC_FIBROSIS'),
        ('deep vein thrombosis', 'VTE_COD'),
        ('depression', 'eFI2_Depression'),
        ('diabetes', 'eFI2_DiabetesEver'),
        ('diabetic retinopathy', 'DRSELIGIBILITY_COD'),
        ('diffuse large b-cell lymphoma', 'C19HAEMCAN_COD'),
        ('dravet syndrome', 'EPIL_COD'),
        ('drug misuse', 'ILLSUBINT_COD'),
        ('dyspepsia', 'eFI2_AbdominalPain'),
        ('epilepsy', 'eFI2_Seizures'),
        ('fallopian tube', 'STERIL_COD'),
        ('follicular lymphoma', 'C19HAEMCAN_COD'),
        ('gastric cancer', 'eFI2_Cancer'),
        ('giant cell arteritis', 'GCA_COD'),
        ('glioma', 'NHAEMCANMORPH_COD'),
        ('gout', 'eFI2_InflammatoryArthritis'),
        ('graft versus host disease', 'GVHD_COD'),
        ('granulomatosis with polyangiitis', 'WEGENERVASC_COD'),
        ('growth hormone deficiency', 'HYPOPITUITARY_COD'),
        ('hand eczema', 'ECZEMA_COD'),
        ('heart failure', 'eFI2_HeartFailure'),
        ('hepatitis b', 'HEPBCVAC_COD'),
        ('hepatocellular carcinoma', 'eFI2_Cancer'),
        ('hiv', 'PREFLANG_COD'),
        ('hodgkin lymphoma', 'HAEMCANMORPH_COD'),
        ('hormone receptor', 'eFI2_ThyroidProblems'),
        ('hypercholesterolaemia', 'CLASSFH_COD'),
        ('immune thrombocytopenia', 'ITP_COD'),
        ('influenza', 'FLUINVITE_COD'),
        ('insomnia', 'eFI2_SleepProblems'),
        ('irritable bowel syndrome', 'IBS_COD'),
        ('ischaemic stroke', 'OSTR_COD'),
        ('juvenile idiopathic arthritis', 'RARTHAD_COD'),
        ('kidney transplant', 'RENALTRANSP_COD'),
        ('leukaemia', 'eFI2_Cancer'),
        ('lung cancer', 'FTCANREF_COD'),
        ('lymphoma', 'C19HAEMCAN_COD'),
        ('macular degeneration', 'CUST_ICB_VISUAL_IMPAIRMENT'),
        ('macular oedema', 'CUST_ICB_VISUAL_IMPAIRMENT'),
        ('major depressive episodes', 'eFI2_Depression'),
        ('malignant melanoma', 'eFI2_Cancer'),
        ('malignant pleural mesothelioma', 'LUNGCAN_COD'),
        ('manic episode', 'MH_COD'),
        ('mantle cell lymphoma', 'HAEMCANMORPH_COD'),
        ('melanoma', 'eFI2_Cancer'),
        ('merkel cell carcinoma', 'C19CAN_COD'),
        ('migraine', 'eFI2_Headache'),
        ('motor neurone disease', 'MND_COD'),
        ('multiple myeloma', 'C19HAEMCAN_COD'),
        ('multiple sclerosis', 'MS_COD'),
        ('myelodysplastic', 'eFI2_AnaemiaEver'),
        ('myelofibrosis', 'MDS_COD'),
        ('myocardial infarction', 'eFI2_IschaemicHeartDisease'),
        ('myotonia', 'CNDATRISK2_COD'),
        ('narcolepsy', 'LD_COD'),
        ('neuroendocrine tumour', 'LUNGCAN_COD'),
        ('non-small cell lung cancer', 'LUNGCAN_COD'),
        ('non-small-cell lung cancer', 'FTCANREF_COD'),
        ('obesity', 'BMI30_COD'),
        ('osteoarthritis', 'CUST_ICB_OSTEOARTHRITIS'),
        ('osteoporosis', 'eFI2_Osteoporosis'),
        ('osteosarcoma', 'NHAEMCANMORPH_COD'),
        ('ovarian cancer', 'C19CAN_COD'),
        ('peripheral arterial disease', 'PADEXC_COD'),
        ('plaque psoriasis', 'PSORIASIS_COD'),
        ('polycystic kidney disease', 'EPPCONGMALF_COD'),
        ('polycythaemia vera', 'C19HAEMCAN_COD'),
        ('pregnancy', 'C19PREG_COD'),
        ('primary biliary cholangitis', 'eFI2_LiverProblems'),
        ('primary hypercholesterolaemia', 'FNFHYP_COD'),
        ('prostate cancer', 'EPPSOLIDCAN_COD'),
        ('psoriasis', 'PSORIASIS_COD'),
        ('psoriatic arthritis', 'RARTHAD_COD'),
        ('pulmonary embolism', 'eFI2_RespiratoryDiseaseTimeSensitive'),
        ('pulmonary fibrosis', 'ILD_COD'),
        ('relapsing multiple sclerosis', 'MS_COD'),
        ('renal cell carcinoma', 'C19CAN_COD'),
        ('renal transplantation', 'RENALTRANSP_COD'),
        ('retinal vein occlusion', 'CUST_ICB_VISUAL_IMPAIRMENT'),
        ('rheumatoid arthritis', 'eFI2_InflammatoryArthritis'),
        ('rivaroxaban', 'DOACCON_COD'),
        ('schizophrenia', 'MH_COD'),
        ('seizures', 'LSZFREQ_COD'),
        ('sepsis', 'C19ACTIVITY_COD'),
        ('asthma', 'SEVAST_COD'),
        ('sickle cell disease', 'SICKLE_COD'),
        ('sleep apnoea', 'CUST_ICB_NON_SEVERE_LDA'),
        ('smoking cessation', 'SMOKINGINT_COD'),
        ('soft tissue sarcoma', 'NHAEMCANMORPH_COD'),
        ('spinal muscular atrophy', 'MND_COD'),
        ('squamous cell', 'C19CAN_COD'),
        ('squamous cell carcinoma', 'C19CAN_COD'),
        ('stem cell transplant', 'ALLOTRANSP_COD'),
        ('stroke', 'eFI2_Stroke'),
        ('systemic lupus erythematosus', 'SLUPUS_COD'),
        ('systemic mastocytosis', 'HAEMCANMORPH_COD'),
        ('thrombocytopenic purpura', 'TTP_COD'),
        ('thrombotic thrombocytopenic purpura', 'TTP_COD'),
        ('thyroid cancer', 'C19CAN_COD'),
        ('tophaceous gout', 'CUST_ICB_OSTEOARTHRITIS'),
        ('transitional cell carcinoma', 'C19CAN_COD'),
        ('type 1 diabetes', 'DMTYPE1_COD'),
        ('type 2 diabetes', 'DMTYPE2_COD'),
        ('ulcerative colitis', 'eFI2_InflammatoryBowelDisease'),
        ('urothelial carcinoma', 'NHAEMCANMORPH_COD'),
        ('urticaria', 'XSAL_COD'),
        ('uveitis', 'CUST_ICB_VISUAL_IMPAIRMENT'),
        ('vascular disease', 'CVDINVITE_COD'),
        ('vasculitis', 'CRYOGLOBVASC_COD')
    ) AS t(Search_Term, Cluster_ID)
),

ClusterCodes AS (
    SELECT
        stc.Search_Term,
        c."SNOMEDCode",
        c."SNOMEDDescription"
    FROM SearchTermClusters stc
    JOIN DATA_HUB.PHM."ClinicalCodingClusterSnomedCodes" c
        ON stc.Cluster_ID = c."Cluster_ID"
    WHERE c."SNOMEDCode" IS NOT NULL
),

ExplicitCodes AS (
    SELECT Search_Term, SNOMEDCode, SNOMEDDescription FROM (VALUES
        ('acute coronary syndrome', '837091000000100', 'Manual mapping'),
        ('ankylosing spondylitis', '162930007', 'Manual mapping'),
        ('ankylosing spondylitis', '239805001', 'Manual mapping'),
        ('ankylosing spondylitis', '239810002', 'Manual mapping'),
        ('ankylosing spondylitis', '239811003', 'Manual mapping'),
        ('ankylosing spondylitis', '394990003', 'Manual mapping'),
        ('ankylosing spondylitis', '429712009', 'Manual mapping'),
        ('ankylosing spondylitis', '441562009', 'Manual mapping'),
        ('ankylosing spondylitis', '441680005', 'Manual mapping'),
        ('ankylosing spondylitis', '441930001', 'Manual mapping'),
        ('axial spondyloarthritis', '723116002', 'Manual mapping'),
        ('choroidal neovascularisation', '380621000000102', 'Manual mapping'),
        ('choroidal neovascularisation', '733124000', 'Manual mapping')
    ) AS t(Search_Term, SNOMEDCode, SNOMEDDescription)
),

AllIndicationCodes AS (
    SELECT Search_Term, "SNOMEDCode" AS SNOMEDCode, "SNOMEDDescription" AS SNOMEDDescription
    FROM ClusterCodes
    UNION ALL
    SELECT Search_Term, SNOMEDCode, SNOMEDDescription
    FROM ExplicitCodes
)
"""

def get_patient_indication_groups(
    patient_pseudonyms: list[str],
    connector: Optional[SnowflakeConnector] = None,
    batch_size: int = 500,
) -> "pd.DataFrame":
    """
    Batch lookup GP diagnosis-based indication groups using Snowflake cluster query.

    This function queries Snowflake directly using the embedded cluster CTE
    (from snomed_indication_mapping_query.sql) to find patients with matching
    GP diagnoses. This is the NEW approach replacing the old SQLite-based lookup.

    The query:
    1. Uses the cluster mapping CTE to get all Search_Term -> SNOMED code mappings
    2. Joins with PrimaryCareClinicalCoding to find patients with matching codes
    3. Returns the most recent match per patient (by EventDateTime)

    Args:
        patient_pseudonyms: List of PseudoNHSNoLinked values (matches PatientPseudonym in GP records)
        connector: Optional SnowflakeConnector (defaults to singleton)
        batch_size: Number of patients per Snowflake query batch (default 500)

    Returns:
        DataFrame with columns:
        - PatientPseudonym: The patient identifier (PseudoNHSNoLinked value)
        - Search_Term: The matched indication (e.g., "rheumatoid arthritis")
        - EventDateTime: Date of the GP diagnosis record

        Patients not found in results have no matching GP diagnosis.
    """
    import pandas as pd

    logger.info(f"Starting Snowflake-direct indication lookup for {len(patient_pseudonyms)} patients...")

    # Handle edge case: empty patient list
    if not patient_pseudonyms:
        logger.warning("Empty patient list provided")
        return pd.DataFrame(columns=['PatientPseudonym', 'Search_Term', 'EventDateTime'])

    # Check Snowflake availability
    if not SNOWFLAKE_AVAILABLE:
        logger.error("Snowflake connector not available - cannot lookup GP records")
        return pd.DataFrame(columns=['PatientPseudonym', 'Search_Term', 'EventDateTime'])

    if not is_snowflake_configured():
        logger.error("Snowflake not configured - cannot lookup GP records")
        return pd.DataFrame(columns=['PatientPseudonym', 'Search_Term', 'EventDateTime'])

    if connector is None:
        connector = get_connector()

    # Results list to collect all matches
    all_results: list[dict] = []

    # Process patients in batches
    total_patients = len(patient_pseudonyms)
    for batch_start in range(0, total_patients, batch_size):
        batch_end = min(batch_start + batch_size, total_patients)
        batch_pseudonyms = patient_pseudonyms[batch_start:batch_end]
        batch_num = batch_start // batch_size + 1
        total_batches = (total_patients + batch_size - 1) // batch_size

        logger.info(f"Batch {batch_num}/{total_batches}: patients {batch_start + 1} to {batch_end}")

        # Build patient IN clause placeholders
        patient_placeholders = ", ".join(["%s"] * len(batch_pseudonyms))

        # Build the full query with cluster CTE
        # This finds the most recent matching diagnosis for each patient
        # Note: Column names must be aliased to ensure consistent casing in results
        query = f"""
{CLUSTER_MAPPING_SQL}
SELECT
    pc."PatientPseudonym" AS "PatientPseudonym",
    aic.Search_Term AS "Search_Term",
    pc."EventDateTime" AS "EventDateTime"
FROM DATA_HUB.PHM."PrimaryCareClinicalCoding" pc
INNER JOIN AllIndicationCodes aic
    ON pc."SNOMEDCode" = aic.SNOMEDCode
WHERE pc."PatientPseudonym" IN ({patient_placeholders})
QUALIFY ROW_NUMBER() OVER (
    PARTITION BY pc."PatientPseudonym"
    ORDER BY pc."EventDateTime" DESC
) = 1
"""

        try:
            results = connector.execute_dict(query, tuple(batch_pseudonyms))

            for row in results:
                all_results.append({
                    'PatientPseudonym': row.get('PatientPseudonym'),
                    'Search_Term': row.get('Search_Term'),
                    'EventDateTime': row.get('EventDateTime'),
                })

            logger.debug(f"Batch {batch_num}: found {len(results)} matches")

        except Exception as e:
            logger.error(f"Error querying GP records for batch {batch_num}: {e}")
            # Continue with other batches - partial results are better than none

    # Build result DataFrame
    result_df = pd.DataFrame(all_results)

    # Log summary statistics
    if len(result_df) > 0:
        matched_count = len(result_df)
        match_rate = 100 * matched_count / total_patients
        unique_terms = result_df['Search_Term'].nunique()
        logger.info(f"Indication lookup complete:")
        logger.info(f"  Total patients queried: {total_patients}")
        logger.info(f"  Patients with GP match: {matched_count} ({match_rate:.1f}%)")
        logger.info(f"  Unique Search_Terms found: {unique_terms}")

        # Log top Search_Terms
        top_terms = result_df['Search_Term'].value_counts().head(5)
        logger.info(f"  Top 5 indications: {dict(top_terms)}")
    else:
        logger.info(f"Indication lookup complete: 0 matches from {total_patients} patients")

    return result_df

# Export public API
__all__ = [
    # Dataclasses
    "ClusterSnomedCodes",
    "IndicationValidationResult",
    "DrugIndicationMatchRate",
    "DrugSnomedMapping",
    "DirectSnomedMatchResult",
    "DirectorateAssignment",
    # Cluster-based lookup functions (existing)
    "get_drug_clusters",
    "get_drug_cluster_ids",
    "get_cluster_snomed_codes",
    "patient_has_indication",
    "validate_indication",
    "get_indication_match_rate",
    "batch_validate_indications",
    "get_available_clusters",
    # Direct SNOMED lookup functions (new)
    "get_drug_snomed_codes",
    "patient_has_indication_direct",
    # Diagnosis-based directorate assignment
    "get_directorate_from_diagnosis",
    # Batch lookup for indication groups
    "batch_lookup_indication_groups",
    # Drug-indication mapping from DimSearchTerm.csv
    "SEARCH_TERM_MERGE_MAP",
    "load_drug_indication_mapping",
    "get_search_terms_for_drug",
    # Snowflake-direct indication lookup (new approach)
    "get_patient_indication_groups",
    "CLUSTER_MAPPING_SQL",
]