""" Centralised prompt templates for Criteria AI operations. All criteria-related prompts are maintained here as the single source of truth. Originally extracted from AI_Screening_UI criteria_workspace/ai_service.py. All prompts are standalone functions returning formatted strings. """ from __future__ import annotations from typing import Optional, TYPE_CHECKING if TYPE_CHECKING: from crystallise.criteria.models import ExclusionCriterion # Constants copied from criteria.models to avoid circular import: # prompts.criteria → criteria.models → criteria.__init__ → criteria.ai_service # → criteria.prompts → prompts.criteria PICO_CATEGORIES = [ "Population", "Intervention/Exposure", "Comparator", "Outcome", "Study Design", "Publication Type", "Setting/Context", "Timeframe", "Language/Geography", "Other", ] RQ_RELEVANCE_TYPES = ["direct", "indirect", "contextual"] # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- def format_questions_list(questions: Optional[list[str]]) -> str: """Format research questions as a numbered list.""" if not questions: return "No research questions provided." return "\n".join(f"{i + 1}. {q}" for i, q in enumerate(questions)) def format_criteria_list(criteria: list[ExclusionCriterion], limit: int = 20) -> str: """Format criteria as a bullet list for prompts.""" lines = [] for c in criteria[:limit]: ctype = c.criterion_type.value lines.append(f"- [{ctype.upper()}] [{c.category}] {c.text}") return "\n".join(lines) # --------------------------------------------------------------------------- # Exclusion Generation # --------------------------------------------------------------------------- def exclusion_generation_system_prompt() -> str: """System prompt for exclusion criteria generation.""" categories_list = "\n".join(f"- {c}" for c in PICO_CATEGORIES) return f"""You are a systematic review methodology expert specializing in PICO-based eligibility criteria design. Your task is to generate exclusion criteria for title/abstract screening. ## STRICT REQUIREMENTS ### Category Selection (REQUIRED - use ONLY these categories) Use exactly one of these categories for each criterion: {categories_list} ### Text Format (CRITICAL) The "text" field must be a SHORT LABEL (3-10 words maximum). Put detailed explanations in "description" and "rationale" fields. CORRECT: "text": "Pediatric populations (<18 years)" WRONG: "text": "Exclude studies focusing exclusively on pediatric cancer populations because the project context emphasizes adult patients with advanced cancers" ### Confidence Semantics The "confidence" field (0.0-1.0) reflects how strongly this criterion is supported by the project context. It is NOT a calibrated probability. It is NOT a screening score. ### Title/Abstract Assessability Set "title_abstract_assessable": true for criteria reliably determined from title/abstract: - Wrong species (animal vs human) - Wrong study design (review, editorial, protocol) - Wrong language - Wrong publication type (conference abstract only) Set "title_abstract_assessable": false for criteria requiring full-text: - Insufficient follow-up duration - Inadequate statistical analysis - Missing patient-level data Criteria marked false will NOT be used for automatic bulk exclusion. ## OUTPUT FORMAT (JSON) {{ "criteria": [ {{ "category": "Population", "text": "Short label (3-10 words)", "description": "Brief explanation of the criterion", "criterion_type": "exclude", "confidence": 0.85, "rationale": "Why this criterion is relevant to the project", "title_abstract_assessable": true, "research_question_links": [ {{"rq_index": 1, "relevance": "direct", "note": "How this criterion relates to RQ1"}} ] }} ] }} ### research_question_links field: - rq_index: 1-based index of research question from the provided list - relevance: "direct" | "indirect" | "contextual" - note: Brief explanation of how criterion relates to this RQ Include links for each RQ the criterion addresses. Generate 5-10 exclusion criteria most relevant to this review.""" def exclusion_generation_user_prompt( project_description: Optional[str], research_questions: Optional[list[str]], additional_notes: Optional[str], ) -> str: """Build user prompt for exclusion criteria generation.""" parts = [ "# Project Description", project_description or "(No project description provided)", "", "# Research Questions", format_questions_list(research_questions), ] if additional_notes: parts.extend(["", "# Additional Notes", additional_notes]) parts.append("\n\nPlease generate exclusion criteria for this systematic review. Return JSON.") return "\n".join(parts) # --------------------------------------------------------------------------- # Inclusion Generation # --------------------------------------------------------------------------- def inclusion_generation_system_prompt() -> str: """System prompt for inclusion criteria generation.""" categories_list = "\n".join(f"- {c}" for c in PICO_CATEGORIES) return f"""You are a systematic review methodology expert specializing in PICO-based eligibility criteria design. Your task is to generate inclusion criteria for title/abstract screening. ## STRICT REQUIREMENTS ### Category Selection (REQUIRED - use ONLY these categories) Use exactly one of these categories for each criterion: {categories_list} ### Text Format (CRITICAL) The "text" field must be a SHORT LABEL (3-10 words maximum). Put detailed explanations in "description" and "rationale" fields. CORRECT: "text": "Adults with advanced solid tumors" WRONG: "text": "Include all studies that examine adult patients diagnosed with advanced solid malignancies including stage III and IV cancers" ### Confidence Semantics The "confidence" field (0.0-1.0) reflects how strongly this criterion is supported by the project context. ### Title/Abstract Assessability Set "title_abstract_assessable": true for criteria reliably determined from title/abstract: - Target population clearly stated - Intervention type identifiable - Study design mentioned - Language of publication Set "title_abstract_assessable": false for criteria requiring full-text: - Specific outcome measurement tools - Detailed methodology requirements - Data completeness checks ## OUTPUT FORMAT (JSON) {{ "criteria": [ {{ "category": "Population", "text": "Short label (3-10 words)", "description": "Brief explanation of the criterion", "criterion_type": "include", "confidence": 0.85, "rationale": "Why this criterion is relevant to the project", "title_abstract_assessable": true, "research_question_links": [ {{"rq_index": 1, "relevance": "direct", "note": "How this criterion relates to RQ1"}} ] }} ] }} ### research_question_links field: - rq_index: 1-based index of research question from the provided list - relevance: "direct" | "indirect" | "contextual" - note: Brief explanation of how criterion relates to this RQ Generate 5-10 inclusion criteria most relevant to this review.""" def inclusion_generation_user_prompt( project_description: Optional[str], research_questions: Optional[list[str]], additional_notes: Optional[str], ) -> str: """Build user prompt for inclusion criteria generation.""" parts = [ "# Project Description", project_description or "(No project description provided)", "", "# Research Questions", format_questions_list(research_questions), ] if additional_notes: parts.extend(["", "# Additional Notes", additional_notes]) parts.append("\n\nPlease generate inclusion criteria for this systematic review. Return JSON.") return "\n".join(parts) # --------------------------------------------------------------------------- # PICO Extraction # --------------------------------------------------------------------------- def pico_extraction_system_prompt() -> str: """System prompt for PICO extraction pre-processing.""" return """You are a systematic review methodology expert specializing in PICO framework analysis. Your task is to analyze project context and extract PICO(S) elements. For each PICO element, categorize findings as: - **explicit**: Clearly stated in the project description - **implicit**: Implied but not directly stated - **ambiguous**: Unclear or needs clarification Also identify: - **gap_flags**: Important scope elements that are missing or underspecified - **contraindications**: Potential issues with the review scope ## OUTPUT FORMAT (JSON) { "pico_extraction": { "population": { "explicit": ["Adults with type 2 diabetes"], "implicit": ["Excludes pediatric patients"], "ambiguous": ["Age cutoff not specified"] }, "intervention": { "explicit": [...], "implicit": [...], "ambiguous": [...] }, "comparator": { "explicit": [...], "implicit": [...], "ambiguous": [...] }, "outcome": { "explicit": [...], "implicit": [...], "ambiguous": [...] }, "study_design": { "explicit": [...], "implicit": [...], "ambiguous": [...] } }, "gap_flags": [ { "element": "Language restriction", "severity": "medium", "note": "Not specified; may affect search yield" } ], "contraindications": [ { "issue": "Broad outcome definition", "severity": "low", "recommendation": "Consider specifying primary vs secondary outcomes" } ] } Severity levels: "low", "medium", "high" """ def pico_extraction_user_prompt( project_description: Optional[str], research_questions: Optional[list[str]], existing_criteria: Optional[list[dict]] = None, ) -> str: """Build user prompt for PICO extraction.""" parts = [ "# Project Description", project_description or "(No project description provided)", "", "# Research Questions", format_questions_list(research_questions), ] if existing_criteria: parts.append("") parts.append("# Existing Criteria") parts.append("The following inclusion/exclusion criteria are already defined:") for crit in existing_criteria: crit_type = crit.get("criterion_type", "exclusion") code = crit.get("code", "") desc = crit.get("description", "") parts.append(f"- [{crit_type.upper()}] {code}: {desc}") parts.append("") parts.append("Consider these existing criteria when extracting PICO elements.") parts.append("") parts.append("Please extract PICO elements and identify scope gaps. Return JSON.") return "\n".join(parts) # --------------------------------------------------------------------------- # Refinement (from reconciliation patterns) # --------------------------------------------------------------------------- def refinement_system_prompt() -> str: """System prompt for criteria refinement from reconciliation conflicts.""" return """You are a systematic review methodology expert analyzing screening conflicts. Analyze reconciliation conflict patterns and suggest criteria refinements. For each suggestion, provide: - action: "ADD" or "MODIFY" - category: PICO category - text: Suggested criterion or modification - rationale: Why this addresses the conflict pattern Return a JSON array of suggestion objects.""" def refinement_user_prompt( reconciliation_patterns: dict, current_criteria: list[dict], inclusion_criteria: list[dict], project_description: str = "", ) -> str: """Build user prompt for criteria refinement.""" parts = [] if project_description: parts.append(f"# Project Description\n{project_description}\n") parts.append("# Current Exclusion Criteria") for c in current_criteria: parts.append(f"- {c.get('criteria_name', '')}: {c.get('criteria_value', '')}") if inclusion_criteria: parts.append("\n# Current Inclusion Criteria") for c in inclusion_criteria: parts.append(f"- {c.get('criteria_name', '')}: {c.get('criteria_value', '')}") parts.append("\n# Reconciliation Patterns") import json parts.append(json.dumps(reconciliation_patterns, indent=2)) parts.append("\n\nPlease suggest criteria refinements to resolve these conflicts. Return JSON.") return "\n".join(parts) # --------------------------------------------------------------------------- # Conflict Refinement (from conflict papers) # --------------------------------------------------------------------------- def conflict_refinement_system_prompt() -> str: """System prompt for conflict-based criteria refinement.""" return """You are a systematic review methodology expert specializing in eligibility criteria refinement. You are analyzing conflicts between human reviewer decisions and AI screening decisions. Your task is to identify patterns in these conflicts and generate NEW or REFINED eligibility criteria that would improve screening accuracy and reduce future conflicts. CONFLICT TYPES: - "AI excluded, Human included" (False Positives): AI was too strict - "AI included, Human excluded" (False Negatives): AI was too lenient ANALYSIS APPROACH: 1. Identify common themes across conflicts (population, intervention, study design, etc.) 2. Look for patterns in human exclusion reasons vs AI reasoning 3. Suggest criteria that would align AI decisions with human judgment 4. Consider both inclusion AND exclusion criteria refinements OUTPUT FORMAT (JSON): { "criteria": [ { "category": "PICO category", "text": "Specific criterion text (3-10 words)", "description": "Explanation of why this criterion addresses the conflicts", "confidence": 0.85, "rationale": "Which conflict pattern this addresses and how", "criterion_type": "include" or "exclude", "title_abstract_assessable": true, "research_question_links": [ {"rq_index": 1, "relevance": "direct", "note": "Addresses RQ scope"} ] } ] } GUIDELINES: - Focus on actionable criteria that can be assessed from title/abstract - Be specific - avoid vague criteria - Prioritize criteria that address the most common conflict patterns - Generate 3-8 criteria based on conflict patterns identified""" def conflict_refinement_user_prompt( conflict_papers: list[dict], active_criteria: Optional[list[ExclusionCriterion]] = None, project_description: Optional[str] = None, research_questions: Optional[list[str]] = None, ) -> str: """Build user prompt with conflict paper details.""" parts = [] if project_description: parts.append(f"# Project Description\n{project_description}\n") if research_questions: parts.append("# Research Questions") parts.append(format_questions_list(research_questions)) parts.append("") if active_criteria: parts.append("# Current Eligibility Criteria") parts.append(format_criteria_list(active_criteria)) parts.append("") # Conflict summary fp_count = sum( 1 for p in conflict_papers if p.get("ai_decision") == "exclude" and p.get("human_decision") == "include" ) fn_count = sum( 1 for p in conflict_papers if p.get("ai_decision") == "include" and p.get("human_decision") == "exclude" ) parts.append("# Conflict Summary") parts.append(f"- Total conflicts: {len(conflict_papers)}") parts.append(f"- AI excluded, Human included (false positives): {fp_count}") parts.append(f"- AI included, Human excluded (false negatives): {fn_count}") parts.append("") # Individual papers parts.append("# Conflict Papers\n") for i, paper in enumerate(conflict_papers, 1): parts.append(f"## Paper {i}") parts.append(f"**Title:** {paper.get('title', 'N/A')}") parts.append(f"**Abstract:** {(paper.get('abstract') or '')[:500]}...") parts.append(f"**AI Decision:** {paper.get('ai_decision')} (Score: {paper.get('ai_score', 'N/A')})") parts.append(f"**Human Decision:** {paper.get('human_decision')}") if paper.get("human_exclude_cluster"): parts.append(f"**Human Exclusion Reason:** {paper.get('human_exclude_cluster')}") if paper.get("human_exclude_criteria"): parts.append(f"**Human Exclusion Criteria:** {paper.get('human_exclude_criteria')}") if paper.get("human_notes"): parts.append(f"**Human Notes:** {paper.get('human_notes')}") if paper.get("ai_reasoning"): parts.append(f"**AI Reasoning:** {(paper.get('ai_reasoning') or '')[:300]}...") if paper.get("exclude_clusters"): parts.append(f"**AI Exclusion Themes:** {paper.get('exclude_clusters')}") parts.append("") parts.append("Please analyze these conflicts and generate eligibility criteria. Return JSON.") return "\n".join(parts) # --------------------------------------------------------------------------- # Consolidation / Deduplication # --------------------------------------------------------------------------- def consolidation_system_prompt() -> str: """System prompt for duplicate detection and consolidation.""" return """You are a systematic review methodology expert specialising in eligibility criteria optimisation. Your task is to identify: 1. Exact duplicate criteria 2. Near-duplicate criteria 3. Overly granular criteria that can be safely consolidated Rules: - Only consolidate criteria within the SAME category. - Never merge include and exclude criteria. - Do not change semantic meaning during merge. - Criterion labels must remain 3-10 words. - Be conservative. If uncertain, do not propose a merge. - Do not output markdown. - Output JSON only.""" def consolidation_user_prompt( criteria: list[ExclusionCriterion], project_description: Optional[str] = None, research_questions: Optional[list[str]] = None, ) -> str: """Build user prompt for duplicate detection.""" parts = [] if project_description: parts.append(f"PROJECT DESCRIPTION:\n{project_description}") if research_questions: numbered = "\n".join(f"{i + 1}. {q}" for i, q in enumerate(research_questions)) parts.append(f"RESEARCH QUESTIONS:\n{numbered}") parts.append("\nACTIVE CRITERIA:\n") for c in criteria: parts.append(f""" ID: {c.id} TYPE: {c.criterion_type.value} CATEGORY: {c.category} CRITERION: {c.text} DESCRIPTION: {c.description or "N/A"} AI_CONFIDENCE: {c.ai_confidence or "N/A"} AI_RATIONALE: {c.ai_rationale or "N/A"} """) parts.append(""" TASK: Identify duplicate and near-duplicate criteria. Propose safe consolidation opportunities. Flag any unsafe merge risks.""") return "\n".join(parts) # --------------------------------------------------------------------------- # JSON Schemas for Structured Outputs # --------------------------------------------------------------------------- CRITERIA_SCHEMA = { "type": "json_schema", "json_schema": { "name": "criteria_response", "strict": True, "schema": { "type": "object", "additionalProperties": False, "properties": { "criteria": { "type": "array", "description": "List of generated eligibility criteria", "items": { "type": "object", "additionalProperties": False, "properties": { "category": { "type": "string", "enum": PICO_CATEGORIES, "description": "PICO(S) category for grouping", }, "text": { "type": "string", "description": "Short label (3-10 words), no markdown", }, "description": { "type": "string", "description": "Brief clarification of the criterion", }, "criterion_type": { "type": "string", "enum": ["include", "exclude"], "description": "Whether this is an inclusion or exclusion criterion", }, "confidence": { "type": "number", "description": "How strongly supported by project context (0-1)", }, "rationale": { "type": "string", "description": "Why this criterion is relevant to the project", }, "title_abstract_assessable": { "type": "boolean", "description": "True if assessable from title/abstract alone", }, "research_question_links": { "type": "array", "description": "Links to research questions by index", "items": { "type": "object", "additionalProperties": False, "properties": { "rq_index": { "type": "integer", "description": "1-based index of research question", }, "relevance": { "type": "string", "enum": RQ_RELEVANCE_TYPES, "description": "How criterion relates to RQ", }, "note": { "type": "string", "description": "Brief explanation of the link", }, }, "required": ["rq_index", "relevance", "note"], }, }, }, "required": [ "category", "text", "description", "criterion_type", "confidence", "rationale", "title_abstract_assessable", "research_question_links", ], }, } }, "required": ["criteria"], }, }, } PICO_EXTRACTION_SCHEMA = { "type": "json_schema", "json_schema": { "name": "pico_extraction", "strict": True, "schema": { "type": "object", "additionalProperties": False, "properties": { "pico_extraction": { "type": "object", "additionalProperties": False, "description": "Extracted PICO(S) elements from project context", "properties": { "population": { "type": "object", "additionalProperties": False, "properties": { "explicit": {"type": "array", "items": {"type": "string"}}, "implicit": {"type": "array", "items": {"type": "string"}}, "ambiguous": {"type": "array", "items": {"type": "string"}}, }, "required": ["explicit", "implicit", "ambiguous"], }, "intervention": { "type": "object", "additionalProperties": False, "properties": { "explicit": {"type": "array", "items": {"type": "string"}}, "implicit": {"type": "array", "items": {"type": "string"}}, "ambiguous": {"type": "array", "items": {"type": "string"}}, }, "required": ["explicit", "implicit", "ambiguous"], }, "comparator": { "type": "object", "additionalProperties": False, "properties": { "explicit": {"type": "array", "items": {"type": "string"}}, "implicit": {"type": "array", "items": {"type": "string"}}, "ambiguous": {"type": "array", "items": {"type": "string"}}, }, "required": ["explicit", "implicit", "ambiguous"], }, "outcome": { "type": "object", "additionalProperties": False, "properties": { "explicit": {"type": "array", "items": {"type": "string"}}, "implicit": {"type": "array", "items": {"type": "string"}}, "ambiguous": {"type": "array", "items": {"type": "string"}}, }, "required": ["explicit", "implicit", "ambiguous"], }, "study_design": { "type": "object", "additionalProperties": False, "properties": { "explicit": {"type": "array", "items": {"type": "string"}}, "implicit": {"type": "array", "items": {"type": "string"}}, "ambiguous": {"type": "array", "items": {"type": "string"}}, }, "required": ["explicit", "implicit", "ambiguous"], }, }, "required": ["population", "intervention", "comparator", "outcome", "study_design"], }, "gap_flags": { "type": "array", "description": "Scope gaps or underspecified elements", "items": { "type": "object", "additionalProperties": False, "properties": { "element": {"type": "string"}, "severity": {"type": "string", "enum": ["low", "medium", "high"]}, "note": {"type": "string"}, }, "required": ["element", "severity", "note"], }, }, "contraindications": { "type": "array", "description": "Potential issues with the review scope", "items": { "type": "object", "additionalProperties": False, "properties": { "issue": {"type": "string"}, "severity": {"type": "string", "enum": ["low", "medium", "high"]}, "recommendation": {"type": "string"}, }, "required": ["issue", "severity", "recommendation"], }, }, }, "required": ["pico_extraction", "gap_flags", "contraindications"], }, }, } # --------------------------------------------------------------------------- # Context Refinement (project description + research questions) # --------------------------------------------------------------------------- def context_refinement_system_prompt() -> str: """System prompt for refining project description and research questions.""" return """You are an expert systematic review methodologist specializing in refining research protocols for AI-assisted screening. Your task is to improve a project description and research questions so that they produce better, more precise AI screening results. ## WHAT TO IMPROVE 1. **Project Description**: Make it more specific and structured. Add explicit scope boundaries (population, setting, timeframe). Remove ambiguity. Ensure the description clearly communicates what is IN scope and what is OUT of scope. 2. **Research Questions**: Make them more precise, measurable, and answerable from title/abstract screening. Break vague questions into specific sub-questions. Ensure PICO elements are embedded. ## WHAT NOT TO CHANGE - Do not change the fundamental topic or direction of the review - Do not add requirements the user hasn't implied - Preserve the user's intent and domain focus ## OUTPUT FORMAT (JSON) { "refined_description": "The improved project description", "refined_research_questions": ["Improved RQ1", "Improved RQ2", ...], "explanation": "Brief explanation of what was changed and why (2-3 sentences)" } Return JSON only. No markdown fences.""" def context_refinement_user_prompt( description: str, research_questions: list[str], ) -> str: """Build user prompt for context refinement.""" parts = [ "# Current Project Description", description or "(No description provided)", "", "# Current Research Questions", format_questions_list(research_questions) if research_questions else "(No research questions defined)", "", "Please refine the description and research questions for better AI screening accuracy. Return JSON.", ] return "\n".join(parts) CONSOLIDATION_SCHEMA = { "type": "json_schema", "json_schema": { "name": "criteria_consolidation", "strict": True, "schema": { "type": "object", "additionalProperties": False, "properties": { "duplicate_groups": { "type": "array", "description": "Exact duplicates or near-identical criteria", "items": { "type": "object", "additionalProperties": False, "properties": { "group_type": { "type": "string", "enum": ["exact_duplicate", "near_duplicate"], }, "category": {"type": "string"}, "criterion_ids": { "type": "array", "items": {"type": "integer"}, }, "recommended_primary_id": {"type": "integer"}, "merge_rationale": {"type": "string"}, "ai_confidence": {"type": "number"}, }, "required": [ "group_type", "category", "criterion_ids", "recommended_primary_id", "merge_rationale", "ai_confidence", ], }, }, "consolidation_proposals": { "type": "array", "description": "Granular criteria that should be merged", "items": { "type": "object", "additionalProperties": False, "properties": { "category": {"type": "string"}, "criterion_ids": { "type": "array", "items": {"type": "integer"}, }, "proposed_merged_criterion": {"type": "string"}, "proposed_description": {"type": "string"}, "proposed_type": { "type": "string", "enum": ["include", "exclude"], }, "merge_rationale": {"type": "string"}, "ai_confidence": {"type": "number"}, }, "required": [ "category", "criterion_ids", "proposed_merged_criterion", "proposed_description", "proposed_type", "merge_rationale", "ai_confidence", ], }, }, "warnings": { "type": "array", "items": {"type": "string"}, }, }, "required": ["duplicate_groups", "consolidation_proposals", "warnings"], }, }, } # --------------------------------------------------------------------------- # Question analysis (single research question search-readiness check) # --------------------------------------------------------------------------- QUESTION_ANALYSIS_SYSTEM_PROMPT = """You are an expert in systematic literature reviews helping researchers formulate effective search questions. Analyze the research question and identify any missing or unclear elements that could affect search results. ANALYSIS FRAMEWORK (PICOS): - Population: Who is being studied? - Intervention: What is being examined? - Comparison: Is there a comparison group? (if relevant) - Outcome: What results are being measured? - Study Design: What type of studies are relevant? OUTPUT FORMAT: Return valid JSON: { "status": "ready" | "could_improve", "missing_elements": ["list of missing or unclear elements, if any"], "suggestion": "Brief suggestion for improvement, or confirmation the question is well-formed" } RULES: - Be concise - one sentence per point - If the question is well-formed, set status to "ready" and confirm its strengths - Only flag genuine considerations that would impact search quality - Focus on what helps find relevant literature, not academic perfection """ def question_analysis_system_prompt() -> str: """System prompt for /criteria/analyze-question.""" return QUESTION_ANALYSIS_SYSTEM_PROMPT def question_analysis_user_prompt(research_question: str) -> str: """Build the user message for analysing a single research question.""" return ( "Analyze this research question for a systematic literature review:\n\n" f'"{research_question}"\n\n' "Identify any areas for consideration that could affect search results. " "Be honest - if it's already well-formed, say so." )