diff --git a/CLAUDE.md b/CLAUDE.md index 5caf510b..9f3bc3a4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -62,9 +62,11 @@ When adding new pages/components: use "ResolutionFlow" branding, ice-cyan gradie ### What's In Progress - ConnectWise PSA Integration (ticket linking, note posting, member mapping, status updates) +- Knowledge Flywheel (Phase 3): AI analysis of FlowPilot sessions → flow proposals, review queue, analytics dashboard ### Recently Completed +- FlowPilot Phase 2: PSA integration, escalation handoff, session pause/resume, mid-session ticket linking - Step Library Foundation - AI chat session conclusion: outcome tracking, AI-generated ticket summaries, resume flow - Survey completion: email-to-self, thank-you page, admin read/unread/archive/delete management @@ -107,12 +109,17 @@ patherly/ │ ├── app/ │ │ ├── main.py # FastAPI entry point │ │ ├── api/endpoints/ # Route handlers (auth, trees, sessions, admin, steps, survey, copilot, assistant_chat, psa_connections) -│ │ ├── api/deps.py # Auth dependencies +│ │ │ ├── flow_proposals.py # Knowledge Flywheel review queue CRUD +│ │ │ └── flowpilot_analytics.py # FlowPilot dashboard metrics +│ │ ├── api/deps.py # Auth dependencies (includes require_team_admin) │ │ ├── api/router.py # Route registration │ │ ├── core/ # config, database, permissions, security, audit, rate_limit -│ │ ├── models/ # SQLAlchemy models +│ │ ├── models/ # SQLAlchemy models (includes FlowProposal) │ │ ├── schemas/ # Pydantic schemas -│ │ └── services/psa/ # PSA provider abstraction (base, connectwise/, cache, encryption, registry, types) +│ │ ├── services/psa/ # PSA provider abstraction (base, connectwise/, cache, encryption, registry, types) +│ │ ├── services/knowledge_flywheel.py # AI session analysis → flow proposals +│ │ ├── services/knowledge_flywheel_scheduler.py # APScheduler job for batch analysis +│ │ └── services/knowledge_gap_service.py # Weak options & escalation signal detection │ ├── alembic/ # Database migrations (001-029+) │ ├── scripts/ # seed_data.py, seed_trees.py │ └── tests/ # pytest integration tests @@ -313,6 +320,16 @@ gh run view --json jobs --jq '.jobs[] | {name: .name, conclusion: .conclusi **66. Dev environment runs on devserver01 (192.168.0.9), not localhost:** Code-server runs in Docker on a LAN server. Frontend/backend are accessed via `192.168.0.9`, not `localhost`. CORS must include `http://192.168.0.9:5173` in `CORS_ORIGINS` and `FRONTEND_URL`. Frontend `.env` must set `VITE_API_URL=http://192.168.0.9:8000`. See [DEV-ENV.md](DEV-ENV.md) for full setup, Docker config, networking, and known issues. +**67. Tree editor route is `/trees/new`:** NOT `/editor/new`. Check `router.tsx` line 156 for the canonical path. Use `getTreeEditorPath()` from `@/lib/routing` when navigating programmatically. + +**68. APScheduler jobs need `max_instances=1`:** Without it, overlapping scheduler runs can process the same records twice (TOCTOU race). Always set `max_instances=1` on interval jobs in `main.py`. + +**69. PostgreSQL `func.sum(case(...))` returns `Decimal` via asyncpg:** Cast to `int()` before storing in Pydantic `dict[str, Any]` fields, or JSON serialization may produce unexpected types. + +**70. Toast library uses `toast.warning()` not `toast.warn()`:** Import from `@/lib/toast`. Methods: `success`, `error`, `warning`, `info`. See `frontend/src/lib/toast.ts`. + +**71. Enhancement/branch_addition proposals cannot be directly approved:** Backend returns 400 — they require `modified_flow_data` via "Edit & Publish" flow. Only `new_flow` proposals support direct approve. + --- ## RBAC & Permissions diff --git a/backend/alembic/versions/3266dd9d8111_add_ai_session_id_to_script_generations.py b/backend/alembic/versions/3266dd9d8111_add_ai_session_id_to_script_generations.py new file mode 100644 index 00000000..9d60eda5 --- /dev/null +++ b/backend/alembic/versions/3266dd9d8111_add_ai_session_id_to_script_generations.py @@ -0,0 +1,40 @@ +"""add ai_session_id to script_generations + +Revision ID: 3266dd9d8111 +Revises: a0b871cb0c5e +Create Date: 2026-03-19 03:52:09.457961 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision: str = '3266dd9d8111' +down_revision: Union[str, None] = 'a0b871cb0c5e' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column('script_generations', sa.Column( + 'ai_session_id', sa.UUID(), nullable=True, + comment='FlowPilot AI session that triggered this generation', + )) + op.create_index( + op.f('ix_script_generations_ai_session_id'), + 'script_generations', ['ai_session_id'], unique=False, + ) + op.create_foreign_key( + 'fk_script_generations_ai_session_id', + 'script_generations', 'ai_sessions', + ['ai_session_id'], ['id'], + ondelete='SET NULL', + ) + + +def downgrade() -> None: + op.drop_constraint('fk_script_generations_ai_session_id', 'script_generations', type_='foreignkey') + op.drop_index(op.f('ix_script_generations_ai_session_id'), table_name='script_generations') + op.drop_column('script_generations', 'ai_session_id') diff --git a/backend/alembic/versions/47c3b4f42e88_add_flow_proposals_table.py b/backend/alembic/versions/47c3b4f42e88_add_flow_proposals_table.py new file mode 100644 index 00000000..5ce48d9f --- /dev/null +++ b/backend/alembic/versions/47c3b4f42e88_add_flow_proposals_table.py @@ -0,0 +1,64 @@ +"""add flow_proposals table + +Revision ID: 47c3b4f42e88 +Revises: cc3201489b72 +Create Date: 2026-03-19 03:11:33.663729 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = '47c3b4f42e88' +down_revision: Union[str, None] = 'cc3201489b72' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table('flow_proposals', + sa.Column('id', sa.UUID(), nullable=False), + sa.Column('account_id', sa.UUID(), nullable=False), + sa.Column('team_id', sa.UUID(), nullable=True), + sa.Column('source_session_id', sa.UUID(), nullable=False), + sa.Column('proposal_type', sa.String(length=30), nullable=False), + sa.Column('target_flow_id', sa.UUID(), nullable=True, comment='For enhancements: which existing flow to modify'), + sa.Column('title', sa.String(length=255), nullable=False, comment='Human-readable title for the proposed flow'), + sa.Column('description', sa.Text(), nullable=True, comment='AI-generated description of what this flow covers'), + sa.Column('proposed_flow_data', postgresql.JSONB(astext_type=sa.Text()), nullable=False, comment='Complete flow/tree_structure definition (nodes, edges, conditions)'), + sa.Column('proposed_diff', postgresql.JSONB(astext_type=sa.Text()), nullable=True, comment='For enhancements: what changed vs existing flow'), + sa.Column('confidence_score', sa.Float(), nullable=False, comment='How confident the system is in this proposal (0.0-1.0)'), + sa.Column('supporting_session_count', sa.Integer(), nullable=False, comment='Number of sessions with similar resolution paths'), + sa.Column('supporting_session_ids', postgresql.JSONB(astext_type=sa.Text()), nullable=False, comment='Array of session IDs that support this proposal'), + sa.Column('problem_domain', sa.String(length=100), nullable=True), + sa.Column('status', sa.String(length=30), nullable=False), + sa.Column('reviewed_by', sa.UUID(), nullable=True), + sa.Column('reviewer_notes', sa.Text(), nullable=True), + sa.Column('published_flow_id', sa.UUID(), nullable=True, comment='The flow that was created/updated when this proposal was approved'), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('reviewed_at', sa.DateTime(timezone=True), nullable=True), + sa.CheckConstraint("proposal_type IN ('new_flow', 'enhancement', 'branch_addition', 'auto_reinforced')", name='ck_flow_proposals_type'), + sa.CheckConstraint("status IN ('pending', 'approved', 'modified', 'rejected', 'dismissed', 'auto_reinforced')", name='ck_flow_proposals_status'), + sa.ForeignKeyConstraint(['account_id'], ['accounts.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['published_flow_id'], ['trees.id'], ondelete='SET NULL'), + sa.ForeignKeyConstraint(['reviewed_by'], ['users.id'], ondelete='SET NULL'), + sa.ForeignKeyConstraint(['source_session_id'], ['ai_sessions.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['target_flow_id'], ['trees.id'], ondelete='SET NULL'), + sa.ForeignKeyConstraint(['team_id'], ['teams.id'], ondelete='SET NULL'), + sa.PrimaryKeyConstraint('id'), + ) + op.create_index(op.f('ix_flow_proposals_account_id'), 'flow_proposals', ['account_id'], unique=False) + op.create_index(op.f('ix_flow_proposals_source_session_id'), 'flow_proposals', ['source_session_id'], unique=False) + op.create_index(op.f('ix_flow_proposals_status'), 'flow_proposals', ['status'], unique=False) + op.create_index(op.f('ix_flow_proposals_team_id'), 'flow_proposals', ['team_id'], unique=False) + + +def downgrade() -> None: + op.drop_index(op.f('ix_flow_proposals_team_id'), table_name='flow_proposals') + op.drop_index(op.f('ix_flow_proposals_status'), table_name='flow_proposals') + op.drop_index(op.f('ix_flow_proposals_source_session_id'), table_name='flow_proposals') + op.drop_index(op.f('ix_flow_proposals_account_id'), table_name='flow_proposals') + op.drop_table('flow_proposals') diff --git a/backend/alembic/versions/a0b871cb0c5e_add_analysis_status_to_ai_sessions.py b/backend/alembic/versions/a0b871cb0c5e_add_analysis_status_to_ai_sessions.py new file mode 100644 index 00000000..4cbd7ad9 --- /dev/null +++ b/backend/alembic/versions/a0b871cb0c5e_add_analysis_status_to_ai_sessions.py @@ -0,0 +1,28 @@ +"""add analysis_status to ai_sessions + +Revision ID: a0b871cb0c5e +Revises: 47c3b4f42e88 +Create Date: 2026-03-19 03:26:26.965134 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision: str = 'a0b871cb0c5e' +down_revision: Union[str, None] = '47c3b4f42e88' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column('ai_sessions', sa.Column( + 'analysis_status', sa.String(length=20), nullable=True, + comment='Knowledge Flywheel status: null (N/A), pending, completed, failed', + )) + + +def downgrade() -> None: + op.drop_column('ai_sessions', 'analysis_status') diff --git a/backend/app/api/deps.py b/backend/app/api/deps.py index f17858de..28536d68 100644 --- a/backend/app/api/deps.py +++ b/backend/app/api/deps.py @@ -145,6 +145,22 @@ async def require_engineer_or_admin( ) +async def require_team_admin( + current_user: Annotated[User, Depends(get_current_active_user)] +) -> User: + """Require team admin, account owner, or super admin role.""" + if current_user.is_super_admin: + return current_user + if current_user.is_team_admin: + return current_user + if current_user.account_role == "owner": + return current_user + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Team admin access required" + ) + + async def require_account_owner( current_user: Annotated[User, Depends(get_current_active_user)] ) -> User: diff --git a/backend/app/api/endpoints/flow_proposals.py b/backend/app/api/endpoints/flow_proposals.py new file mode 100644 index 00000000..6a41b12f --- /dev/null +++ b/backend/app/api/endpoints/flow_proposals.py @@ -0,0 +1,306 @@ +"""Review Queue API — CRUD for flow proposals. + +Endpoints for listing, reviewing, and managing Knowledge Flywheel proposals: + GET /flow-proposals — List proposals (filterable) + GET /flow-proposals/stats — Dashboard stats + GET /flow-proposals/{id} — Get proposal detail + POST /flow-proposals/{id}/review — Approve, reject, modify, or dismiss +""" +import logging +import uuid +from datetime import datetime, timezone, timedelta +from typing import Annotated, Optional +from uuid import UUID + +from fastapi import APIRouter, Depends, HTTPException, Query, Request, status +from sqlalchemy import select, func, case +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import selectinload + +from app.core.rate_limit import limiter +from app.api.deps import get_current_active_user, get_db, require_engineer_or_admin, require_team_admin +from app.models.user import User +from app.models.tree import Tree +from app.models.flow_proposal import FlowProposal +from app.schemas.flow_proposal import ( + FlowProposalSummary, + FlowProposalDetail, + FlowProposalStats, + ReviewProposalRequest, +) + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/flow-proposals", tags=["flow-proposals"]) + + +# ── List proposals ── + +@router.get("", response_model=list[FlowProposalSummary]) +@limiter.limit("30/minute") +async def list_proposals( + request: Request, + current_user: Annotated[User, Depends(get_current_active_user)], + db: Annotated[AsyncSession, Depends(get_db)], + _: None = Depends(require_engineer_or_admin), + proposal_status: Optional[str] = Query(None, alias="status"), + proposal_type: Optional[str] = Query(None, alias="type"), + domain: Optional[str] = Query(None), + sort_by: str = Query("newest", pattern="^(newest|confidence|sessions)$"), + skip: int = Query(0, ge=0), + limit: int = Query(20, ge=1, le=100), +): + """List flow proposals for the current user's account.""" + if not current_user.account_id: + return [] + + query = ( + select(FlowProposal) + .where(FlowProposal.account_id == current_user.account_id) + ) + + if proposal_status: + query = query.where(FlowProposal.status == proposal_status) + if proposal_type: + query = query.where(FlowProposal.proposal_type == proposal_type) + if domain: + query = query.where(FlowProposal.problem_domain == domain) + + # Sorting + if sort_by == "confidence": + query = query.order_by(FlowProposal.confidence_score.desc()) + elif sort_by == "sessions": + query = query.order_by(FlowProposal.supporting_session_count.desc()) + else: # newest + query = query.order_by(FlowProposal.created_at.desc()) + + query = query.offset(skip).limit(limit) + + result = await db.execute(query) + proposals = result.scalars().all() + + return [FlowProposalSummary.model_validate(p) for p in proposals] + + +# ── Stats ── + +@router.get("/stats", response_model=FlowProposalStats) +@limiter.limit("30/minute") +async def get_proposal_stats( + request: Request, + current_user: Annotated[User, Depends(get_current_active_user)], + db: Annotated[AsyncSession, Depends(get_db)], + _: None = Depends(require_engineer_or_admin), +): + """Get review queue dashboard stats.""" + if not current_user.account_id: + return FlowProposalStats( + pending_count=0, approved_this_week=0, rejected_this_week=0, + auto_reinforced_this_week=0, top_domains=[], + ) + + week_ago = datetime.now(timezone.utc) - timedelta(days=7) + + # Count pending + pending_result = await db.execute( + select(func.count(FlowProposal.id)) + .where( + FlowProposal.account_id == current_user.account_id, + FlowProposal.status == "pending", + ) + ) + pending_count = pending_result.scalar() or 0 + + # Reviewed this week (approved/rejected/modified use reviewed_at) + reviewed_result = await db.execute( + select( + FlowProposal.status, + func.count(FlowProposal.id), + ) + .where( + FlowProposal.account_id == current_user.account_id, + FlowProposal.reviewed_at >= week_ago, + FlowProposal.status.in_(["approved", "modified", "rejected", "dismissed"]), + ) + .group_by(FlowProposal.status) + ) + reviewed_counts = {row[0]: row[1] for row in reviewed_result.all()} + + # Auto-reinforced this week (use created_at since they have no review) + reinforced_result = await db.execute( + select(func.count(FlowProposal.id)) + .where( + FlowProposal.account_id == current_user.account_id, + FlowProposal.created_at >= week_ago, + FlowProposal.status == "auto_reinforced", + ) + ) + auto_reinforced_count = reinforced_result.scalar() or 0 + + # Top domains + domain_result = await db.execute( + select( + FlowProposal.problem_domain, + func.count(FlowProposal.id).label("count"), + ) + .where( + FlowProposal.account_id == current_user.account_id, + FlowProposal.status == "pending", + FlowProposal.problem_domain.isnot(None), + ) + .group_by(FlowProposal.problem_domain) + .order_by(func.count(FlowProposal.id).desc()) + .limit(5) + ) + top_domains = [{"domain": row[0], "count": row[1]} for row in domain_result.all()] + + return FlowProposalStats( + pending_count=pending_count, + approved_this_week=reviewed_counts.get("approved", 0) + reviewed_counts.get("modified", 0), + rejected_this_week=reviewed_counts.get("rejected", 0), + auto_reinforced_this_week=auto_reinforced_count, + top_domains=top_domains, + ) + + +# ── Detail ── + +@router.get("/{proposal_id}", response_model=FlowProposalDetail) +@limiter.limit("30/minute") +async def get_proposal( + request: Request, + proposal_id: UUID, + current_user: Annotated[User, Depends(get_current_active_user)], + db: Annotated[AsyncSession, Depends(get_db)], + _: None = Depends(require_engineer_or_admin), +): + """Get full proposal detail.""" + result = await db.execute( + select(FlowProposal).where( + FlowProposal.id == proposal_id, + FlowProposal.account_id == current_user.account_id, + ) + ) + proposal = result.scalar_one_or_none() + if not proposal: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Proposal not found") + + return FlowProposalDetail.model_validate(proposal) + + +# ── Review ── + +@router.post("/{proposal_id}/review", response_model=FlowProposalDetail) +@limiter.limit("10/minute") +async def review_proposal( + request: Request, + proposal_id: UUID, + data: ReviewProposalRequest, + current_user: Annotated[User, Depends(get_current_active_user)], + db: Annotated[AsyncSession, Depends(get_db)], + _: None = Depends(require_team_admin), +): + """Review a proposal: approve, reject, modify, or dismiss.""" + result = await db.execute( + select(FlowProposal).where( + FlowProposal.id == proposal_id, + FlowProposal.account_id == current_user.account_id, + ) + ) + proposal = result.scalar_one_or_none() + if not proposal: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Proposal not found") + + if proposal.status not in ("pending", "dismissed"): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Cannot review proposal in status: {proposal.status}", + ) + + proposal.reviewed_by = current_user.id + proposal.reviewed_at = datetime.now(timezone.utc) + proposal.reviewer_notes = data.reviewer_notes + + if data.action == "approve": + if proposal.proposal_type == "new_flow": + flow_data = proposal.proposed_flow_data + new_tree = await _create_tree_from_proposal(proposal, flow_data, current_user, db) + proposal.status = "approved" + proposal.published_flow_id = new_tree.id + elif proposal.proposal_type in ("enhancement", "branch_addition"): + # Enhancement proposals contain diffs, not complete tree structures. + # Direct approval requires modified_flow_data with the complete merged structure. + # Redirect reviewers to use "Edit & Publish" for enhancements. + if not data.modified_flow_data: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Enhancement proposals require 'Edit & Publish' to merge changes into the existing flow. Use the modify action with modified_flow_data.", + ) + new_tree = await _create_tree_from_proposal(proposal, data.modified_flow_data, current_user, db) + proposal.status = "approved" + proposal.published_flow_id = new_tree.id + else: + # auto_reinforced shouldn't reach here, but handle gracefully + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Cannot approve proposal of type: {proposal.proposal_type}", + ) + + elif data.action == "modify": + if not data.modified_flow_data: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="modified_flow_data is required for modify action", + ) + new_tree = await _create_tree_from_proposal(proposal, data.modified_flow_data, current_user, db) + proposal.status = "modified" + proposal.published_flow_id = new_tree.id + + elif data.action == "reject": + proposal.status = "rejected" + + elif data.action == "dismiss": + proposal.status = "dismissed" + + await db.commit() + + return FlowProposalDetail.model_validate(proposal) + + +async def _create_tree_from_proposal( + proposal: FlowProposal, + flow_data: dict, + user: User, + db: AsyncSession, +) -> Tree: + """Create a new Tree from proposal flow data.""" + tree_structure = flow_data.get("tree_structure", flow_data) + match_keywords = flow_data.get("match_keywords", []) + + if not tree_structure or not isinstance(tree_structure, dict) or not tree_structure.get("id"): + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail="Proposal has no valid tree structure. Use 'Edit & Publish' to build the flow manually.", + ) + + new_tree = Tree( + id=uuid.uuid4(), + name=proposal.title, + description=proposal.description, + tree_type="troubleshooting", + tree_structure=tree_structure, + author_id=user.id, + account_id=proposal.account_id, + team_id=proposal.team_id, + origin="ai_generated" if proposal.proposal_type == "new_flow" else "ai_enhanced", + source_session_id=proposal.source_session_id, + match_keywords=match_keywords, + ) + db.add(new_tree) + await db.flush() + + logger.info( + "Created tree %s from proposal %s (%s)", + new_tree.id, proposal.id, proposal.proposal_type, + ) + return new_tree diff --git a/backend/app/api/endpoints/flowpilot_analytics.py b/backend/app/api/endpoints/flowpilot_analytics.py new file mode 100644 index 00000000..20b095cb --- /dev/null +++ b/backend/app/api/endpoints/flowpilot_analytics.py @@ -0,0 +1,358 @@ +"""FlowPilot Analytics API — MTTR, resolution rates, knowledge coverage. + +Endpoints: + GET /analytics/flowpilot?period=30d — Main dashboard data + GET /analytics/flowpilot/knowledge-gaps — Knowledge gap report +""" +import logging +from datetime import datetime, timezone, timedelta +from typing import Annotated, Optional + +from fastapi import APIRouter, Depends, HTTPException, Query, Request, status +from sqlalchemy import select, func, case, cast, Date, extract +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.rate_limit import limiter +from app.api.deps import get_current_active_user, get_db, require_team_admin +from app.models.user import User +from app.models.tree import Tree +from app.models.ai_session import AISession +from app.models.flow_proposal import FlowProposal +from app.models.psa_post_log import PsaPostLog +from app.schemas.flowpilot_analytics import ( + FlowPilotDashboard, + MTTRDataPoint, + DomainBreakdown, + ConfidenceBreakdown, + KnowledgeCoverage, + DomainCoverage, + PsaMetrics, +) +from app.services.knowledge_gap_service import get_knowledge_gaps, KnowledgeGapReport + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/analytics/flowpilot", tags=["flowpilot-analytics"]) + + +def _get_period_start(period: str) -> datetime: + days = {"7d": 7, "30d": 30, "90d": 90}.get(period, 30) + return datetime.now(timezone.utc) - timedelta(days=days) + + +@router.get("", response_model=FlowPilotDashboard) +@limiter.limit("15/minute") +async def get_dashboard( + request: Request, + current_user: Annotated[User, Depends(get_current_active_user)], + db: Annotated[AsyncSession, Depends(get_db)], + _: None = Depends(require_team_admin), + period: str = Query("30d", pattern="^(7d|30d|90d)$"), +): + """Get FlowPilot analytics dashboard data.""" + if not current_user.account_id: + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="No account") + + account_id = current_user.account_id + period_start = _get_period_start(period) + + # ── Session counts ── + counts_result = await db.execute( + select( + func.count(AISession.id).label("total"), + func.sum(case((AISession.status == "resolved", 1), else_=0)).label("resolved"), + func.sum(case((AISession.status.in_(["escalated", "requesting_escalation"]), 1), else_=0)).label("escalated"), + func.sum(case((AISession.status == "abandoned", 1), else_=0)).label("abandoned"), + func.avg(case((AISession.status == "resolved", AISession.step_count), else_=None)).label("avg_steps"), + func.avg(AISession.session_rating).label("avg_rating"), + ) + .where( + AISession.account_id == account_id, + AISession.created_at >= period_start, + ) + ) + row = counts_result.one() + total = row.total or 0 + resolved = row.resolved or 0 + escalated = row.escalated or 0 + abandoned = row.abandoned or 0 + avg_steps = float(row.avg_steps or 0) + avg_rating = float(row.avg_rating) if row.avg_rating else None + resolution_rate = (resolved / total * 100) if total > 0 else 0.0 + + # ── MTTR ── + mttr_result = await db.execute( + select( + func.avg( + extract("epoch", AISession.resolved_at - AISession.created_at) / 60 + ).label("avg_mttr"), + ) + .where( + AISession.account_id == account_id, + AISession.created_at >= period_start, + AISession.status == "resolved", + AISession.resolved_at.isnot(None), + ) + ) + mttr_row = mttr_result.one() + mttr_minutes = float(mttr_row.avg_mttr) if mttr_row.avg_mttr else None + + # ── Average duration ── + duration_result = await db.execute( + select( + func.avg( + extract("epoch", AISession.resolved_at - AISession.created_at) / 60 + ).label("avg_duration"), + ) + .where( + AISession.account_id == account_id, + AISession.created_at >= period_start, + AISession.resolved_at.isnot(None), + ) + ) + dur_row = duration_result.one() + avg_duration = float(dur_row.avg_duration) if dur_row.avg_duration else 0.0 + + # ── MTTR trend ── + mttr_trend_result = await db.execute( + select( + cast(AISession.resolved_at, Date).label("day"), + func.avg( + extract("epoch", AISession.resolved_at - AISession.created_at) / 60 + ).label("mttr"), + func.count(AISession.id).label("count"), + ) + .where( + AISession.account_id == account_id, + AISession.created_at >= period_start, + AISession.status == "resolved", + AISession.resolved_at.isnot(None), + ) + .group_by(cast(AISession.resolved_at, Date)) + .order_by(cast(AISession.resolved_at, Date)) + ) + mttr_trend = [ + MTTRDataPoint( + date=str(r.day), + mttr_minutes=round(float(r.mttr or 0), 1), + session_count=r.count, + ) + for r in mttr_trend_result.all() + ] + + # ── Domain breakdown ── + domain_result = await db.execute( + select( + AISession.problem_domain, + func.count(AISession.id).label("total"), + func.sum(case((AISession.status == "resolved", 1), else_=0)).label("resolved"), + func.sum(case((AISession.status.in_(["escalated", "requesting_escalation"]), 1), else_=0)).label("escalated"), + ) + .where( + AISession.account_id == account_id, + AISession.created_at >= period_start, + AISession.problem_domain.isnot(None), + ) + .group_by(AISession.problem_domain) + .order_by(func.count(AISession.id).desc()) + ) + sessions_by_domain = [ + DomainBreakdown( + domain=r.problem_domain or "unknown", + total=r.total, + resolved=r.resolved or 0, + escalated=r.escalated or 0, + resolution_rate=round((r.resolved or 0) / r.total * 100, 1) if r.total > 0 else 0.0, + ) + for r in domain_result.all() + ] + + # ── Confidence breakdown ── + confidence_result = await db.execute( + select( + AISession.confidence_tier, + func.count(AISession.id).label("total"), + func.sum(case((AISession.status == "resolved", 1), else_=0)).label("resolved"), + ) + .where( + AISession.account_id == account_id, + AISession.created_at >= period_start, + AISession.status.in_(["resolved", "escalated", "requesting_escalation"]), + ) + .group_by(AISession.confidence_tier) + ) + conf_data = {r.confidence_tier: (r.total or 0, r.resolved or 0) for r in confidence_result.all()} + + guided_total, guided_resolved = conf_data.get("guided", (0, 0)) + exploring_total, exploring_resolved = conf_data.get("exploring", (0, 0)) + discovery_total, discovery_resolved = conf_data.get("discovery", (0, 0)) + + confidence_breakdown = ConfidenceBreakdown( + guided_sessions=guided_total, + guided_resolution_rate=round(guided_resolved / guided_total * 100, 1) if guided_total > 0 else 0.0, + exploring_sessions=exploring_total, + exploring_resolution_rate=round(exploring_resolved / exploring_total * 100, 1) if exploring_total > 0 else 0.0, + discovery_sessions=discovery_total, + discovery_resolution_rate=round(discovery_resolved / discovery_total * 100, 1) if discovery_total > 0 else 0.0, + ) + + # ── Knowledge coverage ── + total_flows_result = await db.execute( + select(func.count(Tree.id)).where(Tree.account_id == account_id) + ) + total_flows = total_flows_result.scalar() or 0 + + ai_flows_result = await db.execute( + select(func.count(Tree.id)).where( + Tree.account_id == account_id, + Tree.origin.in_(["ai_generated", "ai_enhanced"]), + ) + ) + ai_generated_flows = ai_flows_result.scalar() or 0 + + pending_proposals_result = await db.execute( + select(func.count(FlowProposal.id)).where( + FlowProposal.account_id == account_id, + FlowProposal.status == "pending", + ) + ) + total_proposals_pending = pending_proposals_result.scalar() or 0 + + approved_result = await db.execute( + select(func.count(FlowProposal.id)).where( + FlowProposal.account_id == account_id, + FlowProposal.reviewed_at >= period_start, + FlowProposal.status.in_(["approved", "modified"]), + ) + ) + proposals_approved = approved_result.scalar() or 0 + + rejected_result = await db.execute( + select(func.count(FlowProposal.id)).where( + FlowProposal.account_id == account_id, + FlowProposal.reviewed_at >= period_start, + FlowProposal.status == "rejected", + ) + ) + proposals_rejected = rejected_result.scalar() or 0 + + # Domain coverage + domain_coverage_result = await db.execute( + select( + AISession.problem_domain, + func.count(AISession.id).label("session_count"), + func.sum(case((AISession.confidence_tier == "guided", 1), else_=0)).label("guided_count"), + ) + .where( + AISession.account_id == account_id, + AISession.created_at >= period_start, + AISession.problem_domain.isnot(None), + ) + .group_by(AISession.problem_domain) + ) + domain_flow_counts_result = await db.execute( + select( + Tree.tree_type, # Reuse as domain proxy — not ideal but workable + func.count(Tree.id), + ) + .where(Tree.account_id == account_id) + .group_by(Tree.tree_type) + ) + # For now, flow_count per domain isn't directly available since Tree doesn't have problem_domain. + # Use match_keywords or just report 0. We'll improve this in Phase 4 with better flow categorization. + domain_cov_data = {} + for r in domain_coverage_result.all(): + domain = r.problem_domain or "unknown" + sc = r.session_count or 0 + gc = r.guided_count or 0 + domain_cov_data[domain] = DomainCoverage( + domain=domain, + flow_count=0, # TODO: match via category/tags in Phase 4 + session_count=sc, + guided_rate=round(gc / sc * 100, 1) if sc > 0 else 0.0, + ) + + knowledge_coverage = KnowledgeCoverage( + total_flows=total_flows, + ai_generated_flows=ai_generated_flows, + total_proposals_pending=total_proposals_pending, + proposals_approved_this_period=proposals_approved, + proposals_rejected_this_period=proposals_rejected, + coverage_by_domain=list(domain_cov_data.values()), + ) + + # ── PSA metrics ── + psa_metrics = None + psa_linked = await db.execute( + select(func.count(AISession.id)).where( + AISession.account_id == account_id, + AISession.created_at >= period_start, + AISession.psa_ticket_id.isnot(None), + ) + ) + psa_linked_count = psa_linked.scalar() or 0 + + if psa_linked_count > 0 and total > 0: + psa_push_result = await db.execute( + select( + func.count(PsaPostLog.id).label("total_pushes"), + func.sum(case((PsaPostLog.status == "success", 1), else_=0)).label("first_success"), + func.sum(case( + ((PsaPostLog.status == "success") & (PsaPostLog.retry_count > 0), 1), + else_=0 + )).label("retry_success"), + ) + .join(AISession, PsaPostLog.ai_session_id == AISession.id) + .where( + AISession.account_id == account_id, + PsaPostLog.ai_session_id.isnot(None), + PsaPostLog.posted_at >= period_start, + ) + ) + push_row = psa_push_result.one() + total_pushes = push_row.total_pushes or 0 + first_success = push_row.first_success or 0 + retry_success = push_row.retry_success or 0 + + psa_metrics = PsaMetrics( + ticket_link_rate=round(psa_linked_count / total * 100, 1), + auto_push_success_rate=round(first_success / total_pushes * 100, 1) if total_pushes > 0 else 0.0, + auto_push_retry_success_rate=round(retry_success / total_pushes * 100, 1) if total_pushes > 0 else 0.0, + total_time_entries_logged=0, # TODO: track from CW time entries + total_hours_logged=0.0, + ) + + return FlowPilotDashboard( + period=period, + total_sessions=total, + resolved_sessions=resolved, + escalated_sessions=escalated, + abandoned_sessions=abandoned, + resolution_rate=round(resolution_rate, 1), + avg_steps_to_resolution=round(avg_steps, 1), + avg_session_duration_minutes=round(avg_duration, 1), + avg_rating=round(avg_rating, 2) if avg_rating else None, + mttr_minutes=round(mttr_minutes, 1) if mttr_minutes else None, + mttr_trend=mttr_trend, + sessions_by_domain=sessions_by_domain, + confidence_breakdown=confidence_breakdown, + knowledge_coverage=knowledge_coverage, + psa_metrics=psa_metrics, + ) + + +@router.get("/knowledge-gaps", response_model=KnowledgeGapReport) +@limiter.limit("10/minute") +async def get_knowledge_gaps_endpoint( + request: Request, + current_user: Annotated[User, Depends(get_current_active_user)], + db: Annotated[AsyncSession, Depends(get_db)], + _: None = Depends(require_team_admin), + period: str = Query("30d", pattern="^(7d|30d|90d)$"), +): + """Get knowledge gap analysis report.""" + if not current_user.account_id: + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="No account") + + days = {"7d": 7, "30d": 30, "90d": 90}.get(period, 30) + return await get_knowledge_gaps(current_user.account_id, db, period_days=days) diff --git a/backend/app/api/endpoints/scripts.py b/backend/app/api/endpoints/scripts.py index 5f6ab8df..182f837e 100644 --- a/backend/app/api/endpoints/scripts.py +++ b/backend/app/api/endpoints/scripts.py @@ -355,6 +355,7 @@ async def generate_script( user_id=current_user.id, team_id=current_user.team_id, session_id=data.session_id, + ai_session_id=data.ai_session_id, parameters_used=redacted_params, generated_script=rendered_script, ) diff --git a/backend/app/api/router.py b/backend/app/api/router.py index 27d8eee7..38ff2b56 100644 --- a/backend/app/api/router.py +++ b/backend/app/api/router.py @@ -22,6 +22,8 @@ from app.api.endpoints import onboarding from app.api.endpoints import branding from app.api.endpoints import supporting_data from app.api.endpoints import ai_sessions +from app.api.endpoints import flow_proposals +from app.api.endpoints import flowpilot_analytics api_router = APIRouter() @@ -69,3 +71,5 @@ api_router.include_router(onboarding.router) api_router.include_router(branding.router) api_router.include_router(supporting_data.router) api_router.include_router(ai_sessions.router) +api_router.include_router(flow_proposals.router) +api_router.include_router(flowpilot_analytics.router) diff --git a/backend/app/main.py b/backend/app/main.py index 61b5c842..8df118b0 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -190,6 +190,17 @@ async def lifespan(app: FastAPI): replace_existing=True, ) + # Knowledge Flywheel analysis (every 5 minutes) + from app.services.knowledge_flywheel_scheduler import process_pending_analyses + scheduler.add_job( + process_pending_analyses, + trigger="interval", + minutes=5, + id="knowledge_flywheel_analysis", + replace_existing=True, + max_instances=1, + ) + # Auto-seed trees in background on PR environments seed_task = None if settings.SEED_ON_DEPLOY: diff --git a/backend/app/models/__init__.py b/backend/app/models/__init__.py index 5c12b70c..5dde142a 100644 --- a/backend/app/models/__init__.py +++ b/backend/app/models/__init__.py @@ -42,6 +42,7 @@ from .psa_connection import PsaConnection from .psa_post_log import PsaPostLog from .psa_member_mapping import PsaMemberMapping from .supporting_data import SessionSupportingData +from .flow_proposal import FlowProposal __all__ = [ "User", @@ -98,4 +99,5 @@ __all__ = [ "PsaPostLog", "PsaMemberMapping", "SessionSupportingData", + "FlowProposal", ] diff --git a/backend/app/models/ai_session.py b/backend/app/models/ai_session.py index d1c15457..bc023278 100644 --- a/backend/app/models/ai_session.py +++ b/backend/app/models/ai_session.py @@ -156,6 +156,12 @@ class AISession(Base): comment="Optional feedback text from engineer", ) + # ── Knowledge Flywheel ── + analysis_status: Mapped[Optional[str]] = mapped_column( + String(20), nullable=True, + comment="Knowledge Flywheel status: null (N/A), pending, completed, failed", + ) + # ── AI tracking ── total_input_tokens: Mapped[int] = mapped_column( Integer, nullable=False, default=0, diff --git a/backend/app/models/flow_proposal.py b/backend/app/models/flow_proposal.py new file mode 100644 index 00000000..5450e249 --- /dev/null +++ b/backend/app/models/flow_proposal.py @@ -0,0 +1,152 @@ +"""Flow proposal model. + +Generated by the Knowledge Flywheel after AI sessions resolve. +Represents a proposed new flow or enhancement awaiting human review. +""" +import uuid +from datetime import datetime, timezone +from typing import Optional, Any, TYPE_CHECKING + +from sqlalchemy import String, Text, DateTime, ForeignKey, Integer, Float, CheckConstraint +from sqlalchemy.orm import Mapped, mapped_column, relationship +from sqlalchemy.dialects.postgresql import UUID, JSONB + +from app.core.database import Base + +if TYPE_CHECKING: + from app.models.user import User + from app.models.team import Team + from app.models.account import Account + from app.models.tree import Tree + from app.models.ai_session import AISession + + +class FlowProposal(Base): + """A proposed new flow or enhancement generated from an AI session. + + proposal_type: + - new_flow: No similar flow exists. Full flow definition proposed. + - enhancement: Similar flow exists but session discovered new branch/edge case. + - branch_addition: A single new branch to add to an existing flow. + - auto_reinforced: Session matched existing flow exactly (tracking only). + + status: + - pending: Awaiting review + - approved: Reviewed and published to knowledge base + - modified: Reviewer edited before publishing + - rejected: Reviewer decided not to publish (bad quality) + - dismissed: Parked for later — not wrong, just not actionable now. + - auto_reinforced: Session matched existing flow exactly (no review needed) + """ + __tablename__ = "flow_proposals" + __table_args__ = ( + CheckConstraint( + "proposal_type IN ('new_flow', 'enhancement', 'branch_addition', 'auto_reinforced')", + name="ck_flow_proposals_type", + ), + CheckConstraint( + "status IN ('pending', 'approved', 'modified', 'rejected', 'dismissed', 'auto_reinforced')", + name="ck_flow_proposals_status", + ), + ) + + id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), primary_key=True, default=uuid.uuid4 + ) + account_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + ForeignKey("accounts.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + team_id: Mapped[Optional[uuid.UUID]] = mapped_column( + UUID(as_uuid=True), + ForeignKey("teams.id", ondelete="SET NULL"), + nullable=True, + index=True, + ) + source_session_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + ForeignKey("ai_sessions.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + + # ── Proposal details ── + proposal_type: Mapped[str] = mapped_column( + String(30), nullable=False, + ) + target_flow_id: Mapped[Optional[uuid.UUID]] = mapped_column( + UUID(as_uuid=True), + ForeignKey("trees.id", ondelete="SET NULL"), + nullable=True, + comment="For enhancements: which existing flow to modify", + ) + title: Mapped[str] = mapped_column( + String(255), nullable=False, + comment="Human-readable title for the proposed flow", + ) + description: Mapped[Optional[str]] = mapped_column( + Text, nullable=True, + comment="AI-generated description of what this flow covers", + ) + proposed_flow_data: Mapped[dict[str, Any]] = mapped_column( + JSONB, nullable=False, + comment="Complete flow/tree_structure definition (nodes, edges, conditions)", + ) + proposed_diff: Mapped[Optional[dict[str, Any]]] = mapped_column( + JSONB, nullable=True, + comment="For enhancements: what changed vs existing flow", + ) + + # ── Scoring ── + confidence_score: Mapped[float] = mapped_column( + Float, nullable=False, default=0.0, + comment="How confident the system is in this proposal (0.0-1.0)", + ) + supporting_session_count: Mapped[int] = mapped_column( + Integer, nullable=False, default=1, + comment="Number of sessions with similar resolution paths", + ) + supporting_session_ids: Mapped[list] = mapped_column( + JSONB, nullable=False, default=list, + comment="Array of session IDs that support this proposal", + ) + problem_domain: Mapped[Optional[str]] = mapped_column( + String(100), nullable=True, + ) + + # ── Review ── + status: Mapped[str] = mapped_column( + String(30), nullable=False, default="pending", index=True, + ) + reviewed_by: Mapped[Optional[uuid.UUID]] = mapped_column( + UUID(as_uuid=True), + ForeignKey("users.id", ondelete="SET NULL"), + nullable=True, + ) + reviewer_notes: Mapped[Optional[str]] = mapped_column( + Text, nullable=True, + ) + published_flow_id: Mapped[Optional[uuid.UUID]] = mapped_column( + UUID(as_uuid=True), + ForeignKey("trees.id", ondelete="SET NULL"), + nullable=True, + comment="The flow that was created/updated when this proposal was approved", + ) + + # ── Timestamps ── + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) + ) + reviewed_at: Mapped[Optional[datetime]] = mapped_column( + DateTime(timezone=True), nullable=True, + ) + + # ── Relationships ── + account: Mapped["Account"] = relationship("Account") + team: Mapped[Optional["Team"]] = relationship("Team") + source_session: Mapped["AISession"] = relationship("AISession") + target_flow: Mapped[Optional["Tree"]] = relationship("Tree", foreign_keys=[target_flow_id]) + published_flow: Mapped[Optional["Tree"]] = relationship("Tree", foreign_keys=[published_flow_id]) + reviewer: Mapped[Optional["User"]] = relationship("User") diff --git a/backend/app/models/script_template.py b/backend/app/models/script_template.py index c90c2da7..412f72f4 100644 --- a/backend/app/models/script_template.py +++ b/backend/app/models/script_template.py @@ -97,6 +97,10 @@ class ScriptGeneration(Base): session_id: Mapped[Optional[uuid.UUID]] = mapped_column( UUID(as_uuid=True), ForeignKey("sessions.id", ondelete="SET NULL"), nullable=True, index=True ) + ai_session_id: Mapped[Optional[uuid.UUID]] = mapped_column( + UUID(as_uuid=True), ForeignKey("ai_sessions.id", ondelete="SET NULL"), nullable=True, index=True, + comment="FlowPilot AI session that triggered this generation", + ) parameters_used: Mapped[dict] = mapped_column(JSONB, nullable=False, default=dict) generated_script: Mapped[str] = mapped_column(Text, nullable=False) created_at: Mapped[datetime] = mapped_column( diff --git a/backend/app/schemas/flow_proposal.py b/backend/app/schemas/flow_proposal.py new file mode 100644 index 00000000..ca324fb5 --- /dev/null +++ b/backend/app/schemas/flow_proposal.py @@ -0,0 +1,51 @@ +"""Pydantic schemas for flow proposals (Knowledge Flywheel / Review Queue).""" +from __future__ import annotations + +from typing import Optional, Any +from uuid import UUID +from datetime import datetime + +from pydantic import BaseModel, Field + + +class FlowProposalSummary(BaseModel): + """Compact proposal for list views.""" + id: UUID + proposal_type: str + title: str + description: str | None = None + problem_domain: str | None = None + confidence_score: float + supporting_session_count: int + status: str + target_flow_id: UUID | None = None + source_session_id: UUID + created_at: datetime + + model_config = {"from_attributes": True} + + +class FlowProposalDetail(FlowProposalSummary): + """Full proposal detail with flow data.""" + proposed_flow_data: dict[str, Any] + proposed_diff: dict[str, Any] | None = None + supporting_session_ids: list[str] = [] + reviewer_notes: str | None = None + reviewed_by: UUID | None = None + reviewed_at: datetime | None = None + + +class ReviewProposalRequest(BaseModel): + """Review action on a proposal.""" + action: str = Field(..., pattern="^(approve|reject|modify|dismiss)$") + reviewer_notes: str | None = None + modified_flow_data: dict[str, Any] | None = None # Only for "modify" + + +class FlowProposalStats(BaseModel): + """Dashboard stats for the review queue.""" + pending_count: int + approved_this_week: int + rejected_this_week: int + auto_reinforced_this_week: int + top_domains: list[dict[str, Any]] = [] # [{domain, count}] diff --git a/backend/app/schemas/flowpilot_analytics.py b/backend/app/schemas/flowpilot_analytics.py new file mode 100644 index 00000000..2b918992 --- /dev/null +++ b/backend/app/schemas/flowpilot_analytics.py @@ -0,0 +1,72 @@ +"""Pydantic schemas for FlowPilot analytics dashboard.""" +from __future__ import annotations + +from typing import Optional, Any +from datetime import datetime + +from pydantic import BaseModel + + +class MTTRDataPoint(BaseModel): + date: str + mttr_minutes: float + session_count: int + + +class DomainBreakdown(BaseModel): + domain: str + total: int + resolved: int + escalated: int + resolution_rate: float + + +class ConfidenceBreakdown(BaseModel): + guided_sessions: int + guided_resolution_rate: float + exploring_sessions: int + exploring_resolution_rate: float + discovery_sessions: int + discovery_resolution_rate: float + + +class DomainCoverage(BaseModel): + domain: str + flow_count: int + session_count: int + guided_rate: float + + +class KnowledgeCoverage(BaseModel): + total_flows: int + ai_generated_flows: int + total_proposals_pending: int + proposals_approved_this_period: int + proposals_rejected_this_period: int + coverage_by_domain: list[DomainCoverage] = [] + + +class PsaMetrics(BaseModel): + ticket_link_rate: float + auto_push_success_rate: float + auto_push_retry_success_rate: float + total_time_entries_logged: int + total_hours_logged: float + + +class FlowPilotDashboard(BaseModel): + period: str + total_sessions: int + resolved_sessions: int + escalated_sessions: int + abandoned_sessions: int + resolution_rate: float + avg_steps_to_resolution: float + avg_session_duration_minutes: float + avg_rating: float | None = None + mttr_minutes: float | None = None + mttr_trend: list[MTTRDataPoint] = [] + sessions_by_domain: list[DomainBreakdown] = [] + confidence_breakdown: ConfidenceBreakdown + knowledge_coverage: KnowledgeCoverage + psa_metrics: PsaMetrics | None = None diff --git a/backend/app/schemas/script_template.py b/backend/app/schemas/script_template.py index 1dbdc561..de6ec106 100644 --- a/backend/app/schemas/script_template.py +++ b/backend/app/schemas/script_template.py @@ -116,7 +116,8 @@ class ScriptTemplateDetail(ScriptTemplateListItem): class ScriptGenerateRequest(BaseModel): template_id: UUID parameters: dict[str, Any] - session_id: Optional[UUID] = None + session_id: Optional[UUID] = None # Legacy tree-based session + ai_session_id: Optional[UUID] = None # FlowPilot AI session class ScriptGenerateResponse(BaseModel): id: UUID diff --git a/backend/app/services/flowpilot_engine.py b/backend/app/services/flowpilot_engine.py index f3f12980..a4a4ca6d 100644 --- a/backend/app/services/flowpilot_engine.py +++ b/backend/app/services/flowpilot_engine.py @@ -11,7 +11,7 @@ from datetime import datetime, timezone from typing import Any, Optional from uuid import UUID -from sqlalchemy import select +from sqlalchemy import select, or_ from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import selectinload @@ -228,6 +228,11 @@ async def start_session( if ticket_context_block: ticket_prompt_section = f"\n## PSA TICKET CONTEXT\n{ticket_context_block}\n" + # Include available script templates for in-session script generation + script_context = await _build_script_context(team_id, db) + if script_context: + ticket_prompt_section += f"\n{script_context}\n" + system_prompt = FLOWPILOT_SYSTEM_PROMPT.format( structured_output_schema=STRUCTURED_OUTPUT_SCHEMA, team_context=ticket_prompt_section, @@ -448,6 +453,9 @@ async def resolve_session( documentation = _generate_documentation(session) + # Queue for Knowledge Flywheel analysis + session.analysis_status = "pending" + await db.flush() # Push documentation to PSA if ticket is linked @@ -909,6 +917,13 @@ def _create_step_from_parsed( if parsed["type"] == "action": content["action_type"] = parsed.get("action_type", "instruction") content["expected_outcome"] = parsed.get("expected_outcome") + # Script generation fields (populated when FlowPilot suggests a script) + if parsed.get("template_id"): + content["template_id"] = parsed["template_id"] + if parsed.get("pre_filled_params"): + content["pre_filled_params"] = parsed["pre_filled_params"] + if parsed.get("instructions"): + content["instructions"] = parsed["instructions"] elif parsed["type"] == "resolution_suggestion": content["resolution_summary"] = parsed.get("resolution_summary") content["follow_up_recommendations"] = parsed.get("follow_up_recommendations", []) @@ -1066,6 +1081,51 @@ async def _process_ticket_intake( return None, None, "unavailable" +async def _build_script_context( + team_id: Optional[UUID], + db: AsyncSession, +) -> Optional[str]: + """Build script template context for the system prompt. + + Includes available script templates so FlowPilot can suggest + script_generation actions with pre-filled parameters. + """ + try: + from app.models.script_template import ScriptTemplate + + result = await db.execute( + select(ScriptTemplate) + .where( + ScriptTemplate.is_active.is_(True), + or_( + ScriptTemplate.team_id.is_(None), + ScriptTemplate.team_id == team_id, + ), + ) + .order_by(ScriptTemplate.usage_count.desc()) + .limit(20) + ) + templates = result.scalars().all() + + if not templates: + return None + + lines = ["## AVAILABLE SCRIPTS"] + lines.append("When the engineer needs to run a script, suggest an action with action_type='script_generation'.") + lines.append("Include template_id and pre_filled_params based on the diagnostic context.\n") + for t in templates: + params = t.parameters_schema.get("parameters", []) + param_keys = ", ".join(p.get("key", "") for p in params if p.get("key")) + lines.append(f"- {t.name} (ID: {t.id}): {t.description or 'No description'}") + if param_keys: + lines.append(f" Parameters: {param_keys}") + + return "\n".join(lines) + except Exception as e: + logger.warning("Failed to build script context: %s", e) + return None + + async def _build_escalation_package_enhanced( session: AISession, user_id: UUID, diff --git a/backend/app/services/knowledge_flywheel.py b/backend/app/services/knowledge_flywheel.py new file mode 100644 index 00000000..8f0209c1 --- /dev/null +++ b/backend/app/services/knowledge_flywheel.py @@ -0,0 +1,454 @@ +"""Knowledge Flywheel — post-session analysis engine. + +Analyzes resolved AI sessions and generates flow proposals: +- new_flow: Novel resolution path → propose a new troubleshooting flow +- enhancement: Diverged from a matched flow → propose additions +- auto_reinforced: Followed a flow exactly → update flow stats + +Called by the knowledge_flywheel_scheduler (APScheduler) after sessions resolve. +""" +import json +import logging +import uuid +from datetime import datetime, timezone +from typing import Any, Optional +from uuid import UUID + +from sqlalchemy import select, func +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import selectinload + +from app.core.ai_provider import get_ai_provider +from app.core.config import settings +from app.models.ai_session import AISession +from app.models.ai_session_step import AISessionStep +from app.models.flow_proposal import FlowProposal +from app.models.tree import Tree + +logger = logging.getLogger(__name__) + +# Daily budget cap for proposal generation LLM calls per account +MAX_PROPOSALS_PER_DAY = 50 + +FLOW_GENERATION_PROMPT = """\ +You are a knowledge engineer converting a troubleshooting session into a reusable flow definition. + +Given the session transcript below, generate a JSON flow definition that captures the diagnostic logic so other engineers can follow the same path. + +## OUTPUT FORMAT +Respond with ONLY valid JSON: +{ + "title": "Short descriptive title (5-10 words)", + "description": "When to use this flow (1-2 sentences)", + "match_keywords": ["keyword1", "keyword2", ...], + "problem_domain": "active_directory | networking | m365 | hardware | endpoint | virtualization | security | backup | email | printing | cloud | other", + "tree_structure": { + "id": "root", + "type": "decision", + "question": "First diagnostic question", + "help_text": "Context for the engineer", + "options": [ + {"id": "opt1", "label": "Option text", "next_node_id": "node_id"} + ], + "children": [ + { + "id": "node_id", + "type": "decision | action | solution", + "title": "Node title", + "question": "For decision nodes", + "description": "For action/solution nodes", + "options": [], + "next_node_id": "next_id or null for terminal nodes" + } + ] + } +} + +## RULES +- tree_structure uses a flat children array with id-based references via next_node_id +- The root node has type "decision" with a question and options +- Decision nodes have options with next_node_id pointing to child nodes +- Action nodes describe what the engineer should do with a description field +- Solution nodes describe the resolution (terminal — no next_node_id) +- Every decision node must have 2-5 options +- Include the key diagnostic questions that narrowed down the problem +- Skip redundant or dead-end paths from the session +- match_keywords should be symptoms, error messages, and technology names (5-10 keywords) +- Do NOT wrap JSON in markdown code fences\ +""" + +ENHANCEMENT_PROMPT = """\ +You are a knowledge engineer analyzing how a troubleshooting session diverged from an existing flow. + +Given the session transcript and the existing flow structure, identify what should be added or changed. + +## OUTPUT FORMAT +Respond with ONLY valid JSON: +{ + "title": "Enhancement: ", + "description": "Why this enhancement is needed", + "diff_description": "Human-readable summary of changes", + "new_nodes": [ + { + "id": "new_node_id", + "type": "decision | action | solution", + "title": "Node title", + "question": "For decision nodes", + "description": "For action/solution nodes", + "options": [], + "attach_after_node_id": "existing node ID where this branches off", + "new_option_label": "Label for the new option on the parent node" + } + ], + "modified_options": [ + { + "node_id": "existing node ID", + "add_option": {"id": "new_opt", "label": "New option text", "next_node_id": "new_node_id"} + } + ] +} + +## RULES +- Only propose changes supported by the session evidence +- Minimize changes — add branches, don't restructure +- new_nodes should follow the same format as the existing flow +- Do NOT wrap JSON in markdown code fences\ +""" + + +def _build_session_context(session: AISession) -> str: + """Build a text summary of a session for the LLM prompt.""" + parts = [ + f"Problem: {session.problem_summary or 'Unknown'}", + f"Domain: {session.problem_domain or 'Unknown'}", + f"Confidence at resolution: {session.confidence_tier} ({session.confidence_score:.0%})", + f"Resolution: {session.resolution_summary or 'No summary'}", + ] + + if session.escalation_reason: + parts.append(f"Escalation reason: {session.escalation_reason}") + + # Build step-by-step diagnostic trail + steps = sorted(session.steps, key=lambda s: s.step_order) + if steps: + parts.append("\n--- DIAGNOSTIC TRAIL ---") + for step in steps: + content = step.content or {} + step_desc = content.get("text", "") + step_type = content.get("type", step.step_type) + + line = f"Step {step.step_order + 1} [{step_type}]: {step_desc}" + + # Engineer response + if step.was_skipped: + line += "\n → Skipped" + elif step.selected_option: + # Find label from options + label = step.selected_option + if step.options_presented: + for opt in step.options_presented: + if opt.get("value") == step.selected_option: + label = opt.get("label", step.selected_option) + break + line += f"\n → Selected: {label}" + elif step.free_text_input: + line += f"\n → Free text: {step.free_text_input}" + + if step.action_result: + result = step.action_result + outcome = "Succeeded" if result.get("success") else "Did not resolve" + if details := result.get("details"): + outcome += f" — {details}" + line += f"\n → Result: {outcome}" + + parts.append(line) + + return "\n".join(parts) + + +def _has_free_text_escapes(session: AISession) -> bool: + """Check if the session used free-text escapes (diverged from options).""" + return any(step.was_free_text for step in session.steps) + + +async def _check_daily_budget(account_id: UUID, db: AsyncSession) -> bool: + """Check if the account has exceeded the daily proposal generation budget.""" + today_start = datetime.now(timezone.utc).replace(hour=0, minute=0, second=0, microsecond=0) + result = await db.execute( + select(func.count(FlowProposal.id)) + .where( + FlowProposal.account_id == account_id, + FlowProposal.created_at >= today_start, + FlowProposal.status != "auto_reinforced", # Don't count no-LLM proposals + ) + ) + count = result.scalar() or 0 + return count < MAX_PROPOSALS_PER_DAY + + +async def _find_similar_pending_proposal( + title: str, + problem_domain: Optional[str], + account_id: UUID, + db: AsyncSession, +) -> Optional[FlowProposal]: + """Find an existing pending proposal with similar title and domain. + + Uses simple keyword overlap for now. Phase 4 will add embedding similarity. + """ + # Build domain filter — match NULL domain proposals if domain is NULL + domain_filter = ( + FlowProposal.problem_domain == problem_domain + if problem_domain + else FlowProposal.problem_domain.is_(None) + ) + + result = await db.execute( + select(FlowProposal) + .where( + FlowProposal.account_id == account_id, + FlowProposal.status == "pending", + domain_filter, + ) + .limit(20) + ) + candidates = result.scalars().all() + + if not candidates: + return None + + # Simple keyword overlap check + title_words = set(title.lower().split()) + for candidate in candidates: + candidate_words = set(candidate.title.lower().split()) + if len(title_words) > 0 and len(candidate_words) > 0: + overlap = len(title_words & candidate_words) / max(len(title_words), len(candidate_words)) + if overlap > 0.6: + return candidate + + return None + + +async def analyze_session(session: AISession, db: AsyncSession) -> None: + """Analyze a resolved session and create appropriate flow proposal. + + Dispatches to one of three outcomes: + 1. new_flow — novel resolution, no matching flow + 2. enhancement — matched flow but diverged + 3. auto_reinforced — followed existing flow closely + """ + # Re-fetch with eager-loaded steps to avoid async lazy-load errors + result = await db.execute( + select(AISession) + .where(AISession.id == session.id) + .options(selectinload(AISession.steps)) + ) + session = result.scalar_one() + + # Determine which analysis path to take + has_match = session.matched_flow_id is not None + match_score = session.match_score or 0.0 + has_divergence = _has_free_text_escapes(session) + + if has_match and match_score > 0.8 and not has_divergence: + # Path 3: Auto-reinforcement + await _auto_reinforce(session, db) + elif has_match and match_score > 0.5 and has_divergence: + # Path 2: Enhancement proposal + await _propose_enhancement(session, db) + elif not has_match or match_score < 0.5: + # Path 1: New flow proposal + await _propose_new_flow(session, db) + else: + # Edge case: matched but moderate score, no divergence — reinforce + await _auto_reinforce(session, db) + + +async def _auto_reinforce(session: AISession, db: AsyncSession) -> None: + """Update the matched flow's stats and create a tracking record.""" + if session.matched_flow_id: + result = await db.execute( + select(Tree).where(Tree.id == session.matched_flow_id) + ) + flow = result.scalar_one_or_none() + if flow: + # Update flow stats + current_rate = flow.success_rate or 0.0 + # Simple moving average + flow.success_rate = round(current_rate * 0.9 + 1.0 * 0.1, 4) + flow.last_matched_at = datetime.now(timezone.utc) + + # Create tracking record (no review needed) + proposal = FlowProposal( + id=uuid.uuid4(), + account_id=session.account_id, + team_id=session.team_id, + source_session_id=session.id, + proposal_type="auto_reinforced", + title=f"Reinforcement: {session.problem_summary or 'Session'}", + description="Session followed existing flow closely. No changes needed.", + proposed_flow_data={}, + confidence_score=session.confidence_score, + supporting_session_ids=[str(session.id)], + problem_domain=session.problem_domain, + status="auto_reinforced", + target_flow_id=session.matched_flow_id, + ) + db.add(proposal) + logger.info("Auto-reinforced flow %s from session %s", session.matched_flow_id, session.id) + + +async def _propose_new_flow(session: AISession, db: AsyncSession) -> None: + """Generate a new flow proposal from a novel session.""" + if not await _check_daily_budget(session.account_id, db): + logger.warning("Daily proposal budget exceeded for account %s", session.account_id) + return + + session_context = _build_session_context(session) + + try: + provider = get_ai_provider(settings.get_model_for_action("open_chat")) + raw_response, _, _ = await provider.generate_json( + system_prompt=FLOW_GENERATION_PROMPT, + messages=[{"role": "user", "content": session_context}], + max_tokens=4096, + ) + + parsed = _parse_llm_json(raw_response) + except Exception as e: + logger.warning("Knowledge Flywheel LLM call failed for session %s: %s", session.id, e) + return + + title = parsed.get("title", session.problem_summary or "Untitled Flow") + domain = parsed.get("problem_domain", session.problem_domain) + + # Check for similar pending proposals + existing = await _find_similar_pending_proposal(title, domain, session.account_id, db) + if existing: + # Merge into existing proposal + existing.supporting_session_count += 1 + sids = existing.supporting_session_ids or [] + sids.append(str(session.id)) + existing.supporting_session_ids = sids + existing.confidence_score = min(1.0, existing.confidence_score + 0.1) + logger.info( + "Merged session %s into existing proposal %s (now %d supporting)", + session.id, existing.id, existing.supporting_session_count, + ) + return + + proposal = FlowProposal( + id=uuid.uuid4(), + account_id=session.account_id, + team_id=session.team_id, + source_session_id=session.id, + proposal_type="new_flow", + title=title, + description=parsed.get("description"), + proposed_flow_data={ + "tree_structure": parsed.get("tree_structure", {}), + "match_keywords": parsed.get("match_keywords", []), + }, + confidence_score=session.confidence_score, + supporting_session_ids=[str(session.id)], + problem_domain=domain, + status="pending", + ) + db.add(proposal) + logger.info("Created new_flow proposal for session %s: %s", session.id, title) + + +async def _propose_enhancement(session: AISession, db: AsyncSession) -> None: + """Generate an enhancement proposal for an existing flow.""" + if not session.matched_flow_id: + # Fallback to new flow if no match + await _propose_new_flow(session, db) + return + + if not await _check_daily_budget(session.account_id, db): + logger.warning("Daily proposal budget exceeded for account %s", session.account_id) + return + + # Load the matched flow + result = await db.execute( + select(Tree).where(Tree.id == session.matched_flow_id) + ) + matched_flow = result.scalar_one_or_none() + if not matched_flow: + await _propose_new_flow(session, db) + return + + session_context = _build_session_context(session) + flow_json = json.dumps(matched_flow.tree_structure, indent=None) + if len(flow_json) > 4000: + flow_json = flow_json[:4000] + "... [truncated]" + + prompt_content = ( + f"## EXISTING FLOW\n" + f"Name: {matched_flow.name}\n" + f"Structure:\n{flow_json}\n\n" + f"## SESSION THAT DIVERGED\n" + f"{session_context}" + ) + + try: + provider = get_ai_provider(settings.get_model_for_action("open_chat")) + raw_response, _, _ = await provider.generate_json( + system_prompt=ENHANCEMENT_PROMPT, + messages=[{"role": "user", "content": prompt_content}], + max_tokens=4096, + ) + + parsed = _parse_llm_json(raw_response) + except Exception as e: + logger.warning("Knowledge Flywheel enhancement LLM call failed for session %s: %s", session.id, e) + return + + title = parsed.get("title", f"Enhancement: {session.problem_summary or 'Flow update'}") + diff_description = parsed.get("diff_description", "Session diverged from existing flow") + + proposal = FlowProposal( + id=uuid.uuid4(), + account_id=session.account_id, + team_id=session.team_id, + source_session_id=session.id, + proposal_type="enhancement", + target_flow_id=session.matched_flow_id, + title=title, + description=diff_description, + proposed_flow_data={ + "new_nodes": parsed.get("new_nodes", []), + "modified_options": parsed.get("modified_options", []), + }, + proposed_diff={ + "diff_description": diff_description, + "new_nodes": parsed.get("new_nodes", []), + "modified_options": parsed.get("modified_options", []), + }, + confidence_score=session.confidence_score, + supporting_session_ids=[str(session.id)], + problem_domain=session.problem_domain, + status="pending", + ) + db.add(proposal) + logger.info( + "Created enhancement proposal for flow %s from session %s: %s", + session.matched_flow_id, session.id, title, + ) + + +def _parse_llm_json(raw_text: str) -> dict[str, Any]: + """Parse JSON from LLM response, handling common quirks.""" + text = raw_text.strip() + + # Strip markdown code fences if present + if text.startswith("```"): + lines = text.split("\n") + lines = [line for line in lines if not line.strip().startswith("```")] + text = "\n".join(lines).strip() + + try: + return json.loads(text) + except json.JSONDecodeError as e: + logger.warning("Knowledge Flywheel JSON parse failed: %s — raw: %.300s", e, text) + raise ValueError(f"Invalid JSON from LLM: {e}") from e diff --git a/backend/app/services/knowledge_flywheel_scheduler.py b/backend/app/services/knowledge_flywheel_scheduler.py new file mode 100644 index 00000000..c1366b7d --- /dev/null +++ b/backend/app/services/knowledge_flywheel_scheduler.py @@ -0,0 +1,72 @@ +"""Background scheduler for Knowledge Flywheel analysis. + +Runs every 5 minutes via APScheduler, picks up AISession entries +with analysis_status='pending' and runs flow proposal analysis. + +Each session is committed individually to prevent a single failure +from rolling back all progress or causing duplicate proposals. +""" +import logging + +from sqlalchemy import select + +from app.core.database import async_session_maker +from app.models.ai_session import AISession +from app.services.knowledge_flywheel import analyze_session + +logger = logging.getLogger(__name__) + + +async def process_pending_analyses() -> None: + """Process resolved sessions awaiting Knowledge Flywheel analysis.""" + async with async_session_maker() as db: + try: + result = await db.execute( + select(AISession.id) + .where(AISession.analysis_status == "pending") + .order_by(AISession.resolved_at.asc()) + .limit(10) + ) + session_ids = [row[0] for row in result.all()] + except Exception as e: + logger.error("Knowledge Flywheel scheduler query error: %s", e) + return + + if not session_ids: + return + + logger.info("Processing %d pending Knowledge Flywheel analyses", len(session_ids)) + + # Process each session in its own DB session to isolate failures + for session_id in session_ids: + async with async_session_maker() as db: + try: + result = await db.execute( + select(AISession).where(AISession.id == session_id) + ) + session = result.scalar_one_or_none() + if not session or session.analysis_status != "pending": + continue + + await analyze_session(session, db) + session.analysis_status = "completed" + await db.commit() + logger.info("Knowledge Flywheel completed for session %s", session_id) + except Exception as e: + await db.rollback() + logger.warning( + "Knowledge Flywheel failed for session %s: %s", + session_id, e, + ) + # Mark as failed in a separate transaction + try: + async with async_session_maker() as db2: + result = await db2.execute( + select(AISession).where(AISession.id == session_id) + ) + s = result.scalar_one_or_none() + if s: + s.analysis_status = "failed" + await db2.commit() + except Exception: + logger.error("Failed to mark session %s as failed", session_id) diff --git a/backend/app/services/knowledge_gap_service.py b/backend/app/services/knowledge_gap_service.py new file mode 100644 index 00000000..f476efa9 --- /dev/null +++ b/backend/app/services/knowledge_gap_service.py @@ -0,0 +1,334 @@ +"""Knowledge Gap Detection Service. + +Aggregates signals from AI sessions to identify gaps in the knowledge base. +Results are served by the analytics API and cached for 1 hour. + +Signals: +1. Frequent free-text escapes — FlowPilot's options didn't cover a common scenario +2. High escalation rate by domain — domains where engineers can't self-resolve +3. Discovery-mode resolutions — novel problems solved without flow guidance +4. Repeated unmatched patterns — keyword-frequency based (Phase 4: embedding clustering) +""" +import logging +from collections import Counter +from datetime import datetime, timezone, timedelta +from typing import Any, Optional +from uuid import UUID + +from pydantic import BaseModel +from sqlalchemy import select, func, case, text +from sqlalchemy.ext.asyncio import AsyncSession + +from app.models.ai_session import AISession +from app.models.ai_session_step import AISessionStep +from app.models.tree import Tree + +logger = logging.getLogger(__name__) + +# Cache for expensive gap analysis +_cache: dict[str, Any] = {} +_cache_expiry: dict[str, datetime] = {} +CACHE_TTL = timedelta(hours=1) + + +class KnowledgeGap(BaseModel): + gap_type: str # "weak_options" | "high_escalation" | "uncharted_territory" | "repeated_pattern" + domain: str | None = None + severity: str # "high" | "medium" | "low" + title: str + description: str + evidence: dict[str, Any] = {} + suggested_action: str + + +class KnowledgeGapReport(BaseModel): + generated_at: datetime + gaps: list[KnowledgeGap] + + +async def get_knowledge_gaps( + account_id: UUID, + db: AsyncSession, + period_days: int = 30, +) -> KnowledgeGapReport: + """Generate a knowledge gap report for the account. + + Results are cached for 1 hour per account. + """ + cache_key = f"gaps:{account_id}:{period_days}" + now = datetime.now(timezone.utc) + + if cache_key in _cache and _cache_expiry.get(cache_key, now) > now: + return _cache[cache_key] + + period_start = now - timedelta(days=period_days) + + gaps: list[KnowledgeGap] = [] + + # Signal 1: Frequent free-text escapes + signal1 = await _detect_weak_options(account_id, period_start, db) + gaps.extend(signal1) + + # Signal 2: High escalation rate by domain + signal2 = await _detect_high_escalation(account_id, period_start, db) + gaps.extend(signal2) + + # Signal 3: Discovery-mode resolutions + signal3 = await _detect_uncharted_territory(account_id, period_start, db) + gaps.extend(signal3) + + # Signal 4: Repeated unmatched patterns (keyword-based for Phase 3) + signal4 = await _detect_repeated_patterns(account_id, period_start, db) + gaps.extend(signal4) + + # Sort by severity (high > medium > low) + severity_order = {"high": 0, "medium": 1, "low": 2} + gaps.sort(key=lambda g: severity_order.get(g.severity, 3)) + + report = KnowledgeGapReport(generated_at=now, gaps=gaps) + + _cache[cache_key] = report + _cache_expiry[cache_key] = now + CACHE_TTL + + return report + + +async def _detect_weak_options( + account_id: UUID, + period_start: datetime, + db: AsyncSession, +) -> list[KnowledgeGap]: + """Signal 1: Find questions where engineers frequently use free-text escapes.""" + # Count free-text usage per step context_message (the question asked) + result = await db.execute( + select( + AISessionStep.context_message, + func.count(AISessionStep.id).label("total"), + func.sum(case((AISessionStep.was_free_text.is_(True), 1), else_=0)).label("free_text_count"), + ) + .join(AISession, AISessionStep.session_id == AISession.id) + .where( + AISession.account_id == account_id, + AISession.created_at >= period_start, + AISessionStep.step_type == "question", + AISessionStep.context_message.isnot(None), + AISessionStep.responded_at.isnot(None), + ) + .group_by(AISessionStep.context_message) + .having(func.count(AISessionStep.id) >= 3) # Minimum sample size + .order_by(func.sum(case((AISessionStep.was_free_text.is_(True), 1), else_=0)).desc()) + .limit(5) + ) + + gaps = [] + for row in result.all(): + context_msg, total_raw, free_text_raw = row + total = int(total_raw or 0) + free_text_count = int(free_text_raw or 0) + if total == 0 or not free_text_count: + continue + rate = free_text_count / total + if rate < 0.3: + continue + + severity = "high" if rate > 0.6 else "medium" + gaps.append(KnowledgeGap( + gap_type="weak_options", + severity=severity, + title=f"Weak options: {(context_msg or '')[:80]}", + description=( + f"Engineers used free-text input {free_text_count}/{total} times " + f"({rate:.0%}) when asked this question. The predefined options " + f"may not cover common scenarios." + ), + evidence={ + "context_message": context_msg, + "total_responses": total, + "free_text_count": free_text_count, + "free_text_rate": round(rate, 3), + }, + suggested_action="Review the free-text responses and add common answers as options.", + )) + + return gaps + + +async def _detect_high_escalation( + account_id: UUID, + period_start: datetime, + db: AsyncSession, +) -> list[KnowledgeGap]: + """Signal 2: Find domains with >40% escalation rate.""" + result = await db.execute( + select( + AISession.problem_domain, + func.count(AISession.id).label("total"), + func.sum(case( + (AISession.status == "resolved", 1), else_=0 + )).label("resolved"), + func.sum(case( + (AISession.status.in_(["escalated", "requesting_escalation"]), 1), else_=0 + )).label("escalated"), + ) + .where( + AISession.account_id == account_id, + AISession.created_at >= period_start, + AISession.problem_domain.isnot(None), + AISession.status.in_(["resolved", "escalated", "requesting_escalation"]), + ) + .group_by(AISession.problem_domain) + .having(func.count(AISession.id) >= 3) # Minimum sample + ) + + gaps = [] + for row in result.all(): + domain, total_raw, resolved_raw, escalated_raw = row + total = int(total_raw or 0) + resolved = int(resolved_raw or 0) + escalated = int(escalated_raw or 0) + if total == 0 or not escalated: + continue + escalation_rate = escalated / total + if escalation_rate < 0.4: + continue + + severity = "high" if escalation_rate > 0.6 else "medium" + gaps.append(KnowledgeGap( + gap_type="high_escalation", + domain=domain, + severity=severity, + title=f"High escalation rate in {domain}", + description=( + f"{escalated}/{total} sessions ({escalation_rate:.0%}) in {domain} " + f"were escalated. Only {resolved} resolved independently." + ), + evidence={ + "domain": domain, + "total": total, + "resolved": resolved, + "escalated": escalated, + "escalation_rate": round(escalation_rate, 3), + }, + suggested_action=f"Create or improve troubleshooting flows for {domain} issues.", + )) + + return gaps + + +async def _detect_uncharted_territory( + account_id: UUID, + period_start: datetime, + db: AsyncSession, +) -> list[KnowledgeGap]: + """Signal 3: Find discovery-mode resolutions (novel problems solved without flows).""" + result = await db.execute( + select( + AISession.problem_domain, + func.count(AISession.id).label("count"), + ) + .where( + AISession.account_id == account_id, + AISession.created_at >= period_start, + AISession.status == "resolved", + AISession.confidence_tier == "discovery", + ) + .group_by(AISession.problem_domain) + .having(func.count(AISession.id) >= 2) + .order_by(func.count(AISession.id).desc()) + .limit(5) + ) + + gaps = [] + for row in result.all(): + domain, count = row + severity = "high" if count >= 5 else "medium" if count >= 3 else "low" + domain_label = domain or "unknown domain" + gaps.append(KnowledgeGap( + gap_type="uncharted_territory", + domain=domain, + severity=severity, + title=f"Novel resolutions in {domain_label}", + description=( + f"{count} sessions in {domain_label} were resolved in discovery mode " + f"(no matching flow, low confidence). These represent knowledge capture " + f"opportunities — check the Review Queue for auto-generated proposals." + ), + evidence={ + "domain": domain, + "discovery_resolution_count": count, + }, + suggested_action="Review pending flow proposals or create flows from these session patterns.", + )) + + return gaps + + +async def _detect_repeated_patterns( + account_id: UUID, + period_start: datetime, + db: AsyncSession, +) -> list[KnowledgeGap]: + """Signal 4: Find repeated unmatched intake patterns (keyword-frequency based). + + Phase 3 uses keyword frequency on problem_summary. Phase 4 will use + embedding clustering for deeper semantic analysis. + """ + # Get problem summaries from unmatched sessions + result = await db.execute( + select(AISession.problem_summary, AISession.problem_domain) + .where( + AISession.account_id == account_id, + AISession.created_at >= period_start, + AISession.problem_summary.isnot(None), + AISession.matched_flow_id.is_(None), + ) + .limit(200) + ) + rows = result.all() + + if len(rows) < 3: + return [] + + # Extract keywords from summaries and count frequency + word_counts: Counter[str] = Counter() + domain_for_word: dict[str, str | None] = {} + for summary, domain in rows: + if not summary: + continue + words = set(summary.lower().split()) + # Filter out common stop words and short words + stop_words = {"the", "a", "an", "is", "are", "was", "were", "in", "on", "at", + "to", "for", "of", "and", "or", "not", "can", "can't", "with", + "from", "by", "this", "that", "it", "its", "has", "have", "had", + "user", "users", "issue", "error", "problem"} + keywords = {w for w in words if len(w) > 3 and w not in stop_words} + for kw in keywords: + word_counts[kw] += 1 + if kw not in domain_for_word: + domain_for_word[kw] = domain + + gaps = [] + # Find keywords that appear in many unmatched sessions + for keyword, count in word_counts.most_common(3): + if count < 3: + continue + severity = "medium" if count >= 5 else "low" + domain = domain_for_word.get(keyword) + gaps.append(KnowledgeGap( + gap_type="repeated_pattern", + domain=domain, + severity=severity, + title=f"Recurring unmatched pattern: '{keyword}'", + description=( + f"The keyword '{keyword}' appeared in {count} sessions that had no " + f"matching flow. This may indicate a systematic knowledge gap." + ), + evidence={ + "keyword": keyword, + "unmatched_session_count": count, + "domain": domain, + }, + suggested_action=f"Search for '{keyword}' in recent sessions and consider creating a flow.", + )) + + return gaps diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index f15d4410..f6111f7e 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -39,7 +39,7 @@ services: - AI_PROVIDER=anthropic - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY} - GOOGLE_AI_API_KEY=${GOOGLE_AI_API_KEY} - - CORS_ORIGINS=["http://localhost:3000","http://localhost:5173","http://127.0.0.1:3000","http://127.0.0.1:5173"] + - CORS_ORIGINS=["http://localhost:3000","http://localhost:5173","http://127.0.0.1:3000","http://127.0.0.1:5173","http://192.168.0.9:5173","http://192.168.0.9:3000"] depends_on: db: condition: service_healthy @@ -55,7 +55,7 @@ services: - ./frontend:/app - /app/node_modules environment: - - VITE_API_URL=http://localhost:8000 + - VITE_API_URL=http://192.168.0.9:8000 depends_on: - backend diff --git a/docs/2026-03-18-flowpilot-first-pivot-phase3.md b/docs/2026-03-18-flowpilot-first-pivot-phase3.md new file mode 100644 index 00000000..bf71ac15 --- /dev/null +++ b/docs/2026-03-18-flowpilot-first-pivot-phase3.md @@ -0,0 +1,914 @@ +# FlowPilot-First Pivot — Phase 3: Knowledge Flywheel, Script Generator & Analytics + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Close the learning loop. Every resolved AI session automatically proposes new flows or enhancements to existing ones. Team leads curate quality through a Review Queue. FlowPilot can invoke the Script Generator mid-session. Analytics track MTTR, resolution rates, and knowledge coverage to show the ROI. + +**Architecture:** Builds on Phase 1 (AI sessions, FlowPilot Engine, Flow Matching) and Phase 2 (PSA integration, escalation handoff). Introduces the `flow_proposals` model, a post-session analysis pipeline, the Review Queue UI, in-session script generation, and an AI-enhanced analytics dashboard. + +**Tech Stack:** FastAPI, SQLAlchemy 2.0 (async), Anthropic Claude (flow proposal generation), pgvector, React, TypeScript, Tailwind CSS v4 (`@tailwindcss/vite`), Recharts (analytics charts) + +**Prerequisites:** +- Phase 1 complete (AI session core) +- Phase 2 complete (PSA integration, escalation handoff) +- Existing models: `ScriptCategory`, `ScriptTemplate`, `ScriptGeneration` (from Script Generator feature) +- Existing services: `script_template_engine.py`, `session_to_flow_service.py`, `embedding_service.py` +- Existing frontend: `ScriptLibraryPage.tsx`, `ScriptConfigurePane.tsx`, `ScriptParameterForm.tsx`, `ScriptPreview.tsx`, `PowerShellHighlighter.tsx`, `TeamAnalyticsPage.tsx` +- Existing schemas: `schemas/session_to_flow.py`, `schemas/script_template.py` + +**Existing patterns to follow:** + +- Session-to-flow: `app/services/session_to_flow_service.py` — converts legacy `Session` model to tree structures. **NOTE:** This service works with the legacy `Session` model (`session.decisions`, `session.outcome`, `session.scratchpad`), NOT `AISession`. The Knowledge Flywheel must build its own flow generation logic reading from `AISession.conversation_messages` and `AISession.steps`. Use this service as a reference for LLM prompt structure and tree format only. +- Script templates: `app/services/script_template_engine.py` — parameter substitution, validation, sanitization +- Embeddings: `app/services/embedding_service.py` — Voyage AI embeddings for vector search +- Analytics: `app/api/endpoints/analytics.py` — existing team analytics patterns +- Phase 1 engine: `app/services/flowpilot_engine.py` — structured JSON output contracts +- Frontend API pattern: `src/api/aiSessions.ts` uses `aiSessionsApi` object pattern + +**Pivot architecture doc:** `docs/ResolutionFlow_Pivot_Architecture.docx` + +--- + +## Context: What Phase 3 Adds + +Phase 1 built the AI session core. Phase 2 connected it to PSA tickets. Phase 3 makes the system get smarter over time: + +**Knowledge Flywheel:** Every resolved session is analyzed. The system proposes new flows from novel resolutions, suggests enhancements to existing flows when it discovers new branches, and reinforces proven flows when sessions follow known paths. Human-in-the-loop Review Queue ensures quality. + +**In-Session Script Generator:** FlowPilot can invoke the Script Generator contextually during diagnosis. When it detects the engineer needs a PowerShell script (e.g., "reset this user's AD password"), it surfaces the script generator with parameters pre-filled from session context. + +**AI-Enhanced Analytics:** MTTR trends, resolution rates by category, knowledge coverage heatmap, FlowPilot accuracy metrics, knowledge gap detection, and flow quality scoring. + +--- + +## Slice 1: Flow Proposals Model & Post-Session Analysis + +### Task 1: Create FlowProposal model + +**Files:** +- Create: `backend/app/models/flow_proposal.py` + +```python +"""Flow proposal model. + +Generated by the Knowledge Flywheel after AI sessions resolve. +Represents a proposed new flow or enhancement awaiting human review. +""" +import uuid +from datetime import datetime, timezone +from typing import Optional, Any, TYPE_CHECKING + +from sqlalchemy import String, Text, DateTime, ForeignKey, Integer, Float, CheckConstraint +from sqlalchemy.orm import Mapped, mapped_column, relationship +from sqlalchemy.dialects.postgresql import UUID, JSONB + +from app.core.database import Base + +if TYPE_CHECKING: + from app.models.user import User + from app.models.team import Team + from app.models.account import Account + from app.models.tree import Tree + from app.models.ai_session import AISession + + +class FlowProposal(Base): + """A proposed new flow or enhancement generated from an AI session. + + proposal_type: + - new_flow: No similar flow exists. Full flow definition proposed. + - enhancement: Similar flow exists but session discovered new branch/edge case. + - branch_addition: A single new branch to add to an existing flow. + + status: + - pending: Awaiting review + - approved: Reviewed and published to knowledge base + - modified: Reviewer edited before publishing + - rejected: Reviewer decided not to publish (bad quality) + - dismissed: Parked for later — not wrong, just not actionable now. Can resurface if supporting_session_count grows. + - auto_reinforced: Session matched existing flow exactly (no review needed) + """ + __tablename__ = "flow_proposals" + __table_args__ = ( + CheckConstraint( + "proposal_type IN ('new_flow', 'enhancement', 'branch_addition')", + name="ck_flow_proposals_type", + ), + CheckConstraint( + "status IN ('pending', 'approved', 'modified', 'rejected', 'dismissed', 'auto_reinforced')", + name="ck_flow_proposals_status", + ), + ) + + id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), primary_key=True, default=uuid.uuid4 + ) + account_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + ForeignKey("accounts.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + team_id: Mapped[Optional[uuid.UUID]] = mapped_column( + UUID(as_uuid=True), + ForeignKey("teams.id", ondelete="SET NULL"), + nullable=True, + index=True, + ) + source_session_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + ForeignKey("ai_sessions.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + + # ── Proposal details ── + proposal_type: Mapped[str] = mapped_column( + String(30), nullable=False, + ) + target_flow_id: Mapped[Optional[uuid.UUID]] = mapped_column( + UUID(as_uuid=True), + ForeignKey("trees.id", ondelete="SET NULL"), + nullable=True, + comment="For enhancements: which existing flow to modify", + ) + title: Mapped[str] = mapped_column( + String(255), nullable=False, + comment="Human-readable title for the proposed flow", + ) + description: Mapped[Optional[str]] = mapped_column( + Text, nullable=True, + comment="AI-generated description of what this flow covers", + ) + proposed_flow_data: Mapped[dict[str, Any]] = mapped_column( + JSONB, nullable=False, + comment="Complete flow/tree_structure definition (nodes, edges, conditions)", + ) + proposed_diff: Mapped[Optional[dict[str, Any]]] = mapped_column( + JSONB, nullable=True, + comment="For enhancements: what changed vs existing flow", + ) + + # ── Scoring ── + confidence_score: Mapped[float] = mapped_column( + Float, nullable=False, default=0.0, + comment="How confident the system is in this proposal (0.0-1.0)", + ) + supporting_session_count: Mapped[int] = mapped_column( + Integer, nullable=False, default=1, + comment="Number of sessions with similar resolution paths", + ) + supporting_session_ids: Mapped[list] = mapped_column( + JSONB, nullable=False, default=list, + comment="Array of session IDs that support this proposal", + ) + problem_domain: Mapped[Optional[str]] = mapped_column( + String(100), nullable=True, + ) + + # ── Review ── + status: Mapped[str] = mapped_column( + String(30), nullable=False, default="pending", index=True, + ) + reviewed_by: Mapped[Optional[uuid.UUID]] = mapped_column( + UUID(as_uuid=True), + ForeignKey("users.id", ondelete="SET NULL"), + nullable=True, + ) + reviewer_notes: Mapped[Optional[str]] = mapped_column( + Text, nullable=True, + ) + published_flow_id: Mapped[Optional[uuid.UUID]] = mapped_column( + UUID(as_uuid=True), + ForeignKey("trees.id", ondelete="SET NULL"), + nullable=True, + comment="The flow that was created/updated when this proposal was approved", + ) + + # ── Timestamps ── + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) + ) + reviewed_at: Mapped[Optional[datetime]] = mapped_column( + DateTime(timezone=True), nullable=True, + ) + + # ── Relationships ── + account: Mapped["Account"] = relationship("Account") + team: Mapped[Optional["Team"]] = relationship("Team") + source_session: Mapped["AISession"] = relationship("AISession") + target_flow: Mapped[Optional["Tree"]] = relationship("Tree", foreign_keys=[target_flow_id]) + published_flow: Mapped[Optional["Tree"]] = relationship("Tree", foreign_keys=[published_flow_id]) + reviewer: Mapped[Optional["User"]] = relationship("User") +``` + +Register in `app/models/__init__.py`: +```python +from .flow_proposal import FlowProposal +``` + +Add to `__all__`. + +### Task 2: Create Alembic migration + +Generate with: +```bash +cd /projects/patherly/backend +DATABASE_URL=postgresql://postgres:postgres@resolutionflow_postgres:5432/resolutionflow \ + venv/bin/alembic revision --autogenerate -m "add flow_proposals table" +``` + +Indexes: `account_id`, `team_id`, `source_session_id`, `status`, `target_flow_id`, `created_at`. + +**Verification:** Run `alembic upgrade head`. Verify table exists. + +``` +git commit -m "feat(knowledge): add FlowProposal model + migration" +``` + +### Task 3: Build post-session analysis service (Knowledge Flywheel engine) + +**Files:** +- Create: `backend/app/services/knowledge_flywheel.py` + +**Architecture:** + +This service runs after every successful session resolution. It analyzes the session and produces one of three outcomes: + +**1. New Flow Proposal (`new_flow`):** +- Triggered when: Session resolved with `confidence_tier = "discovery"` or `"exploring"`, AND no flow was matched (or match_score < 0.5) +- Process: Make an LLM call (standard tier) with the full session conversation, asking it to: + - Generate a flow title and description + - Convert the diagnostic path into a tree_structure (nodes, edges, conditions) + - Identify the key decision points and branching logic + - Suggest match_keywords for future semantic matching +- Store as a `FlowProposal` with `proposal_type = "new_flow"` and `status = "pending"` + +**2. Flow Enhancement Proposal (`enhancement`):** +- Triggered when: Session matched an existing flow (match_score > 0.5) but diverged at some point (engineer used free-text escape or chose a path not in the flow) +- Process: LLM call comparing the session path with the matched flow, identifying: + - New branches that should be added + - Options that should be added to existing questions + - Steps that should be reordered based on what worked +- Store as `FlowProposal` with `proposal_type = "enhancement"`, `target_flow_id` set, and `proposed_diff` containing the changes + +**3. Flow Reinforcement (`auto_reinforced`):** +- Triggered when: Session followed an existing flow closely (match_score > 0.8, no free-text escapes, resolution matched the flow's expected outcome) +- Process: No LLM call needed. Update the flow's `success_rate` and `last_matched_at`. Create a `FlowProposal` with `status = "auto_reinforced"` for tracking purposes only (no review needed). + +**Key implementation details:** + +- **Do NOT use `asyncio.create_task()`.** Use the APScheduler background job pattern established by `psa_retry_scheduler.py`. Add an `analysis_status` column to `AISession` (values: `null`, `pending`, `completed`, `failed`). Set it to `pending` in `resolve_session()`. A periodic scheduler job picks up pending sessions and runs analysis. This is resilient to server restarts and retryable on failure. +- The LLM call for flow generation should use the existing `ai_provider.generate_json()` with a specific system prompt for flow construction +- The generated `tree_structure` must match the existing tree format used by the Flow Editor (check `models/tree.py` → `tree_structure` JSONB schema). Troubleshooting flows use nested `children` nodes; procedural flows use linear `steps` arrays. The Knowledge Flywheel should generate **troubleshooting** tree format for diagnostic sessions. +- **Data source:** `AISession` uses `conversation_messages` (JSONB list of role/content dicts) and the `steps` relationship (`AISessionStep` with `step_type`, `content`, `selected_option`, `free_text_input`, `action_result`). Build session context from these — do NOT reference `session.decisions` or `session.scratchpad` (those are legacy `Session` model fields). +- For enhancement proposals, also generate a human-readable diff description (e.g., "Added new branch for 'Error code 0x80070005' at step 3") +- Track supporting sessions: if multiple sessions resolve a similar novel problem, increase `supporting_session_count` and `confidence_score` on the existing proposal rather than creating duplicates. Use `problem_domain` match + embedding similarity on `title + description` against existing pending proposals (threshold >0.85 cosine similarity → merge) +- **Rate limiting:** Add a daily budget cap for proposal generation LLM calls (configurable, default 50/day per account) to prevent runaway costs from high-volume teams + +**System prompt for flow generation (excerpt):** + +```python +FLOW_GENERATION_PROMPT = """You are a knowledge engineer converting a troubleshooting session into a reusable flow definition. + +Given the session transcript below, generate a JSON flow definition that captures the diagnostic logic so other engineers can follow the same path. + +## OUTPUT FORMAT +Respond with ONLY valid JSON: +{ + "title": "Short descriptive title", + "description": "When to use this flow", + "match_keywords": ["keyword1", "keyword2", ...], + "problem_domain": "active_directory | networking | m365 | ...", + "tree_structure": { + "id": "root", + "type": "question", + "question": "First diagnostic question", + "children": [ + { + "id": "opt1", + "label": "Option text", + "type": "question | action | solution", + ... + } + ] + } +} + +## RULES +- tree_structure must follow the ResolutionFlow tree format +- Every question node must have 2-5 children (options) +- Action nodes describe what the engineer should do +- Solution nodes describe the resolution +- Include the key diagnostic questions that narrowed down the problem +- Skip redundant or dead-end paths from the session +- match_keywords should be symptoms, error messages, and technology names +""" +``` + +**Verification:** Resolve an AI session (discovery mode, no matched flow). Wait 2-3 seconds. Check `flow_proposals` table — verify a `new_flow` proposal was created with a valid `tree_structure`. Resolve a session that matched an existing flow but diverged — verify an `enhancement` proposal was created. Resolve a session that followed a flow exactly — verify an `auto_reinforced` record was created and the flow's stats were updated. + +``` +git commit -m "feat(knowledge): add Knowledge Flywheel post-session analysis" +``` + +### Task 4: Wire Knowledge Flywheel into session resolution + +**Files:** +- Edit: `backend/app/services/flowpilot_engine.py` — set `analysis_status = "pending"` in `resolve_session()` +- Create: `backend/app/services/knowledge_flywheel_scheduler.py` — APScheduler job +- Edit: `backend/app/main.py` — register the scheduler job +- Migration: add `analysis_status` column to `ai_sessions` table + +**Migration:** Add `analysis_status` column: + +```python +# String(20), nullable=True, default=None +# Values: null (not applicable), "pending", "completed", "failed" +op.add_column('ai_sessions', sa.Column('analysis_status', sa.String(20), nullable=True)) +``` + +**In `resolve_session()`**, after documentation is generated and PSA push is queued: + +```python +session.analysis_status = "pending" +``` + +**Scheduler (`knowledge_flywheel_scheduler.py`):** + +Follow the same pattern as `psa_retry_scheduler.py`: + +```python +async def process_pending_analyses() -> None: + """Process resolved sessions awaiting Knowledge Flywheel analysis.""" + async with async_session_maker() as db: + result = await db.execute( + select(AISession) + .options(selectinload(AISession.steps)) + .where(AISession.analysis_status == "pending") + .limit(10) + ) + sessions = result.scalars().all() + for session in sessions: + try: + await analyze_session(session, db) + session.analysis_status = "completed" + except Exception as e: + logger.warning("Knowledge Flywheel failed for %s: %s", session.id, e) + session.analysis_status = "failed" + await db.commit() +``` + +Register in `main.py` lifespan alongside the existing PSA retry scheduler (5-minute interval). + +**Verification:** Resolve a session. Confirm the response returns immediately. Wait for the scheduler tick (~5 min or trigger manually). Check `flow_proposals` table — confirm the proposal was created and `analysis_status = "completed"`. + +``` +git commit -m "feat(knowledge): wire Knowledge Flywheel into session resolution via scheduler" +``` + +--- + +## Slice 2: Review Queue + +### Task 5: Create Review Queue API endpoints + +**Files:** +- Create: `backend/app/schemas/flow_proposal.py` +- Edit: `backend/app/api/endpoints/ai_sessions.py` (or create a new `flow_proposals.py` router) + +**Schemas:** + +```python +class FlowProposalSummary(BaseModel): + id: UUID + proposal_type: str + title: str + description: str | None + problem_domain: str | None + confidence_score: float + supporting_session_count: int + status: str + target_flow_id: UUID | None + target_flow_name: str | None # Joined from trees table + source_session_id: UUID + created_at: datetime + + model_config = {"from_attributes": True} + +class FlowProposalDetail(FlowProposalSummary): + proposed_flow_data: dict[str, Any] + proposed_diff: dict[str, Any] | None + supporting_session_ids: list[str] + reviewer_notes: str | None + reviewed_by: UUID | None + reviewed_at: datetime | None + +class ReviewProposalRequest(BaseModel): + action: str # "approve" | "reject" | "modify" + reviewer_notes: str | None = None + modified_flow_data: dict[str, Any] | None = None # Only for "modify" + +class FlowProposalStats(BaseModel): + pending_count: int + approved_this_week: int + rejected_this_week: int + auto_reinforced_this_week: int + top_domains: list[dict[str, Any]] # [{domain, count}] +``` + +**Endpoints:** + +``` +GET /api/v1/flow-proposals — List proposals (filterable by status, type, domain) +GET /api/v1/flow-proposals/stats — Dashboard stats for the review queue +GET /api/v1/flow-proposals/{id} — Get proposal detail with full flow data +POST /api/v1/flow-proposals/{id}/review — Approve, reject, or modify a proposal +``` + +**Auth:** `require_engineer_or_admin` for listing/detail. Review actions (approve/reject/modify) require inline check: `if not (current_user.is_super_admin or current_user.is_team_admin): raise HTTPException(403, "Team admin required")`. No existing `require_team_admin` dep exists — add one to `api/deps.py` or use inline checks. + +**Review flow:** + +- **Approve:** Create a new `Tree` from `proposed_flow_data` (for `new_flow`) or update the existing tree (for `enhancement`). Set `tree.origin = "ai_generated"` or `"ai_enhanced"`. Set `tree.source_session_id`. Set proposal `status = "approved"`, `published_flow_id` = new tree ID. +- **Modify:** Same as approve, but use `modified_flow_data` instead of `proposed_flow_data`. Set proposal `status = "modified"`. +- **Reject:** Set proposal `status = "rejected"`. No flow changes. +- **Dismiss:** Set proposal `status = "dismissed"`. Unlike reject (bad quality), dismiss means "not now" — the proposal can resurface if `supporting_session_count` grows. Add `"dismissed"` to the `FlowProposal` status constraint. + +**NOTE on `session_to_flow_service.py`:** This service works with the legacy `Session` model and CANNOT be called directly for `AISession`-based proposals. The Knowledge Flywheel generates `proposed_flow_data` in its own Task 3 — by the time we reach the Review Queue, the flow structure is already in the proposal. The review endpoint just needs to create a `Tree` from the pre-generated `proposed_flow_data` dict (set `tree_type`, `tree_structure`, `origin`, `source_session_id`, etc.). No LLM call needed at review time. + +**Verification:** Create a few proposals via the Knowledge Flywheel. Hit the list endpoint. Review one (approve). Verify a new tree was created. Review another (reject). Verify no tree change. + +``` +git commit -m "feat(knowledge): add Review Queue API endpoints" +``` + +### Task 6: Build Review Queue frontend + +**Files:** +- Create: `frontend/src/pages/ReviewQueuePage.tsx` +- Create: `frontend/src/components/flowpilot/ProposalCard.tsx` +- Create: `frontend/src/components/flowpilot/ProposalDetail.tsx` +- Create: `frontend/src/components/flowpilot/ProposalDiffView.tsx` +- Create: `frontend/src/components/flowpilot/ReviewActions.tsx` +- Create: `frontend/src/api/flowProposals.ts` +- Create: `frontend/src/types/flow-proposal.ts` +- Edit: `frontend/src/router.tsx` +- Edit sidebar navigation + +**Page layout:** Two-panel design similar to the Script Library page. + +**Left panel — Proposal list:** +- Filter tabs: "Pending" (default), "Approved", "Rejected", "Dismissed", "All" +- Filter by domain (dropdown) +- Sort by: newest, highest confidence, most supporting sessions +- Each card shows: title, proposal type badge (`new_flow` green, `enhancement` amber, `branch_addition` blue), domain badge, confidence score, supporting session count, created date + +**Right panel — Proposal detail:** +- Full proposal info: title, description, source session link, confidence +- **For new_flow:** Flow preview — render the proposed `tree_structure` using a simplified read-only version of the flow editor or a tree visualization +- **For enhancement:** Diff view showing what would change on the target flow (added nodes highlighted green, modified nodes highlighted amber) +- Source session link — click to open the session that generated this proposal in read-only mode +- Supporting sessions list (if count > 1) + +**Review actions (bottom bar):** + +- "Approve & Publish" (green) — creates the flow immediately +- "Edit & Publish" — uses `navigate('/editor/new', { state: { preloadedStructure: proposedFlowData, proposalId } })` to open the Flow Editor with the proposed structure pre-loaded (same `location.state` pattern used by `CreateFlowDropdown` → `AIPromptDialog`, see Lesson 46) +- "Dismiss" (muted) — parks the proposal for later; can resurface if supporting sessions grow +- "Reject" (red) — with optional reason textarea +- Reviewer notes input + +**Navigation:** +- Add "Review Queue" to the sidebar under "Knowledge Base" section +- Show a badge with pending count if > 0 (similar to notification badges) + +**Verification:** Navigate to Review Queue. See pending proposals. Click one. See the flow preview. Approve it. Verify a new tree appears in My Trees. Click "Edit & Publish" on another — verify it opens in the Flow Editor with the proposed structure pre-loaded. + +``` +git commit -m "feat(knowledge): add Review Queue frontend" +``` + +--- + +## Slice 3: Knowledge Gap Detection + +### Task 7: Build knowledge gap detection service + +**Files:** +- Create: `backend/app/services/knowledge_gap_service.py` + +**Architecture:** + +This service aggregates signals from AI sessions to identify gaps in the knowledge base: + +**Signal 1 — Frequent free-text escapes:** +- Query `ai_session_steps` where `was_free_text = true` +- Group by the `content` field (the question that was asked) and count +- High counts indicate FlowPilot's options don't cover a common scenario +- Return: list of questions with high free-text rates and the common free-text inputs + +**Signal 2 — High escalation rate by domain:** +- Query `ai_sessions` where `status = "escalated"`, group by `problem_domain` +- Compare escalation rate vs resolution rate per domain +- Domains with >40% escalation rate are flagged as knowledge gaps + +**Signal 3 — Discovery-mode resolutions:** +- Query `ai_sessions` where `status = "resolved"` AND `confidence_tier = "discovery"` at resolution +- These are novel problems that were solved — highest-value knowledge capture opportunities +- Group by `problem_domain` and rank by frequency + +**Signal 4 — Repeated similar intake patterns (DESCOPED to Phase 4):** +~~Use embedding similarity on `intake_content.text` across recent sessions and cluster similar intakes.~~ +**Reason:** The codebase has point-query embedding support (Voyage AI) but no batch embedding or clustering infrastructure. Implementing vector clustering (DBSCAN/k-means) over session embeddings is a significant undertaking. **Phase 3 alternative:** Use keyword frequency analysis on `problem_domain` + `problem_summary` text to find repeated unmatched patterns. Full embedding clustering deferred to Phase 4. + +**Return type:** + +```python +class KnowledgeGapReport(BaseModel): + generated_at: datetime + gaps: list[KnowledgeGap] + +class KnowledgeGap(BaseModel): + gap_type: str # "weak_options" | "high_escalation" | "uncharted_territory" | "repeated_pattern" + domain: str | None + severity: str # "high" | "medium" | "low" + title: str + description: str + evidence: dict[str, Any] # Supporting data (counts, examples, session IDs) + suggested_action: str # What to do about it +``` + +**Endpoint:** + +``` +GET /api/v1/analytics/knowledge-gaps +``` + +Returns the current knowledge gap report. Cache for 1 hour (expensive query). + +**Verification:** Run several AI sessions across different domains. Some should escalate, some should use free-text. Hit the knowledge gaps endpoint. Verify it returns reasonable gap analysis. + +``` +git commit -m "feat(knowledge): add knowledge gap detection service" +``` + +--- + +## Slice 4: In-Session Script Generator + +### Task 8: Enable FlowPilot to invoke Script Generator during sessions + +**Files:** +- Edit: `backend/app/services/flowpilot_engine.py` +- Edit: `frontend/src/components/flowpilot/FlowPilotStepCard.tsx` +- Create: `frontend/src/components/flowpilot/InSessionScriptGenerator.tsx` + +**Backend — System prompt enhancement:** + +Add available script templates to FlowPilot's system prompt context. When building the system prompt in `_build_system_prompt()`, include: + +```python +# Query available script templates +templates = await db.execute( + select(ScriptTemplate) + .where(ScriptTemplate.is_active == True) + .where(or_(ScriptTemplate.team_id == None, ScriptTemplate.team_id == team_id)) + .order_by(ScriptTemplate.usage_count.desc()) + .limit(20) +) +template_list = templates.scalars().all() + +# Add to system prompt +script_context = "\n--- AVAILABLE SCRIPTS ---\n" +for t in template_list: + script_context += f"- {t.name} (ID: {t.id}): {t.description}\n" + script_context += f" Parameters: {', '.join(p['key'] for p in t.parameters_schema.get('parameters', []))}\n" +script_context += "\nWhen the engineer needs to run a script, suggest a script_generation action with the template_id and pre-fill parameters from the diagnostic context.\n" +``` + +**Backend — Structured output for script actions:** + +FlowPilot already supports `action_type: "script_generation"` in its structured output contract (defined in Phase 1). When FlowPilot returns this type, the response includes: + +```json +{ + "type": "action", + "action_type": "script_generation", + "template_id": "uuid-of-the-template", + "pre_filled_params": { + "sam_account_name": "jsmith", + "ou_path": "OU=Users,DC=contoso,DC=com" + }, + "instructions": "Generate a password reset script for this user", + "confidence": 0.85 +} +``` + +The backend should validate that `template_id` exists and is accessible to the user's team. + +**IMPORTANT — Migration needed:** `ScriptGeneration.session_id` currently FKs to legacy `sessions.id`, NOT `ai_sessions.id`. Add a migration to add `ai_session_id` FK column to `script_generations`: + +```python +op.add_column('script_generations', sa.Column( + 'ai_session_id', sa.UUID(), sa.ForeignKey('ai_sessions.id', ondelete='SET NULL'), nullable=True +)) +op.create_index('ix_script_generations_ai_session_id', 'script_generations', ['ai_session_id']) +``` + +Update `ScriptGeneration` model to include `ai_session_id` mapped column. The existing `session_id` FK stays for legacy sessions. + +**Frontend — `InSessionScriptGenerator` component:** + +When `FlowPilotStepCard` receives a step with `action_type === "script_generation"`: + +1. Render the step card with a script generation UI embedded inline +2. Reuse the existing `ScriptParameterForm` component from the Script Library +3. Pre-fill parameters from `pre_filled_params` in the step content +4. Engineer can edit parameters and generate the script +5. On generation, call the existing `POST /api/v1/scripts/generate` endpoint with `ai_session_id` set to the current AI session +6. Display the generated script with the existing `ScriptPreview` component (PowerShell syntax highlighting) +7. Copy/download buttons +8. "Script generated" event is captured in `ai_session_steps` with `step_type = "script_generation"` and `script_generation_id` FK populated +9. After generating, show "Continue" button → engineer reports result back to FlowPilot + +**Key reuse:** Import and compose these existing components from `src/components/scripts/`: + +- `ScriptParameterForm` — dynamic form from parameter schema +- `ScriptPreview` — PowerShell syntax highlighting +- `PowerShellHighlighter` — tokenizer + +**Do NOT rebuild these components.** Wrap them in `InSessionScriptGenerator` which handles the session context (pre-filling, event capture, continue flow). + +**Verification:** Start an AI session about an AD issue. Progress until FlowPilot suggests a script generation action. See the script generator appear inline. Parameters should be pre-filled from conversation context. Generate the script. Copy it. Click continue. Verify the step is captured in session docs with `script_generation_id` populated. + +``` +git commit -m "feat(ai-session): add in-session Script Generator integration" +``` + +--- + +## Slice 5: AI-Enhanced Analytics Dashboard + +### Task 9: Build FlowPilot analytics API endpoints + +**Files:** +- Create: `backend/app/schemas/flowpilot_analytics.py` +- Create: `backend/app/api/endpoints/flowpilot_analytics.py` +- Edit: `backend/app/api/router.py` + +**Schemas:** + +```python +class FlowPilotDashboard(BaseModel): + """Top-level analytics dashboard data.""" + period: str # "7d" | "30d" | "90d" + total_sessions: int + resolved_sessions: int + escalated_sessions: int + abandoned_sessions: int + resolution_rate: float # percentage + avg_steps_to_resolution: float + avg_session_duration_minutes: float + avg_rating: float | None + mttr_minutes: float | None # Mean Time To Resolution + mttr_trend: list[MTTRDataPoint] # For chart + sessions_by_domain: list[DomainBreakdown] + confidence_breakdown: ConfidenceBreakdown + knowledge_coverage: KnowledgeCoverage + psa_metrics: PsaMetrics | None # None if no PSA connection + +class MTTRDataPoint(BaseModel): + date: str # ISO date + mttr_minutes: float + session_count: int + +class DomainBreakdown(BaseModel): + domain: str + total: int + resolved: int + escalated: int + resolution_rate: float + +class ConfidenceBreakdown(BaseModel): + guided_sessions: int + guided_resolution_rate: float + exploring_sessions: int + exploring_resolution_rate: float + discovery_sessions: int + discovery_resolution_rate: float + +class KnowledgeCoverage(BaseModel): + total_flows: int + ai_generated_flows: int + total_proposals_pending: int + proposals_approved_this_period: int + proposals_rejected_this_period: int + coverage_by_domain: list[DomainCoverage] + +class DomainCoverage(BaseModel): + domain: str + flow_count: int + session_count: int + guided_rate: float # % of sessions in this domain that hit "guided" confidence + +class PsaMetrics(BaseModel): + """PSA integration metrics (Phase 2 ROI data).""" + ticket_link_rate: float # % of sessions linked to a PSA ticket + auto_push_success_rate: float # % of pushes that succeeded on first try + auto_push_retry_success_rate: float # % that succeeded after retries + total_time_entries_logged: int + total_hours_logged: float +``` + +**Endpoints:** + +``` +GET /api/v1/analytics/flowpilot?period=30d — Main dashboard data +GET /api/v1/analytics/flowpilot/mttr-trend?period=90d — MTTR trend chart data +GET /api/v1/analytics/flowpilot/knowledge-gaps — Knowledge gap report (from Task 7) +``` + +**Auth:** Team admin or owner. Scope to account. + +**Key queries:** + +- MTTR: `AVG(resolved_at - created_at)` for sessions where `status = "resolved"`, grouped by date +- Confidence breakdown: `COUNT(*) GROUP BY confidence_tier` for resolved sessions +- Domain breakdown: `COUNT(*), SUM(CASE WHEN status='resolved')` grouped by `problem_domain` +- Knowledge coverage: Count flows per domain vs session count per domain. High session count + low flow count = poor coverage. + +**Verification:** Run several AI sessions over different days (can seed test data). Hit the analytics endpoint. Verify all fields are populated with reasonable values. + +``` +git commit -m "feat(analytics): add FlowPilot analytics API" +``` + +### Task 10: Build FlowPilot analytics dashboard frontend + +**Files:** +- Create: `frontend/src/pages/FlowPilotAnalyticsPage.tsx` +- Create: `frontend/src/components/flowpilot/analytics/MTTRChart.tsx` +- Create: `frontend/src/components/flowpilot/analytics/DomainBreakdownChart.tsx` +- Create: `frontend/src/components/flowpilot/analytics/ConfidenceBreakdown.tsx` +- Create: `frontend/src/components/flowpilot/analytics/KnowledgeCoverageMap.tsx` +- Create: `frontend/src/components/flowpilot/analytics/KnowledgeGapsPanel.tsx` +- Create: `frontend/src/api/flowpilotAnalytics.ts` +- Create: `frontend/src/types/flowpilot-analytics.ts` +- Edit: `frontend/src/router.tsx` + +**Design:** Follow existing `TeamAnalyticsPage.tsx` patterns. Use Recharts for charts (already a project dependency). + +**Layout:** + +**Top row — Key metrics cards:** + +- Total sessions (with trend arrow) +- Resolution rate (with trend) +- Average MTTR (with trend) +- Average rating (with star display) +- PSA ticket link rate (% of sessions linked to tickets) + +**Second row — Charts:** + +- MTTR trend area chart (Recharts `AreaChart` — matches existing `TeamAnalyticsPage.tsx` pattern, not `LineChart`) +- Domain breakdown bar chart (Recharts `BarChart`) — resolved vs escalated per domain + +**Third row — Intelligence:** +- Confidence tier donut chart — guided vs exploring vs discovery, with resolution rate overlay +- Knowledge coverage heatmap — domains as rows, columns for flow count / session count / guided rate / gap severity. Color-coded: green (well covered), amber (needs work), red (major gap) + +**Fourth row — Knowledge gaps:** +- `KnowledgeGapsPanel` — renders the knowledge gap report from Task 7 +- Each gap as a card with severity badge, description, and suggested action +- "Create Flow" CTA on high-severity gaps → opens the Flow Editor with suggested structure + +**Period selector:** Dropdown in the page header — 7 days, 30 days, 90 days. + +**Navigation:** Add "FlowPilot Analytics" under the existing "Analytics" section in the sidebar. + +**Verification:** Navigate to the analytics page. Select different periods. Verify charts render with real data. Check knowledge gaps section shows actionable insights. + +``` +git commit -m "feat(analytics): add FlowPilot analytics dashboard" +``` + +--- + +## Summary of All New/Modified Files + +### Backend — New +``` +app/models/flow_proposal.py # FlowProposal model +app/services/knowledge_flywheel.py # Post-session analysis engine +app/services/knowledge_flywheel_scheduler.py # APScheduler job for async analysis +app/services/knowledge_gap_service.py # Knowledge gap detection +app/schemas/flow_proposal.py # Proposal schemas +app/schemas/flowpilot_analytics.py # Analytics schemas +app/api/endpoints/flow_proposals.py # Review Queue API +app/api/endpoints/flowpilot_analytics.py # Analytics API +alembic/versions/xxx_add_flow_proposals.py # Migration (flow_proposals table) +alembic/versions/xxx_add_analysis_status.py # Migration (ai_sessions.analysis_status) +alembic/versions/xxx_add_ai_session_id_to_scripts.py # Migration (script_generations.ai_session_id) +``` + +### Backend — Modified +``` +app/models/__init__.py # Register FlowProposal +app/models/ai_session.py # Add analysis_status column +app/models/script_template.py # Add ai_session_id FK to ScriptGeneration +app/api/deps.py # Add require_team_admin dependency +app/api/router.py # Register new routers +app/main.py # Register knowledge_flywheel_scheduler +app/services/flowpilot_engine.py # Set analysis_status, add script context to system prompt +``` + +### Frontend — New +``` +src/pages/ReviewQueuePage.tsx # Review Queue page +src/pages/FlowPilotAnalyticsPage.tsx # Analytics dashboard +src/components/flowpilot/ProposalCard.tsx # Proposal list card +src/components/flowpilot/ProposalDetail.tsx # Proposal detail panel +src/components/flowpilot/ProposalDiffView.tsx # Enhancement diff viewer +src/components/flowpilot/ReviewActions.tsx # Approve/reject/modify bar +src/components/flowpilot/InSessionScriptGenerator.tsx # Script gen embedded in session +src/components/flowpilot/analytics/MTTRChart.tsx # MTTR trend chart +src/components/flowpilot/analytics/DomainBreakdownChart.tsx +src/components/flowpilot/analytics/ConfidenceBreakdown.tsx +src/components/flowpilot/analytics/KnowledgeCoverageMap.tsx +src/components/flowpilot/analytics/KnowledgeGapsPanel.tsx +src/api/flowProposals.ts # Proposals API client +src/api/flowpilotAnalytics.ts # Analytics API client +src/types/flow-proposal.ts # Proposal types +src/types/flowpilot-analytics.ts # Analytics types +``` + +### Frontend — Modified +``` +src/components/flowpilot/FlowPilotStepCard.tsx # Handle script_generation action type +src/router.tsx # Review Queue + Analytics routes +src/components/sidebar/ # New nav entries +``` + +--- + +## Database Changes + +**Migration:** Create `flow_proposals` table with all columns, constraints, and indexes. + +**Run migration:** +```bash +cd /projects/patherly/backend +DATABASE_URL=postgresql://postgres:postgres@resolutionflow_postgres:5432/resolutionflow \ + venv/bin/alembic upgrade head +``` + +--- + +## Testing Strategy + +### Backend Unit Tests + +**Files:** `backend/tests/test_knowledge_flywheel.py` + +- Test new_flow proposal generation from a discovery-mode session +- Test enhancement proposal generation from a divergent session +- Test auto_reinforcement for matching sessions +- Test duplicate detection (similar proposals get merged) +- Test generated tree_structure matches the expected format + +**Files:** `backend/tests/test_knowledge_gap_service.py` + +- Test free-text escape detection +- Test escalation rate calculation by domain +- Test discovery-mode session grouping + +**Files:** `backend/tests/test_flow_proposals_api.py` + +- Test list/filter proposals +- Test approve → verify tree created +- Test modify → verify modified data used +- Test reject → verify no tree change +- Test RBAC (non-admin can't review) + +### Frontend Manual Testing + +1. Resolve several AI sessions (mix of discovery, exploring, guided) +2. Navigate to Review Queue — verify proposals appear +3. Approve a new_flow proposal — verify tree appears in library +4. Click "Edit & Publish" — verify Flow Editor opens with proposed structure +5. Start a session about an AD issue — progress until FlowPilot suggests a script — generate it inline +6. Navigate to FlowPilot Analytics — verify all charts render with data +7. Check Knowledge Gaps — verify actionable insights appear + +--- + +## What Comes Next (Phase 4+ — NOT in scope here) + +- **Template Marketplace:** Public templates gallery for SEO + lead gen +- **Multi-PSA support:** Autotask, Halo PSA, Datto PSA integrations +- **SSO/SAML:** Enterprise authentication +- **Custom AI training:** Per-account fine-tuning on company procedures +- **Mobile optimization:** Responsive design pass for tablet/phone sessions +- **Webhook integrations:** Slack notifications, Teams alerts on escalation +- **API for third-party tools:** Public API for RMM/PSA vendors to integrate diff --git a/docs/plans/2026-03-16-stack-priorities-and-playwright-plan.md b/docs/plans/2026-03-16-stack-priorities-and-playwright-plan.md index 569c2b5f..c7c19fec 100644 --- a/docs/plans/2026-03-16-stack-priorities-and-playwright-plan.md +++ b/docs/plans/2026-03-16-stack-priorities-and-playwright-plan.md @@ -1,7 +1,7 @@ # Stack Priorities And Playwright Plan > **Date:** 2026-03-16 -> **Updated:** 2026-03-17 +> **Updated:** 2026-03-18 > **Product:** ResolutionFlow > **Purpose:** Turn the recent stack-gap review into a practical, sequenced execution plan diff --git a/docs/plans/2026-03-18-flowpilot-first-pivot-phase1.md b/docs/plans/2026-03-18-flowpilot-first-pivot-phase1.md new file mode 100644 index 00000000..20f7c55d --- /dev/null +++ b/docs/plans/2026-03-18-flowpilot-first-pivot-phase1.md @@ -0,0 +1,1204 @@ +# FlowPilot-First Pivot — Phase 1: AI Session Core + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Build the core AI-powered troubleshooting session — the new flagship product experience. An engineer starts a session by describing a problem (free text, screenshot, log paste), FlowPilot guides them through structured diagnosis with selectable options + free-text escape, and the session closes with auto-generated documentation. Flows are built as a byproduct of real resolutions. + +**Architecture:** New `AISession` and `AISessionStep` models. New `FlowPilotEngine` service orchestrating LLM calls with structured JSON output contracts. New `FlowMatchingEngine` service for semantic matching against existing flows. New frontend session UX with conversational layout. Reuses existing `ai_provider.py`, `embedding_service.py`, `rag_service.py`, and `copilot_service.py` patterns. + +**Tech Stack:** FastAPI, SQLAlchemy 2.0 (async), Anthropic Claude Sonnet 4 (structured JSON output), pgvector (flow matching), React, TypeScript, Tailwind CSS v4, shadcn/ui + +**Pivot architecture doc:** `docs/ResolutionFlow_Pivot_Architecture.docx` (in project root — the full strategic context) + +**Existing patterns to follow:** +- Models: `app/models/copilot_conversation.py`, `app/models/script_template.py` +- Services: `app/services/copilot_service.py`, `app/core/ai_provider.py` +- API: `app/api/endpoints/copilot.py` (auth, quota, error handling patterns) +- Frontend: `src/pages/FlowAssistPage.tsx`, `src/components/copilot/` + +--- + +## Context: What This Pivot Changes + +ResolutionFlow currently requires engineers to build flows manually before getting value. This pivot makes FlowPilot the primary interface — engineers bring a problem, FlowPilot guides diagnosis, and flows get built organically from real resolutions. + +**What stays:** Session runner UX (promoted to core), flow editor (repurposed for curation), script generator (elevated to in-session action), PSA integration (elevated to primary intake channel), all existing models and services. + +**What's new in Phase 1:** AI Session models, FlowPilot Engine service, Flow Matching Engine v1, new session intake + conversational diagnosis frontend, resolve/escalate endpoints with auto-documentation. + +--- + +## Slice 1: Database Models & Migration + +### Task 1: Create AISession model + +**Files:** +- Create: `backend/app/models/ai_session.py` + +```python +"""AI-powered troubleshooting session model. + +Represents a complete FlowPilot interaction from intake to resolution/escalation. +This is the central entity of the FlowPilot-First pivot. +""" +import uuid +from datetime import datetime, timezone +from typing import Optional, Any, TYPE_CHECKING + +from sqlalchemy import String, Text, DateTime, ForeignKey, Boolean, Integer, Float, CheckConstraint +import sqlalchemy as sa +from sqlalchemy.orm import Mapped, mapped_column, relationship +from sqlalchemy.dialects.postgresql import UUID, JSONB + +from app.core.database import Base + +if TYPE_CHECKING: + from app.models.user import User + from app.models.team import Team + from app.models.account import Account + from app.models.tree import Tree + from app.models.psa_connection import PsaConnection + + +class AISession(Base): + """A FlowPilot-guided troubleshooting session. + + Lifecycle: active → resolved | escalated | abandoned + Sessions may be paused and resumed (e.g., escalation handoff). + """ + __tablename__ = "ai_sessions" + __table_args__ = ( + CheckConstraint( + "intake_type IN ('free_text', 'psa_ticket', 'screenshot', 'log_paste', 'combined')", + name="ck_ai_sessions_intake_type", + ), + CheckConstraint( + "status IN ('active', 'paused', 'resolved', 'escalated', 'abandoned')", + name="ck_ai_sessions_status", + ), + CheckConstraint( + "confidence_tier IN ('guided', 'exploring', 'discovery')", + name="ck_ai_sessions_confidence_tier", + ), + ) + + id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), primary_key=True, default=uuid.uuid4 + ) + user_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + ForeignKey("users.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + account_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + ForeignKey("accounts.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + team_id: Mapped[Optional[uuid.UUID]] = mapped_column( + UUID(as_uuid=True), + ForeignKey("teams.id", ondelete="SET NULL"), + nullable=True, + index=True, + ) + + # ── Intake ── + intake_type: Mapped[str] = mapped_column( + String(20), nullable=False, default="free_text" + ) + intake_content: Mapped[dict[str, Any]] = mapped_column( + JSONB, nullable=False, default=dict, + comment="Original intake data: {text, image_urls, log_content, ticket_data}", + ) + problem_summary: Mapped[Optional[str]] = mapped_column( + Text, nullable=True, + comment="AI-generated one-line problem summary from intake", + ) + problem_domain: Mapped[Optional[str]] = mapped_column( + String(100), nullable=True, + comment="Classified domain: active_directory, networking, m365, hardware, etc.", + ) + + # ── Session state ── + status: Mapped[str] = mapped_column( + String(20), nullable=False, default="active", index=True, + ) + confidence_tier: Mapped[str] = mapped_column( + String(20), nullable=False, default="discovery", + comment="Current AI confidence: guided (>80%), exploring (40-80%), discovery (<40%)", + ) + confidence_score: Mapped[float] = mapped_column( + Float, nullable=False, default=0.0, + comment="Numeric confidence 0.0-1.0 for internal tracking", + ) + + # ── Flow matching ── + matched_flow_id: Mapped[Optional[uuid.UUID]] = mapped_column( + UUID(as_uuid=True), + ForeignKey("trees.id", ondelete="SET NULL"), + nullable=True, + comment="If following an existing flow, which one", + ) + match_score: Mapped[Optional[float]] = mapped_column( + Float, nullable=True, + comment="Similarity score of the matched flow (0.0-1.0)", + ) + + # ── PSA link ── + psa_ticket_id: Mapped[Optional[str]] = mapped_column( + String(100), nullable=True, + comment="External PSA ticket ID if session was started from a ticket", + ) + psa_connection_id: Mapped[Optional[uuid.UUID]] = mapped_column( + UUID(as_uuid=True), + ForeignKey("psa_connections.id", ondelete="SET NULL"), + nullable=True, + ) + ticket_data: Mapped[Optional[dict[str, Any]]] = mapped_column( + JSONB, nullable=True, + comment="Snapshot of PSA ticket data at session start", + ) + + # ── Resolution / Escalation ── + resolution_summary: Mapped[Optional[str]] = mapped_column( + Text, nullable=True, + comment="What fixed the issue (set on resolution)", + ) + resolution_action: Mapped[Optional[str]] = mapped_column( + Text, nullable=True, + comment="The specific action/step that resolved the issue", + ) + escalation_reason: Mapped[Optional[str]] = mapped_column( + Text, nullable=True, + comment="Why escalated (set on escalation)", + ) + escalation_package: Mapped[Optional[dict[str, Any]]] = mapped_column( + JSONB, nullable=True, + comment="Context package for receiving engineer: steps_tried, hypotheses, suggestions", + ) + escalated_to_id: Mapped[Optional[uuid.UUID]] = mapped_column( + UUID(as_uuid=True), + ForeignKey("users.id", ondelete="SET NULL"), + nullable=True, + ) + + # ── Feedback ── + session_rating: Mapped[Optional[int]] = mapped_column( + Integer, nullable=True, + comment="1-5 engineer feedback rating", + ) + session_feedback: Mapped[Optional[str]] = mapped_column( + Text, nullable=True, + comment="Optional feedback text from engineer", + ) + + # ── AI tracking ── + total_input_tokens: Mapped[int] = mapped_column( + Integer, nullable=False, default=0, + ) + total_output_tokens: Mapped[int] = mapped_column( + Integer, nullable=False, default=0, + ) + step_count: Mapped[int] = mapped_column( + Integer, nullable=False, default=0, + ) + + # ── Timestamps ── + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + default=lambda: datetime.now(timezone.utc), + onupdate=lambda: datetime.now(timezone.utc), + ) + resolved_at: Mapped[Optional[datetime]] = mapped_column( + DateTime(timezone=True), nullable=True, + ) + + # ── LLM conversation context ── + system_prompt_snapshot: Mapped[Optional[str]] = mapped_column( + Text, nullable=True, + comment="Snapshot of the system prompt used (for debugging/training)", + ) + conversation_messages: Mapped[list[dict[str, Any]]] = mapped_column( + JSONB, nullable=False, default=list, + comment="Full LLM message history for context continuity", + ) + + # ── Relationships ── + user: Mapped["User"] = relationship("User", foreign_keys=[user_id]) + account: Mapped["Account"] = relationship("Account") + team: Mapped[Optional["Team"]] = relationship("Team") + matched_flow: Mapped[Optional["Tree"]] = relationship("Tree", foreign_keys=[matched_flow_id]) + escalated_to: Mapped[Optional["User"]] = relationship("User", foreign_keys=[escalated_to_id]) + psa_connection: Mapped[Optional["PsaConnection"]] = relationship("PsaConnection") + steps: Mapped[list["AISessionStep"]] = relationship( + "AISessionStep", back_populates="session", + cascade="all, delete-orphan", + order_by="AISessionStep.step_order", + ) +``` + +### Task 2: Create AISessionStep model + +**Files:** +- Create: `backend/app/models/ai_session_step.py` + +```python +"""AI session step model. + +Every interaction within an AI session is captured as a step. +Steps are the raw material that becomes flow nodes in the Knowledge Flywheel. +""" +import uuid +from datetime import datetime, timezone +from typing import Optional, Any, TYPE_CHECKING + +from sqlalchemy import String, Text, DateTime, ForeignKey, Integer, Float, CheckConstraint +from sqlalchemy.orm import Mapped, mapped_column, relationship +from sqlalchemy.dialects.postgresql import UUID, JSONB + +from app.core.database import Base + +if TYPE_CHECKING: + from app.models.ai_session import AISession + from app.models.script_template import ScriptGeneration + + +class AISessionStep(Base): + """A single interaction step within a FlowPilot session. + + Step types: + - question: FlowPilot asks a diagnostic question with options + - action: FlowPilot suggests an action for the engineer to perform + - script_generation: FlowPilot invokes the Script Generator + - verification: FlowPilot asks engineer to verify a condition + - info_request: FlowPilot asks engineer to gather specific data + - note: Engineer or FlowPilot adds a contextual note + - intake_analysis: Initial analysis of the intake content + """ + __tablename__ = "ai_session_steps" + __table_args__ = ( + CheckConstraint( + "step_type IN ('question', 'action', 'script_generation', 'verification', " + "'info_request', 'note', 'intake_analysis')", + name="ck_ai_session_steps_step_type", + ), + ) + + id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), primary_key=True, default=uuid.uuid4 + ) + session_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + ForeignKey("ai_sessions.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + step_order: Mapped[int] = mapped_column( + Integer, nullable=False, + comment="Sequential position in the session (0-indexed)", + ) + step_type: Mapped[str] = mapped_column( + String(30), nullable=False, + ) + + # ── Content presented to engineer ── + content: Mapped[dict[str, Any]] = mapped_column( + JSONB, nullable=False, default=dict, + comment="The question/action content rendered in the session UI", + ) + context_message: Mapped[Optional[str]] = mapped_column( + Text, nullable=True, + comment="Why FlowPilot is asking this (shown above the question)", + ) + + # ── Options (for question steps) ── + options_presented: Mapped[Optional[list[dict[str, Any]]]] = mapped_column( + JSONB, nullable=True, + comment="Array of {label, value, followup_hint} options shown to engineer", + ) + + # ── Engineer response ── + selected_option: Mapped[Optional[str]] = mapped_column( + String(500), nullable=True, + comment="Which option the engineer selected (value field)", + ) + free_text_input: Mapped[Optional[str]] = mapped_column( + Text, nullable=True, + comment="If engineer typed a custom response instead of selecting an option", + ) + was_free_text: Mapped[bool] = mapped_column( + default=False, + comment="True if the engineer used the free-text escape hatch", + ) + was_skipped: Mapped[bool] = mapped_column( + default=False, + comment="True if engineer selected 'I don't know / Can't check'", + ) + + # ── Action results ── + action_result: Mapped[Optional[dict[str, Any]]] = mapped_column( + JSONB, nullable=True, + comment="Outcome of action step: {success: bool, details: str, next_hint: str}", + ) + + # ── Script generation link ── + script_generation_id: Mapped[Optional[uuid.UUID]] = mapped_column( + UUID(as_uuid=True), + ForeignKey("script_generations.id", ondelete="SET NULL"), + nullable=True, + ) + + # ── AI internals ── + confidence_at_step: Mapped[float] = mapped_column( + Float, nullable=False, default=0.0, + comment="FlowPilot confidence level at this point (0.0-1.0)", + ) + ai_reasoning: Mapped[Optional[str]] = mapped_column( + Text, nullable=True, + comment="Why FlowPilot chose this step (internal, for debugging/training)", + ) + input_tokens: Mapped[int] = mapped_column( + Integer, nullable=False, default=0, + ) + output_tokens: Mapped[int] = mapped_column( + Integer, nullable=False, default=0, + ) + + # ── Timestamps ── + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) + ) + responded_at: Mapped[Optional[datetime]] = mapped_column( + DateTime(timezone=True), nullable=True, + comment="When the engineer responded to this step", + ) + + # ── Relationships ── + session: Mapped["AISession"] = relationship("AISession", back_populates="steps") + script_generation: Mapped[Optional["ScriptGeneration"]] = relationship("ScriptGeneration") +``` + +### Task 3: Register models in `__init__.py` + +**Files:** +- Edit: `backend/app/models/__init__.py` + +Add these imports after the `ScriptGeneration` line: + +```python +from .ai_session import AISession +from .ai_session_step import AISessionStep +``` + +Add to `__all__`: + +```python + "AISession", + "AISessionStep", +``` + +### Task 4: Create Alembic migration + +Generate with: +```bash +cd backend && alembic revision --autogenerate -m "add ai_sessions and ai_session_steps tables" +``` + +The migration should create both tables with all columns, constraints, and indexes. + +Also add columns to the existing `trees` table for flow matching support: +```python +op.add_column('trees', sa.Column('origin', sa.String(20), nullable=True, comment='manual | ai_generated | ai_enhanced')) +op.add_column('trees', sa.Column('source_session_id', sa.dialects.postgresql.UUID(as_uuid=True), nullable=True)) +op.add_column('trees', sa.Column('match_keywords', sa.dialects.postgresql.JSONB(), nullable=True, server_default='[]')) +op.add_column('trees', sa.Column('success_rate', sa.Float(), nullable=True)) +op.add_column('trees', sa.Column('last_matched_at', sa.DateTime(timezone=True), nullable=True)) +``` + +**Verification:** Run `alembic upgrade head`. Verify both tables exist in psql. Verify trees table has new columns. + +``` +git commit -m "feat(ai-session): add AISession and AISessionStep models + migration" +``` + +--- + +## Slice 2: Pydantic Schemas + +### Task 5: Create AI session schemas + +**Files:** +- Create: `backend/app/schemas/ai_session.py` + +```python +"""Pydantic schemas for FlowPilot AI sessions.""" +from __future__ import annotations + +from typing import Optional, Any +from uuid import UUID +from datetime import datetime + +from pydantic import BaseModel, Field + + +# ── Intake ── + +class AISessionCreateRequest(BaseModel): + """Start a new FlowPilot session.""" + intake_type: str = Field( + "free_text", + pattern="^(free_text|psa_ticket|screenshot|log_paste|combined)$", + ) + intake_content: dict[str, Any] = Field( + ..., + description=( + "Intake payload. Shape depends on intake_type: " + "{text: str} for free_text, " + "{text?: str, image_urls?: list[str]} for screenshot, " + "{text?: str, log_content?: str} for log_paste, " + "{ticket_id: str, psa_connection_id: str} for psa_ticket, " + "any combination for combined." + ), + ) + psa_ticket_id: Optional[str] = None + psa_connection_id: Optional[UUID] = None + + +class AISessionCreateResponse(BaseModel): + """Response after starting a session — includes the first FlowPilot step.""" + session_id: UUID + status: str + confidence_tier: str + problem_summary: str | None = None + problem_domain: str | None = None + matched_flow_id: UUID | None = None + matched_flow_name: str | None = None + match_score: float | None = None + first_step: AISessionStepResponse + + +# ── Step interaction ── + +class StepOptionSchema(BaseModel): + """A selectable option presented to the engineer.""" + label: str + value: str + followup_hint: str | None = None + + +class AISessionStepResponse(BaseModel): + """A FlowPilot step rendered in the session UI.""" + step_id: UUID + step_order: int + step_type: str + content: dict[str, Any] + context_message: str | None = None + options: list[StepOptionSchema] = [] + allow_free_text: bool = True + allow_skip: bool = True + confidence_tier: str + confidence_score: float + + model_config = {"from_attributes": True} + + +class StepResponseRequest(BaseModel): + """Engineer's response to a FlowPilot step.""" + selected_option: str | None = None + free_text_input: str | None = None + was_skipped: bool = False + action_result: dict[str, Any] | None = None + + +class StepResponseResponse(BaseModel): + """FlowPilot's next step after processing the engineer's response.""" + session_id: UUID + status: str + confidence_tier: str + confidence_score: float + next_step: AISessionStepResponse | None = None + resolution_suggested: bool = False + resolution_summary: str | None = None + + +# ── Resolution / Escalation ── + +class ResolveSessionRequest(BaseModel): + """Close a session as resolved.""" + resolution_summary: str = Field(..., min_length=5, max_length=2000) + resolution_action: str | None = None + session_rating: int | None = Field(None, ge=1, le=5) + session_feedback: str | None = None + + +class EscalateSessionRequest(BaseModel): + """Escalate a session to another engineer.""" + escalation_reason: str = Field(..., min_length=5, max_length=2000) + escalated_to_id: UUID | None = None + + +class SessionDocumentation(BaseModel): + """Auto-generated session documentation.""" + problem_summary: str + problem_domain: str | None = None + intake_summary: str + diagnostic_steps: list[DocumentationStep] + resolution_summary: str | None = None + escalation_reason: str | None = None + total_steps: int + duration_display: str | None = None + generated_at: datetime + + +class DocumentationStep(BaseModel): + """A step in the documentation trail.""" + step_number: int + step_type: str + description: str + engineer_response: str | None = None + outcome: str | None = None + + +class SessionCloseResponse(BaseModel): + """Response after resolving or escalating.""" + session_id: UUID + status: str + documentation: SessionDocumentation + + +class RateSessionRequest(BaseModel): + """Submit post-session rating.""" + rating: int = Field(..., ge=1, le=5) + feedback: str | None = None + + +# ── List / Detail ── + +class AISessionSummary(BaseModel): + """Compact session for list views.""" + id: UUID + status: str + intake_type: str + problem_summary: str | None = None + problem_domain: str | None = None + confidence_tier: str + step_count: int + session_rating: int | None = None + created_at: datetime + resolved_at: datetime | None = None + + model_config = {"from_attributes": True} + + +class AISessionDetail(AISessionSummary): + """Full session detail with steps.""" + intake_content: dict[str, Any] + matched_flow_id: UUID | None = None + match_score: float | None = None + resolution_summary: str | None = None + resolution_action: str | None = None + escalation_reason: str | None = None + session_feedback: str | None = None + steps: list[AISessionStepResponse] = [] + + model_config = {"from_attributes": True} +``` + +**Verification:** Import the schemas in a Python shell and construct sample instances to verify validation. + +``` +git commit -m "feat(ai-session): add Pydantic schemas for AI sessions" +``` + +--- + +## Slice 3: FlowPilot Engine Service + +### Task 6: Create FlowPilot Engine service + +This is the brain of the product. It orchestrates the LLM, manages structured output, and drives the diagnostic conversation. + +**Files:** +- Create: `backend/app/services/flowpilot_engine.py` + +**Architecture:** + +``` +┌─────────────────────────────────────────────┐ +│ FlowPilotEngine │ +│ │ +│ start_session(intake) → first_step │ +│ process_response(step_response) → next_step│ +│ resolve_session(summary) → documentation │ +│ escalate_session(reason) → package │ +│ │ +│ Internal: │ +│ _build_system_prompt(session) │ +│ _build_messages(session, new_input) │ +│ _parse_structured_output(llm_response) │ +│ _update_confidence(session, step) │ +│ _generate_documentation(session) │ +│ _classify_intake(intake_content) │ +└─────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────┐ ┌──────────────────┐ +│ ai_provider.py │ │ FlowMatchingEngine│ +│ (Anthropic) │ │ (Slice 4) │ +└─────────────────┘ └──────────────────┘ +``` + +**System Prompt Structure:** + +The system prompt is built dynamically per session and includes: + +1. Role definition (Senior MSP engineer + structured output contract) +2. Response format (STRICT JSON schema — never free-form prose) +3. Team context (if available — client configs, naming conventions) +4. Matched flow context (if a flow matched — full flow definition) +5. Session history (all prior steps for context continuity) +6. Available actions (script templates, verification checks) + +**Structured Output Contract:** + +FlowPilot's LLM responses must ALWAYS be valid JSON matching one of these shapes: + +```json +// Diagnostic question +{ + "type": "question", + "content": "Brief description of what we're checking", + "reasoning": "Internal: why this question matters (stored in ai_reasoning)", + "context_message": "Shown to engineer: why we're asking this", + "options": [ + {"label": "Human-readable option", "value": "machine_value", "followup_hint": "what this implies"}, + {"label": "Another option", "value": "another_value", "followup_hint": null} + ], + "allow_free_text": true, + "allow_skip": true, + "confidence": 0.65 +} + +// Suggested action +{ + "type": "action", + "content": "What the engineer should do", + "reasoning": "Internal: why this action", + "context_message": "Here's what I'd like you to try", + "action_type": "instruction | script_generation | verification | info_request", + "template_id": "uuid | null", + "pre_filled_params": {}, + "expected_outcome": "What success looks like", + "confidence": 0.78 +} + +// Resolution suggestion +{ + "type": "resolution_suggestion", + "content": "Summary of what we did and why it should be resolved", + "reasoning": "Internal: why I think this is resolved", + "resolution_summary": "The issue was caused by X and fixed by Y", + "confidence": 0.92, + "follow_up_recommendations": ["Monitor for 24 hours", "Check event log tomorrow"] +} +``` + +**Key implementation details:** + +- Use `ai_provider.generate_json()` for all LLM calls to enforce JSON output +- Parse the JSON response and validate against the expected shapes +- If parsing fails, retry once with a "please respond in valid JSON" nudge +- Store `reasoning` in `ai_reasoning` column (never shown to engineer) +- Map `confidence` to `confidence_tier`: >0.8 = guided, 0.4-0.8 = exploring, <0.4 = discovery +- Conversation messages accumulate in `session.conversation_messages` JSONB field +- Each step creates both an `AISessionStep` record AND appends to conversation history +- Token counts tracked per step AND accumulated on the session + +**System prompt template** (key excerpt — full prompt will be ~800 tokens): + +```python +FLOWPILOT_SYSTEM_PROMPT = """You are FlowPilot, an expert MSP troubleshooting assistant embedded in ResolutionFlow. You guide engineers through structured diagnosis of IT issues. + +## YOUR ROLE +- Conduct systematic troubleshooting through targeted questions and actions +- Start broad, narrow down based on responses +- Never guess — ask clarifying questions when uncertain +- Suggest specific, actionable steps the engineer can verify +- When confidence is high, suggest resolution; when low, keep investigating + +## RESPONSE FORMAT +You MUST respond with ONLY a valid JSON object. No markdown, no prose, no code fences. +Every response must have a "type" field: "question", "action", or "resolution_suggestion". + +{structured_output_schema} + +## RULES +- Maximum 5 options per question. Options should be the most likely scenarios. +- Always include relevant context in context_message — explain WHY you're asking +- confidence is a float 0.0-1.0 reflecting how certain you are about the diagnosis path +- When multiple symptoms point to one root cause with >90% confidence, suggest resolution +- If you detect the engineer needs a PowerShell script, suggest a script_generation action +- Never suggest restarting or rebooting as a first step — diagnose first +- Be specific: "Check Event Viewer > System > source NTFS" not "check the logs" + +{team_context} + +{matched_flow_context} +""" +``` + +**`start_session` flow:** +1. Receive intake content +2. Call `_classify_intake()` — quick LLM call (haiku-tier) to extract: problem_summary, problem_domain, key symptoms, urgency +3. Call `FlowMatchingEngine.find_matches()` with extracted symptoms +4. Build system prompt with any matched flow context +5. Call LLM with intake as first user message → get first diagnostic step +6. Create `AISession` and first `AISessionStep` records +7. Return session + first step + +**`process_response` flow:** +1. Load session + all steps +2. Append engineer's response to conversation_messages +3. Call LLM with full conversation → get next step +4. Parse structured output +5. Create new `AISessionStep` record +6. Update session confidence, step_count, token counts +7. If type is `resolution_suggestion`, flag it but don't auto-close +8. Return next step + +**`resolve_session` flow:** +1. Set status = resolved, resolved_at = now +2. Call `_generate_documentation()` — constructs a clean doc from all steps +3. Return documentation + +**`_generate_documentation` flow:** +1. Walk all steps in order +2. For each question step: format as "Checked: {context_message} → Response: {selected_option or free_text}" +3. For each action step: format as "Action: {content} → Result: {action_result}" +4. Compile into `SessionDocumentation` schema +5. Calculate duration from created_at to resolved_at + +**Verification:** Write unit tests for `_parse_structured_output` and `_update_confidence`. Test `start_session` with a mock AI provider returning sample JSON. Verify session and step records are created correctly. + +``` +git commit -m "feat(ai-session): add FlowPilot Engine service with structured output" +``` + +--- + +## Slice 4: Flow Matching Engine + +### Task 7: Create Flow Matching Engine v1 + +**Files:** +- Create: `backend/app/services/flow_matching_engine.py` + +**Architecture:** v1 uses keyword matching + existing tree embeddings for semantic search. Deliberately simple — v2 (Phase 3) will add deeper semantic matching. + +**Matching strategy:** +1. **Keyword match:** Extract key terms from intake, match against `trees.match_keywords` JSONB array (new column from migration) +2. **Semantic search:** Embed the intake text via `embedding_service.get_embedding()`, cosine similarity search against `tree_embeddings` table +3. **Category match:** Match `problem_domain` against `trees.category` +4. **Recency boost:** Trees that were recently matched successfully get a score boost + +**Return type:** List of `FlowMatch(tree_id, tree_name, score, match_reason)` sorted by score descending. + +**Threshold:** Only return matches with score > 0.5. Top match with score > 0.8 triggers "guided" confidence tier. + +**Key implementation:** +- Reuse `rag_service.search()` for vector similarity (it already handles pgvector queries) +- Combine keyword score (0.0-1.0) + semantic score (0.0-1.0) + recency score (0.0-0.2) into a weighted composite +- Weights: semantic 0.5, keyword 0.3, recency 0.2 +- Only match against published trees (status = 'published') in the same account + +**Verification:** Seed a few test trees with keywords. Call `find_matches()` with a sample intake and verify reasonable matches are returned. + +``` +git commit -m "feat(ai-session): add Flow Matching Engine v1" +``` + +--- + +## Slice 5: API Endpoints + +### Task 8: Create AI session endpoints + +**Files:** +- Create: `backend/app/api/endpoints/ai_sessions.py` + +Follow the patterns in `copilot.py` exactly: rate limiting, AI quota checks, error handling with AI provider errors, token usage recording. + +**Endpoints:** + +``` +POST /api/v1/ai-sessions — Start a new FlowPilot session +POST /api/v1/ai-sessions/{id}/respond — Submit step response, get next step +POST /api/v1/ai-sessions/{id}/resolve — Resolve the session +POST /api/v1/ai-sessions/{id}/escalate — Escalate the session +GET /api/v1/ai-sessions — List user's sessions (paginated) +GET /api/v1/ai-sessions/{id} — Get session detail with all steps +GET /api/v1/ai-sessions/{id}/documentation — Get auto-generated documentation +POST /api/v1/ai-sessions/{id}/rate — Submit post-session rating +``` + +**Auth:** All endpoints require `get_current_active_user` + `require_engineer_or_admin`. + +**Rate limits:** +- `POST /ai-sessions`: 5/minute (starting sessions is expensive) +- `POST /{id}/respond`: 15/minute (normal conversation pace) +- `GET` endpoints: 30/minute + +**AI quota:** Check quota on `POST /ai-sessions` and `POST /{id}/respond` (both make LLM calls). Record usage after each call using `record_ai_usage()` from `app.core.ai_quota_service`. + +**Key details:** +- `POST /ai-sessions` calls `flowpilot_engine.start_session()`, returns `AISessionCreateResponse` +- `POST /{id}/respond` calls `flowpilot_engine.process_response()`, returns `StepResponseResponse` +- `POST /{id}/resolve` calls `flowpilot_engine.resolve_session()`, returns `SessionCloseResponse` +- `GET /ai-sessions` supports `?status=active` filter and pagination (`?skip=0&limit=20`) +- Ensure the session belongs to the current user (or their team for escalation handoffs) + +### Task 9: Register router + +**Files:** +- Edit: `backend/app/api/router.py` + +Add import and include: +```python +from app.api.endpoints import ai_sessions +api_router.include_router(ai_sessions.router) +``` + +**Verification:** Start the backend. Hit `POST /api/v1/ai-sessions` with a sample intake via curl. Verify session is created, first step is returned. Submit a response, verify next step. Resolve, verify documentation. + +``` +git commit -m "feat(ai-session): add AI session API endpoints" +``` + +--- + +## Slice 6: Frontend — Types & API Client + +### Task 10: Create TypeScript types + +**Files:** +- Create: `frontend/src/types/ai-session.ts` + +Mirror all Pydantic schemas as TypeScript interfaces. Follow the patterns in `types/copilot.ts` and `types/session.ts`. + +Key types: `AISessionCreateRequest`, `AISessionCreateResponse`, `AISessionStepResponse`, `StepOptionSchema`, `StepResponseRequest`, `StepResponseResponse`, `ResolveSessionRequest`, `EscalateSessionRequest`, `SessionCloseResponse`, `SessionDocumentation`, `AISessionSummary`, `AISessionDetail`. + +### Task 11: Create API client functions + +**Files:** +- Create: `frontend/src/api/aiSessions.ts` + +Follow patterns in `api/copilot.ts` — use the existing `client.ts` axios instance. + +```typescript +export const createAISession = (data: AISessionCreateRequest) => + client.post('/ai-sessions', data) + +export const respondToStep = (sessionId: string, data: StepResponseRequest) => + client.post(`/ai-sessions/${sessionId}/respond`, data) + +export const resolveSession = (sessionId: string, data: ResolveSessionRequest) => + client.post(`/ai-sessions/${sessionId}/resolve`, data) + +export const escalateSession = (sessionId: string, data: EscalateSessionRequest) => + client.post(`/ai-sessions/${sessionId}/escalate`, data) + +export const getAISessions = (params?: { status?: string; skip?: number; limit?: number }) => + client.get('/ai-sessions', { params }) + +export const getAISession = (sessionId: string) => + client.get(`/ai-sessions/${sessionId}`) + +export const getSessionDocumentation = (sessionId: string) => + client.get(`/ai-sessions/${sessionId}/documentation`) + +export const rateSession = (sessionId: string, data: { rating: number; feedback?: string }) => + client.post(`/ai-sessions/${sessionId}/rate`, data) +``` + +**Verification:** Import in a component, verify TypeScript compiles with no errors. + +``` +git commit -m "feat(ai-session): add frontend types and API client" +``` + +--- + +## Slice 7: Frontend — FlowPilot Session Page + +### Task 12: Create the FlowPilot session page and components + +This is the most important UX in the product. It needs to feel like a smart conversation, not a form. + +**Files:** +- Create: `frontend/src/pages/FlowPilotSessionPage.tsx` — The main page +- Create: `frontend/src/components/flowpilot/FlowPilotIntake.tsx` — Intake screen +- Create: `frontend/src/components/flowpilot/FlowPilotSession.tsx` — Active session view +- Create: `frontend/src/components/flowpilot/FlowPilotStepCard.tsx` — Individual step card +- Create: `frontend/src/components/flowpilot/FlowPilotOptions.tsx` — Selectable options grid +- Create: `frontend/src/components/flowpilot/FlowPilotActionBar.tsx` — Resolve/Escalate bar +- Create: `frontend/src/components/flowpilot/ConfidenceIndicator.tsx` — Confidence tier badge +- Create: `frontend/src/components/flowpilot/SessionDocView.tsx` — Documentation view +- Create: `frontend/src/components/flowpilot/index.ts` — Barrel export +- Create: `frontend/src/hooks/useFlowPilotSession.ts` — Session state management hook + +**Design requirements (MUST follow existing design system):** +- Dark theme: slate-900/950 backgrounds, glass morphism cards +- Brand cyan (#06b6d4 → #22d3ee) for primary actions and active states +- IBM Plex Sans for body, Bricolage Grotesque for headings, JetBrains Mono for code +- shadcn/ui components as base (Button, Card, Badge, Textarea, etc.) +- Sonner for toast notifications +- Lucide icons throughout + +### Intake Screen (`FlowPilotIntake.tsx`) + +**Layout:** Centered card on the page. Clean, focused, inviting. + +**Elements:** +- Heading: "What are you troubleshooting?" (Bricolage Grotesque) +- Large textarea with placeholder: "Describe the issue, paste an error message, or paste log output..." +- Below textarea: Row of input type pills/badges — "Add Screenshot" (image upload), "Paste Logs" (toggles a monospace textarea), "Pull from Ticket" (opens ticket picker — disabled until PSA integration in Phase 2) +- "Start Session" primary button (cyan, disabled until text has content) +- Below the CTA: subtle text "FlowPilot will analyze your input and guide you through diagnosis" + +**Behavior:** +- On submit: call `createAISession()`, transition to active session view +- Show loading state: "Analyzing your issue..." with a subtle pulse animation +- Screenshot upload stores the file and passes URL in `intake_content.image_urls` + +### Active Session View (`FlowPilotSession.tsx`) + +**Layout:** Conversational scroll view. Steps appear sequentially like a chat, scrolling down as the conversation progresses. + +**Left column (main, ~70%):** The conversation. Each step is a `FlowPilotStepCard`. History steps are collapsed/completed. The current step is expanded and interactive. + +**Right column (sidebar, ~30%):** Session metadata — problem summary, domain badge, confidence indicator, matched flow (if any), step counter, elapsed time. Sticky positioned. + +**Bottom bar:** `FlowPilotActionBar` — always visible. "Resolve" (green) and "Escalate" (amber) buttons. Only enabled after at least one diagnostic step. + +### Step Card (`FlowPilotStepCard.tsx`) + +**For question steps:** +- Context message (if present) shown in a subtle info banner above the question +- Question content as the card heading +- Options rendered as `FlowPilotOptions` — clickable cards in a grid (1-2 columns) +- "None of these — let me describe" link at the bottom → expands a textarea +- "I can't check this right now" skip link +- After responding: card collapses to show the question + selected answer inline + +**For action steps:** +- Action description with checklist-style items if applicable +- "I've completed this action" / "This didn't work" buttons +- If script_generation: embedded script generator (reuse from existing ScriptGeneratorPanel) + +**For resolution_suggestion steps:** +- Summary card with green accent border +- "Yes, this is resolved" button → opens resolve modal +- "No, keep investigating" button → sends response back to FlowPilot + +### Options Grid (`FlowPilotOptions.tsx`) + +**Design:** Cards in a responsive grid (2 columns on desktop, 1 on mobile). Each option card: +- Label text (primary) +- Followup hint (secondary, smaller, muted) +- Hover: cyan border glow + slight scale +- Selected: cyan background with check icon +- Keyboard: Tab through options, Enter to select + +### Confidence Indicator (`ConfidenceIndicator.tsx`) + +Subtle badge in the sidebar: +- **Guided** (>0.8): Green dot + "Proven path" text +- **Exploring** (0.4-0.8): Amber dot + "Investigating" text +- **Discovery** (<0.4): Purple dot + "New territory" text + +Tooltip on hover explains what the tier means. + +### useFlowPilotSession Hook + +Manages all session state: + +```typescript +interface UseFlowPilotSession { + // State + session: AISessionDetail | null + currentStep: AISessionStepResponse | null + allSteps: AISessionStepResponse[] + isLoading: boolean + isProcessing: boolean // waiting for next step from LLM + error: string | null + + // Actions + startSession: (intake: AISessionCreateRequest) => Promise + respondToStep: (response: StepResponseRequest) => Promise + resolveSession: (data: ResolveSessionRequest) => Promise + escalateSession: (data: EscalateSessionRequest) => Promise + rateSession: (rating: number, feedback?: string) => Promise + + // Derived + isActive: boolean + canResolve: boolean + canEscalate: boolean +} +``` + +### Task 13: Add route and navigation + +**Files:** +- Edit: `frontend/src/router.tsx` + +Add lazy import: +```typescript +const FlowPilotSessionPage = lazy(() => import('@/pages/FlowPilotSessionPage')) +``` + +Add routes inside the AppLayout children: +```typescript +{ path: 'pilot', element: page(FlowPilotSessionPage) }, +{ path: 'pilot/:sessionId', element: page(FlowPilotSessionPage) }, +``` + +**Files:** +- Edit sidebar component (in `frontend/src/components/sidebar/`) + +Add a prominent "New Session" entry at the top of the sidebar with the `Sparkles` Lucide icon and cyan highlight. This should be the most visually prominent nav item. Link to `/pilot`. + +**Verification:** Navigate to `/pilot`. See the intake screen. Type a problem, submit. See FlowPilot's first question with selectable options. Select an option, see the next step. Continue for 3-4 steps. Resolve. See documentation. Verify all data is persisted in the database. + +``` +git commit -m "feat(ai-session): add FlowPilot session page, components, and routing" +``` + +--- + +## Slice 8: Session History Integration + +### Task 14: AI sessions in session history + +**Files:** +- Edit: `frontend/src/pages/SessionHistoryPage.tsx` +- Create: `frontend/src/components/flowpilot/AISessionListItem.tsx` + +Add a tab or toggle to switch between "Flow Sessions" (existing) and "AI Sessions" (new). AI sessions show: problem summary, domain badge, status, confidence tier, step count, created_at, resolved_at. + +Click on an AI session navigates to `/pilot/{sessionId}` which loads the session in read-only mode showing the full conversation trail. + +**Verification:** Complete an AI session. Navigate to session history. See the AI session listed. Click it, see the full conversation in read-only mode. + +``` +git commit -m "feat(ai-session): integrate AI sessions into session history" +``` + +--- + +## Summary of All New Files + +### Backend +``` +app/models/ai_session.py # AISession model +app/models/ai_session_step.py # AISessionStep model +app/schemas/ai_session.py # All Pydantic schemas +app/services/flowpilot_engine.py # FlowPilot Engine (LLM orchestration) +app/services/flow_matching_engine.py # Flow Matching Engine v1 +app/api/endpoints/ai_sessions.py # API endpoints +alembic/versions/xxx_add_ai_sessions.py # Migration +``` + +### Backend (edited) +``` +app/models/__init__.py # Register new models +app/api/router.py # Register new router +``` + +### Frontend +``` +src/types/ai-session.ts # TypeScript types +src/api/aiSessions.ts # API client +src/pages/FlowPilotSessionPage.tsx # Main page +src/hooks/useFlowPilotSession.ts # Session state hook +src/components/flowpilot/ + FlowPilotIntake.tsx # Intake screen + FlowPilotSession.tsx # Active session view + FlowPilotStepCard.tsx # Step card + FlowPilotOptions.tsx # Options grid + FlowPilotActionBar.tsx # Resolve/Escalate bar + ConfidenceIndicator.tsx # Confidence badge + SessionDocView.tsx # Documentation view + AISessionListItem.tsx # History list item + index.ts # Barrel export +``` + +### Frontend (edited) +``` +src/router.tsx # New routes +src/components/sidebar/ # New nav entry +src/pages/SessionHistoryPage.tsx # AI session tab +``` + +--- + +## Config Requirements + +No new environment variables needed for Phase 1 — the FlowPilot Engine reuses existing `AI_PROVIDER` and `AI_MODEL_ANTHROPIC` settings (currently `claude-sonnet-4-6`). The `get_ai_provider()` function and `ai_quota_service` work as-is. + +Future phases may add: +- `FLOWPILOT_MODEL_TIER` — route simple questions to haiku, complex diagnosis to sonnet via existing `AI_MODEL_TIERS` config +- `FLOWPILOT_MAX_STEPS` — safety limit on steps per session (default: 30) + +--- + +## Testing Strategy + +### Backend Unit Tests + +**Files:** `backend/tests/test_flowpilot_engine.py` + +- Test `_parse_structured_output` with valid and invalid JSON +- Test `_update_confidence` tier mapping +- Test `_classify_intake` with various intake types +- Test `_generate_documentation` with mock session steps +- Test Flow Matching Engine scoring with mock embeddings + +### Backend Integration Tests + +**Files:** `backend/tests/test_ai_sessions_api.py` + +- Test full session lifecycle: create → respond (3 steps) → resolve +- Test escalation flow: create → respond → escalate +- Test auth enforcement (wrong user can't access another's session) +- Test rate limiting +- Test AI quota enforcement + +### Frontend + +Manual testing of the full flow: +1. Open `/pilot`, type a problem description +2. Verify first step renders with selectable options +3. Select an option, verify next step appears +4. Use the free-text "none of these" escape hatch, verify it works +5. Skip a step with "I can't check this", verify it works +6. Resolve the session, verify documentation generates +7. Check session history page, verify the AI session appears +8. Re-open the completed session, verify read-only conversation view + +--- + +## What Comes Next (Phase 2 — NOT in scope here) + +For context only — do NOT implement these in Phase 1: + +- **PSA ticket intake:** Pull ticket from ConnectWise as session input +- **PSA ticket update:** Push documentation back to ticket on resolution +- **Escalation handoff:** Another engineer picks up a paused session +- **Knowledge Flywheel:** Post-session flow proposal generation +- **Review Queue:** UI for approving AI-generated flow proposals +- **In-session Script Generator:** FlowPilot invokes script generation contextually diff --git a/frontend/src/api/flowProposals.ts b/frontend/src/api/flowProposals.ts new file mode 100644 index 00000000..747b4c6f --- /dev/null +++ b/frontend/src/api/flowProposals.ts @@ -0,0 +1,41 @@ +import apiClient from './client' +import type { + FlowProposalSummary, + FlowProposalDetail, + FlowProposalStats, + ReviewProposalRequest, +} from '@/types/flow-proposal' + +export const flowProposalsApi = { + async list(params?: { + status?: string + type?: string + domain?: string + sort_by?: string + skip?: number + limit?: number + }): Promise { + const response = await apiClient.get('/flow-proposals', { params }) + return response.data + }, + + async getStats(): Promise { + const response = await apiClient.get('/flow-proposals/stats') + return response.data + }, + + async get(id: string): Promise { + const response = await apiClient.get(`/flow-proposals/${id}`) + return response.data + }, + + async review(id: string, data: ReviewProposalRequest): Promise { + const response = await apiClient.post( + `/flow-proposals/${id}/review`, + data + ) + return response.data + }, +} + +export default flowProposalsApi diff --git a/frontend/src/api/flowpilotAnalytics.ts b/frontend/src/api/flowpilotAnalytics.ts new file mode 100644 index 00000000..459e0b6c --- /dev/null +++ b/frontend/src/api/flowpilotAnalytics.ts @@ -0,0 +1,20 @@ +import apiClient from './client' +import type { FlowPilotDashboard, KnowledgeGapReport } from '@/types/flowpilot-analytics' + +export const flowpilotAnalyticsApi = { + async getDashboard(period: string = '30d'): Promise { + const response = await apiClient.get('/analytics/flowpilot', { + params: { period }, + }) + return response.data + }, + + async getKnowledgeGaps(period: string = '30d'): Promise { + const response = await apiClient.get('/analytics/flowpilot/knowledge-gaps', { + params: { period }, + }) + return response.data + }, +} + +export default flowpilotAnalyticsApi diff --git a/frontend/src/api/index.ts b/frontend/src/api/index.ts index 2cd86a7e..9cd86d21 100644 --- a/frontend/src/api/index.ts +++ b/frontend/src/api/index.ts @@ -25,3 +25,5 @@ export { integrationsApi, sessionPsaApi } from './integrations' export { sidebarApi } from './sidebar' export { sessionToFlowApi } from './sessionToFlow' export { aiSessionsApi } from './aiSessions' +export { flowProposalsApi } from './flowProposals' +export { flowpilotAnalyticsApi } from './flowpilotAnalytics' diff --git a/frontend/src/components/flowpilot/FlowPilotSession.tsx b/frontend/src/components/flowpilot/FlowPilotSession.tsx index b3dcb80b..0f5544f5 100644 --- a/frontend/src/components/flowpilot/FlowPilotSession.tsx +++ b/frontend/src/components/flowpilot/FlowPilotSession.tsx @@ -141,6 +141,7 @@ export function FlowPilotSession({ step={step} isCurrentStep={currentStep?.step_id === step.step_id} isProcessing={isProcessing && currentStep?.step_id === step.step_id} + sessionId={session.id} onRespond={onRespond} /> ))} diff --git a/frontend/src/components/flowpilot/FlowPilotStepCard.tsx b/frontend/src/components/flowpilot/FlowPilotStepCard.tsx index 7355cf32..5ed1c02f 100644 --- a/frontend/src/components/flowpilot/FlowPilotStepCard.tsx +++ b/frontend/src/components/flowpilot/FlowPilotStepCard.tsx @@ -4,11 +4,13 @@ import { cn } from '@/lib/utils' import type { AISessionStepResponse, StepResponseRequest } from '@/types/ai-session' import { MarkdownContent } from '@/components/ui/MarkdownContent' import { FlowPilotOptions } from './FlowPilotOptions' +import { InSessionScriptGenerator } from './InSessionScriptGenerator' interface FlowPilotStepCardProps { step: AISessionStepResponse isCurrentStep: boolean isProcessing: boolean + sessionId?: string onRespond: (response: StepResponseRequest) => void } @@ -22,7 +24,7 @@ const STEP_TYPE_ICONS = { note: MessageSquare, } as const -export function FlowPilotStepCard({ step, isCurrentStep, isProcessing, onRespond }: FlowPilotStepCardProps) { +export function FlowPilotStepCard({ step, isCurrentStep, isProcessing, sessionId, onRespond }: FlowPilotStepCardProps) { const [freeText, setFreeText] = useState('') const [showFreeText, setShowFreeText] = useState(false) const [isCollapsed, setIsCollapsed] = useState(!isCurrentStep) @@ -163,8 +165,19 @@ export function FlowPilotStepCard({ step, isCurrentStep, isProcessing, onRespond /> )} + {/* In-session script generator */} + {!isResolutionSuggestion && (content.action_type as string) === 'script_generation' && sessionId && ( + ) || {}} + instructions={(content.instructions as string) || stepText} + sessionId={sessionId} + onRespond={onRespond} + /> + )} + {/* Action step buttons */} - {!isResolutionSuggestion && step.step_type === 'action' && ( + {!isResolutionSuggestion && step.step_type === 'action' && (content.action_type as string) !== 'script_generation' && (
+ + {showParams && Object.keys(params).length > 0 && ( +
+ {Object.entries(params).map(([key, value]) => ( +
+ + setParams(prev => ({ ...prev, [key]: e.target.value }))} + className="w-full rounded-lg border border-border bg-card px-3 py-1.5 text-sm text-foreground placeholder:text-muted-foreground focus:border-[rgba(6,182,212,0.3)] focus:outline-none" + /> +
+ ))} +
+ )} + + + + )} + + {/* Generated script display */} + {generatedScript && ( + <> +
+
+ PowerShell + +
+
+ +
+
+ + {/* Continue buttons */} +
+ + +
+ + )} +
+ ) +} diff --git a/frontend/src/components/flowpilot/ProposalCard.tsx b/frontend/src/components/flowpilot/ProposalCard.tsx new file mode 100644 index 00000000..f86c3f44 --- /dev/null +++ b/frontend/src/components/flowpilot/ProposalCard.tsx @@ -0,0 +1,62 @@ +import { GitBranch, ArrowUpRight, Sparkles, Clock, Hash } from 'lucide-react' +import type { FlowProposalSummary } from '@/types/flow-proposal' + +const TYPE_CONFIG = { + new_flow: { label: 'New Flow', color: 'text-emerald-400 bg-emerald-400/10 border-emerald-400/20', icon: Sparkles }, + enhancement: { label: 'Enhancement', color: 'text-amber-400 bg-amber-400/10 border-amber-400/20', icon: ArrowUpRight }, + branch_addition: { label: 'Branch', color: 'text-blue-400 bg-blue-400/10 border-blue-400/20', icon: GitBranch }, + auto_reinforced: { label: 'Reinforced', color: 'text-muted-foreground bg-card border-border', icon: Sparkles }, +} as const + +interface ProposalCardProps { + proposal: FlowProposalSummary + isSelected: boolean + onClick: () => void +} + +export function ProposalCard({ proposal, isSelected, onClick }: ProposalCardProps) { + const typeConfig = TYPE_CONFIG[proposal.proposal_type] || TYPE_CONFIG.new_flow + const TypeIcon = typeConfig.icon + + return ( + + ) +} diff --git a/frontend/src/components/flowpilot/ProposalDetail.tsx b/frontend/src/components/flowpilot/ProposalDetail.tsx new file mode 100644 index 00000000..5f702ae8 --- /dev/null +++ b/frontend/src/components/flowpilot/ProposalDetail.tsx @@ -0,0 +1,239 @@ +import { useState } from 'react' +import { useNavigate, Link } from 'react-router-dom' +import { toast } from '@/lib/toast' +import { + CheckCircle2, XCircle, Pencil, EyeOff, ChevronDown, ChevronRight, + ExternalLink, Clock, Hash, Sparkles, Loader2, +} from 'lucide-react' +import type { FlowProposalDetail as ProposalDetailType } from '@/types/flow-proposal' + +interface ProposalDetailProps { + proposal: ProposalDetailType + onReview: (action: 'approve' | 'reject' | 'modify' | 'dismiss', notes?: string) => Promise +} + +export function ProposalDetail({ proposal, onReview }: ProposalDetailProps) { + const navigate = useNavigate() + const [reviewNotes, setReviewNotes] = useState('') + const [showFlowData, setShowFlowData] = useState(false) + const [isSubmitting, setIsSubmitting] = useState(false) + const [showRejectConfirm, setShowRejectConfirm] = useState(false) + + const canReview = proposal.status === 'pending' || proposal.status === 'dismissed' + + const handleAction = async (action: 'approve' | 'reject' | 'modify' | 'dismiss') => { + setIsSubmitting(true) + try { + await onReview(action, reviewNotes || undefined) + } finally { + setIsSubmitting(false) + setShowRejectConfirm(false) + } + } + + const handleEditAndPublish = async () => { + // Mark proposal as dismissed so it doesn't stay in the pending queue. + // The tree editor will create the flow independently. + // TODO: Wire tree editor to call review API with action='modify' on save + // when it detects proposalId in location.state, linking the published tree + // back to the proposal. + setIsSubmitting(true) + try { + await onReview('dismiss', 'Opened in Flow Editor for manual editing') + } catch { + // Warn but continue — the editor will still open + toast.warning('Could not update proposal status — it may still appear in the queue') + } finally { + setIsSubmitting(false) + } + + const flowData = proposal.proposed_flow_data + navigate('/trees/new', { + state: { + preloadedStructure: flowData.tree_structure || flowData, + proposalId: proposal.id, + proposalTitle: proposal.title, + }, + }) + } + + return ( +
+ {/* Header */} +
+

{proposal.title}

+ {proposal.description && ( +

{proposal.description}

+ )} +
+ {proposal.problem_domain && ( + + {proposal.problem_domain} + + )} + + + {Math.round(proposal.confidence_score * 100)}% confidence + + + + {proposal.supporting_session_count} supporting session{proposal.supporting_session_count !== 1 ? 's' : ''} + + + + {new Date(proposal.created_at).toLocaleString()} + +
+
+ + {/* Content */} +
+ {/* Source session link */} +
+

Source Session

+ + + View session that generated this proposal + +
+ + {/* Proposed diff (for enhancements) */} + {proposal.proposed_diff && (() => { + const diff = proposal.proposed_diff as { diff_description?: string; new_nodes?: Array<{ title?: string; question?: string; description?: string }> } + return ( +
+

Proposed Changes

+ {diff.diff_description && ( +

{diff.diff_description}

+ )} + {diff.new_nodes && diff.new_nodes.length > 0 && ( +
+

New nodes:

+ {diff.new_nodes.map((node, i) => ( +
+ + + {node.title || node.question || node.description || 'New node'} +
+ ))} +
+ )} +
+ ) + })()} + + {/* Flow data preview */} +
+ + {showFlowData && ( + 
+              {JSON.stringify(proposal.proposed_flow_data, null, 2)}
+
+ )} +
+ + {/* Supporting sessions */} + {proposal.supporting_session_ids.length > 1 && ( +
+

+ Supporting Sessions ({proposal.supporting_session_ids.length}) +

+
+ {proposal.supporting_session_ids.map((sid) => ( + + {sid} + + ))} +
+
+ )} + + {/* Review info (for already-reviewed proposals) */} + {proposal.reviewed_at && ( +
+

Review

+

+ {proposal.status} on{' '} + {new Date(proposal.reviewed_at).toLocaleString()} +

+ {proposal.reviewer_notes && ( +

{proposal.reviewer_notes}

+ )} +
+ )} +
+ + {/* Review actions bar */} + {canReview && ( +
+ {/* Notes input */} + setReviewNotes(e.target.value)} + placeholder="Reviewer notes (optional)" + className="w-full rounded-lg border border-border bg-card px-3 py-2 text-sm text-foreground placeholder:text-muted-foreground focus:border-[rgba(6,182,212,0.3)] focus:outline-none" + /> + + {/* Action buttons */} +
+ {proposal.proposal_type === 'new_flow' ? ( + + ) : ( + + Enhancement proposals require Edit & Publish + + )} + + + +
+
+ )} +
+ ) +} diff --git a/frontend/src/components/flowpilot/index.ts b/frontend/src/components/flowpilot/index.ts index f29c9c3a..77ad5ada 100644 --- a/frontend/src/components/flowpilot/index.ts +++ b/frontend/src/components/flowpilot/index.ts @@ -10,3 +10,6 @@ export { SessionTicketCard } from './SessionTicketCard' export { EscalateModal } from './EscalateModal' export { EscalationQueue } from './EscalationQueue' export { SessionBriefing } from './SessionBriefing' +export { ProposalCard } from './ProposalCard' +export { ProposalDetail } from './ProposalDetail' +export { InSessionScriptGenerator } from './InSessionScriptGenerator' diff --git a/frontend/src/components/layout/Sidebar.tsx b/frontend/src/components/layout/Sidebar.tsx index c85a8d80..3ba63123 100644 --- a/frontend/src/components/layout/Sidebar.tsx +++ b/frontend/src/components/layout/Sidebar.tsx @@ -1,8 +1,8 @@ import { useCallback, useEffect, useState } from 'react' import { useLocation } from 'react-router-dom' import { - LayoutGrid, Network, Wrench, Clock, FileOutput, BarChart3, - Settings, PanelLeftClose, PanelLeftOpen, MessageSquareText, + LayoutGrid, Network, Wrench, Clock, FileOutput, BarChart3, TrendingUp, + Settings, PanelLeftClose, PanelLeftOpen, MessageSquareText, ListChecks, BookOpen, Lightbulb, Code2, Library, Brain, WandSparkles, Sparkles, AlertTriangle, } from 'lucide-react' import { cn } from '@/lib/utils' @@ -94,8 +94,10 @@ export function Sidebar() { + + @@ -161,6 +163,7 @@ export function Sidebar() { + {/* Insights */}
@@ -168,6 +171,7 @@ export function Sidebar() {
+ )} diff --git a/frontend/src/pages/FlowPilotAnalyticsPage.tsx b/frontend/src/pages/FlowPilotAnalyticsPage.tsx new file mode 100644 index 00000000..38ae3c51 --- /dev/null +++ b/frontend/src/pages/FlowPilotAnalyticsPage.tsx @@ -0,0 +1,374 @@ +import { useState, useEffect } from 'react' +import { Link } from 'react-router-dom' +import { + BarChart3, Clock, Star, CheckCircle2, ArrowUpRight, + AlertTriangle, Lightbulb, Loader2, Ticket, +} from 'lucide-react' +import { + AreaChart, Area, BarChart, Bar, + XAxis, YAxis, CartesianGrid, Tooltip, ResponsiveContainer, +} from 'recharts' +import { flowpilotAnalyticsApi } from '@/api' +import { toast } from '@/lib/toast' +import type { FlowPilotDashboard, KnowledgeGapReport, KnowledgeGap } from '@/types/flowpilot-analytics' + +const PERIOD_OPTIONS = [ + { value: '7d', label: 'Last 7 days' }, + { value: '30d', label: 'Last 30 days' }, + { value: '90d', label: 'Last 90 days' }, +] + +const SEVERITY_STYLES = { + high: 'bg-rose-500/10 text-rose-500 border-rose-500/20', + medium: 'bg-amber-400/10 text-amber-400 border-amber-400/20', + low: 'bg-blue-400/10 text-blue-400 border-blue-400/20', +} + +export default function FlowPilotAnalyticsPage() { + const [period, setPeriod] = useState('30d') + const [dashboard, setDashboard] = useState(null) + const [gaps, setGaps] = useState(null) + const [loading, setLoading] = useState(true) + + useEffect(() => { + setLoading(true) + Promise.all([ + flowpilotAnalyticsApi.getDashboard(period), + flowpilotAnalyticsApi.getKnowledgeGaps(period), + ]) + .then(([d, g]) => { + setDashboard(d) + setGaps(g) + }) + .catch(() => toast.error('Failed to load analytics')) + .finally(() => setLoading(false)) + }, [period]) + + if (loading) { + return ( +
+ +
+ ) + } + + if (!dashboard) { + return ( +
+

Failed to load analytics

+
+ ) + } + + const conf = dashboard.confidence_breakdown + + return ( +
+ {/* Header */} +
+
+ + + +

FlowPilot Analytics

+
+
+ + Team Analytics + + +
+
+ + {/* Top row — Key metrics */} +
+ + + + + +
+ + {/* Second row — Charts */} +
+ {/* MTTR Trend */} +
+

+ MTTR Trend +

+ {dashboard.mttr_trend.length > 0 ? ( + + + + new Date(d).toLocaleDateString([], { month: 'short', day: 'numeric' })} + /> + `${Math.round(v)}m`} + /> + [`${Math.round(v ?? 0)}m`, 'MTTR']} + /> + + + + + + + + + + ) : ( +
+ No data for this period +
+ )} +
+ + {/* Domain Breakdown */} +
+

+ Sessions by Domain +

+ {dashboard.sessions_by_domain.length > 0 ? ( + + + + + + + + + + + ) : ( +
+ No domain data +
+ )} +
+
+ + {/* Third row — Confidence + Knowledge Coverage */} +
+ {/* Confidence Breakdown */} +
+

+ Confidence Tiers +

+
+ + + +
+
+ + {/* Knowledge Coverage */} +
+

+ Knowledge Coverage +

+
+
+

Total Flows

+

{dashboard.knowledge_coverage.total_flows}

+
+
+

AI-Generated

+

{dashboard.knowledge_coverage.ai_generated_flows}

+
+
+

Pending Review

+

{dashboard.knowledge_coverage.total_proposals_pending}

+
+
+

Approved This Period

+

{dashboard.knowledge_coverage.proposals_approved_this_period}

+
+
+ {dashboard.knowledge_coverage.total_proposals_pending > 0 && ( + + + Review {dashboard.knowledge_coverage.total_proposals_pending} pending proposals + + )} +
+
+ + {/* Fourth row — Knowledge Gaps */} + {gaps && gaps.gaps.length > 0 && ( +
+
+ +

+ Knowledge Gaps ({gaps.gaps.length}) +

+
+
+ {gaps.gaps.map((gap, i) => ( + + ))} +
+
+ )} +
+ ) +} + +// ── Sub-components ── + +function MetricCard({ + icon: Icon, + label, + value, + iconColor, +}: { + icon: typeof BarChart3 + label: string + value: string | number + iconColor: string +}) { + return ( +
+
+ + + +
+

{label}

+

{value}

+
+
+
+ ) +} + +function ConfidenceTierRow({ + label, + count, + rate, + color, + total, +}: { + label: string + count: number + rate: number + color: string + total: number +}) { + const pct = total > 0 ? (count / total) * 100 : 0 + return ( +
+
+ {label} + + {count} sessions · {rate.toFixed(1)}% resolved + +
+
+
+
+
+ ) +} + +function GapCard({ gap }: { gap: KnowledgeGap }) { + const severityStyle = SEVERITY_STYLES[gap.severity as keyof typeof SEVERITY_STYLES] ?? SEVERITY_STYLES.low + return ( +
+
+

{gap.title}

+ + {gap.severity} + +
+

{gap.description}

+
+ +

{gap.suggested_action}

+
+
+ ) +} diff --git a/frontend/src/pages/ReviewQueuePage.tsx b/frontend/src/pages/ReviewQueuePage.tsx new file mode 100644 index 00000000..9b15c2ad --- /dev/null +++ b/frontend/src/pages/ReviewQueuePage.tsx @@ -0,0 +1,201 @@ +import { useState, useEffect } from 'react' +import { + Lightbulb, Loader2, RefreshCw, CheckCircle2, Clock, Sparkles, +} from 'lucide-react' +import { flowProposalsApi } from '@/api' +import { toast } from '@/lib/toast' +import type { FlowProposalSummary, FlowProposalDetail, FlowProposalStats } from '@/types/flow-proposal' +import { ProposalCard } from '@/components/flowpilot/ProposalCard' +import { ProposalDetail } from '@/components/flowpilot/ProposalDetail' + +const STATUS_TABS = [ + { key: 'pending', label: 'Pending' }, + { key: 'approved', label: 'Approved' }, + { key: 'rejected', label: 'Rejected' }, + { key: 'dismissed', label: 'Dismissed' }, + { key: '', label: 'All' }, +] as const + +export default function ReviewQueuePage() { + const [proposals, setProposals] = useState([]) + const [stats, setStats] = useState(null) + const [selectedId, setSelectedId] = useState(null) + const [detail, setDetail] = useState(null) + const [isLoading, setIsLoading] = useState(true) + const [isLoadingDetail, setIsLoadingDetail] = useState(false) + const [activeTab, setActiveTab] = useState('pending') + const [sortBy, setSortBy] = useState('newest') + + const loadProposals = async () => { + setIsLoading(true) + try { + const [data, statsData] = await Promise.all([ + flowProposalsApi.list({ + status: activeTab || undefined, + sort_by: sortBy, + limit: 50, + }), + flowProposalsApi.getStats(), + ]) + setProposals(data) + setStats(statsData) + } catch { + toast.error('Failed to load proposals') + } finally { + setIsLoading(false) + } + } + + useEffect(() => { + loadProposals() + }, [activeTab, sortBy]) // eslint-disable-line react-hooks/exhaustive-deps + + const loadDetail = async (id: string) => { + setSelectedId(id) + setIsLoadingDetail(true) + try { + const data = await flowProposalsApi.get(id) + setDetail(data) + } catch { + toast.error('Failed to load proposal') + } finally { + setIsLoadingDetail(false) + } + } + + const handleReview = async (action: 'approve' | 'reject' | 'modify' | 'dismiss', notes?: string) => { + if (!selectedId) return + try { + await flowProposalsApi.review(selectedId, { + action, + reviewer_notes: notes || null, + }) + toast.success( + action === 'approve' ? 'Flow published!' : + action === 'dismiss' ? 'Proposal dismissed' : + action === 'reject' ? 'Proposal rejected' : + 'Flow published with modifications' + ) + setSelectedId(null) + setDetail(null) + loadProposals() + } catch { + toast.error('Review action failed') + } + } + + return ( +
+ {/* Left panel — Proposal list */} +
+ {/* Header */} +
+
+
+ +

Review Queue

+
+ +
+ + {/* Stats bar */} + {stats && ( +
+ + + {stats.pending_count} pending + + + + {stats.approved_this_week} approved + + + + {stats.auto_reinforced_this_week} reinforced + +
+ )} + + {/* Tabs */} +
+ {STATUS_TABS.map((tab) => ( + + ))} +
+ + {/* Sort */} + +
+ + {/* Proposal list */} +
+ {isLoading ? ( +
+ +
+ ) : proposals.length === 0 ? ( +
+ +

No proposals found

+
+ ) : ( + proposals.map((proposal) => ( + loadDetail(proposal.id)} + /> + )) + )} +
+
+ + {/* Right panel — Detail */} +
+ {isLoadingDetail ? ( +
+ +
+ ) : detail ? ( + + ) : ( +
+
+ +

Select a proposal to review

+
+
+ )} +
+
+ ) +} diff --git a/frontend/src/router.tsx b/frontend/src/router.tsx index 121c85d8..71acec9a 100644 --- a/frontend/src/router.tsx +++ b/frontend/src/router.tsx @@ -47,6 +47,8 @@ const AssistantChatPage = lazy(() => import('@/pages/AssistantChatPage')) const FlowAssistPage = lazy(() => import('@/pages/FlowAssistPage')) const FlowPilotSessionPage = lazy(() => import('@/pages/FlowPilotSessionPage')) const EscalationQueuePage = lazy(() => import('@/pages/EscalationQueuePage')) +const ReviewQueuePage = lazy(() => import('@/pages/ReviewQueuePage')) +const FlowPilotAnalyticsPage = lazy(() => import('@/pages/FlowPilotAnalyticsPage')) const KBAcceleratorPage = lazy(() => import('@/pages/KBAcceleratorPage')) const GuidesHubPage = lazy(() => import('@/pages/GuidesHubPage')) const GuideDetailPage = lazy(() => import('@/pages/GuideDetailPage')) @@ -174,6 +176,8 @@ export const router = sentryCreateBrowserRouter([ { path: 'pilot', element: page(FlowPilotSessionPage) }, { path: 'pilot/:sessionId', element: page(FlowPilotSessionPage) }, { path: 'escalations', element: page(EscalationQueuePage) }, + { path: 'review-queue', element: page(ReviewQueuePage) }, + { path: 'analytics/flowpilot', element: page(FlowPilotAnalyticsPage) }, { path: 'guides', element: page(GuidesHubPage) }, { path: 'guides/:slug', element: page(GuideDetailPage) }, // Admin routes diff --git a/frontend/src/types/flow-proposal.ts b/frontend/src/types/flow-proposal.ts new file mode 100644 index 00000000..6005e1ff --- /dev/null +++ b/frontend/src/types/flow-proposal.ts @@ -0,0 +1,38 @@ +// ── Flow Proposals (Knowledge Flywheel / Review Queue) ── + +export interface FlowProposalSummary { + id: string + proposal_type: 'new_flow' | 'enhancement' | 'branch_addition' | 'auto_reinforced' + title: string + description: string | null + problem_domain: string | null + confidence_score: number + supporting_session_count: number + status: 'pending' | 'approved' | 'modified' | 'rejected' | 'dismissed' | 'auto_reinforced' + target_flow_id: string | null + source_session_id: string + created_at: string +} + +export interface FlowProposalDetail extends FlowProposalSummary { + proposed_flow_data: Record + proposed_diff: Record | null + supporting_session_ids: string[] + reviewer_notes: string | null + reviewed_by: string | null + reviewed_at: string | null +} + +export interface ReviewProposalRequest { + action: 'approve' | 'reject' | 'modify' | 'dismiss' + reviewer_notes?: string | null + modified_flow_data?: Record | null +} + +export interface FlowProposalStats { + pending_count: number + approved_this_week: number + rejected_this_week: number + auto_reinforced_this_week: number + top_domains: Array<{ domain: string; count: number }> +} diff --git a/frontend/src/types/flowpilot-analytics.ts b/frontend/src/types/flowpilot-analytics.ts new file mode 100644 index 00000000..cd843d94 --- /dev/null +++ b/frontend/src/types/flowpilot-analytics.ts @@ -0,0 +1,79 @@ +export interface MTTRDataPoint { + date: string + mttr_minutes: number + session_count: number +} + +export interface DomainBreakdown { + domain: string + total: number + resolved: number + escalated: number + resolution_rate: number +} + +export interface ConfidenceBreakdown { + guided_sessions: number + guided_resolution_rate: number + exploring_sessions: number + exploring_resolution_rate: number + discovery_sessions: number + discovery_resolution_rate: number +} + +export interface DomainCoverage { + domain: string + flow_count: number + session_count: number + guided_rate: number +} + +export interface KnowledgeCoverage { + total_flows: number + ai_generated_flows: number + total_proposals_pending: number + proposals_approved_this_period: number + proposals_rejected_this_period: number + coverage_by_domain: DomainCoverage[] +} + +export interface PsaMetrics { + ticket_link_rate: number + auto_push_success_rate: number + auto_push_retry_success_rate: number + total_time_entries_logged: number + total_hours_logged: number +} + +export interface FlowPilotDashboard { + period: string + total_sessions: number + resolved_sessions: number + escalated_sessions: number + abandoned_sessions: number + resolution_rate: number + avg_steps_to_resolution: number + avg_session_duration_minutes: number + avg_rating: number | null + mttr_minutes: number | null + mttr_trend: MTTRDataPoint[] + sessions_by_domain: DomainBreakdown[] + confidence_breakdown: ConfidenceBreakdown + knowledge_coverage: KnowledgeCoverage + psa_metrics: PsaMetrics | null +} + +export interface KnowledgeGap { + gap_type: string + domain: string | null + severity: 'high' | 'medium' | 'low' + title: string + description: string + evidence: Record + suggested_action: string +} + +export interface KnowledgeGapReport { + generated_at: string + gaps: KnowledgeGap[] +} diff --git a/frontend/src/types/index.ts b/frontend/src/types/index.ts index 8c86cf00..f8fb643a 100644 --- a/frontend/src/types/index.ts +++ b/frontend/src/types/index.ts @@ -13,6 +13,8 @@ export * from './analytics' export * from './copilot' export type { AssistantChat, AssistantChatMessage, ChatListItem, ChatMessageResponse, RetentionSettings } from './assistant-chat' export * from './ai-session' +export * from './flow-proposal' +export * from './flowpilot-analytics' // API response wrapper types export interface PaginatedResponse { diff --git a/frontend/src/types/scripts.ts b/frontend/src/types/scripts.ts index ab53c41c..92678126 100644 --- a/frontend/src/types/scripts.ts +++ b/frontend/src/types/scripts.ts @@ -73,7 +73,8 @@ export interface ScriptTemplateDetail extends ScriptTemplateListItem { export interface ScriptGenerateRequest { template_id: string parameters: Record - session_id?: string // Phase 3: passed when generating inside a session + session_id?: string // Legacy tree-based session + ai_session_id?: string // FlowPilot AI session } export interface ScriptGenerateResponse {