feat(ai): robust response extraction + structured-output foundation

Harden the Anthropic provider and lay the groundwork for schema-constrained
JSON, optimizing the existing claude-sonnet-4-6 / claude-haiku-4-5 usage
(no model changes).

ai_provider.py:
- _extract_text_from_response replaces fragile response.content[0].text:
  skips non-text leading blocks (e.g. thinking), returns the first text
  block, logs an anthropic.stop_reason warning on max_tokens/refusal
  (truncation now observable), and raises ValueError on a no-text response.
- generate_json gains an optional `schema` param. Anthropic wires it to
  output_config.format (structured outputs); schema=None preserves the exact
  prior call for every existing caller. Gemini accepts-and-ignores it.

kb_conversion_service.py:
- TROUBLESHOOTING_SCHEMA / PROCEDURAL_SCHEMA + _schema_for_target_type(),
  modelled as a strict superset of every field the prompts emit.
- convert_document passes the schema only when the new
  AI_KB_CONVERT_STRUCTURED_OUTPUT setting is True (default False). The
  _try_repair_json fallback stays as belt-and-suspenders.

Tests: 14 provider + 7 schema, TDD (red-green). Live constrained-decoding
smoke-test still required before enabling the flag in production.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

backend/app/core/kb_conversion_service.py

@@ -202,6 +202,115 @@ the engineer attached, NOT from this schema):
 9. Return ONLY valid JSON — no markdown fences, no explanation text."""
 
 
+# ── Structured-output schemas ──
+#
+# These constrain the model's JSON via Anthropic structured outputs
+# (output_config.format) so the response is guaranteed valid and parseable —
+# no markdown fences, no truncated-brace repair. They must be a SUPERSET of
+# every field the corresponding system prompt instructs the model to emit:
+# additionalProperties is False everywhere, so any field the prompt asks for
+# but the schema omits would be impossible to produce.
+#
+# `type`/`field_type` are intentionally left as plain strings (no enum): the
+# downstream parser already normalizes/tolerates the type values, and an enum
+# risks constraining the model away from a value the prompt would yield.
+
+_TROUBLESHOOTING_OPTION_SCHEMA: dict[str, Any] = {
+    "type": "object",
+    "properties": {
+        "label": {"type": "string"},
+        "next_node_id": {"type": "string"},
+    },
+    "required": ["label", "next_node_id"],
+    "additionalProperties": False,
+}
+
+_TROUBLESHOOTING_NODE_SCHEMA: dict[str, Any] = {
+    "type": "object",
+    "properties": {
+        "id": {"type": "string"},
+        "type": {"type": "string"},
+        "question": {"type": "string"},
+        "options": {"type": "array", "items": _TROUBLESHOOTING_OPTION_SCHEMA},
+        "next_node_id": {"type": "string"},
+        "confidence": {"type": "number"},
+        "source_excerpt": {"type": "string"},
+    },
+    # Only the universal fields are required. `question`/`options`/`next_node_id`
+    # vary by node type and stay optional so a resolution node need not carry
+    # options and an action node need not carry a question.
+    "required": ["id", "type", "confidence", "source_excerpt"],
+    "additionalProperties": False,
+}
+
+TROUBLESHOOTING_SCHEMA: dict[str, Any] = {
+    "type": "object",
+    "properties": {
+        "title": {"type": "string"},
+        "description": {"type": "string"},
+        "nodes": {"type": "array", "items": _TROUBLESHOOTING_NODE_SCHEMA},
+    },
+    "required": ["title", "description", "nodes"],
+    "additionalProperties": False,
+}
+
+_PROCEDURAL_STEP_SCHEMA: dict[str, Any] = {
+    "type": "object",
+    "properties": {
+        "id": {"type": "string"},
+        "type": {"type": "string"},
+        "content": {"type": "string"},
+        "confidence": {"type": "number"},
+        "source_excerpt": {"type": "string"},
+    },
+    "required": ["id", "type", "content", "confidence", "source_excerpt"],
+    "additionalProperties": False,
+}
+
+_PROCEDURAL_INTAKE_SCHEMA: dict[str, Any] = {
+    "type": "object",
+    "properties": {
+        "variable_name": {"type": "string"},
+        "label": {"type": "string"},
+        "field_type": {"type": "string"},
+        "required": {"type": "boolean"},
+        "display_order": {"type": "integer"},
+    },
+    "required": [
+        "variable_name",
+        "label",
+        "field_type",
+        "required",
+        "display_order",
+    ],
+    "additionalProperties": False,
+}
+
+PROCEDURAL_SCHEMA: dict[str, Any] = {
+    "type": "object",
+    "properties": {
+        "title": {"type": "string"},
+        "description": {"type": "string"},
+        "steps": {"type": "array", "items": _PROCEDURAL_STEP_SCHEMA},
+        "intake_form": {"type": "array", "items": _PROCEDURAL_INTAKE_SCHEMA},
+    },
+    "required": ["title", "description", "steps", "intake_form"],
+    "additionalProperties": False,
+}
+
+
+def _schema_for_target_type(target_type: str) -> dict[str, Any]:
+    """Return the structured-output schema for a KB conversion target type.
+
+    Mirrors the prompt selection in ``convert_document``: only
+    ``"troubleshooting"`` uses the decision-tree schema; everything else is
+    treated as a procedural flow.
+    """
+    if target_type == "troubleshooting":
+        return TROUBLESHOOTING_SCHEMA
+    return PROCEDURAL_SCHEMA
+
+
 def _build_user_message(
     source_text: str,
     source_metadata: dict[str, Any] | None,
@@ -404,6 +513,16 @@ async def convert_document(
     model = settings.get_model_for_action("kb_convert")
     provider = get_ai_provider(model=model)
 
+    # Structured outputs (flagged): constrain the response to a JSON schema so
+    # the model can't emit fences or truncated JSON. Falls back to prompt-only
+    # JSON (schema=None) when disabled; the parse path below stays intact either
+    # way as a belt-and-suspenders fallback.
+    schema = (
+        _schema_for_target_type(kb_import.target_type)
+        if settings.AI_KB_CONVERT_STRUCTURED_OUTPUT
+        else None
+    )
+
     try:
         raw_text, input_tokens, output_tokens = await provider.generate_json(
             system_prompt=[
@@ -414,6 +533,7 @@ async def convert_document(
             ],
             messages=[{"role": "user", "content": user_message}],
             max_tokens=16384,
+            schema=schema,
         )
     except Exception as e:
         logger.error("AI conversion failed for kb_import=%s: %s", kb_import.id, e)