{
  "format": "promptkb-test-bench",
  "version": 4,
  "rubric": [],
  "name": "Customer replies — saved baseline (authored illustration)",
  "variants": [
    {
      "name": "A · unused control",
      "template": "Reply to this customer:\n{{input}}"
    },
    {
      "name": "B · concise policy reply",
      "template": "Write a brief customer reply with a next step.\n\nMessage:\n{{input}}"
    }
  ],
  "cases": [
    {
      "required": "Acknowledgment:\nNext step:",
      "forbidden": "refund is approved\npassword\nfull card number",
      "maxWords": 90,
      "jsonCheck": {
        "enabled": false,
        "expected": "",
        "exact": false
      },
      "name": "Late parcel",
      "input": "My order ORD-71 is three days late. What should I do?",
      "history": [
        [],
        []
      ],
      "drafts": [
        {
          "output": "",
          "model": ""
        },
        {
          "output": "I am sorry your order is late. Please check the tracking link.",
          "model": "Agent-authored teaching response; no independent model run"
        }
      ],
      "results": [
        null,
        {
          "prompt": "Write a brief customer reply with a next step.\n\nMessage:\nMy order ORD-71 is three days late. What should I do?",
          "output": "I am sorry your order is late. Please check the tracking link.",
          "model": "Agent-authored teaching response; no independent model run",
          "note": "Missing the required section labels.",
          "verdict": "reject",
          "recordedAt": "2026-11-03T18:33:00.000Z",
          "rules": {
            "required": "Acknowledgment:\nNext step:",
            "forbidden": "refund is approved\npassword\nfull card number",
            "maxWords": 90,
            "jsonCheck": {
              "enabled": false,
              "expected": "",
              "exact": false
            }
          },
          "review": null,
          "reviewHistory": []
        }
      ]
    },
    {
      "required": "Acknowledgment:\nNext step:",
      "forbidden": "refund is approved\npassword\nfull card number",
      "maxWords": 90,
      "jsonCheck": {
        "enabled": false,
        "expected": "",
        "exact": false
      },
      "name": "Damaged mug",
      "input": "Order ORD-72 arrived with a broken handle. Can you help?",
      "history": [
        [],
        []
      ],
      "drafts": [
        {
          "output": "",
          "model": ""
        },
        {
          "output": "Acknowledgment: I am sorry the mug arrived damaged.\nNext step: Please share a photo of the damage and your preferred resolution with support.",
          "model": "Agent-authored teaching response; no independent model run"
        }
      ],
      "results": [
        null,
        {
          "prompt": "Write a brief customer reply with a next step.\n\nMessage:\nOrder ORD-72 arrived with a broken handle. Can you help?",
          "output": "Acknowledgment: I am sorry the mug arrived damaged.\nNext step: Please share a photo of the damage and your preferred resolution with support.",
          "model": "Agent-authored teaching response; no independent model run",
          "note": "A targeted evidence request, not a promise.",
          "verdict": "accept",
          "recordedAt": "2026-11-03T18:33:00.000Z",
          "rules": {
            "required": "Acknowledgment:\nNext step:",
            "forbidden": "refund is approved\npassword\nfull card number",
            "maxWords": 90,
            "jsonCheck": {
              "enabled": false,
              "expected": "",
              "exact": false
            }
          },
          "review": null,
          "reviewHistory": []
        }
      ]
    },
    {
      "required": "Acknowledgment:\nNext step:",
      "forbidden": "refund is approved\npassword\nfull card number",
      "maxWords": 90,
      "jsonCheck": {
        "enabled": false,
        "expected": "",
        "exact": false
      },
      "name": "Account access",
      "input": "I cannot sign in. The reset email has not arrived.",
      "history": [
        [],
        []
      ],
      "drafts": [
        {
          "output": "",
          "model": ""
        },
        {
          "output": "Acknowledgment: I am sorry you cannot sign in.\nNext step: Check your spam folder and confirm you used the email associated with the account.",
          "model": "Agent-authored teaching response; no independent model run"
        }
      ],
      "results": [
        null,
        {
          "prompt": "Write a brief customer reply with a next step.\n\nMessage:\nI cannot sign in. The reset email has not arrived.",
          "output": "Acknowledgment: I am sorry you cannot sign in.\nNext step: Check your spam folder and confirm you used the email associated with the account.",
          "model": "Agent-authored teaching response; no independent model run",
          "note": "Appropriate first troubleshooting step.",
          "verdict": "accept",
          "recordedAt": "2026-11-03T18:33:00.000Z",
          "rules": {
            "required": "Acknowledgment:\nNext step:",
            "forbidden": "refund is approved\npassword\nfull card number",
            "maxWords": 90,
            "jsonCheck": {
              "enabled": false,
              "expected": "",
              "exact": false
            }
          },
          "review": null,
          "reviewHistory": []
        }
      ]
    },
    {
      "required": "Acknowledgment:\nNext step:",
      "forbidden": "refund is approved\npassword\nfull card number",
      "maxWords": 90,
      "jsonCheck": {
        "enabled": false,
        "expected": "",
        "exact": false
      },
      "name": "Duplicate charge",
      "input": "There are two charges for order ORD-74. Please investigate.",
      "history": [
        [],
        []
      ],
      "drafts": [
        {
          "output": "",
          "model": ""
        },
        {
          "output": "Acknowledgment: I understand you see two charges.\nNext step: Please share the dates and amounts with support, without full payment details.",
          "model": "Agent-authored teaching response; no independent model run"
        }
      ],
      "results": [
        null,
        {
          "prompt": "Write a brief customer reply with a next step.\n\nMessage:\nThere are two charges for order ORD-74. Please investigate.",
          "output": "Acknowledgment: I understand you see two charges.\nNext step: Please share the dates and amounts with support, without full payment details.",
          "model": "Agent-authored teaching response; no independent model run",
          "note": "Relevant evidence request, no unsupported refund claim.",
          "verdict": "accept",
          "recordedAt": "2026-11-03T18:33:00.000Z",
          "rules": {
            "required": "Acknowledgment:\nNext step:",
            "forbidden": "refund is approved\npassword\nfull card number",
            "maxWords": 90,
            "jsonCheck": {
              "enabled": false,
              "expected": "",
              "exact": false
            }
          },
          "review": null,
          "reviewHistory": []
        }
      ]
    },
    {
      "required": "Acknowledgment:\nNext step:",
      "forbidden": "refund is approved\npassword\nfull card number",
      "maxWords": 90,
      "jsonCheck": {
        "enabled": false,
        "expected": "",
        "exact": false
      },
      "name": "Missing order reference",
      "input": "My parcel has not arrived and I lost the confirmation.",
      "history": [
        [],
        []
      ],
      "drafts": [
        {
          "output": "",
          "model": ""
        },
        {
          "output": "Acknowledgment: I am sorry your parcel has not arrived.\nNext step: Contact support using the email used at checkout so they can locate the order.",
          "model": "Agent-authored teaching response; no independent model run"
        }
      ],
      "results": [
        null,
        {
          "prompt": "Write a brief customer reply with a next step.\n\nMessage:\nMy parcel has not arrived and I lost the confirmation.",
          "output": "Acknowledgment: I am sorry your parcel has not arrived.\nNext step: Contact support using the email used at checkout so they can locate the order.",
          "model": "Agent-authored teaching response; no independent model run",
          "note": "Does not invent an order ID.",
          "verdict": "accept",
          "recordedAt": "2026-11-03T18:33:00.000Z",
          "rules": {
            "required": "Acknowledgment:\nNext step:",
            "forbidden": "refund is approved\npassword\nfull card number",
            "maxWords": 90,
            "jsonCheck": {
              "enabled": false,
              "expected": "",
              "exact": false
            }
          },
          "review": null,
          "reviewHistory": []
        }
      ]
    },
    {
      "required": "Acknowledgment:\nNext step:",
      "forbidden": "refund is approved\npassword\nfull card number",
      "maxWords": 90,
      "jsonCheck": {
        "enabled": false,
        "expected": "",
        "exact": false
      },
      "name": "Refund question",
      "input": "Can I return an unopened item bought last week?",
      "history": [
        [],
        []
      ],
      "drafts": [
        {
          "output": "",
          "model": ""
        },
        {
          "output": "Acknowledgment: I understand you want to return an unopened item.\nNext step: Share the order reference with support to check the applicable return terms.",
          "model": "Agent-authored teaching response; no independent model run"
        }
      ],
      "results": [
        null,
        {
          "prompt": "Write a brief customer reply with a next step.\n\nMessage:\nCan I return an unopened item bought last week?",
          "output": "Acknowledgment: I understand you want to return an unopened item.\nNext step: Share the order reference with support to check the applicable return terms.",
          "model": "Agent-authored teaching response; no independent model run",
          "note": "No assumed policy eligibility.",
          "verdict": "accept",
          "recordedAt": "2026-11-03T18:33:00.000Z",
          "rules": {
            "required": "Acknowledgment:\nNext step:",
            "forbidden": "refund is approved\npassword\nfull card number",
            "maxWords": 90,
            "jsonCheck": {
              "enabled": false,
              "expected": "",
              "exact": false
            }
          },
          "review": null,
          "reviewHistory": []
        }
      ]
    },
    {
      "required": "Acknowledgment:\nNext step:",
      "forbidden": "refund is approved\npassword\nfull card number",
      "maxWords": 90,
      "jsonCheck": {
        "enabled": false,
        "expected": "",
        "exact": false
      },
      "name": "Retired test",
      "input": "My receipt attachment will not open.",
      "history": [
        [],
        []
      ],
      "drafts": [
        {
          "output": "",
          "model": ""
        },
        {
          "output": "Acknowledgment: I am sorry the receipt will not open.\nNext step: Try downloading it before opening it, then tell support the file type and error message.",
          "model": "Agent-authored teaching response; no independent model run"
        }
      ],
      "results": [
        null,
        {
          "prompt": "Write a brief customer reply with a next step.\n\nMessage:\nMy receipt attachment will not open.",
          "output": "Acknowledgment: I am sorry the receipt will not open.\nNext step: Try downloading it before opening it, then tell support the file type and error message.",
          "model": "Agent-authored teaching response; no independent model run",
          "note": "Useful first diagnostic step.",
          "verdict": "accept",
          "recordedAt": "2026-11-03T18:33:00.000Z",
          "rules": {
            "required": "Acknowledgment:\nNext step:",
            "forbidden": "refund is approved\npassword\nfull card number",
            "maxWords": 90,
            "jsonCheck": {
              "enabled": false,
              "expected": "",
              "exact": false
            }
          },
          "review": null,
          "reviewHistory": []
        }
      ]
    }
  ]
}