refs #706: add Smilegate QA history benchmark

ai-web-agent-console/ai_web_agent_console/qa_history.py

@@ -0,0 +1,289 @@
+"""Customer QA benchmark parsing and deterministic SQL evaluation."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+import hashlib
+import json
+from pathlib import Path
+import re
+from typing import Any, Mapping
+
+
+class QaBenchmarkError(RuntimeError):
+    """Raised when the QA benchmark source cannot be used safely."""
+
+
+@dataclass(frozen=True)
+class QaQuestion:
+    question_id: int | None
+    question_code: str
+    category: str
+    title: str
+    question_text: str
+    source_document: str
+    source_sheet: str
+    source_row: int | None
+    source_scenario: str
+    sample_sql: str
+    expected_focus: str
+    baseline_sql: str
+    baseline_answer: str
+    support_level: str
+    evaluation_rule: Mapping[str, Any]
+    last_judgment_status: str = ""
+    last_evaluated_at: str = ""
+
+
+@dataclass(frozen=True)
+class QaJudgment:
+    status: str
+    reason: str
+
+
+def question_fingerprint(question_text: str) -> str:
+    normalized = " ".join(str(question_text or "").split()).casefold()
+    return hashlib.sha256(normalized.encode("utf-8")).hexdigest()
+
+
+def _compact_text(value: object) -> str:
+    return str(value or "").strip()
+
+
+def _string_list(value: object) -> tuple[str, ...]:
+    if not isinstance(value, list):
+        return ()
+    return tuple(_compact_text(item) for item in value if _compact_text(item))
+
+
+def load_benchmark_questions(path: Path) -> tuple[QaQuestion, ...]:
+    try:
+        payload = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, UnicodeError, ValueError) as exc:
+        raise QaBenchmarkError(f"질답 기준 파일을 읽지 못했습니다: {path}") from exc
+    rows = payload.get("scenarios") if isinstance(payload, Mapping) else None
+    if not isinstance(rows, list):
+        raise QaBenchmarkError("질답 기준 파일에 scenarios 배열이 필요합니다.")
+
+    questions: list[QaQuestion] = []
+    seen_codes: set[str] = set()
+    for row in rows:
+        if not isinstance(row, Mapping):
+            raise QaBenchmarkError("질답 기준의 각 시나리오는 객체여야 합니다.")
+        source = row.get("source") if isinstance(row.get("source"), Mapping) else {}
+        history = (
+            row.get("historical_answer")
+            if isinstance(row.get("historical_answer"), Mapping)
+            else {}
+        )
+        code = _compact_text(row.get("case_id")).upper()
+        question_text = _compact_text(row.get("question"))
+        if not code or not question_text:
+            raise QaBenchmarkError("각 질답 기준에는 case_id와 question이 필요합니다.")
+        if code in seen_codes:
+            raise QaBenchmarkError(f"중복된 질답 case_id입니다: {code}")
+        evaluation_rule = row.get("evaluation_rule")
+        if not isinstance(evaluation_rule, Mapping):
+            evaluation_rule = {}
+        questions.append(
+            QaQuestion(
+                question_id=None,
+                question_code=code,
+                category=_compact_text(row.get("category")) or "GENERAL",
+                title=_compact_text(row.get("title")) or code,
+                question_text=question_text,
+                source_document=_compact_text(source.get("workbook")),
+                source_sheet=_compact_text(source.get("sheet")),
+                source_row=_number_or_none(source.get("excel_row")),
+                source_scenario=_compact_text(source.get("scenario")),
+                sample_sql=_compact_text(source.get("sample_query")),
+                expected_focus=_compact_text(row.get("expected_focus")),
+                baseline_sql=_compact_text(history.get("generated_sql")),
+                baseline_answer=_compact_text(history.get("answer_text")),
+                support_level=_compact_text(row.get("support_level")).upper() or "UNKNOWN",
+                evaluation_rule={
+                    "required_sql_terms": list(
+                        _string_list(evaluation_rule.get("required_sql_terms"))
+                    ),
+                    "recommended_sql_terms": list(
+                        _string_list(evaluation_rule.get("recommended_sql_terms"))
+                    ),
+                },
+            )
+        )
+        seen_codes.add(code)
+    return tuple(questions)
+
+
+def question_from_record(record: Mapping[str, Any]) -> QaQuestion:
+    rule = record.get("evaluation_rule")
+    if isinstance(rule, str):
+        try:
+            rule = json.loads(rule)
+        except ValueError:
+            rule = {}
+    if not isinstance(rule, Mapping):
+        rule = {}
+    return QaQuestion(
+        question_id=_number_or_none(record.get("question_id")),
+        question_code=_compact_text(record.get("question_code")),
+        category=_compact_text(record.get("category")) or "GENERAL",
+        title=_compact_text(record.get("title")) or _compact_text(record.get("question_code")),
+        question_text=_compact_text(record.get("question_text")),
+        source_document=_compact_text(record.get("source_document")),
+        source_sheet=_compact_text(record.get("source_sheet")),
+        source_row=_number_or_none(record.get("source_row")),
+        source_scenario=_compact_text(record.get("source_scenario")),
+        sample_sql=_compact_text(record.get("sample_sql")),
+        expected_focus=_compact_text(record.get("expected_focus")),
+        baseline_sql=_compact_text(record.get("baseline_sql")),
+        baseline_answer=_compact_text(record.get("baseline_answer")),
+        support_level=_compact_text(record.get("support_level")).upper() or "UNKNOWN",
+        evaluation_rule={
+            "required_sql_terms": list(
+                _string_list(rule.get("required_sql_terms"))
+            ),
+            "recommended_sql_terms": list(
+                _string_list(rule.get("recommended_sql_terms"))
+            ),
+        },
+        last_judgment_status=_compact_text(record.get("last_judgment_status")),
+        last_evaluated_at=_compact_text(record.get("last_evaluated_at")),
+    )
+
+
+def _number_or_none(value: object) -> int | None:
+    if value is None or value == "":
+        return None
+    try:
+        return int(value)
+    except (TypeError, ValueError):
+        return None
+
+
+def _append_issue(issues: list[str], condition: bool, message: str) -> None:
+    if condition:
+        issues.append(message)
+
+
+def evaluate_sql(
+    question: QaQuestion | None,
+    generated_sql: str,
+    *,
+    execution_succeeded: bool,
+    error_text: str = "",
+) -> QaJudgment:
+    """Evaluate the generated SQL against the customer-approved benchmark rule."""
+    if question is None or not question.question_code:
+        return QaJudgment(
+            status="REVIEW",
+            reason="자유 질의입니다. 고객 기준 정답 시나리오와 연결되지 않아 수동 검토가 필요합니다.",
+        )
+
+    sql = _compact_text(generated_sql)
+    upper_sql = sql.upper()
+    lower_sql = sql.lower()
+    execution_status = "PASS" if execution_succeeded else "FAIL_EXECUTION"
+    issues: list[str] = []
+    failure_markers = ("could not be generated", "exception encountered", "invalid identifier", "ora-")
+    has_failure_text = any(marker in lower_sql for marker in failure_markers)
+    required = _string_list(question.evaluation_rule.get("required_sql_terms"))
+    recommended = _string_list(question.evaluation_rule.get("recommended_sql_terms"))
+    missing_required = [term for term in required if term.upper() not in upper_sql]
+    missing_recommended = [term for term in recommended if term.upper() not in upper_sql]
+
+    if not execution_succeeded:
+        issues.append(f"실행 상태가 {execution_status}입니다.")
+    if not sql:
+        issues.append("생성 SQL이 없습니다.")
+    if has_failure_text:
+        issues.append("생성 SQL에 오류 또는 생성 실패 문구가 포함되어 있습니다.")
+    if missing_required:
+        issues.append("필수 SQL 요소 누락: " + ", ".join(missing_required))
+    if missing_recommended:
+        issues.append("권장 SQL 요소 누락: " + ", ".join(missing_recommended))
+    _append_issue(
+        issues,
+        bool(re.search(r'_[A-Z0-9]*YN"\s*=\s*\'1\'', sql, flags=re.IGNORECASE)),
+        "*_YN 컬럼은 샘플 메타데이터의 Y/N 값으로 비교해야 합니다.",
+    )
+    _append_issue(
+        issues,
+        bool(re.search(r'_[A-Z0-9]*FLAG"\s*=\s*\'Y\'', sql, flags=re.IGNORECASE)),
+        "*_FLAG 컬럼은 샘플 메타데이터의 0/1 값으로 비교해야 합니다.",
+    )
+
+    support = question.support_level
+    if support == "UNSUPPORTED":
+        uses_alias_lookup = "COMN_GAME_ALIAS_BAS" in upper_sql
+        substitutes_sample_game = "STOVE_CHAOSZERO" in upper_sql
+        if not sql and any(marker in error_text.lower() for marker in failure_markers):
+            return QaJudgment("PASS", "미지원 게임 질문이 실행 가능한 SQL로 변환되지 않았습니다. 기대한 안전 차단입니다.")
+        if uses_alias_lookup and not substitutes_sample_game and execution_succeeded:
+            return QaJudgment("PASS", "미지원 게임을 별칭 테이블로만 확인했고 샘플 게임 ID를 임의 대입하지 않았습니다.")
+        return QaJudgment("FAIL", "미지원 게임이 안전한 별칭 조회로 제한되지 않았거나 실행에 실패했습니다.")
+
+    if not execution_succeeded or not sql or has_failure_text or missing_required:
+        return QaJudgment("FAIL", "\n".join(issues) or "필수 SQL 또는 실행 검증에 실패했습니다.")
+
+    _apply_case_specific_rules(question.question_code, sql, upper_sql, issues)
+    if any(issue.startswith("필수") or issue.startswith("월간") or issue.startswith("일별") or issue.startswith("주간") or issue.startswith("CZN-") for issue in issues):
+        return QaJudgment("FAIL", "\n".join(issues))
+    if support == "PARTIAL":
+        issues.append("지원 범위가 일부인 질문이므로 결과 범위를 함께 검토해야 합니다.")
+    if issues:
+        return QaJudgment("WARN", "\n".join(issues))
+    return QaJudgment("PASS", "필수 테이블·컬럼·집계 조건과 실행 결과를 확인했습니다.")
+
+
+def _apply_case_specific_rules(code: str, sql: str, upper_sql: str, issues: list[str]) -> None:
+    if code == "STD-26":
+        _append_issue(issues, "TRUNC(DATE" in upper_sql, "월간 NRU는 DATE 리터럴이 아니라 BASE_DT 기준 월 범위를 사용해야 합니다.")
+        _append_issue(issues, "MAX" not in upper_sql or "BASE_DT" not in upper_sql, "월간 NRU는 최신 MAX(BASE_DT) 스냅샷을 기준으로 해야 합니다.")
+        _append_issue(issues, "RAW_NRU_DT" in upper_sql and "TRUNC(" not in upper_sql, "월간 NRU의 RAW_NRU_DT 기간은 BASE_DT 기준으로 계산해야 합니다.")
+    elif code == "STD-27":
+        _append_issue(issues, "AU_FLAG" in upper_sql, "월간 AU는 AU_FLAG가 아니라 LAST_CONN_DT 전월 조건으로 계산해야 합니다.")
+        _append_issue(issues, any(term not in upper_sql for term in ("MAX", "BASE_DT", "ADD_MONTHS")), "월간 AU는 최신 BASE_DT의 전월을 기준으로 해야 합니다.")
+        _append_issue(issues, any(term not in upper_sql for term in ("STD_USER_YN", "EXPT_USER_YN", "LAST_CONN_DT")), "월간 AU에는 STD_USER_YN, EXPT_USER_YN, LAST_CONN_DT 조건이 필요합니다.")
+    elif code == "STD-22":
+        _append_issue(issues, "NRU_FLAG" not in upper_sql, "일별 NRU는 NRU_FLAG=1을 사용해야 합니다.")
+    elif code == "STD-25":
+        _append_issue(issues, "LAST_CONN_DT" not in upper_sql, "최근 7일 AU는 LAST_CONN_DT 기간 조건을 사용해야 합니다.")
+        _append_issue(issues, "AU_FLAG" in upper_sql, "최근 7일 AU는 AU_FLAG로 제한하면 안 됩니다.")
+        _append_issue(issues, any(term not in upper_sql for term in ("STD_USER_YN", "EXPT_USER_YN")), "최근 7일 AU에는 STD_USER_YN과 EXPT_USER_YN 조건이 필요합니다.")
+        _append_issue(issues, "GROUP BY" in upper_sql, "최근 7일 AU는 일자별 목록이 아니라 단일 집계여야 합니다.")
+    elif code == "STD-28":
+        _append_issue(issues, any(term not in upper_sql for term in ("STD_USER_YN", "EXPT_USER_YN")), "전체 유저 수에는 STD_USER_YN과 EXPT_USER_YN 조건이 필요합니다.")
+        _append_issue(issues, "RU_FLAG" in upper_sql, "이 기준의 전체 유저 수는 RU_FLAG가 아니라 STD_USER_YN으로 계산해야 합니다.")
+    elif code in {"CZN-07", "CZN-08", "CZN-13", "CZN-17"}:
+        _apply_goods_rules(code, sql, upper_sql, issues)
+
+
+def _apply_goods_rules(code: str, sql: str, upper_sql: str, issues: list[str]) -> None:
+    dimension_name_terms = ("DIM_KR_NM", "DIM_EN_NM", "DIM_CTG", "CD_DESC", "CD_DTL_DESC")
+    has_dimension_name = any(term in upper_sql for term in dimension_name_terms)
+    if code in {"CZN-07", "CZN-08", "CZN-17"}:
+        _append_issue(issues, "CZN_COMN_SVC_DIM_BAS" not in upper_sql or not has_dimension_name, f"{code}은 CZN_COMN_SVC_DIM_BAS의 이름/분류 컬럼으로 재화명을 해석해야 합니다.")
+        _append_issue(issues, any(term not in upper_sql for term in ("DIM_TYPE_DIV_CD", "DIM_CTG")) or "크리스탈" not in sql, f"{code}은 GOODS_AGG와 크리스탈 분류 조건을 사용해야 합니다.")
+        _append_issue(issues, "HAVE_CNT" in upper_sql and "<>" not in upper_sql and "!=" not in upper_sql, f"{code}은 HAVE_CNT <> 0으로 0 보유량을 제외해야 합니다.")
+    if code == "CZN-07":
+        _append_issue(issues, any(term not in upper_sql for term in ("CZN_COMN_USER_MST", "EXPT_USER_YN")), "CZN-07은 유저 마스터를 조인하고 제외 유저를 필터링해야 합니다.")
+        _append_issue(issues, "GROUP BY" not in upper_sql or "BASE_DT" not in upper_sql, "CZN-07은 기간별 일자 집계를 위해 BASE_DT GROUP BY가 필요합니다.")
+    elif code == "CZN-08":
+        _append_issue(issues, any(term not in upper_sql for term in ("AU_FLAG", "EXPT_USER_YN")), "CZN-08은 표준 AU와 제외 유저 조건이 필요합니다.")
+        _append_issue(issues, "STD_USER_YN" in upper_sql, "CZN-08 기준에는 STD_USER_YN을 추가하면 모집단이 과도하게 좁아집니다.")
+        _append_issue(issues, "AVG(" in upper_sql, "CZN-08 1인당 평균은 AVG(HAVE_CNT)가 아니라 합계/고유 유저 수여야 합니다.")
+        _append_issue(issues, any(term not in upper_sql for term in ("SUM", "COUNT", "DISTINCT", "/")), "CZN-08 평균은 SUM(HAVE_CNT)/COUNT(DISTINCT GUID)로 계산해야 합니다.")
+    elif code == "CZN-13":
+        _append_issue(issues, "CZN_CUSTOM_GOODS_HAVE_TXN" in upper_sql, "CZN-13은 보유 스냅샷이 아니라 재화 변동 테이블을 사용해야 합니다.")
+        _append_issue(issues, any(term not in upper_sql for term in ("CZN_CUSTOM_GOODS_CHANGE_TXN", "GOODS_CHANGE_CNT")), "CZN-13은 GOODS_CHANGE_CNT를 사용해야 합니다.")
+        _append_issue(issues, "CZN_COMN_SVC_DIM_BAS" not in upper_sql or not has_dimension_name, "CZN-13은 차원 테이블로 에테르를 해석해야 합니다.")
+        _append_issue(issues, any(term not in upper_sql for term in ("DIM_TYPE_DIV_CD", "DIM_CTG", "CHANGE_TYPE_CD", "'USE'")) or "에테르" not in sql, "CZN-13은 에테르 사용 분류와 CHANGE_TYPE_CD='USE' 조건이 필요합니다.")
+        _append_issue(issues, any(term not in upper_sql for term in ("CZN_COMN_USER_MST", "EXPT_USER_YN", "COUNT", "DISTINCT", "GUID", "SUM")), "CZN-13은 대상 유저 조인, 고유 사용 유저 수, 사용량 합계가 필요합니다.")
+    elif code == "CZN-17":
+        _append_issue(issues, any(term not in upper_sql for term in ("CZN_CUSTOM_GOODS_HAVE_TXN", "HAVE_CNT")), "CZN-17은 재화 보유 스냅샷을 사용해야 합니다.")
+        _append_issue(issues, any(term not in upper_sql for term in ("CZN_COMN_CHARACTER_MST", "AU_FLAG", "EXPT_USER_YN")), "CZN-17은 캐릭터 AU와 제외 유저 조건이 필요합니다.")
+        _append_issue(issues, "AVG(" in upper_sql, "CZN-17 평균은 AVG(HAVE_CNT)가 아니라 합계/고유 캐릭터 수여야 합니다.")
+        _append_issue(issues, any(term not in upper_sql for term in ("SUM", "COUNT", "DISTINCT", "/", "CUID")), "CZN-17 평균은 SUM(HAVE_CNT)/COUNT(DISTINCT CUID)로 계산해야 합니다.")
+        _append_issue(issues, "GROUP BY" in upper_sql, "CZN-17은 기준일 단일 집계여야 하므로 상세 GROUP BY를 사용하면 안 됩니다.")
+        _append_issue(issues, any(term in upper_sql for term in ("LEVEL_COL", "ACM_CONN_DCNT", "TDAY_PLAY_TIME")), "CZN-17에는 레벨·접속일·플레이타임 지표가 포함되면 안 됩니다.")

ai-web-agent-console/ai_web_agent_console/qa_history_store.py

@@ -0,0 +1,566 @@
+"""Oracle ADB persistence for the Smilegate customer QA benchmark."""
+
+from __future__ import annotations
+
+from contextlib import contextmanager
+from datetime import datetime, timezone
+import json
+import os
+from pathlib import Path
+from typing import Any, Iterator, Mapping
+from urllib.parse import parse_qs
+
+import oracledb
+
+from src.poc4.qa_history import QaQuestion, load_benchmark_questions, question_fingerprint, question_from_record
+
+
+class QaHistoryStoreError(RuntimeError):
+    """A safe user-facing persistence error."""
+
+
+QUESTION_TABLE = "SG_AI_QA_QUESTION"
+ANSWER_TABLE = "SG_AI_QA_ANSWER"
+HISTORICAL_RUN_KEY = "HISTORICAL:2026-07-21:term-dict-final-v2"
+
+
+def _env_value(name: str, env_file: Path | None = None) -> str:
+    value = os.environ.get(name, "").strip()
+    if value or env_file is None or not env_file.is_file():
+        return value
+    try:
+        lines = env_file.read_text(encoding="utf-8").splitlines()
+    except (OSError, UnicodeError):
+        return ""
+    for line in lines:
+        line = line.strip()
+        if not line or line.startswith("#") or "=" not in line:
+            continue
+        if line.startswith("export "):
+            line = line[7:].lstrip()
+        key, raw = line.split("=", 1)
+        if key.strip() != name:
+            continue
+        raw = raw.strip()
+        if len(raw) >= 2 and raw[0] == raw[-1] and raw[0] in {"'", '"'}:
+            raw = raw[1:-1]
+        return raw.strip()
+    return ""
+
+
+def _normalize_oracle_dsn(raw_dsn: str) -> tuple[str, str]:
+    value = str(raw_dsn or "").strip()
+    if value.startswith("jdbc:oracle:thin:@"):
+        value = value[len("jdbc:oracle:thin:@"):]
+    if "?" not in value:
+        return value, ""
+    dsn, query = value.split("?", 1)
+    parsed = parse_qs(query, keep_blank_values=False)
+    wallet_dir = (parsed.get("TNS_ADMIN") or parsed.get("tns_admin") or [""])[0]
+    return dsn.strip(), wallet_dir.strip()
+
+
+def _read_lob(value: Any) -> Any:
+    return value.read() if hasattr(value, "read") else value
+
+
+def _record_from_cursor(cursor: Any, row: Any) -> dict[str, Any]:
+    names = [column[0].lower() for column in cursor.description]
+    return {name: _read_lob(value) for name, value in zip(names, row)}
+
+
+def _to_json(value: Mapping[str, Any] | None) -> str:
+    payload = dict(value or {})
+    text = json.dumps(payload, ensure_ascii=False, default=str)
+    if len(text) <= 120_000:
+        return text
+    return json.dumps(
+        {
+            "truncated": True,
+            "preview": text[:119_800],
+        },
+        ensure_ascii=False,
+    )
+
+
+def _answer_record(row: Mapping[str, Any]) -> dict[str, Any]:
+    result_json = str(row.get("result_json") or "").strip()
+    try:
+        result = json.loads(result_json) if result_json else {}
+    except ValueError:
+        result = {"raw": result_json}
+    return {
+        "answer_seq": row.get("answer_seq"),
+        "question_id": row.get("question_id"),
+        "answer_kind": str(row.get("answer_kind") or ""),
+        "run_key": str(row.get("run_key") or ""),
+        "conversation_id": str(row.get("conversation_id") or ""),
+        "requested_by": str(row.get("requested_by") or ""),
+        "requested_at": str(row.get("requested_at") or ""),
+        "model_profile": str(row.get("model_profile") or ""),
+        "generated_sql": str(row.get("generated_sql") or ""),
+        "answer_text": str(row.get("answer_text") or ""),
+        "result": result,
+        "execution_output": str(row.get("execution_output") or ""),
+        "execution_status": str(row.get("execution_status") or ""),
+        "judgment_status": str(row.get("judgment_status") or ""),
+        "judgment_reason": str(row.get("judgment_reason") or ""),
+        "duration_ms": row.get("duration_ms"),
+        "created_at": str(row.get("created_at") or ""),
+    }
+
+
+class QaHistoryStore:
+    def __init__(self, *, env_file: Path | None = None) -> None:
+        self._env_file = env_file
+        self._pool: Any | None = None
+
+    def _config(self) -> dict[str, str]:
+        username = (
+            _env_value("POC4_QA_DB_USERNAME", self._env_file)
+            or _env_value("BACKOFFICE_SELECT_AI_DB_USERNAME", self._env_file)
+        )
+        password = (
+            _env_value("POC4_QA_DB_PASSWORD", self._env_file)
+            or _env_value("BACKOFFICE_SELECT_AI_DB_PASSWORD", self._env_file)
+        )
+        raw_dsn = (
+            _env_value("POC4_QA_DB_DSN", self._env_file)
+            or _env_value("BACKOFFICE_SELECT_AI_DB_URL", self._env_file)
+        )
+        dsn, wallet_from_dsn = _normalize_oracle_dsn(raw_dsn)
+        wallet_dir = (
+            _env_value("POC4_QA_DB_WALLET_DIR", self._env_file)
+            or wallet_from_dsn
+            or _env_value("ORACLE_WALLET_DIR", self._env_file)
+        )
+        if not username or not password or not dsn:
+            raise QaHistoryStoreError("질답 이력 DB 접속 설정을 확인해 주세요.")
+        return {
+            "username": username,
+            "password": password,
+            "dsn": dsn,
+            "wallet_dir": wallet_dir,
+        }
+
+    def _get_pool(self) -> Any:
+        if self._pool is not None:
+            return self._pool
+        config = self._config()
+        kwargs: dict[str, Any] = {
+            "user": config["username"],
+            "password": config["password"],
+            "dsn": config["dsn"],
+            "min": 1,
+            "max": 3,
+            "increment": 1,
+            "getmode": oracledb.POOL_GETMODE_WAIT,
+        }
+        wallet_dir = Path(config["wallet_dir"]).expanduser()
+        if config["wallet_dir"]:
+            if not wallet_dir.is_dir():
+                raise QaHistoryStoreError("질답 이력 DB Wallet 경로를 확인해 주세요.")
+            kwargs["config_dir"] = str(wallet_dir)
+            kwargs["wallet_location"] = str(wallet_dir)
+        try:
+            self._pool = oracledb.create_pool(**kwargs)
+            return self._pool
+        except (oracledb.Error, OSError, ValueError) as exc:
+            raise QaHistoryStoreError("질답 이력 DB에 연결하지 못했습니다.") from exc
+
+    @contextmanager
+    def _connection(self) -> Iterator[Any]:
+        try:
+            with self._get_pool().acquire() as connection:
+                yield connection
+        except QaHistoryStoreError:
+            raise
+        except (oracledb.Error, OSError, ValueError) as exc:
+            raise QaHistoryStoreError("질답 이력 DB 작업에 실패했습니다.") from exc
+
+    def list_questions(self, *, limit: int = 200) -> list[QaQuestion]:
+        sql = f"""
+            SELECT q.question_id, q.question_code, q.category, q.title,
+                   q.question_text, q.source_document, q.source_sheet,
+                   q.source_row, q.source_scenario, q.sample_sql,
+                   q.expected_focus, q.baseline_sql, q.baseline_answer,
+                   q.support_level, q.evaluation_rule_json,
+                   latest.judgment_status AS last_judgment_status,
+                   TO_CHAR(latest.evaluated_at AT TIME ZONE 'Asia/Seoul',
+                           'YYYY-MM-DD HH24:MI:SS TZH:TZM') AS last_evaluated_at
+            FROM {QUESTION_TABLE} q
+            LEFT JOIN (
+                SELECT question_id, judgment_status, evaluated_at
+                FROM (
+                    SELECT question_id, judgment_status, evaluated_at,
+                           ROW_NUMBER() OVER (
+                               PARTITION BY question_id ORDER BY answer_seq DESC
+                           ) AS row_no
+                    FROM {ANSWER_TABLE}
+                )
+                WHERE row_no = 1
+            ) latest ON latest.question_id = q.question_id
+            WHERE q.active_yn = 'Y'
+            ORDER BY q.category, q.question_code
+            FETCH FIRST :row_limit ROWS ONLY
+        """
+        with self._connection() as connection:
+            with connection.cursor() as cursor:
+                cursor.execute(sql, {"row_limit": int(limit)})
+                rows = [_record_from_cursor(cursor, row) for row in cursor]
+        return [question_from_record(row) for row in rows]
+
+    def get_question(self, question_id: int) -> QaQuestion | None:
+        sql = f"""
+            SELECT question_id, question_code, category, title, question_text,
+                   source_document, source_sheet, source_row, source_scenario,
+                   sample_sql, expected_focus, baseline_sql, baseline_answer,
+                   support_level, evaluation_rule_json
+            FROM {QUESTION_TABLE}
+            WHERE question_id = :question_id AND active_yn = 'Y'
+        """
+        with self._connection() as connection:
+            with connection.cursor() as cursor:
+                cursor.execute(sql, {"question_id": int(question_id)})
+                row = cursor.fetchone()
+                return question_from_record(_record_from_cursor(cursor, row)) if row else None
+
+    def list_answers(self, question_id: int, *, limit: int = 30) -> list[dict[str, Any]]:
+        sql = f"""
+            SELECT answer_seq, question_id, answer_kind, run_key, conversation_id,
+                   requested_by,
+                   TO_CHAR(requested_at AT TIME ZONE 'Asia/Seoul',
+                           'YYYY-MM-DD HH24:MI:SS TZH:TZM') AS requested_at,
+                   model_profile, generated_sql, answer_text, result_json,
+                   execution_output, execution_status, judgment_status,
+                   judgment_reason, duration_ms,
+                   TO_CHAR(created_at AT TIME ZONE 'Asia/Seoul',
+                           'YYYY-MM-DD HH24:MI:SS TZH:TZM') AS created_at
+            FROM {ANSWER_TABLE}
+            WHERE question_id = :question_id
+            ORDER BY answer_seq DESC
+            FETCH FIRST :row_limit ROWS ONLY
+        """
+        with self._connection() as connection:
+            with connection.cursor() as cursor:
+                cursor.execute(sql, {"question_id": int(question_id), "row_limit": int(limit)})
+                rows = [_record_from_cursor(cursor, row) for row in cursor]
+        return [_answer_record(row) for row in rows]
+
+    def find_or_create_free_text_question(self, question_text: str) -> QaQuestion:
+        normalized = str(question_text or "").strip()
+        if not normalized:
+            raise QaHistoryStoreError("자유 질의가 비어 있습니다.")
+        fingerprint = question_fingerprint(normalized)
+        code = f"ADHOC-{fingerprint[:12].upper()}"
+        merge_sql = f"""
+            MERGE INTO {QUESTION_TABLE} target
+            USING (SELECT :question_hash AS question_hash FROM dual) source
+            ON (target.question_hash = source.question_hash)
+            WHEN NOT MATCHED THEN INSERT (
+                question_code, question_source, question_hash, category, title,
+                question_text, support_level, evaluation_rule_json, active_yn
+            ) VALUES (
+                :question_code, 'FREE_TEXT', :question_hash, 'FREE_TEXT',
+                :title, :question_text, 'REVIEW', '{{}}', 'Y'
+            )
+        """
+        with self._connection() as connection:
+            with connection.cursor() as cursor:
+                cursor.execute(
+                    merge_sql,
+                    {
+                        "question_hash": fingerprint,
+                        "question_code": code,
+                        "title": normalized[:180],
+                        "question_text": normalized,
+                    },
+                )
+                connection.commit()
+                cursor.execute(
+                    f"""SELECT question_id FROM {QUESTION_TABLE}
+                         WHERE question_hash = :question_hash""",
+                    {"question_hash": fingerprint},
+                )
+                row = cursor.fetchone()
+        if not row:
+            raise QaHistoryStoreError("자유 질의 마스터를 저장하지 못했습니다.")
+        question = self.get_question(int(row[0]))
+        if question is None:
+            raise QaHistoryStoreError("자유 질의 마스터를 다시 읽지 못했습니다.")
+        return question
+
+    def record_answer(
+        self,
+        *,
+        question_id: int,
+        answer_kind: str,
+        conversation_id: str,
+        requested_by: str,
+        model_profile: str,
+        generated_sql: str,
+        answer_text: str,
+        result: Mapping[str, Any] | None,
+        execution_output: str,
+        execution_status: str,
+        judgment_status: str,
+        judgment_reason: str,
+        duration_ms: int | None,
+        run_key: str = "",
+    ) -> None:
+        sql = f"""
+            INSERT INTO {ANSWER_TABLE} (
+                question_id, answer_kind, run_key, conversation_id, requested_by,
+                requested_at, model_profile, generated_sql, answer_text, result_json,
+                execution_output, execution_status, judgment_status,
+                judgment_reason, duration_ms
+            ) VALUES (
+                :question_id, :answer_kind, :run_key, :conversation_id,
+                :requested_by, SYSTIMESTAMP, :model_profile, :generated_sql,
+                :answer_text, :result_json, :execution_output, :execution_status,
+                :judgment_status, :judgment_reason, :duration_ms
+            )
+        """
+        with self._connection() as connection:
+            with connection.cursor() as cursor:
+                cursor.execute(
+                    sql,
+                    {
+                        "question_id": int(question_id),
+                        "answer_kind": str(answer_kind)[:20],
+                        "run_key": str(run_key)[:100] or None,
+                        "conversation_id": str(conversation_id)[:100] or None,
+                        "requested_by": str(requested_by)[:100] or None,
+                        "model_profile": str(model_profile)[:100] or None,
+                        "generated_sql": str(generated_sql or ""),
+                        "answer_text": str(answer_text or ""),
+                        "result_json": _to_json(result),
+                        "execution_output": str(execution_output or ""),
+                        "execution_status": str(execution_status)[:40] or None,
+                        "judgment_status": str(judgment_status)[:20],
+                        "judgment_reason": str(judgment_reason or ""),
+                        "duration_ms": duration_ms,
+                    },
+                )
+                connection.commit()
+
+    def seed_benchmark(self, benchmark_file: Path) -> tuple[int, int]:
+        questions = load_benchmark_questions(benchmark_file)
+        raw = json.loads(benchmark_file.read_text(encoding="utf-8"))
+        raw_by_code = {
+            str(item.get("case_id") or "").upper(): item
+            for item in raw.get("scenarios", [])
+            if isinstance(item, Mapping)
+        }
+        seeded_questions = 0
+        seeded_answers = 0
+        for question in questions:
+            question_id = self._upsert_benchmark_question(question)
+            seeded_questions += 1
+            raw_item = raw_by_code[question.question_code]
+            history = raw_item.get("historical_answer") if isinstance(raw_item.get("historical_answer"), Mapping) else {}
+            inserted = self._seed_historical_answer(question_id, history, raw)
+            seeded_answers += 1 if inserted else 0
+        return seeded_questions, seeded_answers
+
+    def _upsert_benchmark_question(self, question: QaQuestion) -> int:
+        sql = f"""
+            MERGE INTO {QUESTION_TABLE} target
+            USING (SELECT :question_code AS question_code FROM dual) source
+            ON (target.question_code = source.question_code)
+            WHEN MATCHED THEN UPDATE SET
+                question_source = 'CUSTOMER_EXCEL',
+                question_hash = :question_hash,
+                category = :category,
+                title = :title,
+                question_text = :question_text,
+                source_document = :source_document,
+                source_sheet = :source_sheet,
+                source_row = :source_row,
+                source_scenario = :source_scenario,
+                sample_sql = :sample_sql,
+                expected_focus = :expected_focus,
+                baseline_sql = :baseline_sql,
+                baseline_answer = :baseline_answer,
+                support_level = :support_level,
+                evaluation_rule_json = :evaluation_rule_json,
+                active_yn = 'Y',
+                updated_at = SYSTIMESTAMP
+            WHEN NOT MATCHED THEN INSERT (
+                question_code, question_source, question_hash, category, title,
+                question_text, source_document, source_sheet, source_row,
+                source_scenario, sample_sql, expected_focus, baseline_sql,
+                baseline_answer, support_level, evaluation_rule_json, active_yn
+            ) VALUES (
+                :question_code, 'CUSTOMER_EXCEL', :question_hash, :category,
+                :title, :question_text, :source_document, :source_sheet,
+                :source_row, :source_scenario, :sample_sql, :expected_focus,
+                :baseline_sql, :baseline_answer, :support_level,
+                :evaluation_rule_json, 'Y'
+            )
+        """
+        binds = {
+            "question_code": question.question_code,
+            "question_hash": question_fingerprint(question.question_text),
+            "category": question.category[:30],
+            "title": question.title[:200],
+            "question_text": question.question_text,
+            "source_document": question.source_document[:255] or None,
+            "source_sheet": question.source_sheet[:255] or None,
+            "source_row": question.source_row,
+            "source_scenario": question.source_scenario,
+            "sample_sql": question.sample_sql,
+            "expected_focus": question.expected_focus,
+            "baseline_sql": question.baseline_sql,
+            "baseline_answer": question.baseline_answer,
+            "support_level": question.support_level[:20],
+            "evaluation_rule_json": json.dumps(question.evaluation_rule, ensure_ascii=False),
+        }
+        with self._connection() as connection:
+            with connection.cursor() as cursor:
+                cursor.execute(sql, binds)
+                connection.commit()
+                cursor.execute(
+                    f"SELECT question_id FROM {QUESTION_TABLE} WHERE question_code = :question_code",
+                    {"question_code": question.question_code},
+                )
+                row = cursor.fetchone()
+        if not row:
+            raise QaHistoryStoreError(f"질문 마스터를 적재하지 못했습니다: {question.question_code}")
+        return int(row[0])
+
+    def _seed_historical_answer(
+        self,
+        question_id: int,
+        history: Mapping[str, Any],
+        benchmark: Mapping[str, Any],
+    ) -> bool:
+        exists_sql = f"""
+            SELECT COUNT(*) FROM {ANSWER_TABLE}
+             WHERE question_id = :question_id AND run_key = :run_key
+        """
+        with self._connection() as connection:
+            with connection.cursor() as cursor:
+                cursor.execute(exists_sql, {"question_id": question_id, "run_key": HISTORICAL_RUN_KEY})
+                if int(cursor.fetchone()[0]) > 0:
+                    return False
+        result = {
+            "source_report": str(benchmark.get("source_report") or ""),
+            "source_redmine": benchmark.get("source_redmine"),
+            "historical_execution_output": str(history.get("execution_output") or ""),
+        }
+        self.record_answer(
+            question_id=question_id,
+            answer_kind="HISTORICAL",
+            run_key=HISTORICAL_RUN_KEY,
+            conversation_id="",
+            requested_by="customer-excel-baseline",
+            model_profile=str(history.get("profile") or ""),
+            generated_sql=str(history.get("generated_sql") or ""),
+            answer_text=str(history.get("answer_text") or ""),
+            result=result,
+            execution_output=str(history.get("execution_output") or ""),
+            execution_status=str(history.get("execution_status") or ""),
+            judgment_status=str(history.get("judgment_status") or "REVIEW"),
+            judgment_reason=str(history.get("judgment_reason") or ""),
+            duration_ms=int(history.get("duration_ms") or 0),
+        )
+        return True
+
+
+def schema_statements() -> tuple[str, ...]:
+    return (
+        f"""
+        CREATE TABLE {QUESTION_TABLE} (
+            question_id NUMBER GENERATED BY DEFAULT ON NULL AS IDENTITY PRIMARY KEY,
+            question_code VARCHAR2(30) UNIQUE,
+            question_source VARCHAR2(30) NOT NULL,
+            question_hash VARCHAR2(64) NOT NULL UNIQUE,
+            category VARCHAR2(30) NOT NULL,
+            title VARCHAR2(200) NOT NULL,
+            question_text CLOB NOT NULL,
+            source_document VARCHAR2(255),
+            source_sheet VARCHAR2(255),
+            source_row NUMBER,
+            source_scenario CLOB,
+            sample_sql CLOB,
+            expected_focus CLOB,
+            baseline_sql CLOB,
+            baseline_answer CLOB,
+            support_level VARCHAR2(20) NOT NULL,
+            evaluation_rule_json CLOB CHECK (evaluation_rule_json IS JSON),
+            active_yn CHAR(1) DEFAULT 'Y' NOT NULL CHECK (active_yn IN ('Y', 'N')),
+            created_at TIMESTAMP WITH TIME ZONE DEFAULT SYSTIMESTAMP NOT NULL,
+            updated_at TIMESTAMP WITH TIME ZONE DEFAULT SYSTIMESTAMP NOT NULL,
+            CONSTRAINT sg_ai_qa_question_source_ck
+                CHECK (question_source IN ('CUSTOMER_EXCEL', 'FREE_TEXT'))
+        )
+        """,
+        f"""
+        CREATE TABLE {ANSWER_TABLE} (
+            answer_seq NUMBER GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
+            question_id NUMBER NOT NULL,
+            answer_kind VARCHAR2(20) NOT NULL,
+            run_key VARCHAR2(100),
+            conversation_id VARCHAR2(100),
+            requested_by VARCHAR2(100),
+            requested_at TIMESTAMP WITH TIME ZONE DEFAULT SYSTIMESTAMP NOT NULL,
+            model_profile VARCHAR2(100),
+            generated_sql CLOB,
+            answer_text CLOB,
+            result_json CLOB CHECK (result_json IS JSON),
+            execution_output CLOB,
+            execution_status VARCHAR2(40),
+            judgment_status VARCHAR2(20) NOT NULL,
+            judgment_reason CLOB,
+            duration_ms NUMBER,
+            created_at TIMESTAMP WITH TIME ZONE DEFAULT SYSTIMESTAMP NOT NULL,
+            CONSTRAINT sg_ai_qa_answer_question_fk
+                FOREIGN KEY (question_id)
+                REFERENCES {QUESTION_TABLE} (question_id)
+                ON DELETE CASCADE,
+            CONSTRAINT sg_ai_qa_answer_kind_ck
+                CHECK (answer_kind IN ('HISTORICAL', 'LIVE')),
+            CONSTRAINT sg_ai_qa_answer_judgment_ck
+                CHECK (judgment_status IN ('PASS', 'WARN', 'FAIL', 'REVIEW'))
+        )
+        """,
+        f"""
+        CREATE INDEX sg_ai_qa_answer_question_ix
+            ON {ANSWER_TABLE} (question_id, answer_seq DESC)
+        """,
+        f"""
+        CREATE UNIQUE INDEX sg_ai_qa_answer_run_uk
+            ON {ANSWER_TABLE} (question_id, run_key)
+        """,
+    )
+
+
+def ensure_schema(store: QaHistoryStore) -> None:
+    objects = (QUESTION_TABLE, ANSWER_TABLE)
+    with store._connection() as connection:
+        with connection.cursor() as cursor:
+            cursor.execute(
+                "SELECT table_name FROM user_tables WHERE table_name IN (:q, :a)",
+                {"q": objects[0], "a": objects[1]},
+            )
+            existing = {str(row[0]) for row in cursor}
+            statements = schema_statements()
+            if QUESTION_TABLE not in existing:
+                cursor.execute(statements[0])
+            if ANSWER_TABLE not in existing:
+                cursor.execute(statements[1])
+            cursor.execute(
+                "SELECT index_name FROM user_indexes WHERE index_name IN (:ix1, :ix2)",
+                {"ix1": "SG_AI_QA_ANSWER_QUESTION_IX", "ix2": "SG_AI_QA_ANSWER_RUN_UK"},
+            )
+            indexes = {str(row[0]) for row in cursor}
+            if "SG_AI_QA_ANSWER_QUESTION_IX" not in indexes:
+                cursor.execute(statements[2])
+            if "SG_AI_QA_ANSWER_RUN_UK" not in indexes:
+                cursor.execute(statements[3])
+            connection.commit()
+
+
+def timestamp_now() -> str:
+    return datetime.now(timezone.utc).isoformat(timespec="seconds")

ai-web-agent-console/app.py

@@ -60,15 +60,9 @@ from ai_web_agent_console.presentation import (
     render_console_header,
     render_login_brand,
 )
-from ai_web_agent_console.audit import render_hmm_audit_tab
-from ai_web_agent_console.profile import AppProfile, AppProfileError, load_app_profile
-from ai_web_agent_console.scenarios import ScenarioConfigError, load_demo_scenarios
-from ai_web_agent_console.query_contracts import (
-    append_query_contract_guidance,
-    evidence_contract_report,
-    matching_query_contracts,
-    missing_evidence_message,
-)
+from src.agent_console.profile import AppProfile, AppProfileError, load_app_profile
+from src.poc4.qa_history import QaJudgment, QaQuestion, evaluate_sql, load_benchmark_questions
+from src.poc4.qa_history_store import QaHistoryStore, QaHistoryStoreError
 
 
 LOG = logging.getLogger(__name__)
@@ -91,7 +85,7 @@ DEFAULT_QUERY_MODEL_PROFILE = "gpt54_mini_oci"
 ENV_FILE = ROOT / ".env"
 MCP_SERVERS_FILE = ROOT / "config" / "mcp_servers.json"
 VPD_TOKEN_PRESETS_FILE = ROOT / "config" / "vpd_token_presets.json"
-DEMO_SCENARIOS_FILE = ROOT / "config" / "smilegate_demo_scenarios.json"
+QA_BENCHMARK_FILE = ROOT / "config" / "smilegate_qa_benchmark.json"
 APP_PROFILE_FILE = ROOT / "config" / "app_profile.json"
 CHAT_DB_FILE = ROOT / "data" / "poc4_mcp_chat.sqlite3"
 DEFAULT_VPD_USER_ID = "1001"
@@ -559,12 +553,180 @@ def new_conversation_id() -> str:
 
 
 def _question_label(question: object) -> str:
-    question_id = str(getattr(question, "question_id"))
-    category = str(getattr(question, "category"))
-    title = str(getattr(question, "title", getattr(question, "text")))
+    question_id = str(
+        getattr(question, "question_code", getattr(question, "question_id", ""))
+    )
+    category = str(getattr(question, "category", ""))
+    title = str(
+        getattr(question, "title", getattr(question, "question_text", ""))
+    )
     return f"{question_id} · {category} · {title}"
 
 
+@st.cache_resource(show_spinner=False)
+def _qa_history_store() -> QaHistoryStore:
+    return QaHistoryStore()
+
+
+def _load_qa_questions() -> tuple[list[QaQuestion], QaHistoryStore | None, str]:
+    try:
+        store = _qa_history_store()
+        questions = store.list_questions(limit=200)
+        if questions:
+            return questions, store, ""
+        return [], store, "질답 이력 DB에 아직 적재된 기준 질문이 없습니다."
+    except QaHistoryStoreError as exc:
+        try:
+            fallback = list(load_benchmark_questions(QA_BENCHMARK_FILE))
+        except Exception:
+            fallback = []
+        return fallback, None, str(exc)
+
+
+def _judgment_label(status: str) -> str:
+    labels = {
+        "PASS": "통과",
+        "WARN": "검토",
+        "FAIL": "실패",
+        "REVIEW": "수동 검토",
+    }
+    return labels.get(str(status or "").upper(), "미실행")
+
+
+def _qa_answer_preview(value: object, limit: int = 180) -> str:
+    text = str(value or "").strip().replace("\n", " ")
+    return text if len(text) <= limit else text[:limit] + "..."
+
+
+def _render_qa_benchmark_panel(
+    questions: list[QaQuestion],
+    store: QaHistoryStore | None,
+    store_error: str,
+) -> QaQuestion | None:
+    st.markdown(
+        '<div class="kb-section-title input" role="heading" aria-level="3">'
+        "고객 질답 검증 시나리오"
+        "</div>",
+        unsafe_allow_html=True,
+    )
+    st.caption(
+        "고객 제공 Excel에서 정리한 47개 후보입니다. 후보를 선택하면 기준 답변과 과거 검증 이력을 확인하고 바로 실행할 수 있습니다."
+    )
+    if store_error:
+        st.warning(
+            "현재는 파일 기준 후보만 표시합니다. ADB 이력 저장은 연결 복구 후 사용할 수 있습니다. "
+            + store_error
+        )
+    if not questions:
+        st.error("표시할 고객 질답 기준이 없습니다.")
+        return None
+
+    category_options = tuple(sorted({question.category for question in questions}))
+    category = st.selectbox(
+        "시나리오 구분",
+        options=("ALL", *category_options),
+        format_func=lambda item: "전체 47개" if item == "ALL" else item,
+        key="poc4_qa_category_filter",
+    )
+    visible_questions = [
+        question for question in questions if category == "ALL" or question.category == category
+    ]
+    st.dataframe(
+        [
+            {
+                "케이스": question.question_code,
+                "구분": question.category,
+                "제목": question.title,
+                "질문": question.question_text,
+                "기대 기준": _qa_answer_preview(question.expected_focus, 150),
+                "최근 판정": _judgment_label(question.last_judgment_status),
+                "최근 실행": question.last_evaluated_at or "과거 기준",
+            }
+            for question in visible_questions
+        ],
+        hide_index=True,
+        width="stretch",
+        height=min(520, 110 + 36 * len(visible_questions)),
+        column_config={
+            "질문": st.column_config.TextColumn(width="large"),
+            "기대 기준": st.column_config.TextColumn(width="large"),
+            "최근 실행": st.column_config.TextColumn(width="medium"),
+        },
+    )
+    by_code = {question.question_code: question for question in visible_questions}
+    selected_code = st.selectbox(
+        "실행할 검증 후보 선택",
+        options=("", *by_code),
+        format_func=lambda code: "선택 안 함 · 자유 질의" if not code else _question_label(by_code[code]),
+        key="poc4_qa_question_code",
+    )
+    selected = by_code.get(selected_code)
+    if selected is None:
+        st.info("자유 텍스트 질문도 실행하고 이력으로 남길 수 있습니다. 자유 질의는 정답 기준이 없어 수동 검토로 표시됩니다.")
+        return None
+
+    with st.expander("선택한 질문의 정답 기준·원본·과거 답변", expanded=True):
+        left, right = st.columns(2)
+        with left:
+            st.markdown("**정답 기준**")
+            st.write(selected.expected_focus or "정답 기준 설명이 없습니다.")
+            st.markdown("**기준 답변**")
+            st.code(selected.baseline_answer or "기준 실행 결과가 없습니다.", language=None)
+        with right:
+            st.markdown("**고객 원본 출처**")
+            st.write(
+                f"{selected.source_document} / {selected.source_sheet} / 행 {selected.source_row or '-'}"
+            )
+            if selected.source_scenario:
+                st.caption(selected.source_scenario)
+            st.markdown("**기준 SQL**")
+            st.code(selected.baseline_sql or selected.sample_sql or "기준 SQL이 없습니다.", language="sql")
+
+        if store is not None and selected.question_id is not None:
+            try:
+                answers = store.list_answers(selected.question_id, limit=30)
+            except QaHistoryStoreError as exc:
+                st.warning(str(exc))
+                answers = []
+            if answers:
+                st.markdown("**과거 답변 이력**")
+                st.dataframe(
+                    [
+                        {
+                            "순번": item["answer_seq"],
+                            "구분": "과거 기준" if item["answer_kind"] == "HISTORICAL" else "이번 실행",
+                            "판정": _judgment_label(item["judgment_status"]),
+                            "실행 상태": item["execution_status"],
+                            "실행 시각": item["requested_at"],
+                            "답변": _qa_answer_preview(item["answer_text"]),
+                        }
+                        for item in answers
+                    ],
+                    hide_index=True,
+                    width="stretch",
+                )
+                answer_by_seq = {str(item["answer_seq"]): item for item in answers}
+                detail_seq = st.selectbox(
+                    "답변 이력 상세",
+                    options=tuple(answer_by_seq),
+                    format_func=lambda value: (
+                        f"#{value} · {_judgment_label(answer_by_seq[value]['judgment_status'])} · "
+                        f"{answer_by_seq[value]['requested_at']}"
+                    ),
+                    key=f"poc4_qa_answer_detail_{selected.question_id}",
+                )
+                detail = answer_by_seq[detail_seq]
+                st.markdown("**판정 근거**")
+                st.write(detail["judgment_reason"] or "판정 근거가 없습니다.")
+                st.markdown("**생성 SQL**")
+                st.code(detail["generated_sql"] or "생성 SQL이 없습니다.", language="sql")
+                st.markdown("**실행 답변**")
+                st.write(detail["answer_text"] or "답변이 없습니다.")
+            else:
+                st.info("아직 저장된 답변 이력이 없습니다.")
+    return selected
+
+
 def _apply_console_theme(profile: AppProfile) -> None:
     apply_console_theme(st, profile)
 
@@ -4824,6 +4986,89 @@ def _render_vpd_operations_tab() -> None:
     )
 
 
+def _select_ai_execution_summary(payload: Any) -> dict[str, Any]:
+    if not isinstance(payload, Mapping):
+        return {
+            "generated_sql": "",
+            "execution_status": "UNKNOWN",
+            "execution_succeeded": False,
+            "result": {},
+        }
+    generated_sql = str(
+        payload.get("generatedSql") or payload.get("generated_sql") or ""
+    ).strip()
+    status = str(payload.get("status") or "").strip().upper()
+    execution = str(payload.get("execution") or "").strip().upper()
+    execution_succeeded = (
+        status == "SHOWSQL_AND_EXECUTED" and execution == "READ_ONLY_EXECUTED"
+    )
+    result = {
+        key: payload.get(key)
+        for key in (
+            "status",
+            "execution",
+            "rowCount",
+            "truncated",
+            "columns",
+            "items",
+            "generatedSql",
+        )
+        if key in payload
+    }
+    return {
+        "generated_sql": generated_sql,
+        "execution_status": status or execution or "UNKNOWN",
+        "execution_succeeded": execution_succeeded,
+        "result": result,
+    }
+
+
+def _record_qa_history(
+    *,
+    store: QaHistoryStore | None,
+    benchmark_question: QaQuestion | None,
+    question_text: str,
+    conversation_id: str,
+    requested_by: str,
+    model_profile: str,
+    answer_text: str,
+    mcp_result: Any,
+    duration_ms: int,
+) -> tuple[QaQuestion | None, QaJudgment | None]:
+    if store is None:
+        return benchmark_question, None
+    try:
+        history_question = benchmark_question or store.find_or_create_free_text_question(
+            question_text
+        )
+        execution = _select_ai_execution_summary(mcp_result)
+        judgment = evaluate_sql(
+            benchmark_question,
+            execution["generated_sql"],
+            execution_succeeded=bool(execution["execution_succeeded"]),
+            error_text=_bounded_json(mcp_result, max_chars=8000),
+        )
+        store.record_answer(
+            question_id=int(history_question.question_id or 0),
+            answer_kind="LIVE",
+            conversation_id=conversation_id,
+            requested_by=requested_by,
+            model_profile=model_profile,
+            generated_sql=execution["generated_sql"],
+            answer_text=answer_text,
+            result=execution["result"],
+            execution_output=_bounded_json(mcp_result, max_chars=24_000),
+            execution_status=str(execution["execution_status"]),
+            judgment_status=judgment.status,
+            judgment_reason=judgment.reason,
+            duration_ms=duration_ms,
+        )
+        return history_question, judgment
+    except QaHistoryStoreError as exc:
+        st.warning(f"질답 이력 저장에 실패했습니다: {exc}")
+        return benchmark_question, None
+
+
 def _process_submitted_question(
     *,
     question: str,
@@ -4839,6 +5084,8 @@ def _process_submitted_question(
     limit: int,
     selected_token_preset: VpdTokenPreset | None,
     execution_mode_override: str,
+    benchmark_question: QaQuestion | None,
+    qa_history_store: QaHistoryStore | None,
 ) -> None:
     normalized_question = question.strip()
     if not normalized_question:
@@ -5395,6 +5642,32 @@ def _process_submitted_question(
         ],
         "discovery_failures": discovery_failures,
     }
+    history_question, history_judgment = _record_qa_history(
+        store=qa_history_store,
+        benchmark_question=benchmark_question,
+        question_text=normalized_question,
+        conversation_id=conversation_id,
+        requested_by=(
+            selected_token_preset.user_id
+            if selected_token_preset is not None
+            else "portal-user"
+        ),
+        model_profile=answer_model_profile,
+        answer_text=str(assistant_message["content"]),
+        mcp_result=answer_source,
+        duration_ms=round((perf_counter() - process_started) * 1000),
+    )
+    if history_judgment is not None:
+        assistant_message["details"]["qa_history"] = {
+            "question_id": history_question.question_id if history_question else None,
+            "question_code": history_question.question_code if history_question else "",
+            "judgment_status": history_judgment.status,
+            "judgment_reason": history_judgment.reason,
+        }
+        st.info(
+            "질답 검증 판정: "
+            f"{_judgment_label(history_judgment.status)} · {history_judgment.reason}"
+        )
     refresh_progress(98, "질의 결과를 저장하고 있습니다.")
     save_chat_turn(
         conversation_id=conversation_id,
@@ -5449,27 +5722,13 @@ def main() -> None:
     if not st.session_state.get(PORTAL_AUTHENTICATED_KEY, False):
         _render_portal_login(profile)
         return
-    try:
-        questions = load_demo_scenarios(DEMO_SCENARIOS_FILE)
-    except ScenarioConfigError as exc:
-        st.error(str(exc))
-        return
-    scenario_key = "poc4_mcp_discovery_scenario"
+    qa_questions, qa_history_store, qa_store_error = _load_qa_questions()
     question_text_key = "poc4_mcp_discovery_question_text"
-    loaded_scenario_key = "poc4_mcp_discovery_loaded_scenario_id"
+    loaded_qa_question_key = "poc4_mcp_discovery_loaded_qa_question_code"
     conversation_id_key = "poc4_mcp_discovery_conversation_id"
     chat_page_key = "poc4_mcp_discovery_page"
     query_progress_notice_key = "poc4_query_progress_notice"
     mcp_cache_generation_key = "poc4_mcp_tools_cache_generation"
-    selected_scenario_state = st.session_state.get(scenario_key)
-    scenario_ids = {question.question_id for question in questions}
-    if str(getattr(selected_scenario_state, "question_id", "")).strip().upper() not in (
-        "",
-        *scenario_ids,
-    ):
-        st.session_state.pop(scenario_key, None)
-        st.session_state.pop(loaded_scenario_key, None)
-        st.session_state[question_text_key] = DEFAULT_QUESTION
     init_chat_store()
     if conversation_id_key not in st.session_state:
         st.session_state[conversation_id_key] = new_conversation_id()
@@ -5757,7 +6016,7 @@ def main() -> None:
                 except (OSError, UnicodeError, ValueError):
                     st.warning("MCP 설정 JSON을 읽지 못했습니다.")
             st.caption(f"config: {MCP_SERVERS_FILE}")
-            st.caption(f"scenario config: {DEMO_SCENARIOS_FILE}")
+            st.caption(f"QA benchmark: {QA_BENCHMARK_FILE}")
             st.caption(f"token presets: {VPD_TOKEN_PRESETS_FILE}")
             st.caption(f"selected LLM model: {selected_query_model_profile}")
             st.caption(f"default model: {default_query_model_profile}")
@@ -5775,35 +6034,36 @@ def main() -> None:
         _render_architecture_tab()
 
     with scenario_tab:
-        st.markdown(
-            '<div class="kb-section-title input" role="heading" aria-level="3">'
-            "질문 입력"
-            "</div>",
-            unsafe_allow_html=True,
-        )
-        selected_scenario = st.selectbox(
-            "업무 데모 질의 샘플",
-            options=(None, *questions),
-            format_func=lambda item: (
-                "선택 안 함 · 직접 질문" if item is None else _question_label(item)
-            ),
-            key=scenario_key,
-        )
-        selected_scenario_id = (
-            None
-            if selected_scenario is None
-            else str(getattr(selected_scenario, "question_id"))
+        selected_qa_question = _render_qa_benchmark_panel(
+            qa_questions,
+            qa_history_store,
+            qa_store_error,
         )
         if question_text_key not in st.session_state:
             st.session_state[question_text_key] = DEFAULT_QUESTION
-        if st.session_state.get(loaded_scenario_key, "") != str(
-            selected_scenario_id or ""
-        ):
-            if selected_scenario is not None:
-                st.session_state[question_text_key] = str(
-                    getattr(selected_scenario, "text")
-                )
-            st.session_state[loaded_scenario_key] = str(selected_scenario_id or "")
+        selected_qa_code = (
+            selected_qa_question.question_code if selected_qa_question is not None else ""
+        )
+        if st.session_state.get(loaded_qa_question_key, "") != selected_qa_code:
+            st.session_state[question_text_key] = (
+                selected_qa_question.question_text
+                if selected_qa_question is not None
+                else DEFAULT_QUESTION
+            )
+            st.session_state[loaded_qa_question_key] = selected_qa_code
+
+        st.markdown(
+            '<div class="kb-section-title input" role="heading" aria-level="3">'
+            "질문 실행"
+            "</div>",
+            unsafe_allow_html=True,
+        )
+        if selected_qa_question is not None:
+            st.caption(
+                "선택한 후보 문장을 수정하지 않고 실행하면 자동 판정합니다. 문장을 수정하면 자유 질의로 저장되어 수동 검토 대상이 됩니다."
+            )
+        else:
+            st.caption("자유 질의는 실행 이력으로 남기며, 고객 기준 정답과 자동 비교하지 않습니다.")
 
         with st.form("poc4_mcp_question_form"):
             question = st.text_area(
@@ -5812,7 +6072,7 @@ def main() -> None:
                 key=question_text_key,
                 height=120,
                 max_chars=1_000,
-                placeholder="업무 질문을 직접 입력하거나 위 질의 샘플을 선택하세요.",
+                placeholder="고객 검증 후보를 선택하거나 업무 질문을 직접 입력하세요.",
             )
             submitted = st.form_submit_button(
                 "질문 전송",
@@ -5889,6 +6149,13 @@ def main() -> None:
             limit=int(limit),
             selected_token_preset=selected_token_preset,
             execution_mode_override=execution_mode_override,
+            benchmark_question=(
+                selected_qa_question
+                if selected_qa_question is not None
+                and question.strip() == selected_qa_question.question_text.strip()
+                else None
+            ),
+            qa_history_store=qa_history_store,
         )
 
 

ai-web-agent-console/scripts/sync_smilegate_qa_history.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+"""Create and seed the Smilegate customer QA benchmark history tables."""
+
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+import sys
+
+
+ROOT = Path(__file__).resolve().parents[1]
+if str(ROOT) not in sys.path:
+    sys.path.insert(0, str(ROOT))
+
+from src.poc4.qa_history_store import QaHistoryStore, ensure_schema
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--benchmark",
+        type=Path,
+        default=ROOT / "config" / "smilegate_qa_benchmark.json",
+        help="Customer Excel benchmark JSON generated from the approved QA report.",
+    )
+    parser.add_argument(
+        "--env-file",
+        type=Path,
+        default=None,
+        help="Optional environment file containing the QA DB connection settings.",
+    )
+    args = parser.parse_args()
+    if not args.benchmark.is_file():
+        raise SystemExit(f"Benchmark file not found: {args.benchmark}")
+    store = QaHistoryStore(env_file=args.env_file)
+    ensure_schema(store)
+    question_count, historical_insert_count = store.seed_benchmark(args.benchmark)
+    print(
+        "qa_history_sync"
+        f" questions={question_count}"
+        f" historical_answers_inserted={historical_insert_count}"
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

ai-web-agent-console/tests/test_qa_history.py

@@ -0,0 +1,80 @@
+from __future__ import annotations
+
+from pathlib import Path
+import unittest
+
+from src.poc4.qa_history import evaluate_sql, load_benchmark_questions
+from src.poc4.qa_history_store import _normalize_oracle_dsn, schema_statements
+
+
+class QaHistoryTest(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls) -> None:
+        benchmark = Path(__file__).parents[1] / "config" / "smilegate_qa_benchmark.json"
+        cls.questions = {item.question_code: item for item in load_benchmark_questions(benchmark)}
+
+    def test_customer_excel_benchmark_contains_all_47_cases(self) -> None:
+        self.assertEqual(47, len(self.questions))
+        self.assertIn("STD-01", self.questions)
+        self.assertIn("CZN-19", self.questions)
+
+    def test_supported_query_passes_when_required_terms_are_present(self) -> None:
+        judgment = evaluate_sql(
+            self.questions["STD-13"],
+            "SELECT SUM(PAYMT_AMT) FROM COMN_SALES_TXN",
+            execution_succeeded=True,
+        )
+        self.assertEqual("PASS", judgment.status)
+
+    def test_monthly_au_with_au_flag_fails(self) -> None:
+        judgment = evaluate_sql(
+            self.questions["STD-27"],
+            """
+            SELECT COUNT(*)
+            FROM CZN_COMN_USER_MST
+            WHERE AU_FLAG = 1
+              AND BASE_DT = (SELECT MAX(BASE_DT) FROM CZN_COMN_USER_MST)
+              AND LAST_CONN_DT >= ADD_MONTHS(BASE_DT, -1)
+              AND STD_USER_YN = 'Y'
+              AND EXPT_USER_YN = 'N'
+            """,
+            execution_succeeded=True,
+        )
+        self.assertEqual("FAIL", judgment.status)
+        self.assertIn("AU_FLAG", judgment.reason)
+
+    def test_unsupported_game_requires_safe_alias_lookup(self) -> None:
+        safe = evaluate_sql(
+            self.questions["STD-02"],
+            "SELECT GAME_ID FROM COMN_GAME_ALIAS_BAS WHERE GAME_NM LIKE '%버블리즈%'",
+            execution_succeeded=True,
+        )
+        unsafe = evaluate_sql(
+            self.questions["STD-02"],
+            "SELECT COUNT(*) FROM CZN_COMN_USER_MST WHERE GAME_ID = 'STOVE_CHAOSZERO'",
+            execution_succeeded=True,
+        )
+        self.assertEqual("PASS", safe.status)
+        self.assertEqual("FAIL", unsafe.status)
+
+    def test_free_text_is_review_not_automatic_pass(self) -> None:
+        judgment = evaluate_sql(None, "SELECT 1 FROM DUAL", execution_succeeded=True)
+        self.assertEqual("REVIEW", judgment.status)
+
+    def test_jdbc_url_wallet_is_normalized_for_python_driver(self) -> None:
+        self.assertEqual(
+            ("sgmpaipoc_medium", "/home/opc/wallet/sgmpaipoc"),
+            _normalize_oracle_dsn(
+                "jdbc:oracle:thin:@sgmpaipoc_medium?TNS_ADMIN=/home/opc/wallet/sgmpaipoc"
+            ),
+        )
+
+    def test_schema_defines_two_history_tables_and_indexes(self) -> None:
+        statements = "\n".join(schema_statements())
+        self.assertIn("CREATE TABLE SG_AI_QA_QUESTION", statements)
+        self.assertIn("CREATE TABLE SG_AI_QA_ANSWER", statements)
+        self.assertIn("answer_seq NUMBER GENERATED ALWAYS AS IDENTITY", statements)
+
+
+if __name__ == "__main__":
+    unittest.main()