From 5409a292970b2719f3bb06b6fc861304b732cb62 Mon Sep 17 00:00:00 2001 From: devmrko Date: Tue, 28 Jul 2026 12:34:40 +0900 Subject: [PATCH] refs #739: seed verified Smilegate QA few-shot candidates --- .../design/739-game-target-contract/README.md | 19 ++ ...3_sgmp_qa_benchmark_fewshot_candidates.sql | 304 ++++++++++++++++++ .../vpdbackoffice/service/McpSseService.java | 6 + .../service/QaVectorService.java | 4 + .../service/SelectAiService.java | 6 + .../service/McpSseServiceTest.java | 3 +- .../service/SelectAiFewShotPromptTest.java | 4 + 7 files changed, 345 insertions(+), 1 deletion(-) create mode 100644 sql/adb/83_sgmp_qa_benchmark_fewshot_candidates.sql diff --git a/docs/design/739-game-target-contract/README.md b/docs/design/739-game-target-contract/README.md index b0b8957..3013c3b 100644 --- a/docs/design/739-game-target-contract/README.md +++ b/docs/design/739-game-target-contract/README.md @@ -141,6 +141,25 @@ Few-shot 원문 SQL은 실행 이력 그대로 검색하지 않는다. 특히 기존 예제는 삭제하지 않는다. 실행 가능성, 게임 범위 일치, 물리 객체 의존성을 전수 검사해 `RETIRED`로 분리하고, 검증된 정규화 예제만 `APPROVED`로 전환한다. +### 고객 질답 47건 후보화·승인 규칙 + +`SG_AI_QA_QUESTION`의 고객 Excel 47건은 `SOURCE_TYPE=CUSTOMER_QA_BENCHMARK`, +`SOURCE_CASE_ID`로 벡터 예제에 한 번씩 적재한다. 초기 상태는 모두 `DRAFT`이고, +질문·기대 포인트·기준 SQL을 embedding 문서로 사용한다. 이 상태의 원문은 운영 검색 +결과에 절대 포함되지 않는다. + +`SG_QA_VECTOR_APPROVE_VERIFIED_BENCHMARK`는 최신 Portal ReAct 실행이 다음을 모두 +만족한 후보만 자동으로 `APPROVED`로 전환한다. + +1. 고객 지원 범위가 `SUPPORTED`일 것 +2. 최신 실행이 `COMPLETED`이며 고객 기준 판정이 `PASS`일 것 +3. 실행 가능한 읽기 전용 SQL이 있을 것 + +승인 시 최신 실행 SQL과 응답으로 embedding을 재생성하고, 실행 이력의 query plan에서 +확인된 `TARGET_TYPE`을 함께 보관한다. `WARN`·`FAIL`·미지원·SQL 미생성 후보는 +`DRAFT / REVIEW`로 남긴다. 따라서 개선 전 SQL이 Few-shot 근거로 노출되지 않으며, +각 검색 결과는 `SOURCE_CASE_ID`로 고객 기준 항목까지 추적할 수 있다. + ## 9. 변경 이력 - Redmine: `#739` diff --git a/sql/adb/83_sgmp_qa_benchmark_fewshot_candidates.sql b/sql/adb/83_sgmp_qa_benchmark_fewshot_candidates.sql new file mode 100644 index 0000000..10658dd --- /dev/null +++ b/sql/adb/83_sgmp_qa_benchmark_fewshot_candidates.sql @@ -0,0 +1,304 @@ +-- Import the 47 customer QA benchmark rows as governed Few-shot candidates. +-- This migration deliberately creates DRAFT records only. Retrieval continues +-- to use APPROVED rows only (see 82_sgmp_qa_fewshot_reference_governance.sql). + +DECLARE + PROCEDURE add_column(p_definition IN VARCHAR2) IS + BEGIN + EXECUTE IMMEDIATE 'ALTER TABLE sg_qa_vector_example ADD (' || p_definition || ')'; + EXCEPTION + WHEN OTHERS THEN + IF SQLCODE != -1430 THEN + RAISE; + END IF; + END; +BEGIN + add_column('source_case_id VARCHAR2(30)'); + add_column('source_type VARCHAR2(30)'); +END; +/ + +BEGIN + EXECUTE IMMEDIATE + 'CREATE UNIQUE INDEX sg_qa_vector_example_source_uk ' + || 'ON sg_qa_vector_example (source_type, source_case_id)'; +EXCEPTION + WHEN OTHERS THEN + IF SQLCODE != -955 THEN + RAISE; + END IF; +END; +/ + +CREATE OR REPLACE FUNCTION sg_qa_vector_seed_benchmark +RETURN NUMBER +AUTHID DEFINER +IS + PRAGMA AUTONOMOUS_TRANSACTION; + v_input CLOB; + v_embedding VECTOR; + v_answer_sql CLOB; + v_answer_text CLOB; + v_exists NUMBER; + v_inserted NUMBER := 0; +BEGIN + FOR item IN ( + SELECT question_code, + question_text, + baseline_sql, + baseline_answer, + expected_focus, + support_level + FROM sg_ai_qa_question + WHERE question_source = 'CUSTOMER_EXCEL' + AND active_yn = 'Y' + ORDER BY question_code + ) LOOP + SELECT CASE + WHEN EXISTS ( + SELECT 1 + FROM sg_qa_vector_example existing + WHERE existing.source_type = 'CUSTOMER_QA_BENCHMARK' + AND existing.source_case_id = item.question_code + ) THEN 1 ELSE 0 + END + INTO v_exists + FROM dual; + + IF v_exists = 0 THEN + v_answer_sql := CASE + WHEN item.baseline_sql IS NULL OR DBMS_LOB.GETLENGTH(TRIM(item.baseline_sql)) = 0 + THEN TO_CLOB('SELECT CAST(NULL AS NUMBER) AS "NO_BASELINE" FROM DUAL WHERE 1 = 0') + ELSE item.baseline_sql + END; + v_answer_text := TO_CLOB('Expected focus: ') || item.expected_focus + || CASE WHEN item.baseline_answer IS NULL THEN NULL + ELSE TO_CLOB(CHR(10) || 'Historical answer: ') || item.baseline_answer END; + v_input := TO_CLOB('Customer QA case: ') || item.question_code + || TO_CLOB(CHR(10) || 'Question: ') || item.question_text + || TO_CLOB(CHR(10) || 'Expected focus: ') || item.expected_focus + || TO_CLOB(CHR(10) || 'Candidate SQL: ') || v_answer_sql; + v_embedding := DBMS_VECTOR.UTL_TO_EMBEDDING( + v_input, + JSON(sg_qa_vector_params('search_document')) + ); + + BEGIN + INSERT INTO sg_qa_vector_example ( + question, answer_sql, answer_text, embedding_input, embedding, embedding_model, + reference_status, reference_kind, target_type, inspection_status, + inspection_note, source_case_id, source_type + ) VALUES ( + item.question_text, v_answer_sql, v_answer_text, v_input, v_embedding, 'cohere.embed-v4.0', + 'DRAFT', + CASE WHEN item.support_level = 'UNSUPPORTED' THEN 'OBJECT_UNAVAILABLE' + ELSE 'SQL_TEMPLATE' END, + 'ANY', + 'PENDING', + 'Imported from the customer benchmark. A candidate cannot be retrieved until scope, logical object role, and SQL safety are reviewed.', + item.question_code, + 'CUSTOMER_QA_BENCHMARK' + ); + v_inserted := v_inserted + 1; + EXCEPTION + WHEN DUP_VAL_ON_INDEX THEN + NULL; + END; + END IF; + END LOOP; + + COMMIT; + RETURN v_inserted; +EXCEPTION + WHEN OTHERS THEN + ROLLBACK; + RAISE; +END; +/ + +CREATE OR REPLACE FUNCTION sg_qa_vector_approve_verified_benchmark +RETURN NUMBER +AUTHID DEFINER +IS + PRAGMA AUTONOMOUS_TRANSACTION; + v_target_type VARCHAR2(16); + v_input CLOB; + v_answer_text CLOB; + v_embedding VECTOR; + v_approved NUMBER := 0; +BEGIN + FOR item IN ( + WITH latest_answer AS ( + SELECT answer.question_id, + answer.generated_sql, + answer.answer_text, + answer.result_json, + answer.judgment_status, + answer.execution_status, + ROW_NUMBER() OVER ( + PARTITION BY answer.question_id ORDER BY answer.answer_seq DESC + ) AS row_rank + FROM sg_ai_qa_answer answer + ) + SELECT example.example_id, + question.question_text, + question.expected_focus, + question.support_level, + latest.generated_sql, + latest.answer_text AS live_answer, + latest.result_json, + latest.judgment_status, + latest.execution_status + FROM sg_qa_vector_example example + JOIN sg_ai_qa_question question + ON question.question_code = example.source_case_id + LEFT JOIN latest_answer latest + ON latest.question_id = question.question_id + AND latest.row_rank = 1 + WHERE example.source_type = 'CUSTOMER_QA_BENCHMARK' + AND example.reference_status = 'DRAFT' + ) LOOP + IF item.support_level = 'SUPPORTED' + AND item.judgment_status = 'PASS' + AND item.execution_status = 'COMPLETED' + AND item.generated_sql IS NOT NULL + AND DBMS_LOB.GETLENGTH(TRIM(item.generated_sql)) > 0 THEN + v_target_type := REGEXP_SUBSTR( + DBMS_LOB.SUBSTR(item.result_json, 32767, 1), + '"targetType"[[:space:]]*:[[:space:]]*"([A-Z]+)"', + 1, 1, NULL, 1 + ); + IF v_target_type NOT IN ('NONE', 'SINGLE', 'MULTI', 'ALL', 'ANY') THEN + v_target_type := 'ANY'; + END IF; + v_answer_text := TO_CLOB('Expected focus: ') || item.expected_focus + || CASE WHEN item.live_answer IS NULL THEN NULL + ELSE TO_CLOB(CHR(10) || 'Verified live answer: ') || item.live_answer END; + v_input := TO_CLOB('Customer QA question: ') || item.question_text + || TO_CLOB(CHR(10) || 'Expected focus: ') || item.expected_focus + || TO_CLOB(CHR(10) || 'Verified SQL template: ') || item.generated_sql; + v_embedding := DBMS_VECTOR.UTL_TO_EMBEDDING( + v_input, + JSON(sg_qa_vector_params('search_document')) + ); + UPDATE sg_qa_vector_example + SET answer_sql = item.generated_sql, + answer_text = v_answer_text, + embedding_input = v_input, + embedding = v_embedding, + reference_status = 'APPROVED', + reference_kind = 'SQL_TEMPLATE', + target_type = v_target_type, + inspection_status = 'VERIFIED', + inspection_note = 'Promoted only after the latest Portal ReAct run completed with PASS for this customer benchmark case.', + verified_at = SYSTIMESTAMP, + verified_by = 'SGMP_POC_BENCHMARK_REVIEW' + WHERE example_id = item.example_id; + v_approved := v_approved + 1; + ELSE + UPDATE sg_qa_vector_example + SET inspection_status = 'REVIEW', + inspection_note = 'Not retrievable: latest customer benchmark execution is unsupported, incomplete, WARN, FAIL, or lacks executable SQL.' + WHERE example_id = item.example_id; + END IF; + END LOOP; + COMMIT; + RETURN v_approved; +EXCEPTION + WHEN OTHERS THEN + ROLLBACK; + RAISE; +END; +/ + +CREATE OR REPLACE FUNCTION sg_qa_vector_search( + p_question IN CLOB, + p_top_k IN PLS_INTEGER DEFAULT 3, + p_target_type IN VARCHAR2 DEFAULT 'ANY' +) RETURN SYS_REFCURSOR +AUTHID DEFINER +IS + v_query_vector VECTOR; + v_results SYS_REFCURSOR; + v_target_type VARCHAR2(16) := UPPER(TRIM(NVL(p_target_type, 'ANY'))); +BEGIN + IF p_question IS NULL THEN + RAISE_APPLICATION_ERROR(-20003, 'question is required.'); + END IF; + IF p_top_k IS NULL OR p_top_k < 1 OR p_top_k > 20 THEN + RAISE_APPLICATION_ERROR(-20004, 'top_k must be between 1 and 20.'); + END IF; + IF v_target_type NOT IN ('NONE', 'SINGLE', 'MULTI', 'ALL', 'ANY') THEN + RAISE_APPLICATION_ERROR(-20005, 'target_type must be NONE, SINGLE, MULTI, ALL, or ANY.'); + END IF; + + v_query_vector := DBMS_VECTOR.UTL_TO_EMBEDDING( + p_question, + JSON(sg_qa_vector_params('search_query')) + ); + + OPEN v_results FOR + SELECT example_id, + question, + answer_sql, + answer_text, + embedding_model, + reference_kind, + target_type, + object_role, + source_case_id, + source_type, + vector_distance(embedding, v_query_vector, COSINE) AS cosine_distance + FROM sg_qa_vector_example + WHERE reference_status = 'APPROVED' + AND (target_type = 'ANY' OR v_target_type = 'ANY' OR target_type = v_target_type) + ORDER BY vector_distance(embedding, v_query_vector, COSINE), example_id + FETCH FIRST p_top_k ROWS ONLY; + RETURN v_results; +END; +/ + +CREATE OR REPLACE FUNCTION sg_qa_vector_context( + p_question IN CLOB, + p_top_k IN PLS_INTEGER DEFAULT 3, + p_target_type IN VARCHAR2 DEFAULT 'ANY' +) RETURN CLOB +AUTHID DEFINER +IS + v_results SYS_REFCURSOR; + v_id NUMBER; + v_q CLOB; + v_sql CLOB; + v_answer CLOB; + v_model VARCHAR2(128); + v_kind VARCHAR2(32); + v_target VARCHAR2(16); + v_role VARCHAR2(64); + v_case_id VARCHAR2(30); + v_source VARCHAR2(30); + v_dist NUMBER; + v_context CLOB := EMPTY_CLOB(); +BEGIN + v_results := sg_qa_vector_search(p_question, p_top_k, p_target_type); + LOOP + FETCH v_results INTO v_id, v_q, v_sql, v_answer, v_model, v_kind, v_target, v_role, + v_case_id, v_source, v_dist; + EXIT WHEN v_results%NOTFOUND; + v_context := v_context + || CASE WHEN DBMS_LOB.GETLENGTH(v_context) = 0 THEN NULL ELSE CHR(10) || CHR(10) END + || '[Example ' || v_id || ', kind=' || v_kind || ', target_type=' || v_target + || CASE WHEN v_case_id IS NULL THEN NULL ELSE ', source_case_id=' || v_case_id END + || ', cosine_distance=' || TO_CHAR(v_dist, 'FM0D000000') || ']' || CHR(10) + || 'Question: ' || v_q || CHR(10) + || 'Answer SQL: ' || v_sql + || CASE WHEN v_answer IS NULL THEN NULL ELSE CHR(10) || 'Answer: ' || v_answer END; + END LOOP; + CLOSE v_results; + RETURN v_context; +END; +/ + +COMMENT ON COLUMN sg_qa_vector_example.source_case_id IS + 'Customer QA benchmark case identifier, such as STD-01 or CZN-01.'; +COMMENT ON COLUMN sg_qa_vector_example.source_type IS + 'Candidate provenance. CUSTOMER_QA_BENCHMARK rows remain DRAFT until reviewed.'; diff --git a/src/main/java/com/cloudhandson/vpdbackoffice/service/McpSseService.java b/src/main/java/com/cloudhandson/vpdbackoffice/service/McpSseService.java index 705cf07..a9e4c01 100644 --- a/src/main/java/com/cloudhandson/vpdbackoffice/service/McpSseService.java +++ b/src/main/java/com/cloudhandson/vpdbackoffice/service/McpSseService.java @@ -260,6 +260,12 @@ public class McpSseService { if (example.objectRole() != null) { item.put("objectRole", example.objectRole()); } + if (example.sourceCaseId() != null) { + item.put("sourceCaseId", example.sourceCaseId()); + } + if (example.sourceType() != null) { + item.put("sourceType", example.sourceType()); + } item.put("cosineDistance", example.cosineDistance()); } return response; diff --git a/src/main/java/com/cloudhandson/vpdbackoffice/service/QaVectorService.java b/src/main/java/com/cloudhandson/vpdbackoffice/service/QaVectorService.java index aaa3626..c08cc20 100644 --- a/src/main/java/com/cloudhandson/vpdbackoffice/service/QaVectorService.java +++ b/src/main/java/com/cloudhandson/vpdbackoffice/service/QaVectorService.java @@ -78,6 +78,8 @@ public class QaVectorService { resultSet.getString("REFERENCE_KIND"), resultSet.getString("TARGET_TYPE"), resultSet.getString("OBJECT_ROLE"), + resultSet.getString("SOURCE_CASE_ID"), + resultSet.getString("SOURCE_TYPE"), resultSet.getDouble("COSINE_DISTANCE") )); } @@ -184,6 +186,8 @@ public class QaVectorService { String referenceKind, String targetType, String objectRole, + String sourceCaseId, + String sourceType, double cosineDistance ) { } diff --git a/src/main/java/com/cloudhandson/vpdbackoffice/service/SelectAiService.java b/src/main/java/com/cloudhandson/vpdbackoffice/service/SelectAiService.java index 3217c46..d6d5fff 100644 --- a/src/main/java/com/cloudhandson/vpdbackoffice/service/SelectAiService.java +++ b/src/main/java/com/cloudhandson/vpdbackoffice/service/SelectAiService.java @@ -292,6 +292,12 @@ public class SelectAiService { if (example.objectRole() != null) { item.put("objectRole", example.objectRole()); } + if (example.sourceCaseId() != null) { + item.put("sourceCaseId", example.sourceCaseId()); + } + if (example.sourceType() != null) { + item.put("sourceType", example.sourceType()); + } item.put("cosineDistance", example.cosineDistance()); } } diff --git a/src/test/java/com/cloudhandson/vpdbackoffice/service/McpSseServiceTest.java b/src/test/java/com/cloudhandson/vpdbackoffice/service/McpSseServiceTest.java index 7384a9c..1abd1a4 100644 --- a/src/test/java/com/cloudhandson/vpdbackoffice/service/McpSseServiceTest.java +++ b/src/test/java/com/cloudhandson/vpdbackoffice/service/McpSseServiceTest.java @@ -363,7 +363,8 @@ class McpSseServiceTest { this.topK = topK; return new VectorSearchResult(question, topK, java.util.List.of(new VectorExample( 42L, "active user count", "SELECT COUNT(*) FROM APP_USER", "AU count", - "cohere.embed-v4.0", "SQL_TEMPLATE", "ANY", null, 0.12 + "cohere.embed-v4.0", "SQL_TEMPLATE", "ANY", null, + "STD-13", "CUSTOMER_QA_BENCHMARK", 0.12 ))); } diff --git a/src/test/java/com/cloudhandson/vpdbackoffice/service/SelectAiFewShotPromptTest.java b/src/test/java/com/cloudhandson/vpdbackoffice/service/SelectAiFewShotPromptTest.java index c34d45e..a6593f8 100644 --- a/src/test/java/com/cloudhandson/vpdbackoffice/service/SelectAiFewShotPromptTest.java +++ b/src/test/java/com/cloudhandson/vpdbackoffice/service/SelectAiFewShotPromptTest.java @@ -20,6 +20,8 @@ class SelectAiFewShotPromptTest { "SQL_TEMPLATE", "SINGLE", "GAME_USER_MASTER", + "STD-16", + "CUSTOMER_QA_BENCHMARK", 0.01 )) ); @@ -46,6 +48,8 @@ class SelectAiFewShotPromptTest { "NO_TARGET", "NONE", null, + null, + null, 0.01 )) );