joungmin/vpd-permission-poc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refs #739: seed verified Smilegate QA few-shot candidates

Browse Source
This commit is contained in:
devmrko
2026-07-28 12:34:40 +09:00
parent ae37235258
commit 5ac9f5d591
7 changed files with 345 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

304
database/adb/83_sgmp_qa_benchmark_fewshot_candidates.sql Normal file
View File

@@ -0,0 +1,304 @@
-- Import the 47 customer QA benchmark rows as governed Few-shot candidates.
-- This migration deliberately creates DRAFT records only. Retrieval continues
-- to use APPROVED rows only (see 82_sgmp_qa_fewshot_reference_governance.sql).
DECLARE
PROCEDURE add_column(p_definition IN VARCHAR2) IS
BEGIN
EXECUTE IMMEDIATE 'ALTER TABLE sg_qa_vector_example ADD (' || p_definition || ')';
EXCEPTION
WHEN OTHERS THEN
IF SQLCODE != -1430 THEN
RAISE;
END IF;
END;
BEGIN
add_column('source_case_id VARCHAR2(30)');
add_column('source_type VARCHAR2(30)');
END;
/
BEGIN
EXECUTE IMMEDIATE
'CREATE UNIQUE INDEX sg_qa_vector_example_source_uk '
|| 'ON sg_qa_vector_example (source_type, source_case_id)';
EXCEPTION
WHEN OTHERS THEN
IF SQLCODE != -955 THEN
RAISE;
END IF;
END;
/
CREATE OR REPLACE FUNCTION sg_qa_vector_seed_benchmark
RETURN NUMBER
AUTHID DEFINER
IS
PRAGMA AUTONOMOUS_TRANSACTION;
v_input CLOB;
v_embedding VECTOR;
v_answer_sql CLOB;
v_answer_text CLOB;
v_exists NUMBER;
v_inserted NUMBER := 0;
BEGIN
FOR item IN (
SELECT question_code,
question_text,
baseline_sql,
baseline_answer,
expected_focus,
support_level
FROM sg_ai_qa_question
WHERE question_source = 'CUSTOMER_EXCEL'
AND active_yn = 'Y'
ORDER BY question_code
) LOOP
SELECT CASE
WHEN EXISTS (
SELECT 1
FROM sg_qa_vector_example existing
WHERE existing.source_type = 'CUSTOMER_QA_BENCHMARK'
AND existing.source_case_id = item.question_code
) THEN 1 ELSE 0
END
INTO v_exists
FROM dual;
IF v_exists = 0 THEN
v_answer_sql := CASE
WHEN item.baseline_sql IS NULL OR DBMS_LOB.GETLENGTH(TRIM(item.baseline_sql)) = 0
THEN TO_CLOB('SELECT CAST(NULL AS NUMBER) AS "NO_BASELINE" FROM DUAL WHERE 1 = 0')
ELSE item.baseline_sql
END;
v_answer_text := TO_CLOB('Expected focus: ') || item.expected_focus
|| CASE WHEN item.baseline_answer IS NULL THEN NULL
ELSE TO_CLOB(CHR(10) || 'Historical answer: ') || item.baseline_answer END;
v_input := TO_CLOB('Customer QA case: ') || item.question_code
|| TO_CLOB(CHR(10) || 'Question: ') || item.question_text
|| TO_CLOB(CHR(10) || 'Expected focus: ') || item.expected_focus
|| TO_CLOB(CHR(10) || 'Candidate SQL: ') || v_answer_sql;
v_embedding := DBMS_VECTOR.UTL_TO_EMBEDDING(
v_input,
JSON(sg_qa_vector_params('search_document'))
);
BEGIN
INSERT INTO sg_qa_vector_example (
question, answer_sql, answer_text, embedding_input, embedding, embedding_model,
reference_status, reference_kind, target_type, inspection_status,
inspection_note, source_case_id, source_type
) VALUES (
item.question_text, v_answer_sql, v_answer_text, v_input, v_embedding, 'cohere.embed-v4.0',
'DRAFT',
CASE WHEN item.support_level = 'UNSUPPORTED' THEN 'OBJECT_UNAVAILABLE'
ELSE 'SQL_TEMPLATE' END,
'ANY',
'PENDING',
'Imported from the customer benchmark. A candidate cannot be retrieved until scope, logical object role, and SQL safety are reviewed.',
item.question_code,
'CUSTOMER_QA_BENCHMARK'
);
v_inserted := v_inserted + 1;
EXCEPTION
WHEN DUP_VAL_ON_INDEX THEN
NULL;
END;
END IF;
END LOOP;
COMMIT;
RETURN v_inserted;
EXCEPTION
WHEN OTHERS THEN
ROLLBACK;
RAISE;
END;
/
CREATE OR REPLACE FUNCTION sg_qa_vector_approve_verified_benchmark
RETURN NUMBER
AUTHID DEFINER
IS
PRAGMA AUTONOMOUS_TRANSACTION;
v_target_type VARCHAR2(16);
v_input CLOB;
v_answer_text CLOB;
v_embedding VECTOR;
v_approved NUMBER := 0;
BEGIN
FOR item IN (
WITH latest_answer AS (
SELECT answer.question_id,
answer.generated_sql,
answer.answer_text,
answer.result_json,
answer.judgment_status,
answer.execution_status,
ROW_NUMBER() OVER (
PARTITION BY answer.question_id ORDER BY answer.answer_seq DESC
) AS row_rank
FROM sg_ai_qa_answer answer
)
SELECT example.example_id,
question.question_text,
question.expected_focus,
question.support_level,
latest.generated_sql,
latest.answer_text AS live_answer,
latest.result_json,
latest.judgment_status,
latest.execution_status
FROM sg_qa_vector_example example
JOIN sg_ai_qa_question question
ON question.question_code = example.source_case_id
LEFT JOIN latest_answer latest
ON latest.question_id = question.question_id
AND latest.row_rank = 1
WHERE example.source_type = 'CUSTOMER_QA_BENCHMARK'
AND example.reference_status = 'DRAFT'
) LOOP
IF item.support_level = 'SUPPORTED'
AND item.judgment_status = 'PASS'
AND item.execution_status = 'COMPLETED'
AND item.generated_sql IS NOT NULL
AND DBMS_LOB.GETLENGTH(TRIM(item.generated_sql)) > 0 THEN
v_target_type := REGEXP_SUBSTR(
DBMS_LOB.SUBSTR(item.result_json, 32767, 1),
'"targetType"[[:space:]]*:[[:space:]]*"([A-Z]+)"',
1, 1, NULL, 1
);
IF v_target_type NOT IN ('NONE', 'SINGLE', 'MULTI', 'ALL', 'ANY') THEN
v_target_type := 'ANY';
END IF;
v_answer_text := TO_CLOB('Expected focus: ') || item.expected_focus
|| CASE WHEN item.live_answer IS NULL THEN NULL
ELSE TO_CLOB(CHR(10) || 'Verified live answer: ') || item.live_answer END;
v_input := TO_CLOB('Customer QA question: ') || item.question_text
|| TO_CLOB(CHR(10) || 'Expected focus: ') || item.expected_focus
|| TO_CLOB(CHR(10) || 'Verified SQL template: ') || item.generated_sql;
v_embedding := DBMS_VECTOR.UTL_TO_EMBEDDING(
v_input,
JSON(sg_qa_vector_params('search_document'))
);
UPDATE sg_qa_vector_example
SET answer_sql = item.generated_sql,
answer_text = v_answer_text,
embedding_input = v_input,
embedding = v_embedding,
reference_status = 'APPROVED',
reference_kind = 'SQL_TEMPLATE',
target_type = v_target_type,
inspection_status = 'VERIFIED',
inspection_note = 'Promoted only after the latest Portal ReAct run completed with PASS for this customer benchmark case.',
verified_at = SYSTIMESTAMP,
verified_by = 'SGMP_POC_BENCHMARK_REVIEW'
WHERE example_id = item.example_id;
v_approved := v_approved + 1;
ELSE
UPDATE sg_qa_vector_example
SET inspection_status = 'REVIEW',
inspection_note = 'Not retrievable: latest customer benchmark execution is unsupported, incomplete, WARN, FAIL, or lacks executable SQL.'
WHERE example_id = item.example_id;
END IF;
END LOOP;
COMMIT;
RETURN v_approved;
EXCEPTION
WHEN OTHERS THEN
ROLLBACK;
RAISE;
END;
/
CREATE OR REPLACE FUNCTION sg_qa_vector_search(
p_question IN CLOB,
p_top_k IN PLS_INTEGER DEFAULT 3,
p_target_type IN VARCHAR2 DEFAULT 'ANY'
) RETURN SYS_REFCURSOR
AUTHID DEFINER
IS
v_query_vector VECTOR;
v_results SYS_REFCURSOR;
v_target_type VARCHAR2(16) := UPPER(TRIM(NVL(p_target_type, 'ANY')));
BEGIN
IF p_question IS NULL THEN
RAISE_APPLICATION_ERROR(-20003, 'question is required.');
END IF;
IF p_top_k IS NULL OR p_top_k < 1 OR p_top_k > 20 THEN
RAISE_APPLICATION_ERROR(-20004, 'top_k must be between 1 and 20.');
END IF;
IF v_target_type NOT IN ('NONE', 'SINGLE', 'MULTI', 'ALL', 'ANY') THEN
RAISE_APPLICATION_ERROR(-20005, 'target_type must be NONE, SINGLE, MULTI, ALL, or ANY.');
END IF;
v_query_vector := DBMS_VECTOR.UTL_TO_EMBEDDING(
p_question,
JSON(sg_qa_vector_params('search_query'))
);
OPEN v_results FOR
SELECT example_id,
question,
answer_sql,
answer_text,
embedding_model,
reference_kind,
target_type,
object_role,
source_case_id,
source_type,
vector_distance(embedding, v_query_vector, COSINE) AS cosine_distance
FROM sg_qa_vector_example
WHERE reference_status = 'APPROVED'
AND (target_type = 'ANY' OR v_target_type = 'ANY' OR target_type = v_target_type)
ORDER BY vector_distance(embedding, v_query_vector, COSINE), example_id
FETCH FIRST p_top_k ROWS ONLY;
RETURN v_results;
END;
/
CREATE OR REPLACE FUNCTION sg_qa_vector_context(
p_question IN CLOB,
p_top_k IN PLS_INTEGER DEFAULT 3,
p_target_type IN VARCHAR2 DEFAULT 'ANY'
) RETURN CLOB
AUTHID DEFINER
IS
v_results SYS_REFCURSOR;
v_id NUMBER;
v_q CLOB;
v_sql CLOB;
v_answer CLOB;
v_model VARCHAR2(128);
v_kind VARCHAR2(32);
v_target VARCHAR2(16);
v_role VARCHAR2(64);
v_case_id VARCHAR2(30);
v_source VARCHAR2(30);
v_dist NUMBER;
v_context CLOB := EMPTY_CLOB();
BEGIN
v_results := sg_qa_vector_search(p_question, p_top_k, p_target_type);
LOOP
FETCH v_results INTO v_id, v_q, v_sql, v_answer, v_model, v_kind, v_target, v_role,
v_case_id, v_source, v_dist;
EXIT WHEN v_results%NOTFOUND;
v_context := v_context
|| CASE WHEN DBMS_LOB.GETLENGTH(v_context) = 0 THEN NULL ELSE CHR(10) || CHR(10) END
|| '[Example ' || v_id || ', kind=' || v_kind || ', target_type=' || v_target
|| CASE WHEN v_case_id IS NULL THEN NULL ELSE ', source_case_id=' || v_case_id END
|| ', cosine_distance=' || TO_CHAR(v_dist, 'FM0D000000') || ']' || CHR(10)
|| 'Question: ' || v_q || CHR(10)
|| 'Answer SQL: ' || v_sql
|| CASE WHEN v_answer IS NULL THEN NULL ELSE CHR(10) || 'Answer: ' || v_answer END;
END LOOP;
CLOSE v_results;
RETURN v_context;
END;
/
COMMENT ON COLUMN sg_qa_vector_example.source_case_id IS
'Customer QA benchmark case identifier, such as STD-01 or CZN-01.';
COMMENT ON COLUMN sg_qa_vector_example.source_type IS
'Candidate provenance. CUSTOMER_QA_BENCHMARK rows remain DRAFT until reviewed.';

19
docs/design/739-game-target-contract/README.md
View File

@@ -141,6 +141,25 @@ Few-shot 원문 SQL은 실행 이력 그대로 검색하지 않는다. 특히
기존 예제는 삭제하지 않는다. 실행 가능성, 게임 범위 일치, 물리 객체 의존성을 전수
검사해 `RETIRED`로 분리하고, 검증된 정규화 예제만 `APPROVED`로 전환한다.
### 고객 질답 47건 후보화·승인 규칙
`SG_AI_QA_QUESTION`의 고객 Excel 47건은 `SOURCE_TYPE=CUSTOMER_QA_BENCHMARK`,
`SOURCE_CASE_ID`로 벡터 예제에 한 번씩 적재한다. 초기 상태는 모두 `DRAFT`이고,
질문·기대 포인트·기준 SQL을 embedding 문서로 사용한다. 이 상태의 원문은 운영 검색
결과에 절대 포함되지 않는다.
`SG_QA_VECTOR_APPROVE_VERIFIED_BENCHMARK`는 최신 Portal ReAct 실행이 다음을 모두
만족한 후보만 자동으로 `APPROVED`로 전환한다.
1. 고객 지원 범위가 `SUPPORTED`일 것
2. 최신 실행이 `COMPLETED`이며 고객 기준 판정이 `PASS`일 것
3. 실행 가능한 읽기 전용 SQL이 있을 것
승인 시 최신 실행 SQL과 응답으로 embedding을 재생성하고, 실행 이력의 query plan에서
확인된 `TARGET_TYPE`을 함께 보관한다. `WARN`·`FAIL`·미지원·SQL 미생성 후보는
`DRAFT / REVIEW`로 남긴다. 따라서 개선 전 SQL이 Few-shot 근거로 노출되지 않으며,
각 검색 결과는 `SOURCE_CASE_ID`로 고객 기준 항목까지 추적할 수 있다.
## 9. 변경 이력
- Redmine: `#739`

6
vpd-backoffice/src/main/java/com/cloudhandson/vpdbackoffice/service/McpSseService.java
View File

@@ -260,6 +260,12 @@ public class McpSseService {
if (example.objectRole() != null) {
item.put("objectRole", example.objectRole());
}
if (example.sourceCaseId() != null) {
item.put("sourceCaseId", example.sourceCaseId());
}
if (example.sourceType() != null) {
item.put("sourceType", example.sourceType());
}
item.put("cosineDistance", example.cosineDistance());
}
return response;

4
vpd-backoffice/src/main/java/com/cloudhandson/vpdbackoffice/service/QaVectorService.java
View File

@@ -78,6 +78,8 @@ public class QaVectorService {
resultSet.getString("REFERENCE_KIND"),
resultSet.getString("TARGET_TYPE"),
resultSet.getString("OBJECT_ROLE"),
resultSet.getString("SOURCE_CASE_ID"),
resultSet.getString("SOURCE_TYPE"),
resultSet.getDouble("COSINE_DISTANCE")
));
}
@@ -184,6 +186,8 @@ public class QaVectorService {
String referenceKind,
String targetType,
String objectRole,
String sourceCaseId,
String sourceType,
double cosineDistance
) {
}

6
vpd-backoffice/src/main/java/com/cloudhandson/vpdbackoffice/service/SelectAiService.java
View File

@@ -292,6 +292,12 @@ public class SelectAiService {
if (example.objectRole() != null) {
item.put("objectRole", example.objectRole());
}
if (example.sourceCaseId() != null) {
item.put("sourceCaseId", example.sourceCaseId());
}
if (example.sourceType() != null) {
item.put("sourceType", example.sourceType());
}
item.put("cosineDistance", example.cosineDistance());
}
}

3
vpd-backoffice/src/test/java/com/cloudhandson/vpdbackoffice/service/McpSseServiceTest.java
View File

@@ -363,7 +363,8 @@ class McpSseServiceTest {
this.topK = topK;
return new VectorSearchResult(question, topK, java.util.List.of(new VectorExample(
42L, "active user count", "SELECT COUNT(*) FROM APP_USER", "AU count",
"cohere.embed-v4.0", "SQL_TEMPLATE", "ANY", null, 0.12
"cohere.embed-v4.0", "SQL_TEMPLATE", "ANY", null,
"STD-13", "CUSTOMER_QA_BENCHMARK", 0.12
)));
}

4
vpd-backoffice/src/test/java/com/cloudhandson/vpdbackoffice/service/SelectAiFewShotPromptTest.java
View File

@@ -20,6 +20,8 @@ class SelectAiFewShotPromptTest {
"SQL_TEMPLATE",
"SINGLE",
"GAME_USER_MASTER",
"STD-16",
"CUSTOMER_QA_BENCHMARK",
0.01
))
);
@@ -46,6 +48,8 @@ class SelectAiFewShotPromptTest {
"NO_TARGET",
"NONE",
null,
null,
null,
0.01
))
);