refs #739: seed verified Smilegate QA few-shot candidates
This commit is contained in:
devmrko
304
database/adb/83_sgmp_qa_benchmark_fewshot_candidates.sql
Normal file
304
database/adb/83_sgmp_qa_benchmark_fewshot_candidates.sql
Normal file
@@ -0,0 +1,304 @@
|
||||
-- Import the 47 customer QA benchmark rows as governed Few-shot candidates.
|
||||
-- This migration deliberately creates DRAFT records only. Retrieval continues
|
||||
-- to use APPROVED rows only (see 82_sgmp_qa_fewshot_reference_governance.sql).
|
||||
|
||||
DECLARE
|
||||
PROCEDURE add_column(p_definition IN VARCHAR2) IS
|
||||
BEGIN
|
||||
EXECUTE IMMEDIATE 'ALTER TABLE sg_qa_vector_example ADD (' || p_definition || ')';
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
IF SQLCODE != -1430 THEN
|
||||
RAISE;
|
||||
END IF;
|
||||
END;
|
||||
BEGIN
|
||||
add_column('source_case_id VARCHAR2(30)');
|
||||
add_column('source_type VARCHAR2(30)');
|
||||
END;
|
||||
/
|
||||
|
||||
BEGIN
|
||||
EXECUTE IMMEDIATE
|
||||
'CREATE UNIQUE INDEX sg_qa_vector_example_source_uk '
|
||||
|| 'ON sg_qa_vector_example (source_type, source_case_id)';
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
IF SQLCODE != -955 THEN
|
||||
RAISE;
|
||||
END IF;
|
||||
END;
|
||||
/
|
||||
|
||||
CREATE OR REPLACE FUNCTION sg_qa_vector_seed_benchmark
|
||||
RETURN NUMBER
|
||||
AUTHID DEFINER
|
||||
IS
|
||||
PRAGMA AUTONOMOUS_TRANSACTION;
|
||||
v_input CLOB;
|
||||
v_embedding VECTOR;
|
||||
v_answer_sql CLOB;
|
||||
v_answer_text CLOB;
|
||||
v_exists NUMBER;
|
||||
v_inserted NUMBER := 0;
|
||||
BEGIN
|
||||
FOR item IN (
|
||||
SELECT question_code,
|
||||
question_text,
|
||||
baseline_sql,
|
||||
baseline_answer,
|
||||
expected_focus,
|
||||
support_level
|
||||
FROM sg_ai_qa_question
|
||||
WHERE question_source = 'CUSTOMER_EXCEL'
|
||||
AND active_yn = 'Y'
|
||||
ORDER BY question_code
|
||||
) LOOP
|
||||
SELECT CASE
|
||||
WHEN EXISTS (
|
||||
SELECT 1
|
||||
FROM sg_qa_vector_example existing
|
||||
WHERE existing.source_type = 'CUSTOMER_QA_BENCHMARK'
|
||||
AND existing.source_case_id = item.question_code
|
||||
) THEN 1 ELSE 0
|
||||
END
|
||||
INTO v_exists
|
||||
FROM dual;
|
||||
|
||||
IF v_exists = 0 THEN
|
||||
v_answer_sql := CASE
|
||||
WHEN item.baseline_sql IS NULL OR DBMS_LOB.GETLENGTH(TRIM(item.baseline_sql)) = 0
|
||||
THEN TO_CLOB('SELECT CAST(NULL AS NUMBER) AS "NO_BASELINE" FROM DUAL WHERE 1 = 0')
|
||||
ELSE item.baseline_sql
|
||||
END;
|
||||
v_answer_text := TO_CLOB('Expected focus: ') || item.expected_focus
|
||||
|| CASE WHEN item.baseline_answer IS NULL THEN NULL
|
||||
ELSE TO_CLOB(CHR(10) || 'Historical answer: ') || item.baseline_answer END;
|
||||
v_input := TO_CLOB('Customer QA case: ') || item.question_code
|
||||
|| TO_CLOB(CHR(10) || 'Question: ') || item.question_text
|
||||
|| TO_CLOB(CHR(10) || 'Expected focus: ') || item.expected_focus
|
||||
|| TO_CLOB(CHR(10) || 'Candidate SQL: ') || v_answer_sql;
|
||||
v_embedding := DBMS_VECTOR.UTL_TO_EMBEDDING(
|
||||
v_input,
|
||||
JSON(sg_qa_vector_params('search_document'))
|
||||
);
|
||||
|
||||
BEGIN
|
||||
INSERT INTO sg_qa_vector_example (
|
||||
question, answer_sql, answer_text, embedding_input, embedding, embedding_model,
|
||||
reference_status, reference_kind, target_type, inspection_status,
|
||||
inspection_note, source_case_id, source_type
|
||||
) VALUES (
|
||||
item.question_text, v_answer_sql, v_answer_text, v_input, v_embedding, 'cohere.embed-v4.0',
|
||||
'DRAFT',
|
||||
CASE WHEN item.support_level = 'UNSUPPORTED' THEN 'OBJECT_UNAVAILABLE'
|
||||
ELSE 'SQL_TEMPLATE' END,
|
||||
'ANY',
|
||||
'PENDING',
|
||||
'Imported from the customer benchmark. A candidate cannot be retrieved until scope, logical object role, and SQL safety are reviewed.',
|
||||
item.question_code,
|
||||
'CUSTOMER_QA_BENCHMARK'
|
||||
);
|
||||
v_inserted := v_inserted + 1;
|
||||
EXCEPTION
|
||||
WHEN DUP_VAL_ON_INDEX THEN
|
||||
NULL;
|
||||
END;
|
||||
END IF;
|
||||
END LOOP;
|
||||
|
||||
COMMIT;
|
||||
RETURN v_inserted;
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
ROLLBACK;
|
||||
RAISE;
|
||||
END;
|
||||
/
|
||||
|
||||
CREATE OR REPLACE FUNCTION sg_qa_vector_approve_verified_benchmark
|
||||
RETURN NUMBER
|
||||
AUTHID DEFINER
|
||||
IS
|
||||
PRAGMA AUTONOMOUS_TRANSACTION;
|
||||
v_target_type VARCHAR2(16);
|
||||
v_input CLOB;
|
||||
v_answer_text CLOB;
|
||||
v_embedding VECTOR;
|
||||
v_approved NUMBER := 0;
|
||||
BEGIN
|
||||
FOR item IN (
|
||||
WITH latest_answer AS (
|
||||
SELECT answer.question_id,
|
||||
answer.generated_sql,
|
||||
answer.answer_text,
|
||||
answer.result_json,
|
||||
answer.judgment_status,
|
||||
answer.execution_status,
|
||||
ROW_NUMBER() OVER (
|
||||
PARTITION BY answer.question_id ORDER BY answer.answer_seq DESC
|
||||
) AS row_rank
|
||||
FROM sg_ai_qa_answer answer
|
||||
)
|
||||
SELECT example.example_id,
|
||||
question.question_text,
|
||||
question.expected_focus,
|
||||
question.support_level,
|
||||
latest.generated_sql,
|
||||
latest.answer_text AS live_answer,
|
||||
latest.result_json,
|
||||
latest.judgment_status,
|
||||
latest.execution_status
|
||||
FROM sg_qa_vector_example example
|
||||
JOIN sg_ai_qa_question question
|
||||
ON question.question_code = example.source_case_id
|
||||
LEFT JOIN latest_answer latest
|
||||
ON latest.question_id = question.question_id
|
||||
AND latest.row_rank = 1
|
||||
WHERE example.source_type = 'CUSTOMER_QA_BENCHMARK'
|
||||
AND example.reference_status = 'DRAFT'
|
||||
) LOOP
|
||||
IF item.support_level = 'SUPPORTED'
|
||||
AND item.judgment_status = 'PASS'
|
||||
AND item.execution_status = 'COMPLETED'
|
||||
AND item.generated_sql IS NOT NULL
|
||||
AND DBMS_LOB.GETLENGTH(TRIM(item.generated_sql)) > 0 THEN
|
||||
v_target_type := REGEXP_SUBSTR(
|
||||
DBMS_LOB.SUBSTR(item.result_json, 32767, 1),
|
||||
'"targetType"[[:space:]]*:[[:space:]]*"([A-Z]+)"',
|
||||
1, 1, NULL, 1
|
||||
);
|
||||
IF v_target_type NOT IN ('NONE', 'SINGLE', 'MULTI', 'ALL', 'ANY') THEN
|
||||
v_target_type := 'ANY';
|
||||
END IF;
|
||||
v_answer_text := TO_CLOB('Expected focus: ') || item.expected_focus
|
||||
|| CASE WHEN item.live_answer IS NULL THEN NULL
|
||||
ELSE TO_CLOB(CHR(10) || 'Verified live answer: ') || item.live_answer END;
|
||||
v_input := TO_CLOB('Customer QA question: ') || item.question_text
|
||||
|| TO_CLOB(CHR(10) || 'Expected focus: ') || item.expected_focus
|
||||
|| TO_CLOB(CHR(10) || 'Verified SQL template: ') || item.generated_sql;
|
||||
v_embedding := DBMS_VECTOR.UTL_TO_EMBEDDING(
|
||||
v_input,
|
||||
JSON(sg_qa_vector_params('search_document'))
|
||||
);
|
||||
UPDATE sg_qa_vector_example
|
||||
SET answer_sql = item.generated_sql,
|
||||
answer_text = v_answer_text,
|
||||
embedding_input = v_input,
|
||||
embedding = v_embedding,
|
||||
reference_status = 'APPROVED',
|
||||
reference_kind = 'SQL_TEMPLATE',
|
||||
target_type = v_target_type,
|
||||
inspection_status = 'VERIFIED',
|
||||
inspection_note = 'Promoted only after the latest Portal ReAct run completed with PASS for this customer benchmark case.',
|
||||
verified_at = SYSTIMESTAMP,
|
||||
verified_by = 'SGMP_POC_BENCHMARK_REVIEW'
|
||||
WHERE example_id = item.example_id;
|
||||
v_approved := v_approved + 1;
|
||||
ELSE
|
||||
UPDATE sg_qa_vector_example
|
||||
SET inspection_status = 'REVIEW',
|
||||
inspection_note = 'Not retrievable: latest customer benchmark execution is unsupported, incomplete, WARN, FAIL, or lacks executable SQL.'
|
||||
WHERE example_id = item.example_id;
|
||||
END IF;
|
||||
END LOOP;
|
||||
COMMIT;
|
||||
RETURN v_approved;
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
ROLLBACK;
|
||||
RAISE;
|
||||
END;
|
||||
/
|
||||
|
||||
CREATE OR REPLACE FUNCTION sg_qa_vector_search(
|
||||
p_question IN CLOB,
|
||||
p_top_k IN PLS_INTEGER DEFAULT 3,
|
||||
p_target_type IN VARCHAR2 DEFAULT 'ANY'
|
||||
) RETURN SYS_REFCURSOR
|
||||
AUTHID DEFINER
|
||||
IS
|
||||
v_query_vector VECTOR;
|
||||
v_results SYS_REFCURSOR;
|
||||
v_target_type VARCHAR2(16) := UPPER(TRIM(NVL(p_target_type, 'ANY')));
|
||||
BEGIN
|
||||
IF p_question IS NULL THEN
|
||||
RAISE_APPLICATION_ERROR(-20003, 'question is required.');
|
||||
END IF;
|
||||
IF p_top_k IS NULL OR p_top_k < 1 OR p_top_k > 20 THEN
|
||||
RAISE_APPLICATION_ERROR(-20004, 'top_k must be between 1 and 20.');
|
||||
END IF;
|
||||
IF v_target_type NOT IN ('NONE', 'SINGLE', 'MULTI', 'ALL', 'ANY') THEN
|
||||
RAISE_APPLICATION_ERROR(-20005, 'target_type must be NONE, SINGLE, MULTI, ALL, or ANY.');
|
||||
END IF;
|
||||
|
||||
v_query_vector := DBMS_VECTOR.UTL_TO_EMBEDDING(
|
||||
p_question,
|
||||
JSON(sg_qa_vector_params('search_query'))
|
||||
);
|
||||
|
||||
OPEN v_results FOR
|
||||
SELECT example_id,
|
||||
question,
|
||||
answer_sql,
|
||||
answer_text,
|
||||
embedding_model,
|
||||
reference_kind,
|
||||
target_type,
|
||||
object_role,
|
||||
source_case_id,
|
||||
source_type,
|
||||
vector_distance(embedding, v_query_vector, COSINE) AS cosine_distance
|
||||
FROM sg_qa_vector_example
|
||||
WHERE reference_status = 'APPROVED'
|
||||
AND (target_type = 'ANY' OR v_target_type = 'ANY' OR target_type = v_target_type)
|
||||
ORDER BY vector_distance(embedding, v_query_vector, COSINE), example_id
|
||||
FETCH FIRST p_top_k ROWS ONLY;
|
||||
RETURN v_results;
|
||||
END;
|
||||
/
|
||||
|
||||
CREATE OR REPLACE FUNCTION sg_qa_vector_context(
|
||||
p_question IN CLOB,
|
||||
p_top_k IN PLS_INTEGER DEFAULT 3,
|
||||
p_target_type IN VARCHAR2 DEFAULT 'ANY'
|
||||
) RETURN CLOB
|
||||
AUTHID DEFINER
|
||||
IS
|
||||
v_results SYS_REFCURSOR;
|
||||
v_id NUMBER;
|
||||
v_q CLOB;
|
||||
v_sql CLOB;
|
||||
v_answer CLOB;
|
||||
v_model VARCHAR2(128);
|
||||
v_kind VARCHAR2(32);
|
||||
v_target VARCHAR2(16);
|
||||
v_role VARCHAR2(64);
|
||||
v_case_id VARCHAR2(30);
|
||||
v_source VARCHAR2(30);
|
||||
v_dist NUMBER;
|
||||
v_context CLOB := EMPTY_CLOB();
|
||||
BEGIN
|
||||
v_results := sg_qa_vector_search(p_question, p_top_k, p_target_type);
|
||||
LOOP
|
||||
FETCH v_results INTO v_id, v_q, v_sql, v_answer, v_model, v_kind, v_target, v_role,
|
||||
v_case_id, v_source, v_dist;
|
||||
EXIT WHEN v_results%NOTFOUND;
|
||||
v_context := v_context
|
||||
|| CASE WHEN DBMS_LOB.GETLENGTH(v_context) = 0 THEN NULL ELSE CHR(10) || CHR(10) END
|
||||
|| '[Example ' || v_id || ', kind=' || v_kind || ', target_type=' || v_target
|
||||
|| CASE WHEN v_case_id IS NULL THEN NULL ELSE ', source_case_id=' || v_case_id END
|
||||
|| ', cosine_distance=' || TO_CHAR(v_dist, 'FM0D000000') || ']' || CHR(10)
|
||||
|| 'Question: ' || v_q || CHR(10)
|
||||
|| 'Answer SQL: ' || v_sql
|
||||
|| CASE WHEN v_answer IS NULL THEN NULL ELSE CHR(10) || 'Answer: ' || v_answer END;
|
||||
END LOOP;
|
||||
CLOSE v_results;
|
||||
RETURN v_context;
|
||||
END;
|
||||
/
|
||||
|
||||
COMMENT ON COLUMN sg_qa_vector_example.source_case_id IS
|
||||
'Customer QA benchmark case identifier, such as STD-01 or CZN-01.';
|
||||
COMMENT ON COLUMN sg_qa_vector_example.source_type IS
|
||||
'Candidate provenance. CUSTOMER_QA_BENCHMARK rows remain DRAFT until reviewed.';
|
||||
Reference in New Issue
Block a user