113 lines
4.4 KiB
SQL
113 lines
4.4 KiB
SQL
-- Read-only benchmark for candidate game-scope extraction profiles.
|
|
-- Korean test questions use UTF-8 base64 reconstruction for SQLcl safety.
|
|
|
|
set serveroutput on size unlimited
|
|
prompt SG_SCOPE_PROFILE_BENCHMARK_LOADED
|
|
|
|
DECLARE
|
|
TYPE t_case IS RECORD (
|
|
case_name VARCHAR2(12),
|
|
question CLOB,
|
|
expected_scope VARCHAR2(20),
|
|
expected_mentions PLS_INTEGER
|
|
);
|
|
TYPE t_cases IS TABLE OF t_case INDEX BY PLS_INTEGER;
|
|
v_cases t_cases;
|
|
v_prompt_prefix CLOB :=
|
|
'Extract only game-name mentions from the user question. '
|
|
|| 'Metrics, acronyms, dates, filters, and database object or column names are not game names unless they are themselves an explicit game title. '
|
|
|| 'When a title-like noun directly qualifies a game data request such as user master, character, sales, AU, NRU, server, or game log, preserve that noun as a game-name mention even when it is not in a catalog. '
|
|
|| 'Do not discard an unknown title merely because it cannot be resolved. General scope words such as common, overall, all, total, or every are not game-name mentions unless they are part of an explicit title. '
|
|
|| 'Return exactly one JSON object with keys game_mentions (array of strings) '
|
|
|| 'and scope_hint (GLOBAL, SINGLE_GAME, MULTI_GAME, ALL_GAMES, UNKNOWN). '
|
|
|| 'Do not resolve names to IDs and do not generate SQL. '
|
|
|| 'Return raw JSON only: no prose, no Markdown, and no code fence. Question: ';
|
|
v_result CLOB;
|
|
v_json JSON_OBJECT_T;
|
|
v_started PLS_INTEGER;
|
|
v_elapsed NUMBER;
|
|
v_scope VARCHAR2(20);
|
|
v_mentions PLS_INTEGER;
|
|
v_raw_json VARCHAR2(5);
|
|
|
|
PROCEDURE run_case(
|
|
p_profile_name IN VARCHAR2,
|
|
p_case t_case
|
|
) IS
|
|
BEGIN
|
|
v_started := DBMS_UTILITY.GET_TIME;
|
|
v_result := DBMS_CLOUD_AI.GENERATE(
|
|
prompt => v_prompt_prefix || p_case.question,
|
|
profile_name => p_profile_name,
|
|
action => 'chat'
|
|
);
|
|
v_elapsed := (DBMS_UTILITY.GET_TIME - v_started) / 100;
|
|
v_json := JSON_OBJECT_T.parse(v_result);
|
|
v_raw_json := 'TRUE';
|
|
v_scope := v_json.get_string('scope_hint');
|
|
v_mentions := v_json.get_array('game_mentions').get_size;
|
|
DBMS_OUTPUT.PUT_LINE(
|
|
p_profile_name || '|' || p_case.case_name
|
|
|| '|seconds=' || TO_CHAR(v_elapsed, 'FM9990D00')
|
|
|| '|raw_json=' || v_raw_json
|
|
|| '|scope=' || NVL(v_scope, 'NULL')
|
|
|| '|mentions=' || v_mentions
|
|
|| '|expected=' || p_case.expected_scope || '/' || p_case.expected_mentions
|
|
);
|
|
EXCEPTION
|
|
WHEN OTHERS THEN
|
|
DBMS_OUTPUT.PUT_LINE(
|
|
p_profile_name || '|' || p_case.case_name
|
|
|| '|seconds=' || TO_CHAR(v_elapsed, 'FM9990D00')
|
|
|| '|ERROR=' || SQLCODE || '|' || SUBSTR(SQLERRM, 1, 180)
|
|
);
|
|
END;
|
|
BEGIN
|
|
DBMS_OUTPUT.PUT_LINE('benchmark_started');
|
|
v_cases(1).case_name := 'NONE';
|
|
v_cases(1).question := utl_i18n.raw_to_char(utl_encode.base64_decode(
|
|
utl_raw.cast_to_raw('7KCE7LK0IOunpOy2nCDslYzroKTspJgu')), 'AL32UTF8');
|
|
v_cases(1).expected_scope := 'GLOBAL';
|
|
v_cases(1).expected_mentions := 0;
|
|
|
|
v_cases(2).case_name := 'SINGLE';
|
|
v_cases(2).question := utl_i18n.raw_to_char(utl_encode.base64_decode(
|
|
utl_raw.cast_to_raw('7Lm07KCc64KYIOy1nOyLoCBBVSDslYzroKTspJgu')), 'AL32UTF8');
|
|
v_cases(2).expected_scope := 'SINGLE_GAME';
|
|
v_cases(2).expected_mentions := 1;
|
|
|
|
v_cases(3).case_name := 'MULTI';
|
|
v_cases(3).question := utl_i18n.raw_to_char(utl_encode.base64_decode(
|
|
utl_raw.cast_to_raw('66Gc65Oc64KY7J247J20656RIOy5tOygnOuCmOydmCAyMDI264WEIDfsm5QgMTXsnbwgQVXrpbwg6rCB6rCBIOyVjOugpOykmC4=')), 'AL32UTF8');
|
|
v_cases(3).expected_scope := 'MULTI_GAME';
|
|
v_cases(3).expected_mentions := 2;
|
|
|
|
v_cases(4).case_name := 'ALL';
|
|
v_cases(4).question := utl_i18n.raw_to_char(utl_encode.base64_decode(
|
|
utl_raw.cast_to_raw('7KCE7LK0IOqyjOyehOydmCDrp6Tstpwg7JWM66Ck7KSYLg==')), 'AL32UTF8');
|
|
v_cases(4).expected_scope := 'ALL_GAMES';
|
|
v_cases(4).expected_mentions := 0;
|
|
|
|
FOR profile_row IN (
|
|
SELECT profile_name
|
|
FROM user_cloud_ai_profiles
|
|
WHERE profile_name IN (
|
|
'SGMP_POC_OCI_GPT54MINI',
|
|
'SGMP_SCOPE_COHERE_VISION',
|
|
'SGMP_SCOPE_COHERE_COMMAND',
|
|
'SGMP_SCOPE_COHERE_PLUS',
|
|
'SGMP_SCOPE_GEMINI_FLASH',
|
|
'SGMP_SCOPE_LLAMA_MAV',
|
|
'SGMP_SCOPE_GROK_NONR'
|
|
)
|
|
ORDER BY profile_name
|
|
) LOOP
|
|
DBMS_OUTPUT.PUT_LINE('profile=' || profile_row.profile_name);
|
|
FOR i IN 1 .. 4 LOOP
|
|
run_case(profile_row.profile_name, v_cases(i));
|
|
END LOOP;
|
|
END LOOP;
|
|
END;
|
|
/
|
|
prompt SG_SCOPE_PROFILE_BENCHMARK_COMPLETED
|