refs #708 #739: use Cohere command and vector game scope

2026-07-29 10:50:38 +09:00
parent 8b04fb4e89
commit 016d1af025
10 changed files with 569 additions and 40 deletions
--- a/sql/adb/115_sgmp_scope_chat_candidate_benchmark.sql
+++ b/sql/adb/115_sgmp_scope_chat_candidate_benchmark.sql
@@ -0,0 +1,112 @@
+-- Read-only benchmark for candidate game-scope extraction profiles.
+-- Korean test questions use UTF-8 base64 reconstruction for SQLcl safety.
+
+set serveroutput on size unlimited
+prompt SG_SCOPE_PROFILE_BENCHMARK_LOADED
+
+DECLARE
+  TYPE t_case IS RECORD (
+    case_name VARCHAR2(12),
+    question CLOB,
+    expected_scope VARCHAR2(20),
+    expected_mentions PLS_INTEGER
+  );
+  TYPE t_cases IS TABLE OF t_case INDEX BY PLS_INTEGER;
+  v_cases t_cases;
+  v_prompt_prefix CLOB :=
+    'Extract only game-name mentions from the user question. '
+    || 'Metrics, acronyms, dates, filters, and database object or column names are not game names unless they are themselves an explicit game title. '
+    || 'When a title-like noun directly qualifies a game data request such as user master, character, sales, AU, NRU, server, or game log, preserve that noun as a game-name mention even when it is not in a catalog. '
+    || 'Do not discard an unknown title merely because it cannot be resolved. General scope words such as common, overall, all, total, or every are not game-name mentions unless they are part of an explicit title. '
+    || 'Return exactly one JSON object with keys game_mentions (array of strings) '
+    || 'and scope_hint (GLOBAL, SINGLE_GAME, MULTI_GAME, ALL_GAMES, UNKNOWN). '
+    || 'Do not resolve names to IDs and do not generate SQL. '
+    || 'Return raw JSON only: no prose, no Markdown, and no code fence. Question: ';
+  v_result CLOB;
+  v_json JSON_OBJECT_T;
+  v_started PLS_INTEGER;
+  v_elapsed NUMBER;
+  v_scope VARCHAR2(20);
+  v_mentions PLS_INTEGER;
+  v_raw_json VARCHAR2(5);
+
+  PROCEDURE run_case(
+    p_profile_name IN VARCHAR2,
+    p_case t_case
+  ) IS
+  BEGIN
+    v_started := DBMS_UTILITY.GET_TIME;
+    v_result := DBMS_CLOUD_AI.GENERATE(
+      prompt => v_prompt_prefix || p_case.question,
+      profile_name => p_profile_name,
+      action => 'chat'
+    );
+    v_elapsed := (DBMS_UTILITY.GET_TIME - v_started) / 100;
+    v_json := JSON_OBJECT_T.parse(v_result);
+    v_raw_json := 'TRUE';
+    v_scope := v_json.get_string('scope_hint');
+    v_mentions := v_json.get_array('game_mentions').get_size;
+    DBMS_OUTPUT.PUT_LINE(
+      p_profile_name || '|' || p_case.case_name
+      || '|seconds=' || TO_CHAR(v_elapsed, 'FM9990D00')
+      || '|raw_json=' || v_raw_json
+      || '|scope=' || NVL(v_scope, 'NULL')
+      || '|mentions=' || v_mentions
+      || '|expected=' || p_case.expected_scope || '/' || p_case.expected_mentions
+    );
+  EXCEPTION
+    WHEN OTHERS THEN
+      DBMS_OUTPUT.PUT_LINE(
+        p_profile_name || '|' || p_case.case_name
+        || '|seconds=' || TO_CHAR(v_elapsed, 'FM9990D00')
+        || '|ERROR=' || SQLCODE || '|' || SUBSTR(SQLERRM, 1, 180)
+      );
+  END;
+BEGIN
+  DBMS_OUTPUT.PUT_LINE('benchmark_started');
+  v_cases(1).case_name := 'NONE';
+  v_cases(1).question := utl_i18n.raw_to_char(utl_encode.base64_decode(
+    utl_raw.cast_to_raw('7KCE7LK0IOunpOy2nCDslYzroKTspJgu')), 'AL32UTF8');
+  v_cases(1).expected_scope := 'GLOBAL';
+  v_cases(1).expected_mentions := 0;
+
+  v_cases(2).case_name := 'SINGLE';
+  v_cases(2).question := utl_i18n.raw_to_char(utl_encode.base64_decode(
+    utl_raw.cast_to_raw('7Lm07KCc64KYIOy1nOyLoCBBVSDslYzroKTspJgu')), 'AL32UTF8');
+  v_cases(2).expected_scope := 'SINGLE_GAME';
+  v_cases(2).expected_mentions := 1;
+
+  v_cases(3).case_name := 'MULTI';
+  v_cases(3).question := utl_i18n.raw_to_char(utl_encode.base64_decode(
+    utl_raw.cast_to_raw('66Gc65Oc64KY7J247J20656RIOy5tOygnOuCmOydmCAyMDI264WEIDfsm5QgMTXsnbwgQVXrpbwg6rCB6rCBIOyVjOugpOykmC4=')), 'AL32UTF8');
+  v_cases(3).expected_scope := 'MULTI_GAME';
+  v_cases(3).expected_mentions := 2;
+
+  v_cases(4).case_name := 'ALL';
+  v_cases(4).question := utl_i18n.raw_to_char(utl_encode.base64_decode(
+    utl_raw.cast_to_raw('7KCE7LK0IOqyjOyehOydmCDrp6Tstpwg7JWM66Ck7KSYLg==')), 'AL32UTF8');
+  v_cases(4).expected_scope := 'ALL_GAMES';
+  v_cases(4).expected_mentions := 0;
+
+  FOR profile_row IN (
+    SELECT profile_name
+      FROM user_cloud_ai_profiles
+     WHERE profile_name IN (
+       'SGMP_POC_OCI_GPT54MINI',
+       'SGMP_SCOPE_COHERE_VISION',
+       'SGMP_SCOPE_COHERE_COMMAND',
+       'SGMP_SCOPE_COHERE_PLUS',
+       'SGMP_SCOPE_GEMINI_FLASH',
+       'SGMP_SCOPE_LLAMA_MAV',
+       'SGMP_SCOPE_GROK_NONR'
+     )
+     ORDER BY profile_name
+  ) LOOP
+    DBMS_OUTPUT.PUT_LINE('profile=' || profile_row.profile_name);
+    FOR i IN 1 .. 4 LOOP
+      run_case(profile_row.profile_name, v_cases(i));
+    END LOOP;
+  END LOOP;
+END;
+/
+prompt SG_SCOPE_PROFILE_BENCHMARK_COMPLETED