refs #749: compare HMM Select AI annotation profiles
This commit is contained in:
devmrko
159
database/adb/85_hmm_hr_select_ai_no_annotation_profile.sql
Normal file
159
database/adb/85_hmm_hr_select_ai_no_annotation_profile.sql
Normal file
@@ -0,0 +1,159 @@
|
||||
-- Redmine #749
|
||||
-- Create an isolated HMM HR Select AI profile with annotations disabled.
|
||||
-- The source profile is never modified.
|
||||
|
||||
SET SERVEROUTPUT ON SIZE UNLIMITED
|
||||
WHENEVER SQLERROR EXIT SQL.SQLCODE
|
||||
|
||||
DECLARE
|
||||
c_source_profile CONSTANT VARCHAR2(128) := 'HMM_HR_DATA_GPT54_PROFILE';
|
||||
c_target_profile CONSTANT VARCHAR2(128) := 'HMM_HR_DATA_GPT54_NOANN_PROFILE';
|
||||
|
||||
l_attributes CLOB;
|
||||
l_object_list CLOB;
|
||||
l_provider VARCHAR2(4000);
|
||||
l_credential_name VARCHAR2(4000);
|
||||
l_model VARCHAR2(4000);
|
||||
l_region VARCHAR2(4000);
|
||||
l_compartment_id VARCHAR2(4000);
|
||||
l_max_tokens NUMBER;
|
||||
l_temperature NUMBER;
|
||||
l_profile_count PLS_INTEGER;
|
||||
l_difference_count PLS_INTEGER;
|
||||
l_annotations VARCHAR2(30);
|
||||
|
||||
FUNCTION attribute_clob(p_name IN VARCHAR2) RETURN CLOB IS
|
||||
l_value CLOB;
|
||||
BEGIN
|
||||
SELECT attribute_value
|
||||
INTO l_value
|
||||
FROM user_cloud_ai_profile_attributes
|
||||
WHERE profile_name = c_source_profile
|
||||
AND attribute_name = p_name;
|
||||
RETURN l_value;
|
||||
END attribute_clob;
|
||||
|
||||
FUNCTION attribute_text(p_name IN VARCHAR2) RETURN VARCHAR2 IS
|
||||
BEGIN
|
||||
RETURN DBMS_LOB.SUBSTR(attribute_clob(p_name), 4000, 1);
|
||||
END attribute_text;
|
||||
BEGIN
|
||||
SELECT COUNT(*)
|
||||
INTO l_profile_count
|
||||
FROM user_cloud_ai_profiles
|
||||
WHERE profile_name = c_source_profile
|
||||
AND status = 'ENABLED';
|
||||
|
||||
IF l_profile_count <> 1 THEN
|
||||
RAISE_APPLICATION_ERROR(-20080, 'Enabled source Select AI profile was not found.');
|
||||
END IF;
|
||||
|
||||
l_provider := attribute_text('provider');
|
||||
l_credential_name := attribute_text('credential_name');
|
||||
l_model := attribute_text('model');
|
||||
l_region := attribute_text('region');
|
||||
l_compartment_id := attribute_text('oci_compartment_id');
|
||||
l_object_list := attribute_clob('object_list');
|
||||
l_max_tokens := TO_NUMBER(
|
||||
attribute_text('max_tokens'),
|
||||
'9999999999',
|
||||
'NLS_NUMERIC_CHARACTERS=''.,'''
|
||||
);
|
||||
l_temperature := TO_NUMBER(
|
||||
attribute_text('temperature'),
|
||||
'9999999990D999999999',
|
||||
'NLS_NUMERIC_CHARACTERS=''.,'''
|
||||
);
|
||||
|
||||
SELECT COUNT(*)
|
||||
INTO l_profile_count
|
||||
FROM user_cloud_ai_profiles
|
||||
WHERE profile_name = c_target_profile;
|
||||
|
||||
IF l_profile_count = 0 THEN
|
||||
SELECT JSON_OBJECT(
|
||||
'provider' VALUE l_provider,
|
||||
'credential_name' VALUE l_credential_name,
|
||||
'model' VALUE l_model,
|
||||
'region' VALUE l_region,
|
||||
'oci_compartment_id' VALUE l_compartment_id,
|
||||
'object_list' VALUE l_object_list FORMAT JSON,
|
||||
'max_tokens' VALUE l_max_tokens,
|
||||
'temperature' VALUE l_temperature,
|
||||
'annotations' VALUE 'false' FORMAT JSON
|
||||
RETURNING CLOB
|
||||
)
|
||||
INTO l_attributes
|
||||
FROM dual;
|
||||
|
||||
DBMS_CLOUD_AI.CREATE_PROFILE(
|
||||
profile_name => c_target_profile,
|
||||
attributes => l_attributes,
|
||||
status => 'enabled',
|
||||
description => 'HMM HR GPT-5.4 mini comparison profile with annotations disabled'
|
||||
);
|
||||
DBMS_OUTPUT.PUT_LINE('PROFILE_CREATED|' || c_target_profile);
|
||||
ELSE
|
||||
DBMS_OUTPUT.PUT_LINE('PROFILE_EXISTS|' || c_target_profile);
|
||||
END IF;
|
||||
|
||||
SELECT COUNT(*)
|
||||
INTO l_difference_count
|
||||
FROM (
|
||||
SELECT attribute_name, attribute_value
|
||||
FROM user_cloud_ai_profile_attributes
|
||||
WHERE profile_name = c_source_profile
|
||||
AND attribute_name <> 'annotations'
|
||||
) source_attributes
|
||||
FULL OUTER JOIN (
|
||||
SELECT attribute_name, attribute_value
|
||||
FROM user_cloud_ai_profile_attributes
|
||||
WHERE profile_name = c_target_profile
|
||||
AND attribute_name <> 'annotations'
|
||||
) target_attributes
|
||||
ON target_attributes.attribute_name = source_attributes.attribute_name
|
||||
WHERE source_attributes.attribute_name IS NULL
|
||||
OR target_attributes.attribute_name IS NULL
|
||||
OR DBMS_LOB.COMPARE(
|
||||
source_attributes.attribute_value,
|
||||
target_attributes.attribute_value
|
||||
) <> 0;
|
||||
|
||||
SELECT LOWER(TRIM(DBMS_LOB.SUBSTR(attribute_value, 30, 1)))
|
||||
INTO l_annotations
|
||||
FROM user_cloud_ai_profile_attributes
|
||||
WHERE profile_name = c_target_profile
|
||||
AND attribute_name = 'annotations';
|
||||
|
||||
IF l_difference_count <> 0 OR l_annotations <> 'false' THEN
|
||||
RAISE_APPLICATION_ERROR(
|
||||
-20081,
|
||||
'Comparison profile differs from source beyond annotations=false.'
|
||||
);
|
||||
END IF;
|
||||
|
||||
DBMS_OUTPUT.PUT_LINE(
|
||||
'PROFILE_VALIDATED|source=' || c_source_profile
|
||||
|| '|target=' || c_target_profile
|
||||
|| '|annotations=false|other_attribute_differences=0'
|
||||
);
|
||||
END;
|
||||
/
|
||||
|
||||
SELECT profile_name, status, description
|
||||
FROM user_cloud_ai_profiles
|
||||
WHERE profile_name IN (
|
||||
'HMM_HR_DATA_GPT54_PROFILE',
|
||||
'HMM_HR_DATA_GPT54_NOANN_PROFILE'
|
||||
)
|
||||
ORDER BY profile_name;
|
||||
|
||||
SELECT profile_name, attribute_name, attribute_value
|
||||
FROM user_cloud_ai_profile_attributes
|
||||
WHERE profile_name IN (
|
||||
'HMM_HR_DATA_GPT54_PROFILE',
|
||||
'HMM_HR_DATA_GPT54_NOANN_PROFILE'
|
||||
)
|
||||
ORDER BY profile_name, attribute_name;
|
||||
|
||||
EXIT SUCCESS
|
||||
103
database/adb/86_hmm_select_ai_annotation_benchmark.sql
Normal file
103
database/adb/86_hmm_select_ai_annotation_benchmark.sql
Normal file
@@ -0,0 +1,103 @@
|
||||
-- Redmine #749
|
||||
-- Compare Select AI prompt and SQL generation with annotations enabled/disabled.
|
||||
-- The Korean prompt is reconstructed from UTF-8 Base64 inside Oracle.
|
||||
|
||||
SET SERVEROUTPUT ON SIZE UNLIMITED
|
||||
SET FEEDBACK OFF
|
||||
SET VERIFY OFF
|
||||
WHENEVER SQLERROR EXIT SQL.SQLCODE
|
||||
|
||||
DECLARE
|
||||
c_annotation_profile CONSTANT VARCHAR2(128) := 'HMM_HR_DATA_GPT54_PROFILE';
|
||||
c_no_annotation_profile CONSTANT VARCHAR2(128) := 'HMM_HR_DATA_GPT54_NOANN_PROFILE';
|
||||
c_prompt_base64 CONSTANT VARCHAR2(4000) :=
|
||||
'7J2067KIIOuLrCDtjIDsm5Drs4Qg7Zy06rCAIOyCrOyaqSDtmITtmansnYQg67O07Jes7KSY';
|
||||
|
||||
l_prompt VARCHAR2(4000) :=
|
||||
UTL_I18N.RAW_TO_CHAR(
|
||||
UTL_ENCODE.BASE64_DECODE(UTL_RAW.CAST_TO_RAW(c_prompt_base64)),
|
||||
'AL32UTF8'
|
||||
);
|
||||
|
||||
PROCEDURE run_one(
|
||||
p_run_no IN PLS_INTEGER,
|
||||
p_action IN VARCHAR2,
|
||||
p_profile_name IN VARCHAR2
|
||||
) IS
|
||||
l_started PLS_INTEGER;
|
||||
l_elapsed_ms PLS_INTEGER;
|
||||
l_result CLOB;
|
||||
l_result_head VARCHAR2(32767);
|
||||
l_result_hash VARCHAR2(128);
|
||||
l_valid_sql VARCHAR2(1) := '-';
|
||||
BEGIN
|
||||
l_started := DBMS_UTILITY.GET_TIME;
|
||||
l_result := DBMS_CLOUD_AI.GENERATE(
|
||||
l_prompt,
|
||||
p_profile_name,
|
||||
p_action
|
||||
);
|
||||
l_elapsed_ms := (DBMS_UTILITY.GET_TIME - l_started) * 10;
|
||||
l_result_head := DBMS_LOB.SUBSTR(l_result, 32767, 1);
|
||||
SELECT RAWTOHEX(STANDARD_HASH(l_result_head, 'SHA256'))
|
||||
INTO l_result_hash
|
||||
FROM dual;
|
||||
|
||||
IF p_action = 'showsql' THEN
|
||||
IF REGEXP_LIKE(LTRIM(l_result_head), '^(SELECT|WITH)[[:space:]]', 'i') THEN
|
||||
l_valid_sql := 'Y';
|
||||
ELSE
|
||||
l_valid_sql := 'N';
|
||||
END IF;
|
||||
END IF;
|
||||
|
||||
DBMS_OUTPUT.PUT_LINE(
|
||||
'BENCHMARK|run=' || p_run_no
|
||||
|| '|action=' || p_action
|
||||
|| '|profile=' || p_profile_name
|
||||
|| '|elapsed_ms=' || l_elapsed_ms
|
||||
|| '|result_chars=' || DBMS_LOB.GETLENGTH(l_result)
|
||||
|| '|valid_sql=' || l_valid_sql
|
||||
|| '|result_sha256=' || l_result_hash
|
||||
);
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
l_elapsed_ms := (DBMS_UTILITY.GET_TIME - l_started) * 10;
|
||||
DBMS_OUTPUT.PUT_LINE(
|
||||
'BENCHMARK_ERROR|run=' || p_run_no
|
||||
|| '|action=' || p_action
|
||||
|| '|profile=' || p_profile_name
|
||||
|| '|elapsed_ms=' || l_elapsed_ms
|
||||
|| '|error=' || REPLACE(SUBSTR(SQLERRM, 1, 500), '|', '/')
|
||||
);
|
||||
END run_one;
|
||||
|
||||
PROCEDURE run_pair(p_run_no IN PLS_INTEGER, p_action IN VARCHAR2) IS
|
||||
BEGIN
|
||||
IF MOD(p_run_no, 2) = 1 THEN
|
||||
run_one(p_run_no, p_action, c_annotation_profile);
|
||||
run_one(p_run_no, p_action, c_no_annotation_profile);
|
||||
ELSE
|
||||
run_one(p_run_no, p_action, c_no_annotation_profile);
|
||||
run_one(p_run_no, p_action, c_annotation_profile);
|
||||
END IF;
|
||||
END run_pair;
|
||||
BEGIN
|
||||
DBMS_OUTPUT.PUT_LINE(
|
||||
'BENCHMARK_START|prompt_utf8_bytes=' ||
|
||||
UTL_RAW.LENGTH(UTL_I18N.STRING_TO_RAW(l_prompt, 'AL32UTF8'))
|
||||
);
|
||||
|
||||
FOR run_no IN 1 .. 3 LOOP
|
||||
run_pair(run_no, 'showprompt');
|
||||
END LOOP;
|
||||
|
||||
FOR run_no IN 1 .. 3 LOOP
|
||||
run_pair(run_no, 'showsql');
|
||||
END LOOP;
|
||||
|
||||
DBMS_OUTPUT.PUT_LINE('BENCHMARK_END');
|
||||
END;
|
||||
/
|
||||
|
||||
EXIT SUCCESS
|
||||
Reference in New Issue
Block a user