feat #565: add vector knowledge ingestion demo
This commit is contained in:
devmrko
@@ -0,0 +1,269 @@
|
||||
package com.cloudhandson.vpdbackoffice.service;
|
||||
|
||||
import com.cloudhandson.vpdbackoffice.domain.probe.ProbeCommand;
|
||||
import com.cloudhandson.vpdbackoffice.domain.probe.ProbeResult;
|
||||
import com.cloudhandson.vpdbackoffice.domain.protectedobject.ProtectedObject;
|
||||
import com.cloudhandson.vpdbackoffice.domain.token.IssuedToken;
|
||||
import com.cloudhandson.vpdbackoffice.domain.vector.VectorChunk;
|
||||
import com.cloudhandson.vpdbackoffice.domain.vector.VectorIngestCommand;
|
||||
import com.cloudhandson.vpdbackoffice.domain.vector.VectorIngestResult;
|
||||
import com.cloudhandson.vpdbackoffice.domain.vector.VectorKnowledgeSummary;
|
||||
import com.cloudhandson.vpdbackoffice.domain.vector.VectorSearchResult;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Pattern;
|
||||
import org.springframework.dao.DataAccessException;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
@Service
|
||||
public class VectorKnowledgeService {
|
||||
|
||||
public static final String VECTOR_OBJECT = "CB_VECTOR_SEARCH_DOCUMENTS";
|
||||
public static final String DEMO_MODE = "DEMO";
|
||||
public static final String AI_MODE = "AI";
|
||||
|
||||
private static final Pattern TAG_PATTERN = Pattern.compile("[A-Z0-9_-]+");
|
||||
private static final int MAX_DOCUMENT_LENGTH = 500_000;
|
||||
|
||||
private final JdbcTemplate jdbcTemplate;
|
||||
private final ObjectMapper objectMapper;
|
||||
private final OpenAiCompatibleClient embeddingClient;
|
||||
private final ProtectedObjectService protectedObjectService;
|
||||
private final BearerTokenService tokenService;
|
||||
private final OrdsProbeService ordsProbeService;
|
||||
|
||||
public VectorKnowledgeService(
|
||||
JdbcTemplate jdbcTemplate,
|
||||
ObjectMapper objectMapper,
|
||||
OpenAiCompatibleClient embeddingClient,
|
||||
ProtectedObjectService protectedObjectService,
|
||||
BearerTokenService tokenService,
|
||||
OrdsProbeService ordsProbeService
|
||||
) {
|
||||
this.jdbcTemplate = jdbcTemplate;
|
||||
this.objectMapper = objectMapper;
|
||||
this.embeddingClient = embeddingClient;
|
||||
this.protectedObjectService = protectedObjectService;
|
||||
this.tokenService = tokenService;
|
||||
this.ordsProbeService = ordsProbeService;
|
||||
}
|
||||
|
||||
public VectorKnowledgeSummary summary() {
|
||||
int documents = count("SELECT COUNT(DISTINCT document_id) FROM cb_vector_document_chunk");
|
||||
int chunks = count("SELECT COUNT(*) FROM cb_vector_document_chunk");
|
||||
int tags = count("SELECT COUNT(*) FROM cb_vector_document_tag");
|
||||
boolean registered = protectedObjectService.findEnabled().stream()
|
||||
.anyMatch(object -> VECTOR_OBJECT.equalsIgnoreCase(object.objectName()));
|
||||
return new VectorKnowledgeSummary(
|
||||
documents,
|
||||
chunks,
|
||||
tags,
|
||||
registered,
|
||||
embeddingClient.embeddingConfigured() ? embeddingClient.embeddingModelName() : "DEMO-4D"
|
||||
);
|
||||
}
|
||||
|
||||
public boolean aiEmbeddingConfigured() {
|
||||
return embeddingClient.embeddingConfigured();
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public VectorIngestResult ingest(VectorIngestCommand command) {
|
||||
String documentId = requiredDocumentId(command.documentId());
|
||||
String title = required(command.title(), "문서 제목");
|
||||
String content = required(command.content(), "문서 본문");
|
||||
if (content.length() > MAX_DOCUMENT_LENGTH) {
|
||||
throw new AppException("문서 본문은 " + MAX_DOCUMENT_LENGTH + "자 이내로 입력하세요.");
|
||||
}
|
||||
String sourceUri = command.sourceUri() == null || command.sourceUri().isBlank()
|
||||
? "kb://backoffice/" + documentId
|
||||
: command.sourceUri().trim();
|
||||
Set<String> tags = normalizeTags(command.techTags());
|
||||
String mode = normalizeMode(command.embeddingMode());
|
||||
List<VectorChunk> chunks = VectorChunker.chunk(content, command.chunkSize());
|
||||
|
||||
replaceDocument(documentId);
|
||||
long nextChunkId = nextChunkId(chunks.size());
|
||||
int tagCount = 0;
|
||||
for (VectorChunk chunk : chunks) {
|
||||
List<Double> embedding = embed(chunk.text(), mode);
|
||||
String embeddingJson = json(embedding);
|
||||
long chunkId = nextChunkId++;
|
||||
jdbcTemplate.update("""
|
||||
INSERT INTO cb_vector_document_chunk
|
||||
(chunk_id, document_id, chunk_no, title, chunk_text, source_uri, embedding)
|
||||
VALUES (?, ?, ?, ?, ?, ?, TO_VECTOR(?))
|
||||
""", chunkId, documentId, chunk.chunkNo(), title, chunk.text(), sourceUri, embeddingJson);
|
||||
for (String tag : tags) {
|
||||
jdbcTemplate.update("""
|
||||
INSERT INTO cb_vector_document_tag (chunk_id, tech_tag)
|
||||
VALUES (?, ?)
|
||||
""", chunkId, tag);
|
||||
tagCount++;
|
||||
}
|
||||
}
|
||||
return new VectorIngestResult(documentId, chunks.size(), tagCount, mode, embeddingModel(mode));
|
||||
}
|
||||
|
||||
public VectorSearchResult search(long userId, String query, int limit, String embeddingMode) {
|
||||
String normalizedQuery = required(query, "검색 질문");
|
||||
ProtectedObject vectorObject = protectedObjectService.findEnabled().stream()
|
||||
.filter(object -> VECTOR_OBJECT.equalsIgnoreCase(object.objectName()))
|
||||
.findFirst()
|
||||
.orElseThrow(() -> new AppException(
|
||||
"CB_VECTOR_SEARCH_DOCUMENTS 보호 객체가 없습니다. 28_agent_ords_vector_tag_vpd_setup.sql을 먼저 실행하세요."));
|
||||
String mode = normalizeMode(embeddingMode);
|
||||
String requestBody = "{\"embedding\":" + json(embed(normalizedQuery, mode)) + "}";
|
||||
IssuedToken temporary = tokenService.issueTemporaryToken(userId, "벡터 지식자료 검색 임시 실행");
|
||||
try {
|
||||
ProbeResult probe = ordsProbeService.runProbe(new ProbeCommand(
|
||||
temporary.keyId(), vectorObject.objectId(), temporary.plainToken(), normalizeLimit(limit), requestBody));
|
||||
return new VectorSearchResult(normalizedQuery, mode, embeddingModel(mode), probe);
|
||||
} finally {
|
||||
tokenService.revokeToken(temporary.keyId(), "temporary vector search completed");
|
||||
}
|
||||
}
|
||||
|
||||
private void replaceDocument(String documentId) {
|
||||
jdbcTemplate.update("""
|
||||
DELETE FROM cb_vector_document_tag
|
||||
WHERE chunk_id IN (
|
||||
SELECT chunk_id FROM cb_vector_document_chunk WHERE document_id = ?
|
||||
)
|
||||
""", documentId);
|
||||
jdbcTemplate.update("DELETE FROM cb_vector_document_chunk WHERE document_id = ?", documentId);
|
||||
}
|
||||
|
||||
private long nextChunkId(int chunkCount) {
|
||||
try {
|
||||
Long sequenceValue = jdbcTemplate.queryForObject(
|
||||
"SELECT cb_vector_chunk_seq.NEXTVAL FROM dual", Long.class);
|
||||
if (sequenceValue != null) {
|
||||
return sequenceValue;
|
||||
}
|
||||
} catch (DataAccessException ignored) {
|
||||
// Older installations may not have the optional sequence yet. The
|
||||
// fallback keeps the demonstration usable; the setup SQL creates it.
|
||||
}
|
||||
Long max = jdbcTemplate.queryForObject(
|
||||
"SELECT NVL(MAX(chunk_id), 28000) FROM cb_vector_document_chunk", Long.class);
|
||||
return (max == null ? 28000 : max) + 1;
|
||||
}
|
||||
|
||||
private List<Double> embed(String input, String mode) {
|
||||
if (AI_MODE.equals(mode)) {
|
||||
return embeddingClient.embedding(input);
|
||||
}
|
||||
return demoEmbedding(input);
|
||||
}
|
||||
|
||||
private List<Double> demoEmbedding(String input) {
|
||||
double[] vector = new double[4];
|
||||
String normalized = input.toLowerCase(Locale.ROOT);
|
||||
addIfContains(vector, normalized, 0, "spring", "boot", "java", "security");
|
||||
addIfContains(vector, normalized, 1, "oracle", "vpd", "policy", "database", "db");
|
||||
addIfContains(vector, normalized, 2, "ords", "rest", "handler", "http", "api");
|
||||
addIfContains(vector, normalized, 3, "mcp", "tool", "권한", "tag", "태그");
|
||||
for (String token : normalized.split("[^a-z0-9가-힣]+")) {
|
||||
if (token.length() >= 3) {
|
||||
vector[Math.floorMod(token.hashCode(), vector.length)] += 0.03;
|
||||
}
|
||||
}
|
||||
double length = 0;
|
||||
for (double value : vector) {
|
||||
length += value * value;
|
||||
}
|
||||
if (length == 0) {
|
||||
vector[0] = 1;
|
||||
length = 1;
|
||||
}
|
||||
double scale = Math.sqrt(length);
|
||||
List<Double> result = new ArrayList<>(vector.length);
|
||||
for (double value : vector) {
|
||||
result.add(value / scale);
|
||||
}
|
||||
return List.copyOf(result);
|
||||
}
|
||||
|
||||
private void addIfContains(double[] vector, String input, int index, String... terms) {
|
||||
for (String term : terms) {
|
||||
if (input.contains(term)) {
|
||||
vector[index] += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private Set<String> normalizeTags(String value) {
|
||||
if (value == null || value.isBlank()) {
|
||||
throw new AppException("기술 태그를 하나 이상 입력하세요.");
|
||||
}
|
||||
Set<String> tags = new LinkedHashSet<>();
|
||||
for (String raw : value.split("[,\\s]+")) {
|
||||
String tag = raw.trim().toUpperCase(Locale.ROOT);
|
||||
if (tag.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
if (!TAG_PATTERN.matcher(tag).matches()) {
|
||||
throw new AppException("기술 태그는 영문 대문자, 숫자, '_' 또는 '-'만 사용할 수 있습니다: " + tag);
|
||||
}
|
||||
tags.add(tag);
|
||||
}
|
||||
if (tags.isEmpty()) {
|
||||
throw new AppException("기술 태그를 하나 이상 입력하세요.");
|
||||
}
|
||||
return Set.copyOf(tags);
|
||||
}
|
||||
|
||||
private String normalizeMode(String mode) {
|
||||
String normalized = mode == null ? DEMO_MODE : mode.trim().toUpperCase(Locale.ROOT);
|
||||
if (!DEMO_MODE.equals(normalized) && !AI_MODE.equals(normalized)) {
|
||||
throw new AppException("임베딩 방식은 DEMO 또는 AI만 사용할 수 있습니다.");
|
||||
}
|
||||
if (AI_MODE.equals(normalized) && !embeddingClient.embeddingConfigured()) {
|
||||
throw new AppException("AI 임베딩이 설정되지 않았습니다. 설정에서 embedding model/API key를 추가하거나 DEMO 임베딩을 선택하세요.");
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
private String embeddingModel(String mode) {
|
||||
return AI_MODE.equals(mode) ? embeddingClient.embeddingModelName() : "DEMO-4D";
|
||||
}
|
||||
|
||||
private String json(List<Double> values) {
|
||||
try {
|
||||
return objectMapper.writeValueAsString(values);
|
||||
} catch (Exception exception) {
|
||||
throw new AppException("임베딩 JSON 생성에 실패했습니다: " + exception.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private int count(String sql) {
|
||||
Integer count = jdbcTemplate.queryForObject(sql, Integer.class);
|
||||
return count == null ? 0 : count;
|
||||
}
|
||||
|
||||
private int normalizeLimit(int limit) {
|
||||
return Math.min(Math.max(limit, 1), 100);
|
||||
}
|
||||
|
||||
private String required(String value, String label) {
|
||||
if (value == null || value.isBlank()) {
|
||||
throw new AppException(label + "을 입력하세요.");
|
||||
}
|
||||
return value.trim();
|
||||
}
|
||||
|
||||
private String requiredDocumentId(String value) {
|
||||
String documentId = required(value, "문서 ID");
|
||||
if (documentId.length() > 200 || !documentId.matches("[A-Za-z0-9_.:-]+")) {
|
||||
throw new AppException("문서 ID는 영문·숫자와 '.', '_', ':', '-'만 사용할 수 있습니다.");
|
||||
}
|
||||
return documentId;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user