Add video management endpoints and fix LLM extraction pipeline
- Add missing endpoints: fetch-transcript, extract, bulk-extract/pending,
bulk-transcript/pending, manual restaurant add, restaurant update
- Add OCI HTTP client dependency (jersey3) for GenAI SDK compatibility
- Fix Oracle null parameter ORA-17004 with jdbcType=CLOB in MyBatis
- Fix evaluation IS JSON constraint by storing as valid JSON
- Add @JsonProperty("transcript") for frontend compatibility
- Add Korean-only rule to LLM extraction prompt
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -43,6 +43,7 @@ dependencies {
|
|||||||
// OCI SDK (GenAI for LLM + Embeddings)
|
// OCI SDK (GenAI for LLM + Embeddings)
|
||||||
implementation 'com.oracle.oci.sdk:oci-java-sdk-generativeaiinference:3.49.0'
|
implementation 'com.oracle.oci.sdk:oci-java-sdk-generativeaiinference:3.49.0'
|
||||||
implementation 'com.oracle.oci.sdk:oci-java-sdk-common:3.49.0'
|
implementation 'com.oracle.oci.sdk:oci-java-sdk-common:3.49.0'
|
||||||
|
implementation 'com.oracle.oci.sdk:oci-java-sdk-common-httpclient-jersey3:3.49.0'
|
||||||
|
|
||||||
// Jackson for JSON
|
// Jackson for JSON
|
||||||
implementation 'com.fasterxml.jackson.core:jackson-databind'
|
implementation 'com.fasterxml.jackson.core:jackson-databind'
|
||||||
|
|||||||
@@ -3,12 +3,13 @@ package com.tasteby.controller;
|
|||||||
import com.tasteby.domain.VideoDetail;
|
import com.tasteby.domain.VideoDetail;
|
||||||
import com.tasteby.domain.VideoSummary;
|
import com.tasteby.domain.VideoSummary;
|
||||||
import com.tasteby.security.AuthUtil;
|
import com.tasteby.security.AuthUtil;
|
||||||
import com.tasteby.service.CacheService;
|
import com.tasteby.service.*;
|
||||||
import com.tasteby.service.VideoService;
|
import com.tasteby.util.JsonUtil;
|
||||||
import org.springframework.http.HttpStatus;
|
import org.springframework.http.HttpStatus;
|
||||||
import org.springframework.web.bind.annotation.*;
|
import org.springframework.web.bind.annotation.*;
|
||||||
import org.springframework.web.server.ResponseStatusException;
|
import org.springframework.web.server.ResponseStatusException;
|
||||||
|
|
||||||
|
import java.util.HashMap;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
|
||||||
@@ -18,10 +19,23 @@ public class VideoController {
|
|||||||
|
|
||||||
private final VideoService videoService;
|
private final VideoService videoService;
|
||||||
private final CacheService cache;
|
private final CacheService cache;
|
||||||
|
private final YouTubeService youTubeService;
|
||||||
|
private final PipelineService pipelineService;
|
||||||
|
private final ExtractorService extractorService;
|
||||||
|
private final RestaurantService restaurantService;
|
||||||
|
private final GeocodingService geocodingService;
|
||||||
|
|
||||||
public VideoController(VideoService videoService, CacheService cache) {
|
public VideoController(VideoService videoService, CacheService cache,
|
||||||
|
YouTubeService youTubeService, PipelineService pipelineService,
|
||||||
|
ExtractorService extractorService, RestaurantService restaurantService,
|
||||||
|
GeocodingService geocodingService) {
|
||||||
this.videoService = videoService;
|
this.videoService = videoService;
|
||||||
this.cache = cache;
|
this.cache = cache;
|
||||||
|
this.youTubeService = youTubeService;
|
||||||
|
this.pipelineService = pipelineService;
|
||||||
|
this.extractorService = extractorService;
|
||||||
|
this.restaurantService = restaurantService;
|
||||||
|
this.geocodingService = geocodingService;
|
||||||
}
|
}
|
||||||
|
|
||||||
@GetMapping
|
@GetMapping
|
||||||
@@ -73,7 +87,157 @@ public class VideoController {
|
|||||||
return Map.of("ok", true);
|
return Map.of("ok", true);
|
||||||
}
|
}
|
||||||
|
|
||||||
// NOTE: SSE streaming endpoints (bulk-transcript, bulk-extract, remap-cuisine,
|
@PostMapping("/{id}/fetch-transcript")
|
||||||
// remap-foods, rebuild-vectors) and process/extract/fetch-transcript endpoints
|
public Map<String, Object> fetchTranscript(@PathVariable String id,
|
||||||
// will be added in VideoSseController after core pipeline services are migrated.
|
@RequestParam(defaultValue = "auto") String mode) {
|
||||||
|
AuthUtil.requireAdmin();
|
||||||
|
var video = videoService.findDetail(id);
|
||||||
|
if (video == null) throw new ResponseStatusException(HttpStatus.NOT_FOUND, "Video not found");
|
||||||
|
|
||||||
|
var result = youTubeService.getTranscript(video.getVideoId(), mode);
|
||||||
|
if (result == null || result.text() == null) {
|
||||||
|
throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "No transcript available");
|
||||||
|
}
|
||||||
|
|
||||||
|
videoService.updateTranscript(id, result.text());
|
||||||
|
return Map.of("ok", true, "length", result.text().length(), "source", result.source());
|
||||||
|
}
|
||||||
|
|
||||||
|
@GetMapping("/extract/prompt")
|
||||||
|
public Map<String, Object> getExtractPrompt() {
|
||||||
|
return Map.of("prompt", extractorService.getPrompt());
|
||||||
|
}
|
||||||
|
|
||||||
|
@PostMapping("/{id}/extract")
|
||||||
|
public Map<String, Object> extract(@PathVariable String id,
|
||||||
|
@RequestBody(required = false) Map<String, String> body) {
|
||||||
|
AuthUtil.requireAdmin();
|
||||||
|
var video = videoService.findDetail(id);
|
||||||
|
if (video == null) throw new ResponseStatusException(HttpStatus.NOT_FOUND, "Video not found");
|
||||||
|
if (video.getTranscriptText() == null || video.getTranscriptText().isBlank()) {
|
||||||
|
throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "No transcript");
|
||||||
|
}
|
||||||
|
|
||||||
|
String customPrompt = body != null ? body.get("prompt") : null;
|
||||||
|
var videoMap = Map.<String, Object>of("id", id, "video_id", video.getVideoId(), "title", video.getTitle());
|
||||||
|
int count = pipelineService.processExtract(videoMap, video.getTranscriptText(), customPrompt);
|
||||||
|
if (count > 0) cache.flush();
|
||||||
|
return Map.of("ok", true, "restaurants_extracted", count);
|
||||||
|
}
|
||||||
|
|
||||||
|
@GetMapping("/bulk-extract/pending")
|
||||||
|
public Map<String, Object> bulkExtractPending() {
|
||||||
|
var videos = videoService.findVideosForBulkExtract();
|
||||||
|
var summary = videos.stream().map(v -> Map.of("id", v.get("id"), "title", v.get("title"))).toList();
|
||||||
|
return Map.of("count", videos.size(), "videos", summary);
|
||||||
|
}
|
||||||
|
|
||||||
|
@GetMapping("/bulk-transcript/pending")
|
||||||
|
public Map<String, Object> bulkTranscriptPending() {
|
||||||
|
var videos = videoService.findVideosWithoutTranscript();
|
||||||
|
var summary = videos.stream().map(v -> Map.of("id", v.get("id"), "title", v.get("title"))).toList();
|
||||||
|
return Map.of("count", videos.size(), "videos", summary);
|
||||||
|
}
|
||||||
|
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
@PostMapping("/{videoId}/restaurants/manual")
|
||||||
|
public Map<String, Object> addManualRestaurant(@PathVariable String videoId,
|
||||||
|
@RequestBody Map<String, Object> body) {
|
||||||
|
AuthUtil.requireAdmin();
|
||||||
|
String name = (String) body.get("name");
|
||||||
|
if (name == null || name.isBlank()) {
|
||||||
|
throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "name is required");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Geocode
|
||||||
|
var geo = geocodingService.geocodeRestaurant(name, (String) body.get("address"));
|
||||||
|
var data = new HashMap<String, Object>();
|
||||||
|
data.put("name", name);
|
||||||
|
data.put("address", geo != null ? geo.get("formatted_address") : body.get("address"));
|
||||||
|
data.put("region", body.get("region"));
|
||||||
|
data.put("latitude", geo != null ? geo.get("latitude") : null);
|
||||||
|
data.put("longitude", geo != null ? geo.get("longitude") : null);
|
||||||
|
data.put("cuisine_type", body.get("cuisine_type"));
|
||||||
|
data.put("price_range", body.get("price_range"));
|
||||||
|
data.put("google_place_id", geo != null ? geo.get("google_place_id") : null);
|
||||||
|
data.put("phone", geo != null ? geo.get("phone") : null);
|
||||||
|
data.put("website", geo != null ? geo.get("website") : null);
|
||||||
|
data.put("business_status", geo != null ? geo.get("business_status") : null);
|
||||||
|
data.put("rating", geo != null ? geo.get("rating") : null);
|
||||||
|
data.put("rating_count", geo != null ? geo.get("rating_count") : null);
|
||||||
|
|
||||||
|
String restId = restaurantService.upsert(data);
|
||||||
|
|
||||||
|
// Parse foods and guests
|
||||||
|
List<String> foods = null;
|
||||||
|
Object foodsRaw = body.get("foods_mentioned");
|
||||||
|
if (foodsRaw instanceof List<?>) {
|
||||||
|
foods = ((List<?>) foodsRaw).stream().map(Object::toString).toList();
|
||||||
|
} else if (foodsRaw instanceof String s && !s.isBlank()) {
|
||||||
|
foods = List.of(s.split("\\s*,\\s*"));
|
||||||
|
}
|
||||||
|
|
||||||
|
List<String> guests = null;
|
||||||
|
Object guestsRaw = body.get("guests");
|
||||||
|
if (guestsRaw instanceof List<?>) {
|
||||||
|
guests = ((List<?>) guestsRaw).stream().map(Object::toString).toList();
|
||||||
|
} else if (guestsRaw instanceof String s && !s.isBlank()) {
|
||||||
|
guests = List.of(s.split("\\s*,\\s*"));
|
||||||
|
}
|
||||||
|
|
||||||
|
String evaluation = body.get("evaluation") instanceof String s ? s : null;
|
||||||
|
|
||||||
|
restaurantService.linkVideoRestaurant(videoId, restId, foods, evaluation, guests);
|
||||||
|
cache.flush();
|
||||||
|
return Map.of("ok", true, "restaurant_id", restId);
|
||||||
|
}
|
||||||
|
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
@PutMapping("/{videoId}/restaurants/{restaurantId}")
|
||||||
|
public Map<String, Object> updateVideoRestaurant(@PathVariable String videoId,
|
||||||
|
@PathVariable String restaurantId,
|
||||||
|
@RequestBody Map<String, Object> body) {
|
||||||
|
AuthUtil.requireAdmin();
|
||||||
|
|
||||||
|
// Update link fields (foods_mentioned, evaluation, guests)
|
||||||
|
List<String> foods = null;
|
||||||
|
Object foodsRaw = body.get("foods_mentioned");
|
||||||
|
if (foodsRaw instanceof List<?>) {
|
||||||
|
foods = ((List<?>) foodsRaw).stream().map(Object::toString).toList();
|
||||||
|
} else if (foodsRaw instanceof String s && !s.isBlank()) {
|
||||||
|
foods = List.of(s.split("\\s*,\\s*"));
|
||||||
|
}
|
||||||
|
|
||||||
|
List<String> guests = null;
|
||||||
|
Object guestsRaw = body.get("guests");
|
||||||
|
if (guestsRaw instanceof List<?>) {
|
||||||
|
guests = ((List<?>) guestsRaw).stream().map(Object::toString).toList();
|
||||||
|
} else if (guestsRaw instanceof String s && !s.isBlank()) {
|
||||||
|
guests = List.of(s.split("\\s*,\\s*"));
|
||||||
|
}
|
||||||
|
|
||||||
|
// evaluation must be valid JSON for DB IS JSON constraint
|
||||||
|
String evaluationJson = null;
|
||||||
|
Object evalRaw = body.get("evaluation");
|
||||||
|
if (evalRaw instanceof String s && !s.isBlank()) {
|
||||||
|
evaluationJson = s.trim().startsWith("{") || s.trim().startsWith("\"") ? s : JsonUtil.toJson(s);
|
||||||
|
} else if (evalRaw instanceof Map<?, ?>) {
|
||||||
|
evaluationJson = JsonUtil.toJson(evalRaw);
|
||||||
|
}
|
||||||
|
String foodsJson = foods != null ? JsonUtil.toJson(foods) : null;
|
||||||
|
String guestsJson = guests != null ? JsonUtil.toJson(guests) : null;
|
||||||
|
videoService.updateVideoRestaurantFields(videoId, restaurantId, foodsJson, evaluationJson, guestsJson);
|
||||||
|
|
||||||
|
// Update restaurant fields if provided
|
||||||
|
var restFields = new HashMap<String, Object>();
|
||||||
|
for (var key : List.of("name", "address", "region", "cuisine_type", "price_range")) {
|
||||||
|
if (body.containsKey(key)) restFields.put(key, body.get(key));
|
||||||
|
}
|
||||||
|
if (!restFields.isEmpty()) {
|
||||||
|
restaurantService.update(restaurantId, restFields);
|
||||||
|
}
|
||||||
|
|
||||||
|
cache.flush();
|
||||||
|
return Map.of("ok", true);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
package com.tasteby.domain;
|
package com.tasteby.domain;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||||
import lombok.AllArgsConstructor;
|
import lombok.AllArgsConstructor;
|
||||||
import lombok.Builder;
|
import lombok.Builder;
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
@@ -23,6 +24,7 @@ public class VideoDetail {
|
|||||||
private boolean hasLlm;
|
private boolean hasLlm;
|
||||||
private int restaurantCount;
|
private int restaurantCount;
|
||||||
private int matchedCount;
|
private int matchedCount;
|
||||||
|
@JsonProperty("transcript")
|
||||||
private String transcriptText;
|
private String transcriptText;
|
||||||
private List<VideoRestaurantLink> restaurants;
|
private List<VideoRestaurantLink> restaurants;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -65,4 +65,12 @@ public interface VideoMapper {
|
|||||||
@Param("llmResponse") String llmResponse);
|
@Param("llmResponse") String llmResponse);
|
||||||
|
|
||||||
List<Map<String, Object>> findVideosForBulkExtract();
|
List<Map<String, Object>> findVideosForBulkExtract();
|
||||||
|
|
||||||
|
List<Map<String, Object>> findVideosWithoutTranscript();
|
||||||
|
|
||||||
|
void updateVideoRestaurantFields(@Param("videoId") String videoId,
|
||||||
|
@Param("restaurantId") String restaurantId,
|
||||||
|
@Param("foodsJson") String foodsJson,
|
||||||
|
@Param("evaluation") String evaluation,
|
||||||
|
@Param("guestsJson") String guestsJson);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -25,6 +25,7 @@ public class ExtractorService {
|
|||||||
- 각 식당에 대해 아래 필드를 JSON 배열로 반환
|
- 각 식당에 대해 아래 필드를 JSON 배열로 반환
|
||||||
- 확실하지 않은 정보는 null
|
- 확실하지 않은 정보는 null
|
||||||
- 추가 설명 없이 JSON만 반환
|
- 추가 설명 없이 JSON만 반환
|
||||||
|
- 무조건 한글로 만들어주세요
|
||||||
|
|
||||||
필드:
|
필드:
|
||||||
- name: 식당 이름 (string, 필수)
|
- name: 식당 이름 (string, 필수)
|
||||||
@@ -50,6 +51,10 @@ public class ExtractorService {
|
|||||||
this.genAi = genAi;
|
this.genAi = genAi;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public String getPrompt() {
|
||||||
|
return EXTRACT_PROMPT;
|
||||||
|
}
|
||||||
|
|
||||||
public record ExtractionResult(List<Map<String, Object>> restaurants, String rawResponse) {}
|
public record ExtractionResult(List<Map<String, Object>> restaurants, String rawResponse) {}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
package com.tasteby.service;
|
package com.tasteby.service;
|
||||||
|
|
||||||
|
import com.tasteby.util.JsonUtil;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
@@ -118,12 +119,22 @@ public class PipelineService {
|
|||||||
|
|
||||||
// Link video <-> restaurant
|
// Link video <-> restaurant
|
||||||
var foods = restData.get("foods_mentioned");
|
var foods = restData.get("foods_mentioned");
|
||||||
var evaluation = restData.get("evaluation");
|
var evaluationRaw = restData.get("evaluation");
|
||||||
var guests = restData.get("guests");
|
var guests = restData.get("guests");
|
||||||
|
|
||||||
|
// evaluation must be stored as valid JSON (DB has IS JSON check constraint)
|
||||||
|
// Store as JSON string literal: "평가 내용" (valid JSON)
|
||||||
|
String evaluationJson = null;
|
||||||
|
if (evaluationRaw instanceof Map<?, ?>) {
|
||||||
|
evaluationJson = JsonUtil.toJson(evaluationRaw);
|
||||||
|
} else if (evaluationRaw instanceof String s && !s.isBlank()) {
|
||||||
|
evaluationJson = JsonUtil.toJson(s);
|
||||||
|
}
|
||||||
|
|
||||||
restaurantService.linkVideoRestaurant(
|
restaurantService.linkVideoRestaurant(
|
||||||
videoDbId, restId,
|
videoDbId, restId,
|
||||||
foods instanceof List<?> ? (List<String>) foods : null,
|
foods instanceof List<?> ? (List<String>) foods : null,
|
||||||
evaluation instanceof String ? (String) evaluation : null,
|
evaluationJson,
|
||||||
guests instanceof List<?> ? (List<String>) guests : null
|
guests instanceof List<?> ? (List<String>) guests : null
|
||||||
);
|
);
|
||||||
|
|
||||||
|
|||||||
@@ -98,7 +98,21 @@ public class VideoService {
|
|||||||
}).toList();
|
}).toList();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public void updateTranscript(String id, String transcript) {
|
||||||
|
mapper.updateTranscript(id, transcript);
|
||||||
|
}
|
||||||
|
|
||||||
public void updateVideoFields(String id, String status, String transcript, String llmResponse) {
|
public void updateVideoFields(String id, String status, String transcript, String llmResponse) {
|
||||||
mapper.updateVideoFields(id, status, transcript, llmResponse);
|
mapper.updateVideoFields(id, status, transcript, llmResponse);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public List<Map<String, Object>> findVideosWithoutTranscript() {
|
||||||
|
var rows = mapper.findVideosWithoutTranscript();
|
||||||
|
return rows.stream().map(JsonUtil::lowerKeys).toList();
|
||||||
|
}
|
||||||
|
|
||||||
|
public void updateVideoRestaurantFields(String videoId, String restaurantId,
|
||||||
|
String foodsJson, String evaluation, String guestsJson) {
|
||||||
|
mapper.updateVideoRestaurantFields(videoId, restaurantId, foodsJson, evaluation, guestsJson);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -166,7 +166,7 @@
|
|||||||
|
|
||||||
<insert id="linkVideoRestaurant">
|
<insert id="linkVideoRestaurant">
|
||||||
INSERT INTO video_restaurants (id, video_id, restaurant_id, foods_mentioned, evaluation, guests)
|
INSERT INTO video_restaurants (id, video_id, restaurant_id, foods_mentioned, evaluation, guests)
|
||||||
VALUES (#{id}, #{videoId}, #{restaurantId}, #{foods}, #{evaluation}, #{guests})
|
VALUES (#{id}, #{videoId}, #{restaurantId}, #{foods,jdbcType=CLOB}, #{evaluation,jdbcType=CLOB}, #{guests,jdbcType=CLOB})
|
||||||
</insert>
|
</insert>
|
||||||
|
|
||||||
<!-- ===== Lookups ===== -->
|
<!-- ===== Lookups ===== -->
|
||||||
|
|||||||
@@ -216,4 +216,20 @@
|
|||||||
ORDER BY v.published_at DESC
|
ORDER BY v.published_at DESC
|
||||||
</select>
|
</select>
|
||||||
|
|
||||||
|
<select id="findVideosWithoutTranscript" resultType="map">
|
||||||
|
SELECT id, video_id, title, url
|
||||||
|
FROM videos
|
||||||
|
WHERE (transcript_text IS NULL OR dbms_lob.getlength(transcript_text) = 0)
|
||||||
|
AND status != 'skip'
|
||||||
|
ORDER BY created_at
|
||||||
|
</select>
|
||||||
|
|
||||||
|
<update id="updateVideoRestaurantFields">
|
||||||
|
UPDATE video_restaurants
|
||||||
|
SET foods_mentioned = #{foodsJson,jdbcType=CLOB},
|
||||||
|
evaluation = #{evaluation,jdbcType=CLOB},
|
||||||
|
guests = #{guestsJson,jdbcType=CLOB}
|
||||||
|
WHERE video_id = #{videoId} AND restaurant_id = #{restaurantId}
|
||||||
|
</update>
|
||||||
|
|
||||||
</mapper>
|
</mapper>
|
||||||
|
|||||||
Reference in New Issue
Block a user