From 1088b23790118679abbe5a296b0aea7716147779 Mon Sep 17 00:00:00 2001 From: joungmin Date: Mon, 13 Apr 2026 07:34:18 +0000 Subject: [PATCH] Add Notes, Voice Clone TTS, fix auth persistence and maxTokens MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Notes: - notes table with TEXT/AUDIO types, category support - Audio upload → OpenRouter Gemini STT → OCI GenAI polish/summary - Raw STT saved separately in raw_content column - Polish/summary button for manual re-processing - Async processing with real-time polling Voice Clone TTS: - Qwen3-TTS 1.7B model on A10 GPU via FastAPI server - Voice profile registration (record/upload → save embedding) - Profile-based TTS generation API - TTS web page with recording, profile management, generation Auth fixes: - Store both access + refresh tokens in localStorage - Initialize state from localStorage synchronously (no flash) - Request interceptor reads token from localStorage every request - Refresh via body (not just cookie) Other fixes: - maxTokens 4096 → 65536 (OCI GenAI Gemini supports up to 65536) - Fix broken Korean chars in source files - OpenRouter config for STT - ffmpeg installed for audio conversion - Ollama + Gemma 4 E4B installed (STT fallback) - nginx proxy for TTS server (/api/tts/) Co-Authored-By: Claude Opus 4.6 (1M context) --- .gitignore | 3 + .../com/sundol/controller/AuthController.java | 15 +- .../com/sundol/controller/NoteController.java | 561 ++++++++++++++++++ .../com/sundol/repository/NoteRepository.java | 148 +++++ .../com/sundol/service/OciGenAiService.java | 2 +- .../src/main/resources/application.yml | 4 + sundol-frontend/src/app/notes/[id]/page.tsx | 275 +++++++++ sundol-frontend/src/app/notes/new/page.tsx | 166 ++++++ sundol-frontend/src/app/notes/page.tsx | 104 ++++ sundol-frontend/src/app/tts/page.tsx | 302 ++++++++++ sundol-frontend/src/components/nav-bar.tsx | 2 + sundol-frontend/src/lib/api.ts | 57 +- sundol-frontend/src/lib/auth-context.tsx | 133 ++--- tts-server.py | 211 +++++++ 14 files changed, 1863 insertions(+), 120 deletions(-) create mode 100644 sundol-backend/src/main/java/com/sundol/controller/NoteController.java create mode 100644 sundol-backend/src/main/java/com/sundol/repository/NoteRepository.java create mode 100644 sundol-frontend/src/app/notes/[id]/page.tsx create mode 100644 sundol-frontend/src/app/notes/new/page.tsx create mode 100644 sundol-frontend/src/app/notes/page.tsx create mode 100644 sundol-frontend/src/app/tts/page.tsx create mode 100644 tts-server.py diff --git a/.gitignore b/.gitignore index 4a843e7..49bafea 100644 --- a/.gitignore +++ b/.gitignore @@ -68,3 +68,6 @@ oracle_data/ # ======================== .claude/ cookies.txt +audio-uploads/ +voice-profiles/ +*.wav diff --git a/sundol-backend/src/main/java/com/sundol/controller/AuthController.java b/sundol-backend/src/main/java/com/sundol/controller/AuthController.java index c07e0ad..69eb986 100644 --- a/sundol-backend/src/main/java/com/sundol/controller/AuthController.java +++ b/sundol-backend/src/main/java/com/sundol/controller/AuthController.java @@ -45,11 +45,18 @@ public class AuthController { } @PostMapping("/refresh") - public Mono> refresh(ServerHttpRequest request, ServerHttpResponse response) { - HttpCookie cookie = request.getCookies().getFirst("refreshToken"); - String refreshToken = cookie != null ? cookie.getValue() : null; + public Mono> refresh( + ServerHttpRequest request, ServerHttpResponse response, + @RequestBody(required = false) Map body) { + // 1차: body에서 refreshToken + String refreshToken = (body != null) ? body.get("refreshToken") : null; + // 2차: cookie에서 refreshToken + if (refreshToken == null || refreshToken.isBlank()) { + HttpCookie cookie = request.getCookies().getFirst("refreshToken"); + refreshToken = cookie != null ? cookie.getValue() : null; + } - if (refreshToken == null) { + if (refreshToken == null || refreshToken.isBlank()) { return Mono.just(ResponseEntity.status(401).build()); } diff --git a/sundol-backend/src/main/java/com/sundol/controller/NoteController.java b/sundol-backend/src/main/java/com/sundol/controller/NoteController.java new file mode 100644 index 0000000..6aa2135 --- /dev/null +++ b/sundol-backend/src/main/java/com/sundol/controller/NoteController.java @@ -0,0 +1,561 @@ +package com.sundol.controller; + +import com.sundol.repository.CategoryRepository; +import com.sundol.repository.NoteRepository; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.ResponseEntity; +import org.springframework.http.codec.multipart.FilePart; +import org.springframework.security.core.annotation.AuthenticationPrincipal; +import org.springframework.web.bind.annotation.*; +import reactor.core.publisher.Mono; +import reactor.core.scheduler.Schedulers; + +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Base64; +import java.util.List; +import java.util.Map; + +@RestController +@RequestMapping("/api/notes") +public class NoteController { + + private static final Logger log = LoggerFactory.getLogger(NoteController.class); + private static final Path AUDIO_DIR = Path.of(System.getProperty("user.dir"), "audio-uploads"); + private static final HttpClient httpClient = HttpClient.newHttpClient(); + + @Value("${openrouter.api-key:}") + private String openRouterApiKey; + + @Value("${openrouter.model:google/gemini-2.5-flash}") + private String openRouterModel; + + private final NoteRepository noteRepository; + private final CategoryRepository categoryRepository; + private final com.sundol.service.OciGenAiService genAiService; + + public NoteController(NoteRepository noteRepository, CategoryRepository categoryRepository, + com.sundol.service.OciGenAiService genAiService) { + this.noteRepository = noteRepository; + this.categoryRepository = categoryRepository; + this.genAiService = genAiService; + try { Files.createDirectories(AUDIO_DIR); } catch (Exception ignored) {} + } + + @GetMapping + public Mono>>> list( + @AuthenticationPrincipal String userId, + @RequestParam(required = false) String categoryId) { + return Mono.fromCallable(() -> noteRepository.list(userId, categoryId)) + .subscribeOn(Schedulers.boundedElastic()) + .map(ResponseEntity::ok); + } + + @GetMapping("/{id}") + public Mono>> getById( + @AuthenticationPrincipal String userId, + @PathVariable String id) { + return Mono.fromCallable(() -> { + Map note = noteRepository.findById(userId, id); + if (note == null) return ResponseEntity.notFound().>build(); + return ResponseEntity.ok(note); + }).subscribeOn(Schedulers.boundedElastic()); + } + + @PostMapping + public Mono>> create( + @AuthenticationPrincipal String userId, + @RequestBody Map body) { + return Mono.fromCallable(() -> { + String title = body.getOrDefault("title", ""); + String content = body.getOrDefault("content", ""); + String categoryId = body.get("categoryId"); + String id = noteRepository.insert(userId, title, content, "TEXT", null, categoryId); + return ResponseEntity.ok(Map.of("id", id)); + }).subscribeOn(Schedulers.boundedElastic()); + } + + @PatchMapping("/{id}") + public Mono>> update( + @AuthenticationPrincipal String userId, + @PathVariable String id, + @RequestBody Map body) { + return Mono.fromCallable(() -> { + String title = body.get("title"); + String content = body.get("content"); + String categoryId = body.get("categoryId"); + noteRepository.update(id, userId, title, content, categoryId); + return ResponseEntity.ok(noteRepository.findById(userId, id)); + }).subscribeOn(Schedulers.boundedElastic()); + } + + /** + * 기존 노트의 내용을 LLM으로 교정 + 요약 재실행 + */ + @PostMapping("/{id}/polish") + public Mono>> polishNote( + @AuthenticationPrincipal String userId, + @PathVariable String id) { + return Mono.fromCallable(() -> { + Map note = noteRepository.findById(userId, id); + if (note == null) return ResponseEntity.notFound().>build(); + + String content = note.get("CONTENT") != null ? note.get("CONTENT").toString() : ""; + if (content.isBlank()) return ResponseEntity.badRequest().>build(); + + // raw_content가 있으면 그걸 사용, 없으면 content에서 전문 추출 + Object rawObj = note.get("RAW_CONTENT"); + String rawText = (rawObj != null && !rawObj.toString().isBlank()) ? rawObj.toString() : content; + if (rawText.contains("# 전문")) { + int idx = rawText.indexOf("# 전문"); + rawText = rawText.substring(idx + "# 전문".length()).strip(); + } + + String noteType = note.get("NOTE_TYPE") != null ? note.get("NOTE_TYPE").toString() : "TEXT"; + boolean isAudio = "AUDIO".equals(noteType) || note.get("AUDIO_PATH") != null; + + noteRepository.updateNoteType(id, "TRANSCRIBING"); + + final String finalRawText = rawText; + Schedulers.boundedElastic().schedule(() -> { + try { + noteRepository.updateContent(id, finalRawText + "\n\n--- 텍스트 교정 중... ---"); + String polished = polishTranscription(finalRawText); + + noteRepository.updateContent(id, polished + "\n\n--- 요약 생성 중... ---"); + String summary = summarizeTranscription(polished); + + String result = "# 요약\n\n" + summary + "\n\n---\n\n# 전문\n\n" + polished; + String newTitle = generateAudioTitle(summary, java.time.LocalDateTime.now()); + noteRepository.update(id, null, newTitle, result, null); + noteRepository.updateNoteType(id, isAudio ? "AUDIO" : "TEXT"); + log.info("Polish + summary complete for note {}", id); + } catch (Exception e) { + log.error("Polish failed for note {}", id, e); + noteRepository.updateNoteType(id, isAudio ? "AUDIO" : "TEXT"); + } + }); + + return ResponseEntity.ok(Map.of("status", "processing")); + }).subscribeOn(Schedulers.boundedElastic()); + } + + @DeleteMapping("/{id}") + public Mono> delete( + @AuthenticationPrincipal String userId, + @PathVariable String id) { + return Mono.fromRunnable(() -> noteRepository.delete(id, userId)) + .subscribeOn(Schedulers.boundedElastic()) + .then(Mono.just(ResponseEntity.ok().build())); + } + + /** + * 오디오 파일 업로드 → Gemma 4 STT → 텍스트 노트 생성 + */ + @PostMapping("/audio") + public Mono>> uploadAudio( + @AuthenticationPrincipal String userId, + @RequestPart("file") FilePart filePart, + @RequestPart(value = "title", required = false) String title, + @RequestPart(value = "categoryId", required = false) String categoryId) { + final String inputTitle = (title != null && !title.isBlank()) ? title : "음성 변환 중..."; + final String inputCategoryId = categoryId; + return Mono.fromCallable(() -> { + // 1. 파일 저장 + String fileName = System.currentTimeMillis() + "_" + filePart.filename(); + Path audioFile = AUDIO_DIR.resolve(fileName); + filePart.transferTo(audioFile).block(); + log.info("Audio file saved: {} ({} bytes)", audioFile, Files.size(audioFile)); + + // 2. 노트 즉시 생성 (TRANSCRIBING 상태) + String id = noteRepository.insert(userId, inputTitle, "음성 변환을 시작합니다...", "TRANSCRIBING", fileName, inputCategoryId); + + // 3. 백그라운드에서 STT 실행 + Schedulers.boundedElastic().schedule(() -> { + try { + transcribeAsync(id, audioFile, inputTitle); + } catch (Exception e) { + log.error("Async transcription failed for note {}", id, e); + noteRepository.updateContent(id, "음성 변환에 실패했습니다: " + e.getMessage()); + noteRepository.updateNoteType(id, "AUDIO_FAILED"); + } + }); + + return ResponseEntity.ok(Map.of("id", id)); + }).subscribeOn(Schedulers.boundedElastic()); + } + + /** + * 비동기 STT 처리. + * Step 1: OpenRouter (Gemini) STT → raw_content에 저장 + * Step 2: OCI GenAI로 교정 → content에 저장 + * Step 3: OCI GenAI로 요약 → content 앞에 추가 + */ + private void transcribeAsync(String noteId, Path audioFile, String inputTitle) throws IOException, InterruptedException { + // === Step 1: STT (OpenRouter Gemini) === + String rawResult = null; + + if (openRouterApiKey != null && !openRouterApiKey.isBlank()) { + try { + noteRepository.updateContent(noteId, "Gemini로 음성 변환 중..."); + rawResult = transcribeWithOpenRouter(audioFile); + log.info("OpenRouter STT: {} chars", rawResult != null ? rawResult.length() : 0); + } catch (Exception e) { + log.warn("OpenRouter STT failed: {}", e.getMessage()); + noteRepository.updateContent(noteId, "Gemini STT 실패: " + e.getMessage()); + } + } + + // Gemma fallback (OpenRouter 실패 시) + if (rawResult == null || rawResult.isBlank()) { + try { + noteRepository.updateContent(noteId, "Gemma로 음성 변환 중..."); + Path wavFile = convertToWav(audioFile); + rawResult = transcribeChunk(wavFile); + cleanup(wavFile, audioFile); + } catch (Exception e) { + log.error("All STT failed for note {}", noteId, e); + noteRepository.updateContent(noteId, "모든 음성 변환 실패: " + e.getMessage()); + noteRepository.updateNoteType(noteId, "AUDIO_FAILED"); + return; + } + } + + if (rawResult == null || rawResult.isBlank()) { + noteRepository.updateContent(noteId, "음성 변환 결과가 비어있습니다."); + noteRepository.updateNoteType(noteId, "AUDIO_FAILED"); + return; + } + + // raw 텍스트를 별도 컬럼에 저장 + content에도 일단 저장 + noteRepository.updateRawContent(noteId, rawResult); + noteRepository.updateContent(noteId, rawResult); + String sttTitle = java.time.LocalDateTime.now().format( + java.time.format.DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm")) + " 음성 메모"; + noteRepository.update(noteId, null, + inputTitle.equals("음성 변환 중...") ? sttTitle : inputTitle, + rawResult, null); + noteRepository.updateNoteType(noteId, "AUDIO"); + log.info("STT raw saved: {} chars", rawResult.length()); + + // === Step 2: 교정 (OCI GenAI) === + try { + noteRepository.updateNoteType(noteId, "TRANSCRIBING"); + noteRepository.updateContent(noteId, rawResult + "\n\n--- 텍스트 교정 중 (OCI GenAI)... ---"); + String polished = polishTranscription(rawResult); + log.info("Polish complete: {} chars", polished.length()); + + // === Step 3: 요약 (OCI GenAI) === + noteRepository.updateContent(noteId, polished + "\n\n--- 요약 생성 중... ---"); + String summary = summarizeTranscription(polished); + log.info("Summary complete: {} chars", summary.length()); + + // 최종 결과 저장 + String result = "# 요약\n\n" + summary + "\n\n---\n\n# 전문\n\n" + polished; + String finalTitle = inputTitle.equals("음성 변환 중...") + ? generateAudioTitle(summary, java.time.LocalDateTime.now()) + : inputTitle; + noteRepository.update(noteId, null, finalTitle, result, null); + log.info("Final note saved: {} chars", result.length()); + } catch (Exception e) { + log.warn("Polish/summary failed, keeping raw STT text: {}", e.getMessage()); + } + + noteRepository.updateNoteType(noteId, "AUDIO"); + } + + /** + * Gemma 4 E4B를 사용하여 오디오 파일을 텍스트로 변환 + */ + /** + * 오디오 파일을 wav로 변환한다 (Ollama 호환성). + */ + private Path convertToWav(Path audioFile) throws IOException, InterruptedException { + String name = audioFile.getFileName().toString(); + if (name.toLowerCase().endsWith(".wav")) return audioFile; + + Path wavFile = audioFile.getParent().resolve(name.replaceAll("\\.[^.]+$", "") + ".wav"); + ProcessBuilder pb = new ProcessBuilder( + "ffmpeg", "-i", audioFile.toString(), + "-ar", "16000", "-ac", "1", "-y", + wavFile.toString() + ); + pb.redirectErrorStream(true); + Process proc = pb.start(); + String output = new String(proc.getInputStream().readAllBytes()); + int exitCode = proc.waitFor(); + if (exitCode != 0) { + log.error("ffmpeg conversion failed (exit {}): {}", exitCode, output.substring(0, Math.min(500, output.length()))); + throw new IOException("오디오 변환 실패 (ffmpeg exit " + exitCode + ")"); + } + log.info("Converted {} to wav: {} bytes", name, Files.size(wavFile)); + return wavFile; + } + + private static final int CHUNK_SECONDS = 180; // 3분 단위 분할 + + private String transcribeWithGemma(Path audioFile) throws IOException, InterruptedException { + Path wavFile = convertToWav(audioFile); + double duration = getAudioDuration(wavFile); + log.info("Audio duration: {}s", duration); + + if (duration <= CHUNK_SECONDS) { + String result = transcribeChunk(wavFile); + cleanup(wavFile, audioFile); + return result; + } + + // 긴 오디오: 3분 단위로 분할 + int chunks = (int) Math.ceil(duration / CHUNK_SECONDS); + log.info("Splitting audio into {} chunks of {}s", chunks, CHUNK_SECONDS); + + StringBuilder fullText = new StringBuilder(); + for (int i = 0; i < chunks; i++) { + int start = i * CHUNK_SECONDS; + Path chunkFile = wavFile.getParent().resolve("chunk_" + i + "_" + System.currentTimeMillis() + ".wav"); + + ProcessBuilder pb = new ProcessBuilder( + "ffmpeg", "-i", wavFile.toString(), + "-ss", String.valueOf(start), "-t", String.valueOf(CHUNK_SECONDS), + "-ar", "16000", "-ac", "1", "-y", chunkFile.toString() + ); + pb.redirectErrorStream(true); + Process proc = pb.start(); + proc.getInputStream().readAllBytes(); + proc.waitFor(); + + log.info("Transcribing chunk {}/{} ({}s-{}s)", i + 1, chunks, start, Math.min(start + CHUNK_SECONDS, (int) duration)); + + try { + String chunkText = transcribeChunk(chunkFile); + if (!chunkText.isBlank()) { + if (fullText.length() > 0) fullText.append("\n\n"); + fullText.append(chunkText); + } + } catch (Exception e) { + log.warn("Chunk {} failed: {}", i + 1, e.getMessage()); + fullText.append("\n\n[chunk ").append(i + 1).append(" 변환 실패]"); + } finally { + try { Files.deleteIfExists(chunkFile); } catch (Exception ignored) {} + } + } + + cleanup(wavFile, audioFile); + String result = fullText.toString().strip(); + if (result.isBlank()) throw new IOException("Gemma STT returned empty for all chunks"); + return result; + } + + private String transcribeChunk(Path wavFile) throws IOException, InterruptedException { + byte[] audioBytes = Files.readAllBytes(wavFile); + String base64Audio = Base64.getEncoder().encodeToString(audioBytes); + log.info("Chunk base64: {} chars ({} MB)", base64Audio.length(), audioBytes.length / 1024 / 1024); + + String payload = """ + { + "model": "gemma4:e4b", + "messages": [{"role": "user", "content": "Transcribe the following audio to text accurately. Output only the spoken content in its original language. Do not add any description or translation.", "images": ["%s"]}], + "stream": false, + "options": {"num_ctx": 8000} + } + """.formatted(base64Audio); + + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create("http://localhost:11434/api/chat")) + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString(payload)) + .timeout(java.time.Duration.ofMinutes(10)) + .build(); + + HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); + if (response.statusCode() != 200) { + log.error("Gemma STT error {}: {}", response.statusCode(), response.body().substring(0, Math.min(500, response.body().length()))); + throw new IOException("Gemma STT failed: HTTP " + response.statusCode()); + } + + var root = new com.fasterxml.jackson.databind.ObjectMapper().readTree(response.body()); + return root.path("message").path("content").asText("").strip(); + } + + /** + * STT 결과를 LLM으로 교정한다. + * 발음 오인식 보정, 문장 구분, 불필요한 추임새 제거, 가독성 향상. + */ + private String polishTranscription(String rawText) { + if (!genAiService.isConfigured()) { + log.info("GenAI not configured, skipping polish"); + return rawText; + } + + try { + String systemMsg = + "당신은 전문 속기사입니다. 음성 인식(STT) 텍스트를 교정해주세요.\n\n" + + "## 규칙\n" + + "1. 발음 오인식 단어를 문맥에 맞게 보정하세요.\n" + + "2. 추임새(어, 음, 그, 아, 뭐, 이제, 근데)를 제거하세요.\n" + + "3. 문장 부호를 넣고 단락을 나누세요.\n" + + "4. 절대 요약하지 마세요. 원문의 모든 내용을 빠짐없이 유지하세요.\n" + + "5. 내용을 추가하거나 삭제하지 마세요. 교정만 하세요.\n" + + "6. 전문 용어와 고유 명사는 올바르게 표기하세요.\n" + + "7. 입력 텍스트와 비슷한 분량으로 출력하세요. 줄이지 마세요.\n" + + "8. Markdown 형식으로 출력하세요."; + + // maxTokens 65536이므로 대부분 한 번에 처리 가능 + if (rawText.length() <= 30000) { + log.info("Polishing in single call: {} chars", rawText.length()); + return genAiService.chat(systemMsg, + "아래 STT 텍스트를 교정해주세요. 전체 내용을 빠짐없이 유지하세요:\n\n" + rawText, null).strip(); + } + + // 30000자 이상만 분할 + StringBuilder polished = new StringBuilder(); + int chunkSize = 20000; + int totalChunks = (int) Math.ceil((double) rawText.length() / chunkSize); + for (int i = 0; i < rawText.length(); i += chunkSize) { + int chunkNum = (i / chunkSize) + 1; + String chunk = rawText.substring(i, Math.min(i + chunkSize, rawText.length())); + log.info("Polishing chunk {}/{}: {} chars", chunkNum, totalChunks, chunk.length()); + String result = genAiService.chat(systemMsg, + "아래 STT 텍스트를 교정해주세요. 전체 내용을 빠짐없이 유지하세요:\n\n" + chunk, null).strip(); + if (polished.length() > 0) polished.append("\n\n"); + polished.append(result); + } + return polished.toString(); + } catch (Exception e) { + log.warn("Polish transcription failed, returning raw text: {}", e.getMessage()); + return rawText; + } + } + + /** + * 교정된 텍스트를 요약한다. + */ + private String summarizeTranscription(String polishedText) { + if (!genAiService.isConfigured()) return ""; + + try { + String systemMsg = + "당신은 회의록/녹음 요약 전문가입니다. 아래 텍스트를 요약해주세요.\n\n" + + "## 규칙\n" + + "1. 주요 논의 주제별로 소제목(##)을 나누어 요약하세요.\n" + + "2. 각 주제 아래 핵심 내용을 불릿 포인트로 정리하세요.\n" + + "3. 주요 결정 사항, 액션 아이템이 있다면 별도로 표시하세요.\n" + + "4. 원문과 같은 언어로 작성하세요.\n" + + "5. Markdown 형식으로 작성하세요.\n" + + "6. 원본 길이에 비례하여 요약하세요. 긴 내용은 상세하게, 짧은 내용은 간결하게.\n" + + "7. 중요한 수치, 이름, 기술명은 빠뜨리지 마세요."; + + String content = polishedText.length() > 15000 + ? polishedText.substring(0, 15000) : polishedText; + return genAiService.chat(systemMsg, "아래 내용을 요약해주세요:\n\n" + content, null).strip(); + } catch (Exception e) { + log.warn("Summarization failed: {}", e.getMessage()); + return ""; + } + } + + /** + * LLM으로 음성 메모 제목을 생성한다. "일시 - 핵심 주제" 형태. + */ + private String generateAudioTitle(String summary, java.time.LocalDateTime dateTime) { + if (!genAiService.isConfigured() || summary.isBlank()) { + return dateTime.format(java.time.format.DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm")) + " 음성 메모"; + } + + try { + String systemMsg = "음성 메모의 요약을 보고 10자 이내의 짧은 제목을 생성해주세요. " + + "제목만 출력하세요. 따옴표, 설명, 접두사 없이 제목만."; + String title = genAiService.chat(systemMsg, summary, null).strip() + .replaceAll("^\"|\"$", "").replaceAll("^'|'$", ""); + if (title.length() > 40) title = title.substring(0, 40); + return dateTime.format(java.time.format.DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm")) + " " + title; + } catch (Exception e) { + log.warn("Title generation failed: {}", e.getMessage()); + return dateTime.format(java.time.format.DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm")) + " 음성 메모"; + } + } + + /** + * OpenRouter API (Gemini 2.5 Flash)를 사용하여 오디오 STT. 한 번에 전체 파일 처리 가능. + */ + private String transcribeWithOpenRouter(Path audioFile) throws IOException, InterruptedException { + byte[] audioBytes = Files.readAllBytes(audioFile); + String base64Audio = Base64.getEncoder().encodeToString(audioBytes); + + String mimeType = "audio/wav"; + String name = audioFile.getFileName().toString().toLowerCase(); + if (name.endsWith(".mp3")) mimeType = "audio/mpeg"; + else if (name.endsWith(".m4a")) mimeType = "audio/mp4"; + else if (name.endsWith(".ogg")) mimeType = "audio/ogg"; + else if (name.endsWith(".webm")) mimeType = "audio/webm"; + else if (name.endsWith(".flac")) mimeType = "audio/flac"; + + log.info("OpenRouter STT: {} ({} MB, {})", name, audioBytes.length / 1024 / 1024, mimeType); + + // OpenRouter chat/completions API with audio input + String payload = new com.fasterxml.jackson.databind.ObjectMapper().writeValueAsString(Map.of( + "model", openRouterModel, + "messages", List.of(Map.of( + "role", "user", + "content", List.of( + Map.of("type", "input_audio", "input_audio", Map.of( + "data", base64Audio, + "format", mimeType.substring(mimeType.indexOf('/') + 1) + )), + Map.of("type", "text", "text", + "Transcribe the audio accurately. Output only the spoken content in its original language. " + + "Do not add description, annotation, timestamps, or translation. " + + "If the audio contains Korean, output in Korean.") + ) + )) + )); + + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create("https://openrouter.ai/api/v1/chat/completions")) + .header("Content-Type", "application/json") + .header("Authorization", "Bearer " + openRouterApiKey) + .POST(HttpRequest.BodyPublishers.ofString(payload)) + .timeout(java.time.Duration.ofMinutes(10)) + .build(); + + HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); + + if (response.statusCode() != 200) { + log.error("OpenRouter STT error {}: {}", response.statusCode(), + response.body().substring(0, Math.min(500, response.body().length()))); + throw new IOException("OpenRouter STT failed: HTTP " + response.statusCode()); + } + + var root = new com.fasterxml.jackson.databind.ObjectMapper().readTree(response.body()); + String text = root.path("choices").path(0).path("message").path("content").asText("").strip(); + + if (text.isBlank()) { + throw new IOException("OpenRouter STT returned empty result"); + } + + return text; + } + + private double getAudioDuration(Path audioFile) throws IOException, InterruptedException { + ProcessBuilder pb = new ProcessBuilder("ffprobe", "-i", audioFile.toString(), + "-show_entries", "format=duration", "-v", "quiet", "-of", "csv=p=0"); + pb.redirectErrorStream(true); + Process proc = pb.start(); + String output = new String(proc.getInputStream().readAllBytes()).strip(); + proc.waitFor(); + try { return Double.parseDouble(output); } catch (NumberFormatException e) { return 0; } + } + + private void cleanup(Path wavFile, Path originalFile) { + if (!wavFile.equals(originalFile)) { + try { Files.deleteIfExists(wavFile); } catch (Exception ignored) {} + } + } +} diff --git a/sundol-backend/src/main/java/com/sundol/repository/NoteRepository.java b/sundol-backend/src/main/java/com/sundol/repository/NoteRepository.java new file mode 100644 index 0000000..323ce7f --- /dev/null +++ b/sundol-backend/src/main/java/com/sundol/repository/NoteRepository.java @@ -0,0 +1,148 @@ +package com.sundol.repository; + +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Repository; + +import java.sql.Clob; +import java.util.List; +import java.util.Map; + +@Repository +public class NoteRepository { + + private final JdbcTemplate jdbcTemplate; + + public NoteRepository(JdbcTemplate jdbcTemplate) { + this.jdbcTemplate = jdbcTemplate; + } + + public String insert(String userId, String title, String content, String noteType, String audioPath, String categoryId) { + if (categoryId != null) { + jdbcTemplate.update( + "INSERT INTO notes (id, user_id, title, content, note_type, audio_path, category_id, created_at, updated_at) " + + "VALUES (SYS_GUID(), HEXTORAW(?), ?, ?, ?, ?, HEXTORAW(?), SYSTIMESTAMP, SYSTIMESTAMP)", + new Object[]{userId, title, content, noteType, audioPath, categoryId}, + new int[]{java.sql.Types.VARCHAR, java.sql.Types.VARCHAR, java.sql.Types.CLOB, java.sql.Types.VARCHAR, java.sql.Types.VARCHAR, java.sql.Types.VARCHAR} + ); + } else { + jdbcTemplate.update( + "INSERT INTO notes (id, user_id, title, content, note_type, audio_path, created_at, updated_at) " + + "VALUES (SYS_GUID(), HEXTORAW(?), ?, ?, ?, ?, SYSTIMESTAMP, SYSTIMESTAMP)", + new Object[]{userId, title, content, noteType, audioPath}, + new int[]{java.sql.Types.VARCHAR, java.sql.Types.VARCHAR, java.sql.Types.CLOB, java.sql.Types.VARCHAR, java.sql.Types.VARCHAR} + ); + } + var result = jdbcTemplate.queryForList( + "SELECT RAWTOHEX(id) AS id FROM notes WHERE user_id = HEXTORAW(?) ORDER BY created_at DESC FETCH FIRST 1 ROW ONLY", + userId + ); + return (String) result.get(0).get("ID"); + } + + public List> list(String userId, String categoryId) { + if (categoryId != null && !categoryId.isBlank()) { + return jdbcTemplate.queryForList( + "SELECT RAWTOHEX(n.id) AS id, n.title, n.note_type, n.audio_path, " + + " RAWTOHEX(n.category_id) AS category_id, c.full_path AS category_path, " + + " n.created_at, n.updated_at " + + "FROM notes n LEFT JOIN categories c ON c.id = n.category_id " + + "WHERE n.user_id = HEXTORAW(?) AND n.category_id = HEXTORAW(?) " + + "ORDER BY n.created_at DESC", + userId, categoryId + ); + } + return jdbcTemplate.queryForList( + "SELECT RAWTOHEX(n.id) AS id, n.title, n.note_type, n.audio_path, " + + " RAWTOHEX(n.category_id) AS category_id, c.full_path AS category_path, " + + " n.created_at, n.updated_at " + + "FROM notes n LEFT JOIN categories c ON c.id = n.category_id " + + "WHERE n.user_id = HEXTORAW(?) " + + "ORDER BY n.created_at DESC", + userId + ); + } + + public Map findById(String userId, String id) { + var results = jdbcTemplate.queryForList( + "SELECT RAWTOHEX(n.id) AS id, n.title, n.content, n.raw_content, n.note_type, n.audio_path, " + + " RAWTOHEX(n.category_id) AS category_id, c.full_path AS category_path, " + + " n.created_at, n.updated_at " + + "FROM notes n LEFT JOIN categories c ON c.id = n.category_id " + + "WHERE RAWTOHEX(n.id) = ? AND n.user_id = HEXTORAW(?)", + id, userId + ); + if (results.isEmpty()) return null; + return convertClobFields(results.get(0)); + } + + public void update(String id, String userId, String title, String content, String categoryId) { + if (userId != null) { + if (categoryId != null) { + jdbcTemplate.update( + "UPDATE notes SET title = ?, content = ?, category_id = HEXTORAW(?), updated_at = SYSTIMESTAMP " + + "WHERE RAWTOHEX(id) = ? AND user_id = HEXTORAW(?)", + new Object[]{title, content, categoryId, id, userId}, + new int[]{java.sql.Types.VARCHAR, java.sql.Types.CLOB, java.sql.Types.VARCHAR, java.sql.Types.VARCHAR, java.sql.Types.VARCHAR} + ); + } else { + jdbcTemplate.update( + "UPDATE notes SET title = ?, content = ?, updated_at = SYSTIMESTAMP " + + "WHERE RAWTOHEX(id) = ? AND user_id = HEXTORAW(?)", + new Object[]{title, content, id, userId}, + new int[]{java.sql.Types.VARCHAR, java.sql.Types.CLOB, java.sql.Types.VARCHAR, java.sql.Types.VARCHAR} + ); + } + } else { + // 내부 호출 (userId 없음) + jdbcTemplate.update( + "UPDATE notes SET title = ?, content = ?, updated_at = SYSTIMESTAMP WHERE RAWTOHEX(id) = ?", + new Object[]{title, content, id}, + new int[]{java.sql.Types.VARCHAR, java.sql.Types.CLOB, java.sql.Types.VARCHAR} + ); + } + } + + public void updateRawContent(String id, String rawContent) { + jdbcTemplate.update( + "UPDATE notes SET raw_content = ?, updated_at = SYSTIMESTAMP WHERE RAWTOHEX(id) = ?", + new Object[]{rawContent, id}, + new int[]{java.sql.Types.CLOB, java.sql.Types.VARCHAR} + ); + } + + public void updateContent(String id, String content) { + jdbcTemplate.update( + "UPDATE notes SET content = ?, updated_at = SYSTIMESTAMP WHERE RAWTOHEX(id) = ?", + new Object[]{content, id}, + new int[]{java.sql.Types.CLOB, java.sql.Types.VARCHAR} + ); + } + + public void updateNoteType(String id, String noteType) { + jdbcTemplate.update( + "UPDATE notes SET note_type = ?, updated_at = SYSTIMESTAMP WHERE RAWTOHEX(id) = ?", + noteType, id + ); + } + + public void delete(String id, String userId) { + jdbcTemplate.update( + "DELETE FROM notes WHERE RAWTOHEX(id) = ? AND user_id = HEXTORAW(?)", + id, userId + ); + } + + private Map convertClobFields(Map row) { + for (var entry : row.entrySet()) { + Object val = entry.getValue(); + if (val instanceof Clob clob) { + try { + entry.setValue(clob.getSubString(1, (int) clob.length())); + } catch (Exception e) { + entry.setValue(null); + } + } + } + return row; + } +} diff --git a/sundol-backend/src/main/java/com/sundol/service/OciGenAiService.java b/sundol-backend/src/main/java/com/sundol/service/OciGenAiService.java index 3600c3b..82afa4c 100644 --- a/sundol-backend/src/main/java/com/sundol/service/OciGenAiService.java +++ b/sundol-backend/src/main/java/com/sundol/service/OciGenAiService.java @@ -96,7 +96,7 @@ public class OciGenAiService { Map.of("role", "SYSTEM", "content", List.of(Map.of("type", "TEXT", "text", systemMessage))), Map.of("role", "USER", "content", List.of(Map.of("type", "TEXT", "text", userMessage))) ), - "maxTokens", 4096, + "maxTokens", 65536, "temperature", 0.3 ) ); diff --git a/sundol-backend/src/main/resources/application.yml b/sundol-backend/src/main/resources/application.yml index ae70c0f..a0f5cdb 100644 --- a/sundol-backend/src/main/resources/application.yml +++ b/sundol-backend/src/main/resources/application.yml @@ -29,6 +29,10 @@ oci: model: ${OCI_GENAI_MODEL:google.gemini-2.5-flash} base-url: ${OCI_GENAI_BASE_URL:https://inference.generativeai.us-chicago-1.oci.oraclecloud.com/20231130/actions} +openrouter: + api-key: ${OPENROUTER_API_KEY:} + model: ${OPENROUTER_MODEL:google/gemini-2.5-flash} + jina: reader: api-key: ${JINA_READER_API_KEY:} diff --git a/sundol-frontend/src/app/notes/[id]/page.tsx b/sundol-frontend/src/app/notes/[id]/page.tsx new file mode 100644 index 0000000..4f3736d --- /dev/null +++ b/sundol-frontend/src/app/notes/[id]/page.tsx @@ -0,0 +1,275 @@ +"use client"; + +import { useEffect, useState } from "react"; +import { useParams, useRouter } from "next/navigation"; +import AuthGuard from "@/components/auth-guard"; +import NavBar from "@/components/nav-bar"; +import { useApi } from "@/lib/use-api"; +import ReactMarkdown from "react-markdown"; + +interface NoteDetail { + ID: string; + TITLE: string; + CONTENT: string; + RAW_CONTENT: string | null; + NOTE_TYPE: string; + AUDIO_PATH: string | null; + CATEGORY_PATH: string | null; + CREATED_AT: string; + UPDATED_AT: string; +} + +export default function NoteDetailPage() { + const { request } = useApi(); + const router = useRouter(); + const params = useParams(); + const id = params.id as string; + + const [note, setNote] = useState(null); + const [loading, setLoading] = useState(true); + const [editing, setEditing] = useState(false); + const [editTitle, setEditTitle] = useState(""); + const [editContent, setEditContent] = useState(""); + const [saving, setSaving] = useState(false); + const [deleting, setDeleting] = useState(false); + const [polishing, setPolishing] = useState(false); + const [showRaw, setShowRaw] = useState(false); + + const fetchNote = async () => { + try { + const data = await request({ method: "GET", url: `/api/notes/${id}` }); + setNote(data); + setEditTitle(data.TITLE || ""); + setEditContent(data.CONTENT || ""); + } catch (err) { + console.error("Failed to load note:", err); + } finally { + setLoading(false); + } + }; + + useEffect(() => { + fetchNote(); + }, [id]); + + // TRANSCRIBING 상태면 3초 폴링 + useEffect(() => { + if (!note || note.NOTE_TYPE !== "TRANSCRIBING") return; + const interval = setInterval(fetchNote, 3000); + return () => clearInterval(interval); + }, [note?.NOTE_TYPE]); + + const handleSave = async () => { + setSaving(true); + try { + const updated = await request({ + method: "PATCH", + url: `/api/notes/${id}`, + data: { title: editTitle, content: editContent }, + }); + setNote(updated); + setEditing(false); + } catch (err) { + console.error("Failed to update note:", err); + alert("저장에 실패했습니다."); + } finally { + setSaving(false); + } + }; + + const handleDelete = async () => { + if (!confirm("정말 삭제하시겠습니까?")) return; + setDeleting(true); + try { + await request({ method: "DELETE", url: `/api/notes/${id}` }); + router.push("/notes"); + } catch (err) { + console.error("Failed to delete note:", err); + setDeleting(false); + } + }; + + if (loading) { + return ( + +
+

Loading...

+
+ + ); + } + + if (!note) { + return ( + +
+

노트를 찾을 수 없습니다.

+ +
+ + ); + } + + return ( + + +
+ + + {/* 헤더 */} +
+
+ + {note.NOTE_TYPE === "TRANSCRIBING" ? "변환 중..." : + note.NOTE_TYPE === "AUDIO_FAILED" ? "변환 실패" : + note.NOTE_TYPE === "AUDIO" ? "음성" : "텍스트"} + + {note.CATEGORY_PATH && ( + + {note.CATEGORY_PATH} + + )} +
+ + {editing ? ( + setEditTitle(e.target.value)} + className="w-full px-3 py-1 rounded-lg bg-[var(--color-bg-hover)] border border-[var(--color-border)] focus:border-[var(--color-primary)] focus:outline-none text-xl font-bold mb-2" + /> + ) : ( +

{note.TITLE || "제목 없음"}

+ )} + +
+ 생성: {new Date(note.CREATED_AT).toLocaleString("ko-KR")} + 수정: {new Date(note.UPDATED_AT).toLocaleString("ko-KR")} +
+
+ + {/* 변환 중 인디케이터 */} + {note.NOTE_TYPE === "TRANSCRIBING" && ( +
+
+ 음성 변환 중입니다. 이 페이지에서 실시간으로 진행 상태를 확인할 수 있습니다. +
+ )} + + {/* 내용 */} +
+ {editing ? ( +