Add admin features, responsive UI, user reviews, visit stats, and channel-colored markers
- Admin: video management with Google Maps match status, manual restaurant mapping, restaurant remap on name change - Admin: user management tab with favorites/reviews detail - Admin: channel deletion fix for IDs with slashes - Frontend: responsive mobile layout (map top, list bottom, 2-row header) - Frontend: channel-colored map markers with legend - Frontend: my reviews list, favorites toggle, visit counter overlay - Frontend: force light mode for dark theme devices - Backend: visit tracking (site_visits table), user reviews endpoint - Backend: bulk transcript/extract streaming, geocode key fixes - Nginx config for production deployment Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
joungmin
93
backend/api/routes/admin_users.py
Normal file
93
backend/api/routes/admin_users.py
Normal file
@@ -0,0 +1,93 @@
|
||||
"""Admin user management API routes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, Query
|
||||
|
||||
from core.db import conn
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.get("")
|
||||
def list_users(
|
||||
limit: int = Query(50, le=200),
|
||||
offset: int = Query(0, ge=0),
|
||||
):
|
||||
"""List all users with favorite/review counts."""
|
||||
sql = """
|
||||
SELECT u.id, u.email, u.nickname, u.avatar_url, u.provider, u.created_at,
|
||||
NVL(fav.cnt, 0) AS favorite_count,
|
||||
NVL(rev.cnt, 0) AS review_count
|
||||
FROM tasteby_users u
|
||||
LEFT JOIN (
|
||||
SELECT user_id, COUNT(*) AS cnt FROM user_favorites GROUP BY user_id
|
||||
) fav ON fav.user_id = u.id
|
||||
LEFT JOIN (
|
||||
SELECT user_id, COUNT(*) AS cnt FROM user_reviews GROUP BY user_id
|
||||
) rev ON rev.user_id = u.id
|
||||
ORDER BY u.created_at DESC
|
||||
OFFSET :off ROWS FETCH NEXT :lim ROWS ONLY
|
||||
"""
|
||||
count_sql = "SELECT COUNT(*) FROM tasteby_users"
|
||||
with conn() as c:
|
||||
cur = c.cursor()
|
||||
cur.execute(count_sql)
|
||||
total = cur.fetchone()[0]
|
||||
cur.execute(sql, {"off": offset, "lim": limit})
|
||||
cols = [d[0].lower() for d in cur.description]
|
||||
rows = [dict(zip(cols, row)) for row in cur.fetchall()]
|
||||
for r in rows:
|
||||
if r.get("created_at"):
|
||||
r["created_at"] = r["created_at"].isoformat()
|
||||
return {"users": rows, "total": total}
|
||||
|
||||
|
||||
@router.get("/{user_id}/favorites")
|
||||
def get_user_favorites(user_id: str):
|
||||
"""Get a user's favorite restaurants."""
|
||||
sql = """
|
||||
SELECT r.id, r.name, r.address, r.region, r.cuisine_type,
|
||||
r.rating, r.business_status, f.created_at
|
||||
FROM user_favorites f
|
||||
JOIN restaurants r ON r.id = f.restaurant_id
|
||||
WHERE f.user_id = :u
|
||||
ORDER BY f.created_at DESC
|
||||
"""
|
||||
with conn() as c:
|
||||
cur = c.cursor()
|
||||
cur.execute(sql, {"u": user_id})
|
||||
cols = [d[0].lower() for d in cur.description]
|
||||
rows = [dict(zip(cols, row)) for row in cur.fetchall()]
|
||||
for r in rows:
|
||||
if r.get("created_at"):
|
||||
r["created_at"] = r["created_at"].isoformat()
|
||||
return rows
|
||||
|
||||
|
||||
@router.get("/{user_id}/reviews")
|
||||
def get_user_reviews(user_id: str):
|
||||
"""Get a user's reviews with restaurant names."""
|
||||
sql = """
|
||||
SELECT r.id, r.restaurant_id, r.rating, r.review_text,
|
||||
r.visited_at, r.created_at,
|
||||
rest.name AS restaurant_name
|
||||
FROM user_reviews r
|
||||
LEFT JOIN restaurants rest ON rest.id = r.restaurant_id
|
||||
WHERE r.user_id = :u
|
||||
ORDER BY r.created_at DESC
|
||||
"""
|
||||
with conn() as c:
|
||||
cur = c.cursor()
|
||||
cur.execute(sql, {"u": user_id})
|
||||
cols = [d[0].lower() for d in cur.description]
|
||||
rows = [dict(zip(cols, row)) for row in cur.fetchall()]
|
||||
for r in rows:
|
||||
# Handle CLOB
|
||||
if hasattr(r.get("review_text"), "read"):
|
||||
r["review_text"] = r["review_text"].read()
|
||||
if r.get("visited_at"):
|
||||
r["visited_at"] = r["visited_at"].isoformat()
|
||||
if r.get("created_at"):
|
||||
r["created_at"] = r["created_at"].isoformat()
|
||||
return rows
|
||||
@@ -1,16 +1,24 @@
|
||||
"""Channel API routes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from pydantic import BaseModel
|
||||
|
||||
from core import youtube
|
||||
|
||||
_executor = ThreadPoolExecutor(max_workers=4)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
class ChannelCreate(BaseModel):
|
||||
channel_id: str
|
||||
channel_name: str
|
||||
title_filter: str | None = None
|
||||
|
||||
|
||||
@router.get("")
|
||||
@@ -21,7 +29,7 @@ def list_channels():
|
||||
@router.post("", status_code=201)
|
||||
def create_channel(body: ChannelCreate):
|
||||
try:
|
||||
row_id = youtube.add_channel(body.channel_id, body.channel_name)
|
||||
row_id = youtube.add_channel(body.channel_id, body.channel_name, body.title_filter)
|
||||
return {"id": row_id, "channel_id": body.channel_id}
|
||||
except Exception as e:
|
||||
if "UQ_CHANNELS_CID" in str(e).upper():
|
||||
@@ -29,18 +37,53 @@ def create_channel(body: ChannelCreate):
|
||||
raise
|
||||
|
||||
|
||||
@router.post("/{channel_id}/scan")
|
||||
def scan_channel(channel_id: str):
|
||||
"""Trigger a scan for new videos from this channel."""
|
||||
def _do_scan(channel_id: str, full: bool):
|
||||
"""Sync scan logic, runs in thread pool."""
|
||||
channels = youtube.get_active_channels()
|
||||
ch = next((c for c in channels if c["channel_id"] == channel_id), None)
|
||||
if not ch:
|
||||
raise HTTPException(404, "Channel not found")
|
||||
return None
|
||||
|
||||
videos = youtube.fetch_channel_videos(channel_id, max_results=50)
|
||||
new_count = 0
|
||||
for v in videos:
|
||||
row_id = youtube.save_video(ch["id"], v)
|
||||
if row_id:
|
||||
new_count += 1
|
||||
return {"total_fetched": len(videos), "new_videos": new_count}
|
||||
after = None if full else youtube.get_latest_video_date(ch["id"])
|
||||
title_filter = ch.get("title_filter")
|
||||
existing_vids = youtube.get_existing_video_ids(ch["id"])
|
||||
|
||||
candidates = []
|
||||
total_fetched = 0
|
||||
for videos_page in youtube.fetch_channel_videos_iter(channel_id, published_after=after):
|
||||
total_fetched += len(videos_page)
|
||||
new_in_page = 0
|
||||
for v in videos_page:
|
||||
if title_filter and title_filter not in v["title"]:
|
||||
continue
|
||||
if v["video_id"] in existing_vids:
|
||||
continue
|
||||
candidates.append(v)
|
||||
new_in_page += 1
|
||||
if not full and new_in_page == 0 and total_fetched > 50:
|
||||
break
|
||||
|
||||
new_count = youtube.save_videos_batch(ch["id"], candidates)
|
||||
return {"total_fetched": total_fetched, "new_videos": new_count}
|
||||
|
||||
|
||||
@router.post("/{channel_id}/scan")
|
||||
async def scan_channel(channel_id: str, full: bool = False):
|
||||
"""Trigger a scan for new videos from this channel (non-blocking)."""
|
||||
loop = asyncio.get_event_loop()
|
||||
result = await loop.run_in_executor(_executor, _do_scan, channel_id, full)
|
||||
if result is None:
|
||||
raise HTTPException(404, "Channel not found")
|
||||
return result
|
||||
|
||||
|
||||
@router.delete("/{channel_id:path}")
|
||||
def delete_channel(channel_id: str):
|
||||
"""Deactivate a channel. Accepts channel_id or DB id."""
|
||||
deleted = youtube.deactivate_channel(channel_id)
|
||||
if not deleted:
|
||||
# Try by DB id
|
||||
deleted = youtube.deactivate_channel_by_db_id(channel_id)
|
||||
if not deleted:
|
||||
raise HTTPException(404, "Channel not found")
|
||||
return {"ok": True}
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
"""Restaurant API routes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Query
|
||||
|
||||
from core import restaurant
|
||||
@@ -13,8 +15,9 @@ def list_restaurants(
|
||||
offset: int = Query(0, ge=0),
|
||||
cuisine: str | None = None,
|
||||
region: str | None = None,
|
||||
channel: str | None = None,
|
||||
):
|
||||
return restaurant.get_all(limit=limit, offset=offset, cuisine=cuisine, region=region)
|
||||
return restaurant.get_all(limit=limit, offset=offset, cuisine=cuisine, region=region, channel=channel)
|
||||
|
||||
|
||||
@router.get("/{restaurant_id}")
|
||||
@@ -25,6 +28,45 @@ def get_restaurant(restaurant_id: str):
|
||||
return r
|
||||
|
||||
|
||||
@router.put("/{restaurant_id}")
|
||||
def update_restaurant(restaurant_id: str, body: dict):
|
||||
from core.db import conn
|
||||
r = restaurant.get_by_id(restaurant_id)
|
||||
if not r:
|
||||
raise HTTPException(404, "Restaurant not found")
|
||||
|
||||
allowed = ("name", "address", "region", "cuisine_type", "price_range",
|
||||
"phone", "website", "latitude", "longitude")
|
||||
sets = []
|
||||
params: dict = {"rid": restaurant_id}
|
||||
for field in allowed:
|
||||
if field in body:
|
||||
sets.append(f"{field} = :{field}")
|
||||
params[field] = body[field] if body[field] != "" else None
|
||||
if not sets:
|
||||
raise HTTPException(400, "No fields to update")
|
||||
sets.append("updated_at = SYSTIMESTAMP")
|
||||
sql = f"UPDATE restaurants SET {', '.join(sets)} WHERE id = :rid"
|
||||
with conn() as c:
|
||||
c.cursor().execute(sql, params)
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
@router.delete("/{restaurant_id}")
|
||||
def delete_restaurant(restaurant_id: str):
|
||||
from core.db import conn
|
||||
r = restaurant.get_by_id(restaurant_id)
|
||||
if not r:
|
||||
raise HTTPException(404, "Restaurant not found")
|
||||
with conn() as c:
|
||||
cur = c.cursor()
|
||||
cur.execute("DELETE FROM restaurant_vectors WHERE restaurant_id = :rid", {"rid": restaurant_id})
|
||||
cur.execute("DELETE FROM user_reviews WHERE restaurant_id = :rid", {"rid": restaurant_id})
|
||||
cur.execute("DELETE FROM video_restaurants WHERE restaurant_id = :rid", {"rid": restaurant_id})
|
||||
cur.execute("DELETE FROM restaurants WHERE id = :rid", {"rid": restaurant_id})
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
@router.get("/{restaurant_id}/videos")
|
||||
def get_restaurant_videos(restaurant_id: str):
|
||||
r = restaurant.get_by_id(restaurant_id)
|
||||
|
||||
@@ -95,3 +95,74 @@ def list_my_reviews(
|
||||
):
|
||||
"""List current user's reviews."""
|
||||
return review.get_user_reviews(current_user["sub"], limit=limit, offset=offset)
|
||||
|
||||
|
||||
# --- Favorites ---
|
||||
|
||||
@router.get("/restaurants/{restaurant_id}/favorite")
|
||||
def get_favorite_status(
|
||||
restaurant_id: str,
|
||||
current_user: dict = Depends(get_current_user),
|
||||
):
|
||||
"""Check if current user has favorited this restaurant."""
|
||||
from core.db import conn
|
||||
with conn() as c:
|
||||
cur = c.cursor()
|
||||
cur.execute(
|
||||
"SELECT id FROM user_favorites WHERE user_id = :u AND restaurant_id = :r",
|
||||
{"u": current_user["sub"], "r": restaurant_id},
|
||||
)
|
||||
return {"favorited": cur.fetchone() is not None}
|
||||
|
||||
|
||||
@router.post("/restaurants/{restaurant_id}/favorite")
|
||||
def toggle_favorite(
|
||||
restaurant_id: str,
|
||||
current_user: dict = Depends(get_current_user),
|
||||
):
|
||||
"""Toggle favorite. Returns new state."""
|
||||
from core.db import conn
|
||||
user_id = current_user["sub"]
|
||||
with conn() as c:
|
||||
cur = c.cursor()
|
||||
cur.execute(
|
||||
"SELECT id FROM user_favorites WHERE user_id = :u AND restaurant_id = :r",
|
||||
{"u": user_id, "r": restaurant_id},
|
||||
)
|
||||
row = cur.fetchone()
|
||||
if row:
|
||||
cur.execute("DELETE FROM user_favorites WHERE id = :fid", {"fid": row[0]})
|
||||
return {"favorited": False}
|
||||
else:
|
||||
cur.execute(
|
||||
"INSERT INTO user_favorites (user_id, restaurant_id) VALUES (:u, :r)",
|
||||
{"u": user_id, "r": restaurant_id},
|
||||
)
|
||||
return {"favorited": True}
|
||||
|
||||
|
||||
@router.get("/users/me/favorites")
|
||||
def list_my_favorites(
|
||||
current_user: dict = Depends(get_current_user),
|
||||
):
|
||||
"""List current user's favorite restaurants."""
|
||||
from core.db import conn
|
||||
sql = """
|
||||
SELECT r.id, r.name, r.address, r.region, r.latitude, r.longitude,
|
||||
r.cuisine_type, r.price_range, r.google_place_id,
|
||||
r.business_status, r.rating, r.rating_count,
|
||||
f.created_at
|
||||
FROM user_favorites f
|
||||
JOIN restaurants r ON r.id = f.restaurant_id
|
||||
WHERE f.user_id = :u
|
||||
ORDER BY f.created_at DESC
|
||||
"""
|
||||
with conn() as c:
|
||||
cur = c.cursor()
|
||||
cur.execute(sql, {"u": current_user["sub"]})
|
||||
cols = [d[0].lower() for d in cur.description]
|
||||
rows = [dict(zip(cols, row)) for row in cur.fetchall()]
|
||||
for r in rows:
|
||||
if r.get("created_at"):
|
||||
r["created_at"] = r["created_at"].isoformat()
|
||||
return rows
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
"""Search API routes — keyword + semantic search."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, Query
|
||||
|
||||
from core import restaurant, vector
|
||||
|
||||
43
backend/api/routes/stats.py
Normal file
43
backend/api/routes/stats.py
Normal file
@@ -0,0 +1,43 @@
|
||||
"""Site visit statistics API."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter
|
||||
|
||||
from core.db import conn
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.post("/visit")
|
||||
def record_visit():
|
||||
"""Record a page visit. Increments today's count."""
|
||||
sql = """
|
||||
MERGE INTO site_visits sv
|
||||
USING (SELECT TRUNC(SYSDATE) AS d FROM dual) src
|
||||
ON (sv.visit_date = src.d)
|
||||
WHEN MATCHED THEN UPDATE SET sv.visit_count = sv.visit_count + 1
|
||||
WHEN NOT MATCHED THEN INSERT (visit_date, visit_count) VALUES (src.d, 1)
|
||||
"""
|
||||
with conn() as c:
|
||||
c.cursor().execute(sql)
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
@router.get("/visits")
|
||||
def get_visits():
|
||||
"""Return today's visit count and all-time total."""
|
||||
sql_today = """
|
||||
SELECT NVL(visit_count, 0) FROM site_visits WHERE visit_date = TRUNC(SYSDATE)
|
||||
"""
|
||||
sql_total = """
|
||||
SELECT NVL(SUM(visit_count), 0) FROM site_visits
|
||||
"""
|
||||
with conn() as c:
|
||||
cur = c.cursor()
|
||||
cur.execute(sql_today)
|
||||
row = cur.fetchone()
|
||||
today = int(row[0]) if row else 0
|
||||
cur.execute(sql_total)
|
||||
total = int(cur.fetchone()[0])
|
||||
return {"today": today, "total": total}
|
||||
@@ -1,17 +1,29 @@
|
||||
"""Video API routes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json as _json
|
||||
import logging
|
||||
import random
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
from fastapi import APIRouter, Query
|
||||
from fastapi.responses import StreamingResponse
|
||||
|
||||
from core.db import conn
|
||||
from core.pipeline import process_pending
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter()
|
||||
_executor = ThreadPoolExecutor(max_workers=4)
|
||||
|
||||
|
||||
@router.get("")
|
||||
def list_videos(
|
||||
status: str | None = None,
|
||||
limit: int = Query(50, le=200),
|
||||
limit: int = Query(50, le=500),
|
||||
offset: int = Query(0, ge=0),
|
||||
):
|
||||
conditions = []
|
||||
@@ -23,7 +35,11 @@ def list_videos(
|
||||
where = ("WHERE " + " AND ".join(conditions)) if conditions else ""
|
||||
sql = f"""
|
||||
SELECT v.id, v.video_id, v.title, v.url, v.status,
|
||||
v.published_at, c.channel_name
|
||||
v.published_at, c.channel_name,
|
||||
CASE WHEN v.transcript_text IS NOT NULL AND dbms_lob.getlength(v.transcript_text) > 0 THEN 1 ELSE 0 END as has_transcript,
|
||||
CASE WHEN v.llm_raw_response IS NOT NULL AND dbms_lob.getlength(v.llm_raw_response) > 0 THEN 1 ELSE 0 END as has_llm,
|
||||
(SELECT COUNT(*) FROM video_restaurants vr WHERE vr.video_id = v.id) as restaurant_count,
|
||||
(SELECT COUNT(*) FROM video_restaurants vr JOIN restaurants r ON r.id = vr.restaurant_id WHERE vr.video_id = v.id AND r.google_place_id IS NOT NULL) as matched_count
|
||||
FROM videos v
|
||||
JOIN channels c ON c.id = v.channel_id
|
||||
{where}
|
||||
@@ -41,12 +57,604 @@ def list_videos(
|
||||
d = dict(zip(cols, row))
|
||||
if d.get("published_at"):
|
||||
d["published_at"] = d["published_at"].isoformat()
|
||||
d["has_transcript"] = bool(d.get("has_transcript"))
|
||||
d["has_llm"] = bool(d.get("has_llm"))
|
||||
d["restaurant_count"] = d.get("restaurant_count", 0)
|
||||
d["matched_count"] = d.get("matched_count", 0)
|
||||
results.append(d)
|
||||
return results
|
||||
|
||||
|
||||
def _get_unprocessed_videos() -> list[dict]:
|
||||
"""Get videos that have transcripts but no LLM extraction."""
|
||||
sql = """
|
||||
SELECT v.id, v.video_id, v.title, v.url, v.transcript_text
|
||||
FROM videos v
|
||||
WHERE v.transcript_text IS NOT NULL
|
||||
AND dbms_lob.getlength(v.transcript_text) > 0
|
||||
AND (v.llm_raw_response IS NULL OR dbms_lob.getlength(v.llm_raw_response) = 0)
|
||||
AND v.status != 'skip'
|
||||
ORDER BY v.published_at DESC
|
||||
"""
|
||||
with conn() as c:
|
||||
cur = c.cursor()
|
||||
cur.execute(sql)
|
||||
rows = cur.fetchall()
|
||||
result = []
|
||||
for r in rows:
|
||||
transcript = r[4]
|
||||
if hasattr(transcript, "read"):
|
||||
transcript = transcript.read()
|
||||
result.append({
|
||||
"id": r[0], "video_id": r[1], "title": r[2], "url": r[3],
|
||||
"transcript": transcript,
|
||||
})
|
||||
return result
|
||||
|
||||
|
||||
@router.get("/bulk-extract/pending")
|
||||
def bulk_extract_pending_count():
|
||||
"""Get count of videos pending LLM extraction."""
|
||||
videos = _get_unprocessed_videos()
|
||||
return {"count": len(videos), "videos": [{"id": v["id"], "title": v["title"]} for v in videos]}
|
||||
|
||||
|
||||
@router.post("/bulk-extract")
|
||||
def bulk_extract():
|
||||
"""Process all unextracted videos with random delays. Streams SSE progress."""
|
||||
from core.pipeline import process_video_extract
|
||||
|
||||
videos = _get_unprocessed_videos()
|
||||
|
||||
def generate():
|
||||
total = len(videos)
|
||||
total_restaurants = 0
|
||||
yield f"data: {_json.dumps({'type': 'start', 'total': total})}\n\n"
|
||||
|
||||
for i, v in enumerate(videos):
|
||||
# Random delay (3-8 seconds) between requests to avoid bot detection
|
||||
if i > 0:
|
||||
delay = random.uniform(3.0, 8.0)
|
||||
yield f"data: {_json.dumps({'type': 'wait', 'index': i, 'delay': round(delay, 1)})}\n\n"
|
||||
time.sleep(delay)
|
||||
|
||||
yield f"data: {_json.dumps({'type': 'processing', 'index': i, 'title': v['title']})}\n\n"
|
||||
|
||||
try:
|
||||
count = process_video_extract(
|
||||
{"id": v["id"], "video_id": v["video_id"], "title": v["title"], "url": v["url"]},
|
||||
v["transcript"],
|
||||
)
|
||||
total_restaurants += count
|
||||
yield f"data: {_json.dumps({'type': 'done', 'index': i, 'title': v['title'], 'restaurants': count})}\n\n"
|
||||
except Exception as e:
|
||||
logger.error("Bulk extract error for %s: %s", v["video_id"], e)
|
||||
yield f"data: {_json.dumps({'type': 'error', 'index': i, 'title': v['title'], 'message': str(e)})}\n\n"
|
||||
|
||||
yield f"data: {_json.dumps({'type': 'complete', 'total': total, 'total_restaurants': total_restaurants})}\n\n"
|
||||
|
||||
return StreamingResponse(generate(), media_type="text/event-stream")
|
||||
|
||||
|
||||
def _get_no_transcript_videos() -> list[dict]:
|
||||
"""Get videos that have no transcript yet."""
|
||||
sql = """
|
||||
SELECT v.id, v.video_id, v.title
|
||||
FROM videos v
|
||||
WHERE (v.transcript_text IS NULL OR dbms_lob.getlength(v.transcript_text) = 0)
|
||||
AND v.status != 'skip'
|
||||
ORDER BY v.published_at DESC
|
||||
"""
|
||||
with conn() as c:
|
||||
cur = c.cursor()
|
||||
cur.execute(sql)
|
||||
return [{"id": r[0], "video_id": r[1], "title": r[2]} for r in cur.fetchall()]
|
||||
|
||||
|
||||
@router.get("/bulk-transcript/pending")
|
||||
def bulk_transcript_pending_count():
|
||||
"""Get count of videos without transcripts."""
|
||||
videos = _get_no_transcript_videos()
|
||||
return {"count": len(videos), "videos": [{"id": v["id"], "title": v["title"]} for v in videos]}
|
||||
|
||||
|
||||
@router.post("/bulk-transcript")
|
||||
def bulk_transcript():
|
||||
"""Fetch transcripts for all videos missing them. Streams SSE progress."""
|
||||
from core.youtube import get_transcript
|
||||
|
||||
videos = _get_no_transcript_videos()
|
||||
|
||||
def generate():
|
||||
total = len(videos)
|
||||
success = 0
|
||||
yield f"data: {_json.dumps({'type': 'start', 'total': total})}\n\n"
|
||||
|
||||
for i, v in enumerate(videos):
|
||||
# Random delay (5-15 seconds) to avoid bot detection — longer than LLM since this hits YouTube
|
||||
if i > 0:
|
||||
delay = random.uniform(5.0, 15.0)
|
||||
yield f"data: {_json.dumps({'type': 'wait', 'index': i, 'delay': round(delay, 1)})}\n\n"
|
||||
time.sleep(delay)
|
||||
|
||||
yield f"data: {_json.dumps({'type': 'processing', 'index': i, 'title': v['title']})}\n\n"
|
||||
|
||||
try:
|
||||
transcript, source = get_transcript(v["video_id"])
|
||||
if transcript:
|
||||
with conn() as c:
|
||||
cur = c.cursor()
|
||||
cur.execute(
|
||||
"UPDATE videos SET transcript_text = :txt WHERE id = :vid",
|
||||
{"txt": transcript, "vid": v["id"]},
|
||||
)
|
||||
success += 1
|
||||
yield f"data: {_json.dumps({'type': 'done', 'index': i, 'title': v['title'], 'source': source, 'length': len(transcript)})}\n\n"
|
||||
else:
|
||||
yield f"data: {_json.dumps({'type': 'error', 'index': i, 'title': v['title'], 'message': 'No transcript available'})}\n\n"
|
||||
except Exception as e:
|
||||
logger.error("Bulk transcript error for %s: %s", v["video_id"], e)
|
||||
yield f"data: {_json.dumps({'type': 'error', 'index': i, 'title': v['title'], 'message': str(e)})}\n\n"
|
||||
|
||||
yield f"data: {_json.dumps({'type': 'complete', 'total': total, 'success': success})}\n\n"
|
||||
|
||||
return StreamingResponse(generate(), media_type="text/event-stream")
|
||||
|
||||
|
||||
@router.get("/extract/prompt")
|
||||
def get_extract_prompt():
|
||||
"""Get the current LLM extraction prompt template."""
|
||||
from core.extractor import _EXTRACT_PROMPT
|
||||
return {"prompt": _EXTRACT_PROMPT}
|
||||
|
||||
|
||||
def _do_process(limit: int):
|
||||
return {"restaurants_extracted": process_pending(limit)}
|
||||
|
||||
|
||||
@router.post("/process")
|
||||
def trigger_processing(limit: int = Query(5, le=20)):
|
||||
"""Manually trigger processing of pending videos."""
|
||||
count = process_pending(limit)
|
||||
return {"restaurants_extracted": count}
|
||||
async def trigger_processing(limit: int = Query(5, le=20)):
|
||||
"""Manually trigger processing of pending videos (non-blocking)."""
|
||||
loop = asyncio.get_event_loop()
|
||||
return await loop.run_in_executor(_executor, _do_process, limit)
|
||||
|
||||
|
||||
@router.get("/{video_db_id}")
|
||||
def get_video_detail(video_db_id: str):
|
||||
"""Get video detail including transcript and extracted restaurants."""
|
||||
from fastapi import HTTPException
|
||||
import json
|
||||
|
||||
with conn() as c:
|
||||
cur = c.cursor()
|
||||
cur.execute("""
|
||||
SELECT v.id, v.video_id, v.title, v.url, v.status,
|
||||
v.published_at, v.transcript_text,
|
||||
c.channel_name
|
||||
FROM videos v
|
||||
JOIN channels c ON c.id = v.channel_id
|
||||
WHERE v.id = :vid
|
||||
""", {"vid": video_db_id})
|
||||
row = cur.fetchone()
|
||||
if not row:
|
||||
raise HTTPException(404, "Video not found")
|
||||
|
||||
transcript = row[6]
|
||||
if hasattr(transcript, "read"):
|
||||
transcript = transcript.read()
|
||||
|
||||
# Get extracted restaurants for this video
|
||||
cur.execute("""
|
||||
SELECT r.id, r.name, r.address, r.cuisine_type, r.price_range, r.region,
|
||||
vr.foods_mentioned, vr.evaluation, vr.guests,
|
||||
r.google_place_id, r.latitude, r.longitude
|
||||
FROM video_restaurants vr
|
||||
JOIN restaurants r ON r.id = vr.restaurant_id
|
||||
WHERE vr.video_id = :vid
|
||||
""", {"vid": video_db_id})
|
||||
|
||||
restaurants = []
|
||||
for rr in cur.fetchall():
|
||||
foods_raw = rr[6].read() if hasattr(rr[6], "read") else rr[6]
|
||||
eval_raw = rr[7].read() if hasattr(rr[7], "read") else rr[7]
|
||||
guests_raw = rr[8].read() if hasattr(rr[8], "read") else rr[8]
|
||||
|
||||
def parse_json(val, default):
|
||||
if val is None:
|
||||
return default
|
||||
if isinstance(val, (list, dict)):
|
||||
return val
|
||||
try:
|
||||
return json.loads(val)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
return default
|
||||
|
||||
restaurants.append({
|
||||
"restaurant_id": rr[0],
|
||||
"name": rr[1],
|
||||
"address": rr[2],
|
||||
"cuisine_type": rr[3],
|
||||
"price_range": rr[4],
|
||||
"region": rr[5],
|
||||
"foods_mentioned": parse_json(foods_raw, []),
|
||||
"evaluation": parse_json(eval_raw, {}),
|
||||
"guests": parse_json(guests_raw, []),
|
||||
"google_place_id": rr[9],
|
||||
"has_location": rr[10] is not None and rr[11] is not None,
|
||||
})
|
||||
|
||||
return {
|
||||
"id": row[0],
|
||||
"video_id": row[1],
|
||||
"title": row[2],
|
||||
"url": row[3],
|
||||
"status": row[4],
|
||||
"published_at": row[5].isoformat() if row[5] else None,
|
||||
"transcript": transcript,
|
||||
"channel_name": row[7],
|
||||
"restaurants": restaurants,
|
||||
}
|
||||
|
||||
|
||||
def _do_fetch_transcript(video_db_id: str, mode: str):
|
||||
from core.youtube import get_transcript
|
||||
|
||||
with conn() as c:
|
||||
cur = c.cursor()
|
||||
cur.execute("SELECT video_id FROM videos WHERE id = :vid", {"vid": video_db_id})
|
||||
row = cur.fetchone()
|
||||
if not row:
|
||||
return {"error": 404, "detail": "Video not found"}
|
||||
video_id = row[0]
|
||||
|
||||
transcript, source = get_transcript(video_id, mode=mode)
|
||||
if not transcript:
|
||||
return {"error": 422, "detail": "Transcript unavailable for this video"}
|
||||
|
||||
with conn() as c:
|
||||
cur = c.cursor()
|
||||
cur.execute(
|
||||
"UPDATE videos SET transcript_text = :txt WHERE id = :vid",
|
||||
{"txt": transcript, "vid": video_db_id},
|
||||
)
|
||||
|
||||
return {"ok": True, "length": len(transcript), "source": source}
|
||||
|
||||
|
||||
@router.post("/{video_db_id}/fetch-transcript")
|
||||
async def fetch_transcript(video_db_id: str, mode: str = Query("auto")):
|
||||
"""Fetch and save transcript for a video (non-blocking)."""
|
||||
from fastapi import HTTPException
|
||||
|
||||
if mode not in ("auto", "manual", "generated"):
|
||||
raise HTTPException(400, "mode must be auto, manual, or generated")
|
||||
|
||||
loop = asyncio.get_event_loop()
|
||||
result = await loop.run_in_executor(_executor, _do_fetch_transcript, video_db_id, mode)
|
||||
if "error" in result:
|
||||
raise HTTPException(result["error"], result["detail"])
|
||||
return result
|
||||
|
||||
|
||||
def _do_extract(video_db_id: str, custom_prompt: str | None):
|
||||
from core.pipeline import process_video_extract
|
||||
|
||||
with conn() as c:
|
||||
cur = c.cursor()
|
||||
cur.execute(
|
||||
"SELECT id, video_id, title, url, transcript_text FROM videos WHERE id = :vid",
|
||||
{"vid": video_db_id},
|
||||
)
|
||||
row = cur.fetchone()
|
||||
if not row:
|
||||
return {"error": 404, "detail": "Video not found"}
|
||||
transcript = row[4]
|
||||
if hasattr(transcript, "read"):
|
||||
transcript = transcript.read()
|
||||
if not transcript:
|
||||
return {"error": 422, "detail": "No transcript available for this video"}
|
||||
|
||||
count = process_video_extract(
|
||||
{"id": row[0], "video_id": row[1], "title": row[2], "url": row[3]},
|
||||
transcript,
|
||||
custom_prompt=custom_prompt,
|
||||
)
|
||||
return {"ok": True, "restaurants_extracted": count}
|
||||
|
||||
|
||||
@router.post("/{video_db_id}/extract")
|
||||
async def extract_restaurants_from_video(video_db_id: str, body: dict = None):
|
||||
"""Run LLM extraction on an existing transcript (non-blocking)."""
|
||||
from fastapi import HTTPException
|
||||
custom_prompt = body.get("prompt") if body else None
|
||||
loop = asyncio.get_event_loop()
|
||||
result = await loop.run_in_executor(_executor, _do_extract, video_db_id, custom_prompt)
|
||||
if "error" in result:
|
||||
raise HTTPException(result["error"], result["detail"])
|
||||
return result
|
||||
|
||||
|
||||
@router.post("/{video_db_id}/skip")
|
||||
def skip_video(video_db_id: str):
|
||||
"""Mark a video as skipped."""
|
||||
from fastapi import HTTPException
|
||||
with conn() as c:
|
||||
cur = c.cursor()
|
||||
cur.execute(
|
||||
"UPDATE videos SET status = 'skip' WHERE id = :vid",
|
||||
{"vid": video_db_id},
|
||||
)
|
||||
if cur.rowcount == 0:
|
||||
raise HTTPException(404, "Video not found")
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
@router.delete("/{video_db_id}")
|
||||
def delete_video(video_db_id: str):
|
||||
"""Delete a video and its related data."""
|
||||
from core.db import conn as get_conn
|
||||
with get_conn() as c:
|
||||
cur = c.cursor()
|
||||
# Delete vector embeddings for restaurants only linked to this video
|
||||
cur.execute("""
|
||||
DELETE FROM restaurant_vectors
|
||||
WHERE restaurant_id IN (
|
||||
SELECT vr.restaurant_id FROM video_restaurants vr
|
||||
WHERE vr.video_id = :vid
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM video_restaurants vr2
|
||||
WHERE vr2.restaurant_id = vr.restaurant_id
|
||||
AND vr2.video_id != :vid
|
||||
)
|
||||
)
|
||||
""", {"vid": video_db_id})
|
||||
# Delete reviews for restaurants only linked to this video
|
||||
cur.execute("""
|
||||
DELETE FROM user_reviews
|
||||
WHERE restaurant_id IN (
|
||||
SELECT vr.restaurant_id FROM video_restaurants vr
|
||||
WHERE vr.video_id = :vid
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM video_restaurants vr2
|
||||
WHERE vr2.restaurant_id = vr.restaurant_id
|
||||
AND vr2.video_id != :vid
|
||||
)
|
||||
)
|
||||
""", {"vid": video_db_id})
|
||||
# Delete restaurants only linked to this video
|
||||
cur.execute("""
|
||||
DELETE FROM restaurants
|
||||
WHERE id IN (
|
||||
SELECT vr.restaurant_id FROM video_restaurants vr
|
||||
WHERE vr.video_id = :vid
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM video_restaurants vr2
|
||||
WHERE vr2.restaurant_id = vr.restaurant_id
|
||||
AND vr2.video_id != :vid
|
||||
)
|
||||
)
|
||||
""", {"vid": video_db_id})
|
||||
# Delete video-restaurant links
|
||||
cur.execute("DELETE FROM video_restaurants WHERE video_id = :vid", {"vid": video_db_id})
|
||||
# Delete the video
|
||||
cur.execute("DELETE FROM videos WHERE id = :vid", {"vid": video_db_id})
|
||||
if cur.rowcount == 0:
|
||||
from fastapi import HTTPException
|
||||
raise HTTPException(404, "Video not found")
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
@router.put("/{video_db_id}")
|
||||
def update_video(video_db_id: str, body: dict):
|
||||
"""Update video title."""
|
||||
from fastapi import HTTPException
|
||||
title = body.get("title")
|
||||
if not title:
|
||||
raise HTTPException(400, "title is required")
|
||||
with conn() as c:
|
||||
cur = c.cursor()
|
||||
cur.execute(
|
||||
"UPDATE videos SET title = :title WHERE id = :vid",
|
||||
{"title": title, "vid": video_db_id},
|
||||
)
|
||||
if cur.rowcount == 0:
|
||||
raise HTTPException(404, "Video not found")
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
@router.delete("/{video_db_id}/restaurants/{restaurant_id}")
|
||||
def delete_video_restaurant(video_db_id: str, restaurant_id: str):
|
||||
"""Delete a video-restaurant mapping. Also cleans up orphaned restaurant."""
|
||||
from fastapi import HTTPException
|
||||
with conn() as c:
|
||||
cur = c.cursor()
|
||||
cur.execute(
|
||||
"DELETE FROM video_restaurants WHERE video_id = :vid AND restaurant_id = :rid",
|
||||
{"vid": video_db_id, "rid": restaurant_id},
|
||||
)
|
||||
if cur.rowcount == 0:
|
||||
raise HTTPException(404, "Mapping not found")
|
||||
# Clean up orphaned restaurant (no other video links)
|
||||
cur.execute("""
|
||||
DELETE FROM restaurant_vectors WHERE restaurant_id = :rid
|
||||
AND NOT EXISTS (SELECT 1 FROM video_restaurants WHERE restaurant_id = :rid)
|
||||
""", {"rid": restaurant_id})
|
||||
cur.execute("""
|
||||
DELETE FROM user_reviews WHERE restaurant_id = :rid
|
||||
AND NOT EXISTS (SELECT 1 FROM video_restaurants WHERE restaurant_id = :rid)
|
||||
""", {"rid": restaurant_id})
|
||||
cur.execute("""
|
||||
DELETE FROM restaurants WHERE id = :rid
|
||||
AND NOT EXISTS (SELECT 1 FROM video_restaurants WHERE restaurant_id = :rid)
|
||||
""", {"rid": restaurant_id})
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
@router.post("/{video_db_id}/restaurants/manual")
|
||||
def add_manual_restaurant(video_db_id: str, body: dict):
|
||||
"""Manually add a restaurant and link it to a video."""
|
||||
from fastapi import HTTPException
|
||||
from core import restaurant as rest_mod
|
||||
from core.geocoding import geocode_restaurant
|
||||
|
||||
name = body.get("name", "").strip()
|
||||
if not name:
|
||||
raise HTTPException(400, "Restaurant name is required")
|
||||
|
||||
address = body.get("address", "").strip() or None
|
||||
region = body.get("region", "").strip() or None
|
||||
cuisine_type = body.get("cuisine_type", "").strip() or None
|
||||
price_range = body.get("price_range", "").strip() or None
|
||||
foods = body.get("foods_mentioned", [])
|
||||
evaluation = body.get("evaluation", "").strip() or None
|
||||
guests = body.get("guests", [])
|
||||
|
||||
# Geocode to get lat/lng and Google data
|
||||
geo = geocode_restaurant(name, address or region or "")
|
||||
if not geo:
|
||||
raise HTTPException(400, f"'{name}' 위치를 찾을 수 없습니다. 주소를 입력해주세요.")
|
||||
|
||||
rid = rest_mod.upsert(
|
||||
name=name,
|
||||
address=geo.get("formatted_address") or address,
|
||||
region=region,
|
||||
latitude=geo["latitude"],
|
||||
longitude=geo["longitude"],
|
||||
cuisine_type=cuisine_type,
|
||||
price_range=price_range,
|
||||
google_place_id=geo.get("google_place_id"),
|
||||
phone=geo.get("phone"),
|
||||
website=geo.get("website"),
|
||||
business_status=geo.get("business_status"),
|
||||
rating=geo.get("rating"),
|
||||
rating_count=geo.get("rating_count"),
|
||||
)
|
||||
|
||||
link_id = rest_mod.link_video_restaurant(
|
||||
video_db_id=video_db_id,
|
||||
restaurant_id=rid,
|
||||
foods=foods if isinstance(foods, list) else [],
|
||||
evaluation=evaluation,
|
||||
guests=guests if isinstance(guests, list) else [],
|
||||
)
|
||||
|
||||
return {"ok": True, "restaurant_id": rid, "link_id": link_id}
|
||||
|
||||
|
||||
@router.put("/{video_db_id}/restaurants/{restaurant_id}")
|
||||
def update_video_restaurant(video_db_id: str, restaurant_id: str, body: dict):
|
||||
"""Update restaurant info linked to a video.
|
||||
|
||||
If name changed, re-geocode and remap to a new restaurant record.
|
||||
"""
|
||||
from fastapi import HTTPException
|
||||
import json as _json
|
||||
|
||||
# Check if name changed — need to remap
|
||||
new_name = body.get("name", "").strip() if "name" in body else None
|
||||
if new_name:
|
||||
with conn() as c:
|
||||
cur = c.cursor()
|
||||
cur.execute("SELECT name FROM restaurants WHERE id = :rid", {"rid": restaurant_id})
|
||||
row = cur.fetchone()
|
||||
old_name = row[0] if row else ""
|
||||
|
||||
if old_name != new_name:
|
||||
# Name changed: geocode new restaurant, remap
|
||||
from core import restaurant as rest_mod
|
||||
from core.geocoding import geocode_restaurant
|
||||
|
||||
address = body.get("address", "").strip() or body.get("region", "").strip() or ""
|
||||
geo = geocode_restaurant(new_name, address)
|
||||
if not geo:
|
||||
raise HTTPException(400, f"'{new_name}' 위치를 찾을 수 없습니다.")
|
||||
|
||||
new_rid = rest_mod.upsert(
|
||||
name=new_name,
|
||||
address=geo.get("formatted_address") or body.get("address"),
|
||||
region=body.get("region"),
|
||||
latitude=geo["latitude"],
|
||||
longitude=geo["longitude"],
|
||||
cuisine_type=body.get("cuisine_type"),
|
||||
price_range=body.get("price_range"),
|
||||
google_place_id=geo.get("google_place_id"),
|
||||
phone=geo.get("phone"),
|
||||
website=geo.get("website"),
|
||||
business_status=geo.get("business_status"),
|
||||
rating=geo.get("rating"),
|
||||
rating_count=geo.get("rating_count"),
|
||||
)
|
||||
|
||||
# Read existing mapping data, delete old, create new
|
||||
with conn() as c:
|
||||
cur = c.cursor()
|
||||
cur.execute(
|
||||
"SELECT foods_mentioned, evaluation, guests FROM video_restaurants WHERE video_id = :vid AND restaurant_id = :rid",
|
||||
{"vid": video_db_id, "rid": restaurant_id},
|
||||
)
|
||||
old_vr = cur.fetchone()
|
||||
|
||||
cur.execute(
|
||||
"DELETE FROM video_restaurants WHERE video_id = :vid AND restaurant_id = :rid",
|
||||
{"vid": video_db_id, "rid": restaurant_id},
|
||||
)
|
||||
|
||||
# Build new mapping values from body or old data
|
||||
def _parse(val, default):
|
||||
if val is None:
|
||||
return default
|
||||
if hasattr(val, "read"):
|
||||
val = val.read()
|
||||
if isinstance(val, (list, dict)):
|
||||
return val
|
||||
try:
|
||||
return _json.loads(val)
|
||||
except Exception:
|
||||
return default
|
||||
|
||||
old_foods = _parse(old_vr[0], []) if old_vr else []
|
||||
old_eval = _parse(old_vr[1], {}) if old_vr else {}
|
||||
old_guests = _parse(old_vr[2], []) if old_vr else []
|
||||
|
||||
foods = body.get("foods_mentioned", old_foods)
|
||||
evaluation = body.get("evaluation", old_eval)
|
||||
guests = body.get("guests", old_guests)
|
||||
|
||||
eval_text = evaluation.get("text", "") if isinstance(evaluation, dict) else str(evaluation or "")
|
||||
|
||||
rest_mod.link_video_restaurant(
|
||||
video_db_id=video_db_id,
|
||||
restaurant_id=new_rid,
|
||||
foods=foods if isinstance(foods, list) else [],
|
||||
evaluation=eval_text or None,
|
||||
guests=guests if isinstance(guests, list) else [],
|
||||
)
|
||||
|
||||
return {"ok": True, "remapped": True, "new_restaurant_id": new_rid}
|
||||
|
||||
# No name change — update in place
|
||||
with conn() as c:
|
||||
cur = c.cursor()
|
||||
r_sets = []
|
||||
r_params: dict = {"rid": restaurant_id}
|
||||
for field in ("name", "address", "region", "cuisine_type", "price_range"):
|
||||
if field in body:
|
||||
r_sets.append(f"{field} = :{field}")
|
||||
r_params[field] = body[field]
|
||||
if r_sets:
|
||||
r_sets.append("updated_at = SYSTIMESTAMP")
|
||||
sql = f"UPDATE restaurants SET {', '.join(r_sets)} WHERE id = :rid"
|
||||
cur.execute(sql, r_params)
|
||||
|
||||
vr_params: dict = {"vid": video_db_id, "rid": restaurant_id}
|
||||
vr_sets = []
|
||||
for field in ("foods_mentioned", "evaluation", "guests"):
|
||||
if field in body:
|
||||
vr_sets.append(f"{field} = :{field}")
|
||||
val = body[field]
|
||||
vr_params[field] = _json.dumps(val, ensure_ascii=False) if isinstance(val, (list, dict)) else val
|
||||
if vr_sets:
|
||||
sql = f"UPDATE video_restaurants SET {', '.join(vr_sets)} WHERE video_id = :vid AND restaurant_id = :rid"
|
||||
cur.execute(sql, vr_params)
|
||||
|
||||
return {"ok": True}
|
||||
|
||||
Reference in New Issue
Block a user