colcaroliandClaude Opus 5
97568ba90e
Vectors from two different embedding models are not comparable — Titan v2 has 1024 dimensions where nomic-embed-text has 768 — and _cosine_similarity zips them, so a mismatch produced quietly meaningless scores rather than an error. Nothing forced a re-embed either: sync_local_kb only looked at the content hash, so switching models left every cached vector in place. local_knowledge_docs now records the model each vector came from, and a row is re-embedded when its content hash OR its model differs. Saving a new LOCAL_KB_EMBEDDING_MODEL from the settings panel starts a background pass over every file; the knowledge base panel also has a button for it. While that pass runs, the per-request sync steps aside so no file is embedded twice. The panel now shows what the embedding model is doing: model name, vector size, how many files are embedded with the current model, how many are still on an older one, a progress bar while a pass runs, and the last error. A "接続テスト" button embeds one short string so a broken or unreachable model can be told apart from one that is merely slow to warm up. Also read TAVILY_API_KEY from os.environ at call time. It was imported as a config constant, which is evaluated before load_into_env() runs, so a key saved from the settings panel was stored but never used — not even after a restart. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
113 lines
4.1 KiB
Python
113 lines
4.1 KiB
Python
"""ローカルナレッジベース（backend/knowledge_base/**.md）を GUI から閲覧するための読み取り専用 API。
|
|
|
|
コンテキスト注入に使われるファイルそのものを一覧・表示するだけで、ここからの編集はできない。
|
|
埋め込み済みかどうか（indexed）は local_knowledge_docs テーブルを引いて付ける。
|
|
"""
|
|
from datetime import datetime
|
|
|
|
from fastapi import APIRouter, Depends, HTTPException, Query
|
|
from pydantic import BaseModel
|
|
from sqlalchemy.orm import Session as DBSession
|
|
|
|
from app.core.memory_kb import local_knowledge
|
|
from app.core.memory_kb.local_knowledge import list_kb_files, read_kb_file
|
|
from app.db.database import get_db
|
|
from app.db.models import LocalKnowledgeDoc
|
|
|
|
router = APIRouter(prefix="/api/kb", tags=["knowledge-base"])
|
|
|
|
|
|
class KbFileInfo(BaseModel):
|
|
path: str
|
|
name: str
|
|
# ナレッジベース直下のファイルは空文字。UI ではこの単位でグループ表示する
|
|
directory: str
|
|
title: str
|
|
preview: str
|
|
size: int
|
|
updated_at: datetime
|
|
# 埋め込み済み（＝話題に応じた自動注入の対象になっている）か
|
|
indexed: bool
|
|
|
|
|
|
class KbFileContent(BaseModel):
|
|
path: str
|
|
name: str
|
|
title: str
|
|
content: str
|
|
size: int
|
|
updated_at: datetime
|
|
|
|
|
|
@router.get("/files", response_model=list[KbFileInfo])
|
|
def list_files(db: DBSession = Depends(get_db)):
|
|
"""ナレッジベース配下の Markdown を一覧で返す（本文は含めず、先頭のプレビューのみ）。"""
|
|
# 「埋め込み済み」は現在のモデルで埋め込まれている行だけを指す（別モデルの行は埋め直し待ち）
|
|
current_model = local_knowledge.current_embedding_model()
|
|
indexed_paths = {
|
|
# DB 側は OS 依存の区切り文字で保存されているため、比較用に posix 形式へ寄せる
|
|
path.replace("\\", "/")
|
|
for (path,) in db.query(LocalKnowledgeDoc.path)
|
|
.filter(LocalKnowledgeDoc.embedding_model == current_model)
|
|
.all()
|
|
}
|
|
return [
|
|
KbFileInfo(**info, indexed=info["path"] in indexed_paths)
|
|
for info in list_kb_files()
|
|
]
|
|
|
|
|
|
@router.get("/file", response_model=KbFileContent)
|
|
def get_file(path: str = Query(..., description="ナレッジベースディレクトリからの相対パス")):
|
|
"""1ファイルの全文を返す。"""
|
|
file = read_kb_file(path)
|
|
if file is None:
|
|
raise HTTPException(status_code=404, detail="Knowledge base file not found")
|
|
return KbFileContent(**file)
|
|
|
|
|
|
class EmbeddingStatus(BaseModel):
|
|
"""埋め込みモデルの現状。GUI の状態表示・進捗バーに使う。"""
|
|
model: str
|
|
# 現在のモデルで埋め込まれたベクトルの次元数（1件も無ければ None）
|
|
dimensions: int | None
|
|
file_count: int
|
|
indexed: int
|
|
# 別のモデルで埋め込まれていて埋め直しが必要な件数
|
|
stale: int
|
|
running: bool
|
|
total: int
|
|
done: int
|
|
failed: int
|
|
started_at: datetime | None
|
|
finished_at: datetime | None
|
|
last_error: str | None
|
|
|
|
|
|
class EmbeddingProbe(BaseModel):
|
|
"""短いテキストを実際に埋め込んでみた結果。"""
|
|
ok: bool
|
|
model: str
|
|
dimensions: int | None
|
|
latency_ms: float
|
|
error: str | None
|
|
|
|
|
|
@router.get("/embedding", response_model=EmbeddingStatus)
|
|
def get_embedding_status(db: DBSession = Depends(get_db)):
|
|
"""現在の埋め込みモデルと、埋め込み済み件数・実行中の進捗を返す。"""
|
|
return local_knowledge.embedding_status(db)
|
|
|
|
|
|
@router.post("/embedding/test", response_model=EmbeddingProbe)
|
|
async def test_embedding():
|
|
"""埋め込みモデルに実際に1回リクエストして、応答するかどうかを確かめる。"""
|
|
return await local_knowledge.probe_embedding()
|
|
|
|
|
|
@router.post("/embedding/reindex", response_model=EmbeddingStatus)
|
|
async def reindex_embeddings(db: DBSession = Depends(get_db)):
|
|
"""全ファイルの再埋め込みをバックグラウンドで開始する。実行中なら何もしない（進捗をそのまま返す）。"""
|
|
local_knowledge.start_reembed_all()
|
|
return local_knowledge.embedding_status(db)
|