Reorganize core into subpackages, add local KB, ruff/pyright tooling
- Split backend/app/core into weather/, discord/, memory_kb/, search/ subpackages by responsibility, keeping config.py and llm.py as the shared foundation; updated all import sites accordingly. - Add local Markdown knowledge base support (local_knowledge.py, knowledge_base/fgo.md, LOCAL_KB_* settings) that embeds and injects relevant docs into the system prompt by topic similarity. - Add ruff and pyright as backend dev dependencies with basic config in pyproject.toml for linting and type checking. - Document local machine quirks in CLAUDE.md (Postgres on port 8080, SSLKEYLOGFILE breaking litellm/pyright via OPENSSL_Uplink crashes). - Add backend/run_dev.ps1 dev launcher and docs/discord-bot.md.
This commit is contained in:
@@ -17,6 +17,12 @@ TAVILY_API_KEY: str | None = os.getenv("TAVILY_API_KEY")
|
||||
# 蓄積した知識をシステムプロンプトに注入する際の最大件数
|
||||
KNOWLEDGE_CONTEXT_LIMIT: int = int(os.getenv("KNOWLEDGE_CONTEXT_LIMIT", "15"))
|
||||
|
||||
# ローカルMarkdownナレッジベース（話題に応じて関連ファイルをシステムプロンプトに注入）
|
||||
LOCAL_KB_DIR: str = os.getenv("LOCAL_KB_DIR", "knowledge_base")
|
||||
LOCAL_KB_EMBEDDING_MODEL: str = os.getenv("LOCAL_KB_EMBEDDING_MODEL", "text-embedding-3-small")
|
||||
LOCAL_KB_TOP_K: int = int(os.getenv("LOCAL_KB_TOP_K", "3"))
|
||||
LOCAL_KB_MIN_SCORE: float = float(os.getenv("LOCAL_KB_MIN_SCORE", "0.75"))
|
||||
|
||||
# Amazon Bedrock
|
||||
# 以下はすべて LiteLLM(boto3) が os.environ から直接読み取るため、.env に書くだけで自動的に有効になる。
|
||||
# IAM経由（推奨）: AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY を設定する。
|
||||
|
||||
@@ -11,11 +11,11 @@ from app.core.config import (
|
||||
WEATHER_NOTIFY_MODEL,
|
||||
)
|
||||
from app.core.llm import complete
|
||||
from app.core.scheduler import send_useless_tip
|
||||
from app.core.tagging import find_related_tag, generate_tags
|
||||
from app.core.weather import fetch_tomorrow_weather, judge_laundry
|
||||
from app.core.weather_cache import set_latest
|
||||
from app.core.word_memory import record_message
|
||||
from app.core.discord.scheduler import send_useless_tip
|
||||
from app.core.memory_kb.tagging import find_related_tag, generate_tags
|
||||
from app.core.weather.weather import fetch_tomorrow_weather, judge_laundry
|
||||
from app.core.weather.weather_cache import set_latest
|
||||
from app.core.memory_kb.word_memory import record_message
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -20,10 +20,10 @@ from app.core.config import (
|
||||
WEATHER_NOTIFY_MODEL,
|
||||
)
|
||||
from app.core.llm import complete
|
||||
from app.core.notify import send_discord_message
|
||||
from app.core.weather import fetch_tomorrow_weather, judge_laundry
|
||||
from app.core.weather_cache import set_latest
|
||||
from app.core.word_memory import pick_random_word
|
||||
from app.core.discord.notify import send_discord_message
|
||||
from app.core.weather.weather import fetch_tomorrow_weather, judge_laundry
|
||||
from app.core.weather.weather_cache import set_latest
|
||||
from app.core.memory_kb.word_memory import pick_random_word
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -7,7 +7,7 @@ from contextlib import asynccontextmanager
|
||||
|
||||
import litellm
|
||||
from app.core.config import DEFAULT_MODEL, DEBUG
|
||||
from app.core.websearch import WEB_SEARCH_TOOL, search_web
|
||||
from app.core.search.websearch import WEB_SEARCH_TOOL, search_web
|
||||
|
||||
litellm.drop_params = True
|
||||
if DEBUG:
|
||||
@@ -97,6 +97,12 @@ async def complete(
|
||||
return result
|
||||
|
||||
|
||||
async def embed(texts: list[str], model: str) -> list[list[float]]:
|
||||
"""テキスト群を埋め込みベクトルに変換する。"""
|
||||
response = await litellm.aembedding(model=model, input=texts)
|
||||
return [item["embedding"] for item in response.data]
|
||||
|
||||
|
||||
async def resolve_web_search(
|
||||
messages: list[dict],
|
||||
model: str | None = None,
|
||||
|
||||
@@ -34,4 +34,7 @@ def get_knowledge_context(db: DBSession) -> str | None:
|
||||
urls = ", ".join(s.url for s in entry.sources) or "出典不明"
|
||||
blocks.append(f"- 「{entry.query}」について: {entry.summary}\n (出典: {urls})")
|
||||
|
||||
return "以下はこれまでのWeb検索で得た知識です。関連する話題が出たら活用してください:\n\n" + "\n".join(blocks)
|
||||
return (
|
||||
"以下はこれまでのWeb検索で得た知識です。関連する話題が出たら活用してください。"
|
||||
"ただしローカルナレッジベースの資料が別途提示されている場合は、そちらを優先してください:\n\n" + "\n".join(blocks)
|
||||
)
|
||||
@@ -0,0 +1,131 @@
|
||||
"""backend/knowledge_base/ 配下のMarkdownファイルから、話題に応じて関連知識をシステムプロンプトに注入する機能。
|
||||
|
||||
- sync_local_kb(): ディレクトリをスキャンし、新規/変更されたファイルだけ埋め込みを計算してDBに保存する
|
||||
- get_local_kb_context(): 直近のユーザー発言に埋め込み類似度の高いファイルを探し、システムプロンプト用テキストにする
|
||||
"""
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy.orm import Session as DBSession
|
||||
|
||||
from app.core import llm
|
||||
from app.core.config import LOCAL_KB_DIR, LOCAL_KB_EMBEDDING_MODEL, LOCAL_KB_MIN_SCORE, LOCAL_KB_TOP_K
|
||||
from app.db.models import LocalKnowledgeDoc
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _kb_dir() -> Path:
|
||||
return Path(LOCAL_KB_DIR)
|
||||
|
||||
|
||||
def _hash(content: str) -> str:
|
||||
return hashlib.sha256(content.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _title_from_content(content: str, fallback: str) -> str:
|
||||
for line in content.splitlines():
|
||||
stripped = line.strip()
|
||||
if stripped.startswith("#"):
|
||||
return stripped.lstrip("#").strip() or fallback
|
||||
return fallback
|
||||
|
||||
|
||||
def _cosine_similarity(a: list[float], b: list[float]) -> float:
|
||||
dot = sum(x * y for x, y in zip(a, b))
|
||||
norm_a = math.sqrt(sum(x * x for x in a))
|
||||
norm_b = math.sqrt(sum(y * y for y in b))
|
||||
if norm_a == 0 or norm_b == 0:
|
||||
return 0.0
|
||||
return dot / (norm_a * norm_b)
|
||||
|
||||
|
||||
async def sync_local_kb(db: DBSession) -> None:
|
||||
"""ナレッジベースディレクトリをスキャンし、新規/変更ファイルの埋め込みを更新する。
|
||||
削除されたファイルに対応するレコードも削除する。
|
||||
"""
|
||||
kb_dir = _kb_dir()
|
||||
if not kb_dir.is_dir():
|
||||
return
|
||||
|
||||
md_files = sorted(kb_dir.rglob("*.md"))
|
||||
seen_paths: set[str] = set()
|
||||
|
||||
for file_path in md_files:
|
||||
rel_path = str(file_path.relative_to(kb_dir))
|
||||
seen_paths.add(rel_path)
|
||||
content = file_path.read_text(encoding="utf-8")
|
||||
content_hash = _hash(content)
|
||||
|
||||
existing = db.query(LocalKnowledgeDoc).filter(LocalKnowledgeDoc.path == rel_path).first()
|
||||
if existing and existing.content_hash == content_hash:
|
||||
continue
|
||||
|
||||
try:
|
||||
[embedding] = await llm.embed([content], model=LOCAL_KB_EMBEDDING_MODEL)
|
||||
except Exception:
|
||||
logger.exception("Failed to embed local knowledge file: %s", rel_path)
|
||||
continue
|
||||
|
||||
title = _title_from_content(content, fallback=file_path.stem)
|
||||
if existing:
|
||||
existing.content_hash = content_hash
|
||||
existing.title = title
|
||||
existing.embedding_json = json.dumps(embedding)
|
||||
else:
|
||||
db.add(LocalKnowledgeDoc(
|
||||
path=rel_path,
|
||||
title=title,
|
||||
content_hash=content_hash,
|
||||
embedding_json=json.dumps(embedding),
|
||||
))
|
||||
db.commit()
|
||||
|
||||
stale = db.query(LocalKnowledgeDoc).filter(LocalKnowledgeDoc.path.notin_(seen_paths)).all()
|
||||
for doc in stale:
|
||||
db.delete(doc)
|
||||
if stale:
|
||||
db.commit()
|
||||
|
||||
|
||||
async def get_local_kb_context(db: DBSession, query_text: str) -> str | None:
|
||||
"""直近のユーザー発言に関連するローカルMarkdown知識をシステムプロンプト用テキストにして返す。"""
|
||||
if not query_text.strip():
|
||||
return None
|
||||
|
||||
await sync_local_kb(db)
|
||||
|
||||
docs = db.query(LocalKnowledgeDoc).all()
|
||||
if not docs:
|
||||
return None
|
||||
|
||||
try:
|
||||
[query_embedding] = await llm.embed([query_text], model=LOCAL_KB_EMBEDDING_MODEL)
|
||||
except Exception:
|
||||
logger.exception("Failed to embed query for local knowledge search")
|
||||
return None
|
||||
|
||||
scored = [
|
||||
(doc, _cosine_similarity(query_embedding, json.loads(doc.embedding_json)))
|
||||
for doc in docs
|
||||
]
|
||||
scored.sort(key=lambda pair: pair[1], reverse=True)
|
||||
top = [(doc, score) for doc, score in scored[:LOCAL_KB_TOP_K] if score >= LOCAL_KB_MIN_SCORE]
|
||||
if not top:
|
||||
return None
|
||||
|
||||
kb_dir = _kb_dir()
|
||||
blocks = []
|
||||
for doc, _ in top:
|
||||
content = (kb_dir / doc.path).read_text(encoding="utf-8")
|
||||
blocks.append(f"### {doc.title}\n{content}")
|
||||
|
||||
return (
|
||||
"以下はローカルのナレッジベースから話題に関連すると判断された資料です。"
|
||||
"この資料はWeb検索結果より優先度の高い、信頼できる情報源として扱ってください。"
|
||||
"ここに必要な情報が含まれている場合はWeb検索を行わず、この内容のみで回答してください。"
|
||||
"情報が不足する場合に限り、補足としてWeb検索を使ってください:\n\n" + "\n\n".join(blocks)
|
||||
)
|
||||
@@ -2,11 +2,12 @@ from sqlalchemy.orm import Session as DBSession
|
||||
from app.db.models import Memory, Message, Session
|
||||
from app.core.config import MEMORY_SUMMARIZE_THRESHOLD
|
||||
from app.core import llm
|
||||
from app.core import knowledge
|
||||
from app.core.tagging import tag_memory
|
||||
from app.core.memory_kb import knowledge
|
||||
from app.core.memory_kb import local_knowledge
|
||||
from app.core.memory_kb.tagging import tag_memory
|
||||
|
||||
|
||||
def get_context_messages(db: DBSession, session_id: str) -> list[dict]:
|
||||
async def get_context_messages(db: DBSession, session_id: str) -> list[dict]:
|
||||
"""蓄積知識・長期メモリ（要約）・短期メモリ（最近のメッセージ）を結合してコンテキストを返す。"""
|
||||
memories = db.query(Memory).filter(Memory.session_id == session_id).order_by(Memory.created_at).all()
|
||||
messages = db.query(Message).filter(Message.session_id == session_id).order_by(Message.created_at).all()
|
||||
@@ -17,6 +18,12 @@ def get_context_messages(db: DBSession, session_id: str) -> list[dict]:
|
||||
if knowledge_context:
|
||||
context.append({"role": "system", "content": knowledge_context})
|
||||
|
||||
if messages:
|
||||
latest_user_message = next((m.content for m in reversed(messages) if m.role == "user"), "")
|
||||
local_kb_context = await local_knowledge.get_local_kb_context(db, latest_user_message)
|
||||
if local_kb_context:
|
||||
context.append({"role": "system", "content": local_kb_context})
|
||||
|
||||
if memories:
|
||||
combined_summary = "\n\n".join(m.summary for m in memories)
|
||||
context.append({
|
||||
@@ -38,9 +45,10 @@ async def maybe_summarize(db: DBSession, session: Session) -> None:
|
||||
half = MEMORY_SUMMARIZE_THRESHOLD // 2
|
||||
to_summarize = messages[:half]
|
||||
|
||||
conversation_text = "\n".join(f"{m.role}: {m.content}" for m in to_summarize)
|
||||
summary_prompt = [
|
||||
{"role": "system", "content": "以下の会話を簡潔に要約してください。重要な事実・決定・文脈を保持してください。"},
|
||||
*[{"role": m.role, "content": m.content} for m in to_summarize],
|
||||
{"role": "user", "content": conversation_text},
|
||||
]
|
||||
summary_text = await llm.complete(summary_prompt, model=session.model, session_id=session.id)
|
||||
|
||||
@@ -11,7 +11,7 @@ import random
|
||||
|
||||
from sqlalchemy.orm import Session as DBSession
|
||||
|
||||
from app.core import word_memory
|
||||
from app.core.memory_kb import word_memory
|
||||
from app.core.config import TAGGING_MODEL
|
||||
from app.core.llm import complete
|
||||
from app.db.database import SessionLocal
|
||||
@@ -12,7 +12,8 @@ WEB_SEARCH_TOOL = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "web_search",
|
||||
"description": "最新情報・事実確認・知らない固有名詞など、自分の知識だけでは答えられない可能性があるときにWebを検索する。",
|
||||
"description": "最新情報・事実確認・知らない固有名詞など、自分の知識だけでは答えられない可能性があるときにWebを検索する。"
|
||||
"ただし、システムプロンプトに提示されたローカルナレッジベースの資料で回答できる場合は使わないこと。",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
Reference in New Issue
Block a user