AI コンテンツモデレーション — Claude によるマルチモーダルな Trust & Safety

import json
from openai import OpenAI
client = OpenAI(api_key="sk-kn-...", base_url="https://api.kunavo.com/v1")

POLICY = """You are a content moderator. Judge the content against these
categories: toxic, spam, harassment, sexual, violence, self_harm, deception,
brand_unsafe. Be conservative — when in doubt, flag for human review."""

# Claude enforces a json_schema response_format; json_object it does not.
VERDICT = {
    "type": "object",
    "properties": {
        "verdict": {"type": "string", "enum": ["allow", "review", "block"]},
        "categories": {"type": "array", "items": {"type": "string", "enum": [
            "toxic", "spam", "harassment", "sexual", "violence",
            "self_harm", "deception", "brand_unsafe"]}},
        "confidence": {"type": "number"},  # 0.0-1.0
        "rationale": {"type": "string"},   # one sentence
    },
    "required": ["verdict", "categories", "confidence", "rationale"],
    "additionalProperties": False,
}
RESPONSE_FORMAT = {"type": "json_schema",
                   "json_schema": {"name": "verdict", "schema": VERDICT}}

def moderate_text(text: str) -> dict:
    resp = client.chat.completions.create(
        model="claude-haiku-4-5",  # cheap + fast for high-volume
        messages=[
            {"role": "system", "content": POLICY},
            {"role": "user", "content": text},
        ],
        response_format=RESPONSE_FORMAT,
        max_tokens=200,
    )
    return json.loads(resp.choices[0].message.content)

def moderate_image(image_url: str) -> dict:
    resp = client.chat.completions.create(
        model="claude-haiku-4-5",  # vision + cheap
        messages=[
            {"role": "system", "content": POLICY},
            {"role": "user", "content": [
                {"type": "text", "text": "Moderate this image:"},
                {"type": "image_url", "image_url": {"url": image_url}},
            ]},
        ],
        response_format=RESPONSE_FORMAT,
        max_tokens=200,
    )
    return json.loads(resp.choices[0].message.content)