使用 LangChain 实现语义搜索

pip install langchain openai meilisearch python-dotenv

# .env

MEILI_HTTP_ADDR="您的 Meilisearch 主机地址"
MEILI_API_KEY="您的 Meilisearch API 密钥"
OPENAI_API_KEY="您的 OpenAI API 密钥"

# setup.py

import os
from dotenv import load_dotenv # 如不使用 dotenv 请删除此行
from langchain.vectorstores import Meilisearch
from langchain.embeddings.openai import OpenAIEmbeddings
from langchain.document_loaders import JSONLoader

load_dotenv() # 如不使用 dotenv 请删除此行


# 缺少环境变量时退出
if "MEILI_HTTP_ADDR" not in os.environ:
    raise Exception("缺少 MEILI_HTTP_ADDR 环境变量")
if "MEILI_API_KEY" not in os.environ:
    raise Exception("缺少 MEILI_API_KEY 环境变量")
if "OPENAI_API_KEY" not in os.environ:
    raise Exception("缺少 OPENAI_API_KEY 环境变量")


# 后续设置代码将在此处添加 👇

# setup.py


# 之前的代码


# 加载文档
loader = JSONLoader(
    file_path="./movies-lite.json",
    jq_schema=".[] | {id: .id, overview: .overview, title: .title}",
    text_content=False,
)
documents = loader.load()
print("已加载 {} 个文档".format(len(documents)))


# 将文档存储到 Meilisearch
embeddings = OpenAIEmbeddings()
embedders = {
        "custom": {
            "source": "userProvided",
            "dimensions": 1536
        }
    }
embedder_name = "custom"
vector_store = Meilisearch.from_documents(documents=documents, embedding=embeddings, embedders=embedders, embedder_name=embedder_name)

print("开始导入文档")

# search.py

import os
from dotenv import load_dotenv
from langchain.vectorstores import Meilisearch
from langchain.embeddings.openai import OpenAIEmbeddings
import meilisearch

load_dotenv()


# 可以使用与 `setup.py` 相同的代码来检查缺失的环境变量


# 创建向量存储
client = meilisearch.Client(
    url=os.environ.get("MEILI_HTTP_ADDR"),
    api_key=os.environ.get("MEILI_API_KEY"),
)
embeddings = OpenAIEmbeddings()
vector_store = Meilisearch(client=client, embedding=embeddings)


# 执行相似性搜索
embedder_name = "custom"
query = "superhero fighting evil in a city at night"
results = vector_store.similarity_search(
    query=query,
    embedder_name=embedder_name,
    k=3,
)

# 显示搜索结果
for result in results:
    print(result.page_content)

{"id": 155, "title": "The Dark Knight", "overview": "Batman raises the stakes in his war on crime. With the help of Lt. Jim Gordon and District Attorney Harvey Dent, Batman sets out to dismantle the remaining criminal organizations that plague the streets. The partnership proves to be effective, but they soon find themselves prey to a reign of chaos unleashed by a rising criminal mastermind known to the terrified citizens of Gotham as the Joker."}
{"id": 314, "title": "Catwoman", "overview": "Liquidated after discovering a corporate conspiracy, mild-mannered graphic artist Patience Phillips washes up on an island, where she's resurrected and endowed with the prowess of a cat -- and she's eager to use her new skills ... as a vigilante. Before you can say \"cat and mouse,\" handsome gumshoe Tom Lone is on her tail."}
{"id": 268, "title": "Batman", "overview": "Batman must face his most ruthless nemesis when a deformed madman calling himself \"The Joker\" seizes control of Gotham's criminal underworld."}