How to Ingest Reddit Discussions into RAG Pipelines (LangChain & Claude)

import os
import requests
from langchain_core.documents import Document
from langchain_community.vectorstores import Chroma
from langchain_openai import OpenAIEmbeddings

SUBSCRAPER_KEY = os.getenv("SUBSCRAPER_API_KEY")

def load_reddit_discussions(topic: str, limit: int = 20):
    url = "https://www.subscraper.dev/api/v1/search"
    headers = {
        "Authorization": f"Bearer {SUBSCRAPER_KEY}",
        "Content-Type": "application/json"
    }
    
    # Query high-value discussions
    payload = {
        "query": f"{topic} review OR problems OR alternatives",
        "sort": "top",
        "time": "year",
        "limit": limit
    }
    
    res = requests.post(url, json=payload, headers=headers).json()
    documents = []
    
    for item in res.get("data", []):
        # Format clean text representation for embeddings
        content = f"Title: {item['title']}\nSubreddit: r/{item['subreddit']}\nScore: {item['score']}\nContent: {item.get('selftext', '')}"
        
        doc = Document(
            page_content=content,
            metadata={
                "id": item["id"],
                "url": item["url"],
                "score": item["score"],
                "subreddit": item["subreddit"]
            }
        )
        documents.append(doc)
        
    return documents

# Index into vector database
docs = load_reddit_discussions("Supabase vs Firebase")
vectorstore = Chroma.from_documents(docs, OpenAIEmbeddings())
print(f"Indexed {len(docs)} Reddit discussions into Chroma vectorstore!")