네이버 블로그 크롤링

import os, re, requests
 
HEADERS = {
    "X-Naver-Client-Id": os.environ["NAVER_CLIENT_ID"],
    "X-Naver-Client-Secret": os.environ["NAVER_CLIENT_SECRET"],
}
 
def search_blog(query, pages=2, display=100):
    items = []
    for page in range(pages):
        res = requests.get(
            "https://openapi.naver.com/v1/search/blog.json",
            headers=HEADERS,
            params={"query": query, "display": display, "start": page * display + 1, "sort": "date"},
            timeout=10,
        )
        res.raise_for_status()
        items += res.json()["items"]
    return [
        {"url": it["link"], "title": re.sub(r"<[^>]+>", "", it["title"]), "date": it["postdate"]}
        for it in items
    ]

from bs4 import BeautifulSoup
 
def post_view_url(url):
    blog_id, log_no = url.rstrip("/").split("/")[-2:]
    return f"https://blog.naver.com/PostView.naver?blogId={blog_id}&logNo={log_no}"
 
def fetch_body(url):
    html = requests.get(post_view_url(url), headers={"User-Agent": "Mozilla/5.0"}, timeout=10).text
    soup = BeautifulSoup(html, "html.parser")
    node = soup.select_one("div.se-main-container") or soup.select_one("#postViewArea")
    if node is None:
        return None
    text = node.get_text(" ", strip=True)
    return re.sub(r"\s+", " ", text.replace("​", "")).strip()

import time
import pandas as pd
 
rows = []
for post in search_blog("파이썬 크롤링"):
    try:
        post["body"] = fetch_body(post["url"])
    except requests.RequestException as e:
        post["body"] = None
        post["error"] = str(e)
    rows.append(post)
    time.sleep(0.5)                     # 요청 간격을 둔다
 
pd.DataFrame(rows).to_csv("blog.csv", index=False, encoding="utf-8-sig")