Pengambilan-data-fisip

import json
import requests
import pandas as pd

from pathlib import Path


# ========================================
# 1. Folder project
# ========================================

PROJECT = Path(
    "/media/bgoode/Data4/Unpad/Kuliah/BigDataFisip"
)

RAW_DIR = PROJECT / "data" / "raw"

RAW_DIR.mkdir(
    parents=True,
    exist_ok=True
)


# ========================================
# 2. API
# ========================================

url = (
    "https://api.bsky.app/"
    "xrpc/app.bsky.feed.searchPosts"
)

params = {
    "q": '"climate change"',
    "limit": 100,
    "sort": "latest",
    "lang": "en"
}


# ========================================
# 3. Request
# ========================================

response = requests.get(
    url,
    params=params,
    headers={
        "Accept":
            "application/json",

        "User-Agent":
            "BigDataFisip-Class/1.0"
    },
    timeout=30
)


print(
    "HTTP Status:",
    response.status_code
)


# ========================================
# 4. Validasi response
# ========================================

if response.status_code != 200:

    raise RuntimeError(
        f"HTTP {response.status_code}: "
        f"{response.text[:300]}"
    )


data = response.json()

posts = data.get(
    "posts",
    []
)

print(
    "Jumlah posting:",
    len(posts)
)


# ========================================
# 5. Simpan JSON asli
# ========================================

json_file = (
    RAW_DIR /
    "bluesky_climate_change_raw.json"
)

with open(
    json_file,
    "w",
    encoding="utf-8"
) as f:

    json.dump(
        data,
        f,
        ensure_ascii=False,
        indent=2
    )


# ========================================
# 6. Ubah ke tabel
# ========================================

rows = []

for post in posts:

    record = post.get(
        "record",
        {}
    )

    author = post.get(
        "author",
        {}
    )

    rows.append({

        "uri":
            post.get("uri"),

        "created_at":
            record.get("createdAt"),

        "username":
            author.get("handle"),

        "display_name":
            author.get("displayName"),

        "text":
            record.get("text"),

        "language":
            record.get("langs"),

        "reply_count":
            post.get(
                "replyCount",
                0
            ),

        "repost_count":
            post.get(
                "repostCount",
                0
            ),

        "like_count":
            post.get(
                "likeCount",
                0
            )
    })


df_raw = pd.DataFrame(
    rows
)


# ========================================
# 7. Simpan CSV raw
# ========================================

csv_file = (
    RAW_DIR /
    "bluesky_climate_change_raw.csv"
)

df_raw.to_csv(
    csv_file,
    index=False,
    encoding="utf-8-sig"
)


# ========================================
# 8. Pemeriksaan
# ========================================

print(
    "Ukuran dataset:",
    df_raw.shape
)

display(
    df_raw.head()
)