|
|
| Line 1: |
Line 1: |
| − | Ada dua bentuk data mentah:
| + | Dari contoh yang didapat, kita akan melakukan hal berikut |
| | | | |
| − | 1. `JSON` → salinan respons API, sedekat mungkin dengan data asli dari Bluesky.
| + | <pre> |
| − | 2. `CSV` → field penting sudah dijadikan tabel, tetapi **belum dibersihkan** sehingga cocok untuk latihan mahasiswa.
| + | bluesky_climate_change_raw.csv |
| − | | + | ↓ |
| − | Secara resmi, `app.bsky.feed.searchPosts` mewajibkan parameter `q`, mendukung `sort=latest/top`, filter `lang`, dan `limit` maksimum **100** per request. Dokumentasi juga mengingatkan bahwa sintaks pencarian frasa/Boolean tidak sepenuhnya dispesifikasikan, walaupun sintaks Lucene direkomendasikan.
| + | DATA CLEANING |
| − | | + | ↓ |
| − | ---
| + | PREPROCESSING |
| − | | + | ↓ |
| − | # A. Target akhir praktikum
| |
| − | | |
| − | Kita akan menghasilkan:
| |
| − | | |
| − | ```text
| |
| − | BigDataFisip/
| |
| − | │
| |
| − | ├── venvs/
| |
| − | │ └── jupyter-polsci/
| |
| − | │
| |
| − | └── data/
| |
| − | ├── raw/
| |
| − | │ ├── bluesky_climate_change_raw.json
| |
| − | │ └── bluesky_climate_change_raw.csv
| |
| − | │
| |
| − | └── processed/
| |
| − | ```
| |
| − | | |
| − | Mahasiswa nantinya **tidak mengubah file di `raw/`**.
| |
| − | | |
| − | Hasil cleaning disimpan ke:
| |
| − | | |
| − | ```text
| |
| − | data/processed/
| |
| − | ```
| |
| − | | |
| − | ---
| |
| − | | |
| − | # B. Langkah 1 — Pastikan Jupyter menggunakan environment yang benar
| |
| − | | |
| − | Buat cell:
| |
| − | | |
| − | ```python
| |
| − | import sys
| |
| − | | |
| − | print(sys.executable)
| |
| − | print(sys.version)
| |
| − | ```
| |
| − | | |
| − | Pada komputer seharusnya kurang lebih:
| |
| − | | |
| − | ```text
| |
| − | /venvs/jupyter-polsci/bin/python
| |
| − | | |
| − | 3.10.12 ...
| |
| − | ```
| |
| − | | |
| − | Kalau demikian, environment sudah benar.
| |
| − | | |
| − | ---
| |
| − | | |
| − | # C. Langkah 2 — Import library
| |
| − | | |
| − | Buat cell baru:
| |
| − | | |
| − | ```python
| |
| − | import json
| |
| − | import requests
| |
| − | import pandas as pd
| |
| − | | |
| − | from pathlib import Path
| |
| − | ```
| |
| − | | |
| − | Kita menggunakan:
| |
| − | | |
| − | ```text
| |
| − | requests → mengambil data API
| |
| − | json → menyimpan respons asli
| |
| − | pandas → mengubah data menjadi tabel
| |
| − | Path → mengatur folder/file
| |
| − | ```
| |
| − | | |
| − | ---
| |
| − | | |
| − | # D. Langkah 3 — Tentukan folder penyimpanan
| |
| − | | |
| − | ```python
| |
| − | PROJECT = Path(
| |
| − | "/media/bgoode/Data4/Unpad/Kuliah/BigDataFisip"
| |
| − | )
| |
| − | | |
| − | RAW_DIR = PROJECT / "data" / "raw"
| |
| − | PROCESSED_DIR = PROJECT / "data" / "processed"
| |
| − | | |
| − | RAW_DIR.mkdir(
| |
| − | parents=True,
| |
| − | exist_ok=True
| |
| − | )
| |
| − | | |
| − | PROCESSED_DIR.mkdir(
| |
| − | parents=True,
| |
| − | exist_ok=True
| |
| − | )
| |
| − | | |
| − | print(RAW_DIR)
| |
| − | ```
| |
| − | | |
| − | Hasil:
| |
| − | | |
| − | ```text
| |
| − | /data/raw
| |
| − | ```
| |
| − | | |
| − | ---
| |
| − | | |
| − | # E. Langkah 4 — Tentukan topik (misalnya Climate Change)
| |
| − | | |
| − | Endpoint:
| |
| − | | |
| − | ```python
| |
| − | url = "https://api.bsky.app/xrpc/app.bsky.feed.searchPosts"
| |
| − | ```
| |
| − | | |
| − | Kemudian query:
| |
| − | | |
| − | ```python
| |
| − | params = {
| |
| − | "q": '"climate change"',
| |
| − | "limit": 100,
| |
| − | "sort": "latest",
| |
| − | "lang": "en"
| |
| − | }
| |
| − | ```
| |
| − | | |
| − | Artinya:
| |
| − | | |
| − | ```text
| |
| − | q
| |
| − | ↓ | |
| − | mencari "climate change"
| |
| − | | |
| − | limit
| |
| − | ↓
| |
| − | maksimal 100 posting
| |
| − | | |
| − | sort
| |
| − | ↓ | |
| − | latest = hasil terbaru
| |
| − | | |
| − | lang
| |
| − | ↓
| |
| − | en = bahasa Inggris
| |
| − | ```
| |
| − | | |
| − | Nilai `100` adalah batas maksimum `limit` menurut Lexicon resmi Bluesky.
| |
| − | | |
| − | Catatan:
| |
| − | | |
| − | > `limit=100` berarti **meminta maksimal 100**, bukan jaminan server selalu memberikan tepat 100.
| |
| − | | |
| − | ---
| |
| − | | |
| − | # F. Langkah 5 — Kirim request
| |
| − | | |
| − | ```python
| |
| − | response = requests.get(
| |
| − | url,
| |
| − | params=params,
| |
| − | headers={
| |
| − | "Accept": "application/json",
| |
| − | "User-Agent": "BigDataFisip-Class/1.0"
| |
| − | },
| |
| − | timeout=30
| |
| − | )
| |
| − | | |
| − | print(
| |
| − | "HTTP Status:",
| |
| − | response.status_code
| |
| − | )
| |
| − | | |
| − | print(
| |
| − | "Content-Type:",
| |
| − | response.headers.get("content-type")
| |
| − | )
| |
| − | ```
| |
| − | | |
| − | Harapannya:
| |
| − | | |
| − | ```text
| |
| − | HTTP Status: 200
| |
| − | Content-Type: application/json; charset=utf-8
| |
| − | ```
| |
| − | | |
| − | ---
| |
| − | | |
| − | # G. Langkah 6 — Cek hasil JSON
| |
| − | | |
| − | Gunakan:
| |
| − | | |
| − | ```python
| |
| − | if response.status_code != 200:
| |
| − | | |
| − | print("Request gagal")
| |
| − | | |
| − | print(
| |
| − | response.text[:1000]
| |
| − | )
| |
| − | | |
| − | raise RuntimeError(
| |
| − | "Data belum berhasil diambil"
| |
| − | )
| |
| − | ```
| |
| − | | |
| − | Kemudian pastikan hasil memang JSON:
| |
| − | | |
| − | ```python
| |
| − | content_type = response.headers.get(
| |
| − | "content-type",
| |
| − | ""
| |
| − | )
| |
| − | | |
| − | if "application/json" not in content_type:
| |
| − | | |
| − | raise RuntimeError(
| |
| − | "Respons server bukan JSON"
| |
| − | )
| |
| − | ```
| |
| − | | |
| − | Baru setelah itu:
| |
| − | | |
| − | ```python
| |
| − | data = response.json()
| |
| − | ```
| |
| − | | |
| − | Ini mencegah error:
| |
| − | | |
| − | ```text
| |
| − | JSONDecodeError
| |
| − | ```
| |
| − | | |
| − | ketika server sebenarnya memberikan HTML error.
| |
| − | | |
| − | ---
| |
| − | | |
| − | # H. Langkah 7 — Ambil daftar posting
| |
| − | | |
| − | ```python
| |
| − | posts = data.get(
| |
| − | "posts",
| |
| − | []
| |
| − | )
| |
| − | | |
| − | print(
| |
| − | "Jumlah posting:",
| |
| − | len(posts)
| |
| − | )
| |
| − | ```
| |
| − | | |
| − | Target:
| |
| − | | |
| − | ```text
| |
| − | Jumlah posting: 100
| |
| − | ```
| |
| − | | |
| − | Jika hasilnya misalnya:
| |
| − | | |
| − | ```text
| |
| − | Jumlah posting: 86
| |
| − | ```
| |
| − | | |
| − | jangan mengubahnya menjadi 100, biarkan saja.
| |
| − | | |
| − | Itu berarti API memang hanya mengembalikan 86 pada request tersebut.
| |
| − | | |
| − | ---
| |
| − | | |
| − | # I. Langkah 8 — Lihat bentuk JSON
| |
| − | | |
| − | Sekarang:
| |
| − | | |
| − | ```python
| |
| − | print(
| |
| − | data.keys()
| |
| − | )
| |
| − | ```
| |
| − | | |
| − | Lalu:
| |
| − | | |
| − | ```python
| |
| − | print(
| |
| − | posts[0].keys()
| |
| − | )
| |
| − | ```
| |
| − | | |
| − | Dan lihat posting pertama:
| |
| − | | |
| − | ```python
| |
| − | posts[0]
| |
| − | ```
| |
| − | | |
| − | Mahasiswa kemungkinan akan melihat struktur yang cukup kompleks.
| |
| − | | |
| − | Secara sederhana:
| |
| − | | |
| − | ```text
| |
| − | post
| |
| − | │
| |
| − | ├── uri
| |
| − | ├── cid
| |
| − | │
| |
| − | ├── author
| |
| − | │ ├── handle
| |
| − | │ ├── displayName
| |
| − | │ └── ...
| |
| − | │
| |
| − | ├── record
| |
| − | │ ├── text
| |
| − | │ ├── createdAt
| |
| − | │ ├── langs
| |
| − | │ └── ...
| |
| − | │
| |
| − | ├── replyCount
| |
| − | ├── repostCount
| |
| − | ├── likeCount
| |
| − | └── ...
| |
| − | ```
| |
| − | | |
| − | > **Data dari API belum tentu berbentuk tabel seperti Excel.**
| |
| − | | |
| − | Data API sering berbentuk:
| |
| − | | |
| − | ```text
| |
| − | JSON
| |
| − | ↓
| |
| − | nested / bertingkat
| |
| − | ```
| |
| − | | |
| − | ---
| |
| − | | |
| − | # J. Langkah 9 — Simpan JSON mentah terlebih dahulu
| |
| − | | |
| − | **Lakukan ini sebelum cleaning.**
| |
| − | | |
| − | ```python
| |
| − | json_file = (
| |
| − | RAW_DIR /
| |
| − | "bluesky_climate_change_raw.json"
| |
| − | )
| |
| − | | |
| − | with open(
| |
| − | json_file,
| |
| − | "w",
| |
| − | encoding="utf-8"
| |
| − | ) as f:
| |
| − | | |
| − | json.dump(
| |
| − | data,
| |
| − | f,
| |
| − | ensure_ascii=False,
| |
| − | indent=2
| |
| − | )
| |
| − | | |
| − | print(
| |
| − | "Tersimpan:",
| |
| − | json_file
| |
| − | )
| |
| − | ```
| |
| − | | |
| − | Sekarang kita memiliki:
| |
| − | | |
| − | ```text
| |
| − | data/raw/
| |
| − | └── bluesky_climate_change_raw.json
| |
| − | ```
| |
| − | | |
| − | File ini **jangan diedit**.
| |
| − | | |
| − | Anggap sebagai:
| |
| − | | |
| − | > salinan data asli yang diterima dari API.
| |
| − | | |
| − | ---
| |
| − | | |
| − | # K. Langkah 10 — Pilih field yang akan dipelajari mahasiswa
| |
| − | | |
| − | Sekarang JSON bertingkat kita ubah menjadi tabel.
| |
| − | | |
| − | ```python
| |
| − | rows = []
| |
| − | | |
| − | for post in posts:
| |
| − | | |
| − | record = post.get(
| |
| − | "record",
| |
| − | {}
| |
| − | )
| |
| − | | |
| − | author = post.get(
| |
| − | "author",
| |
| − | {}
| |
| − | )
| |
| − | | |
| − | rows.append({
| |
| − | | |
| − | "uri":
| |
| − | post.get("uri"), | |
| − | | |
| − | "created_at":
| |
| − | record.get("createdAt"),
| |
| − | | |
| − | "username":
| |
| − | author.get("handle"),
| |
| − | | |
| − | "display_name":
| |
| − | author.get("displayName"),
| |
| − | | |
| − | "text":
| |
| − | record.get("text"),
| |
| − | | |
| − | "language":
| |
| − | record.get("langs"),
| |
| − | | |
| − | "reply_count":
| |
| − | post.get(
| |
| − | "replyCount",
| |
| − | 0
| |
| − | ),
| |
| − | | |
| − | "repost_count":
| |
| − | post.get(
| |
| − | "repostCount",
| |
| − | 0
| |
| − | ),
| |
| − | | |
| − | "like_count":
| |
| − | post.get(
| |
| − | "likeCount",
| |
| − | 0
| |
| − | )
| |
| − | })
| |
| − | ```
| |
| − | | |
| − | Kemudian:
| |
| − | | |
| − | ```python
| |
| − | df_raw = pd.DataFrame(
| |
| − | rows
| |
| − | )
| |
| − | ```
| |
| − | | |
| − | ---
| |
| − | | |
| − | # L. Langkah 11 — Lihat hasil tabel
| |
| − | | |
| − | ```python
| |
| − | df_raw.head()
| |
| − | ```
| |
| − | | |
| − | Kurang lebih mahasiswa akan memperoleh:
| |
| − | | |
| − | | created_at | username | text | language | reply_count | repost_count | like_count |
| |
| − | | ---------- | -------- | ----------------- | -------- | ----------: | -----------: | ---------: |
| |
| − | | ... | user1 | Climate change... | ['en'] | 1 | 3 | 12 |
| |
| − | | ... | user2 | New report... | ['en'] | 0 | 1 | 6 |
| |
| − | | ... | user3 | ... | ['en'] | 2 | 0 | 9 |
| |
| − | | |
| − | Kemudian:
| |
| − | | |
| − | ```python
| |
| − | df_raw.shape
| |
| − | ```
| |
| − | | |
| − | Harapannya:
| |
| − | | |
| − | ```text
| |
| − | (100, 9)
| |
| − | ```
| |
| − | | |
| − | ---
| |
| − | | |
| − | # M. Langkah 12 — Periksa kondisi data, tetapi jangan dibersihkan dahulu
| |
| − | | |
| − | Jalankan:
| |
| − | | |
| − | ```python
| |
| − | df_raw.info()
| |
| − | ```
| |
| − | | |
| − | Kemudian:
| |
| − | | |
| − | ```python
| |
| − | df_raw.isna().sum()
| |
| − | ```
| |
| − | | |
| − | Cek juga:
| |
| − | | |
| − | ```python
| |
| − | df_raw.head(10)
| |
| − | ```
| |
| − | | |
| − | Dan:
| |
| − | | |
| − | ```python
| |
| − | df_raw["text"].head(10)
| |
| − | ```
| |
| − | | |
| − | Pada tahap ini mahasiswa hanya **mengamati**.
| |
| − | | |
| − | Belum:
| |
| − | | |
| − | ```python
| |
| − | dropna()
| |
| − | drop_duplicates()
| |
| − | replace()
| |
| − | ```
| |
| − | | |
| − | dan sebagainya.
| |
| − | | |
| − | ---
| |
| − | | |
| − | # N. Langkah 13 — Simpan CSV mentah
| |
| − | | |
| − | ```python
| |
| − | csv_file = (
| |
| − | RAW_DIR /
| |
| − | "bluesky_climate_change_raw.csv"
| |
| − | )
| |
| − | | |
| − | df_raw.to_csv(
| |
| − | csv_file,
| |
| − | index=False,
| |
| − | encoding="utf-8-sig"
| |
| − | )
| |
| − | | |
| − | print(
| |
| − | "Tersimpan:",
| |
| − | csv_file
| |
| − | )
| |
| − | ```
| |
| − | | |
| − | Hasil akhirnya:
| |
| − | | |
| − | ```text
| |
| − | BigDataFisip/
| |
| − | └── data/
| |
| − | │
| |
| − | ├── raw/
| |
| − | │ ├── bluesky_climate_change_raw.json
| |
| − | │ └── bluesky_climate_change_raw.csv
| |
| − | │
| |
| − | └── processed/
| |
| − | ```
| |
| − | | |
| − | Saya menggunakan:
| |
| − | | |
| − | ```python
| |
| − | encoding="utf-8-sig"
| |
| − | ```
| |
| − | | |
| − | supaya karakter seperti emoji dan karakter non-ASCII relatif aman jika CSV nantinya dibuka menggunakan Excel.
| |
| − | | |
| − | ---
| |
| − | | |
| − | # O. Perbedaan dua file tersebut
| |
| − | | |
| − | | |
| − | ### `raw.json`
| |
| − | | |
| − | ```text
| |
| − | Bluesky API
| |
| − | ↓
| |
| − | JSON asli
| |
| − | ```
| |
| − | | |
| − | Belum kita ubah strukturnya.
| |
| − | | |
| − | ### `raw.csv`
| |
| − | | |
| − | ```text
| |
| − | JSON
| |
| − | ↓
| |
| − | pilih field
| |
| − | ↓
| |
| − | tabel
| |
| − | ```
| |
| − | | |
| − | Sudah mengalami **transformasi struktur**, tetapi belum mengalami cleaning isi.
| |
| − | | |
| − | ---
| |
| − | | |
| − | # P. Sekarang mahasiswa mendapatkan dataset untuk cleaning
| |
| − | | |
| − | Setelah raw CSV tersedia, praktik berikutnya bisa dimulai.
| |
| − | | |
| − | ### 1. Cek tipe data
| |
| − | | |
| − | ```python
| |
| − | df_raw.info()
| |
| − | ```
| |
| − | | |
| − | ---
| |
| − | | |
| − | ### 2. Cari missing value
| |
| − | | |
| − | ```python
| |
| − | df_raw.isna().sum()
| |
| − | ```
| |
| − | | |
| − | Mahasiswa menentukan:
| |
| − | | |
| − | ```text
| |
| − | mana yang kosong?
| |
| − | kenapa bisa kosong?
| |
| − | perlukah dihapus?
| |
| − | ```
| |
| − | | |
| − | ---
| |
| − | | |
| − | ### 3. Periksa duplikasi
| |
| − | | |
| − | ```python
| |
| − | df_raw.duplicated().sum()
| |
| − | ```
| |
| − | | |
| − | Lebih baik juga:
| |
| − | | |
| − | ```python
| |
| − | df_raw.duplicated(
| |
| − | subset=["uri"]
| |
| − | ).sum()
| |
| − | ```
| |
| − | | |
| − | Diskusikan:
| |
| − | | |
| − | > Mengapa `uri` lebih cocok untuk memeriksa posting yang sama dibandingkan seluruh row?
| |
| − | | |
| − | ---
| |
| − | | |
| − | ### 4. Periksa teks
| |
| − | | |
| − | ```python
| |
| − | display(
| |
| − | df_raw[["text"]]
| |
| − | .head(20)
| |
| − | )
| |
| − | ```
| |
| − | | |
| − | Kemungkinan ditemukan:
| |
| − | | |
| − | ```text
| |
| − | URL
| |
| − | emoji
| |
| − | hashtag
| |
| − | newline
| |
| − | spasi berlebih
| |
| − | mention
| |
| − | ```
| |
| − | | |
| − | Itulah bahan untuk text cleaning.
| |
| − | | |
| − | ---
| |
| − | | |
| − | ### 5. Periksa relevansi
| |
| − | | |
| − | | |
| − | Cari:
| |
| − | | |
| − | ```python
| |
| − | df_raw[
| |
| − | ["text"]
| |
| − | ].head(20)
| |
| − | ```
| |
| − | | |
| − | Tanyakan:
| |
| − | | |
| − | > Apakah semuanya benar-benar membahas climate change?
| |
| − | | |
| − | Tidak boleh otomatis diasumsikan iya, karena dokumentasi Bluesky sendiri menyatakan sintaks dan perilaku query tidak sepenuhnya dispesifikasikan.
| |
| − | | |
| − | Ini bisa menjadi contoh nyata:
| |
| − | | |
| − | ```text
| |
| − | Search API
| |
| − | ↓
| |
| − | candidate data
| |
| − | | |
| − | bukan
| |
| − | | |
| − | Search API
| |
| − | ↓
| |
| − | ground truth
| |
| − | ```
| |
| − | | |
| − | ---
| |
| − | | |
| − | # Q. Tugas cleaning
| |
| − | | |
| − | Mahasiswa diminta menghasilkan:
| |
| − | | |
| − | ```text
| |
| | bluesky_climate_change_clean.csv | | bluesky_climate_change_clean.csv |
| − | ```
| + | ↓ |
| | + | ANALISIS |
| | + | </pre> |
| | | | |
| − | dari:
| + | Step by step nya adalah |
| | | | |
| − | ```text
| + | ### Import library dan membaca data |
| − | bluesky_climate_change_raw.csv
| + | <pre> |
| − | ```
| |
| − | | |
| − | Dengan tugas:
| |
| − | | |
| − | **1. Konversi tanggal**
| |
| − | | |
| − | ```text
| |
| − | created_at
| |
| − | string → datetime
| |
| − | ```
| |
| − | | |
| − | **2. Missing value**
| |
| − | | |
| − | Identifikasi dan tentukan perlakuannya.
| |
| − | | |
| − | **3. Duplikasi**
| |
| − | | |
| − | Gunakan `uri` untuk membantu mendeteksi posting yang sama.
| |
| − | | |
| − | **4. Cleaning text**
| |
| − | | |
| − | Misalnya:
| |
| − | | |
| − | ```text
| |
| − | newline
| |
| − | spasi berlebih
| |
| − | URL
| |
| − | ```
| |
| − | | |
| − | Namun mahasiswa harus menjelaskan **mengapa suatu elemen dihapus**, bukan sekadar menghapus semuanya.
| |
| − | | |
| − | **5. Relevansi topik**
| |
| − | | |
| − | Periksa apakah posting benar-benar terkait perubahan iklim.
| |
| − | | |
| − | **6. Engagement**
| |
| − | | |
| − | Buat variabel:
| |
| − | | |
| − | ```text
| |
| − | engagement
| |
| − | =
| |
| − | reply_count
| |
| − | +
| |
| − | repost_count
| |
| − | +
| |
| − | like_count
| |
| − | ```
| |
| − | | |
| − | ---
| |
| − | | |
| − | # R. Kode lengkap pengambilan data
| |
| − | | |
| − | ```python
| |
| − | import json
| |
| − | import requests
| |
| | import pandas as pd | | import pandas as pd |
| | + | import re |
| | | | |
| − | from pathlib import Path
| + | df = pd.read_csv( |
| − | | |
| − | | |
| − | # ========================================
| |
| − | # 1. Folder project
| |
| − | # ========================================
| |
| − | | |
| − | PROJECT = Path(
| |
| − | "/media/bgoode/Data4/Unpad/Kuliah/BigDataFisip"
| |
| − | )
| |
| − | | |
| − | RAW_DIR = PROJECT / "data" / "raw"
| |
| − | | |
| − | RAW_DIR.mkdir(
| |
| − | parents=True,
| |
| − | exist_ok=True
| |
| − | )
| |
| − | | |
| − | | |
| − | # ========================================
| |
| − | # 2. API
| |
| − | # ========================================
| |
| − | | |
| − | url = (
| |
| − | "https://api.bsky.app/"
| |
| − | "xrpc/app.bsky.feed.searchPosts"
| |
| − | )
| |
| − | | |
| − | params = {
| |
| − | "q": '"climate change"',
| |
| − | "limit": 100,
| |
| − | "sort": "latest",
| |
| − | "lang": "en"
| |
| − | }
| |
| − | | |
| − | | |
| − | # ========================================
| |
| − | # 3. Request
| |
| − | # ========================================
| |
| − | | |
| − | response = requests.get(
| |
| − | url,
| |
| − | params=params,
| |
| − | headers={
| |
| − | "Accept":
| |
| − | "application/json",
| |
| − | | |
| − | "User-Agent":
| |
| − | "BigDataFisip-Class/1.0"
| |
| − | },
| |
| − | timeout=30
| |
| − | )
| |
| − | | |
| − | | |
| − | print(
| |
| − | "HTTP Status:",
| |
| − | response.status_code
| |
| − | )
| |
| − | | |
| − | | |
| − | # ========================================
| |
| − | # 4. Validasi response
| |
| − | # ========================================
| |
| − | | |
| − | if response.status_code != 200:
| |
| − | | |
| − | raise RuntimeError(
| |
| − | f"HTTP {response.status_code}: "
| |
| − | f"{response.text[:300]}"
| |
| − | )
| |
| − | | |
| − | | |
| − | data = response.json()
| |
| − | | |
| − | posts = data.get(
| |
| − | "posts",
| |
| − | []
| |
| − | )
| |
| − | | |
| − | print(
| |
| − | "Jumlah posting:",
| |
| − | len(posts)
| |
| − | )
| |
| − | | |
| − | | |
| − | # ========================================
| |
| − | # 5. Simpan JSON asli
| |
| − | # ========================================
| |
| − | | |
| − | json_file = (
| |
| − | RAW_DIR /
| |
| − | "bluesky_climate_change_raw.json"
| |
| − | )
| |
| − | | |
| − | with open(
| |
| − | json_file,
| |
| − | "w",
| |
| − | encoding="utf-8"
| |
| − | ) as f:
| |
| − | | |
| − | json.dump(
| |
| − | data,
| |
| − | f,
| |
| − | ensure_ascii=False,
| |
| − | indent=2
| |
| − | )
| |
| − | | |
| − | | |
| − | # ========================================
| |
| − | # 6. Ubah ke tabel
| |
| − | # ========================================
| |
| − | | |
| − | rows = []
| |
| − | | |
| − | for post in posts:
| |
| − | | |
| − | record = post.get(
| |
| − | "record",
| |
| − | {}
| |
| − | )
| |
| − | | |
| − | author = post.get(
| |
| − | "author",
| |
| − | {}
| |
| − | )
| |
| − | | |
| − | rows.append({
| |
| − | | |
| − | "uri":
| |
| − | post.get("uri"),
| |
| − | | |
| − | "created_at":
| |
| − | record.get("createdAt"),
| |
| − | | |
| − | "username":
| |
| − | author.get("handle"),
| |
| − | | |
| − | "display_name":
| |
| − | author.get("displayName"),
| |
| − | | |
| − | "text":
| |
| − | record.get("text"),
| |
| − | | |
| − | "language":
| |
| − | record.get("langs"),
| |
| − | | |
| − | "reply_count":
| |
| − | post.get(
| |
| − | "replyCount",
| |
| − | 0
| |
| − | ),
| |
| − | | |
| − | "repost_count":
| |
| − | post.get(
| |
| − | "repostCount",
| |
| − | 0
| |
| − | ),
| |
| − | | |
| − | "like_count":
| |
| − | post.get(
| |
| − | "likeCount",
| |
| − | 0
| |
| − | )
| |
| − | })
| |
| − | | |
| − | | |
| − | df_raw = pd.DataFrame(
| |
| − | rows
| |
| − | )
| |
| − | | |
| − | | |
| − | # ========================================
| |
| − | # 7. Simpan CSV raw
| |
| − | # ========================================
| |
| − | | |
| − | csv_file = (
| |
| − | RAW_DIR /
| |
| | "bluesky_climate_change_raw.csv" | | "bluesky_climate_change_raw.csv" |
| | ) | | ) |
| | | | |
| − | df_raw.to_csv(
| + | df.head() |
| − | csv_file,
| + | </pre> |
| − | index=False,
| |
| − | encoding="utf-8-sig"
| |
| − | )
| |
| − | | |
| − | | |
| − | # ========================================
| |
| − | # 8. Pemeriksaan
| |
| − | # ========================================
| |
| − | | |
| − | print(
| |
| − | "Ukuran dataset:",
| |
| − | df_raw.shape
| |
| − | )
| |
| − | | |
| − | display(
| |
| − | df_raw.head()
| |
| − | )
| |
| − | ```
| |
| − | | |
| − | ---
| |
| − | | |
| − | ```text
| |
| − | 100 data mentah
| |
| − | ↓
| |
| − | identifikasi masalah
| |
| − | ↓
| |
| − | data cleaning
| |
| − | ↓
| |
| − | data processed
| |
| − | ↓
| |
| − | analisis
| |
| − | ```
| |
| − | | |
| − | [1]: https://bsky.network/docs/api-directory/?utm_source=chatgpt.com "API Hosts and Auth | Bluesky Protocol Services"
| |
