Add Weibo and Weibo comments modules

README.md

@@ -27,6 +27,7 @@ Currently, it supports the following platforms:
 * [Truth Social](https://truth.social)
 * [Pinterest](https://pinterest.com)
 * [RedNote/Xiaohongshu](https://xiaohongshu.com) (posts and comments)
+* [Weibo](https://weibo.com) (posts and comments)
 
 Platform support requires regular maintenance to keep up with changes to the platforms. If something does not work, we
 welcome issues and pull requests. See 'Limitations' below for some known limitations to data capture.
@@ -90,13 +91,23 @@ platform. The following limitations are known:
   * Live streams
 * *Douyin*
   * douyin.com/search/ page is untested (requires login) 
+* *Weibo* is only supported on the desktop site, weibo.com, and is untested while logged in. Logged out, Weibo itself
+  does not show:
+  * Search results
+  * A profile's timeline (its video tab does work)
+  * Reposts of a post
+* *Weibo* items that cannot be captured:
+  * Posts on video pages (weibo.com/tv); their comments are captured
 
 For some platforms, the level of detail of the data that can be collected depends on the page it is captured from:
 
 * *Pinterest* items may lack some metadata unless captured from the individual post's page, most notably the timestamp 
   of the post.
 * *RedNote/Xiaohongshu* items will often lack the item's post description, timestamp, and video URL, unless captured by 
   opening the post's own page/clicking it in an overview.
+* *Weibo* shortens long posts in feeds and lists. Opening the post's own page captures its full text; with 'Skip 
+  duplicates' or 'Update existing' selected, the full post replaces the shortened one. Mapped items say whether the 
+  text was shortened.
 
 Note that these are *known* limitations; data capture may break or change based on platform changes. Always 
 cross-reference captured data with what you are seeing in your browser.

modules/_loader.js

@@ -17,7 +17,9 @@ async function load() {
         await import("./rednote.js"),
         await import("./rednote-comments.js"),
         await import("./twitter.js"),
-        await import("./gab.js")
+        await import("./gab.js"),
+        await import("./weibo.js"),
+        await import("./weibo-comments.js")
     ];
 
     for(const module of imported_modules) {

modules/weibo-comments.js

@@ -0,0 +1,162 @@
+export const MODULE_NAME = 'Weibo (comments)';
+export const DOMAIN = 'weibo.com';
+export const MODULE_ID = 'weibo-comments';
+
+/**
+ * Collect comments on Weibo posts from weibo.com
+ *
+ * Comments load from /ajax/statuses/buildComments, on a post's own page and
+ * on video pages. The first requests for a post list its top-level comments,
+ * each with up to two of its replies in `comments`; opening a comment's
+ * replies lists all of them. Replies are collected as items of their own.
+ *
+ * @param {string} response  Content of the captured response
+ * @param {string} source_platform_url  URL of the page the user is on
+ * @param {string} source_url  URL of the captured response
+ * @returns {Array}  Weibo comments
+ */
+export function capture(response, source_platform_url, source_url) {
+    const domain = source_platform_url.split("/")[2].toLowerCase().replace(/^www\./, '');
+    if (!domain.endsWith('weibo.com')) {
+        return [];
+    }
+
+    if (!response) {
+        return [];
+    }
+
+    let data;
+    try {
+        data = JSON.parse(response);
+    } catch (e) {
+        return [];
+    }
+
+    if (!data || !Array.isArray(data.data)) {
+        return [];
+    }
+
+    const comments = [];
+    for (const comment of data.data) {
+        if (!is_comment(comment)) {
+            continue;
+        }
+
+        comments.push(comment);
+        for (const reply of Array.isArray(comment.comments) ? comment.comments : []) {
+            if (is_comment(reply)) {
+                comments.push(reply);
+            }
+        }
+    }
+
+    return comments;
+}
+
+// capture() uses this helper, and map_item() uses it too. It stays outside
+// the block 4CAT's map_item sync replaces, so that a sync cannot remove it.
+
+/**
+ * Whether an object is a Weibo comment
+ *
+ * Comments have their own ID, the ID of the comment their thread starts with
+ * (their own, for a top-level comment), an author and a creation date.
+ *
+ * @param {*} item
+ * @returns {boolean}
+ */
+function is_comment(item) {
+    return typeof item === 'object' && item !== null
+        && typeof item.idstr === 'string'
+        && typeof item.rootidstr === 'string'
+        && typeof item.user === 'object' && item.user !== null
+        && typeof item.created_at === 'string';
+}
+
+// === auto-generated by 4cat map_item sync — BLOCK REPLACED AUTOMATICALLY ===
+// (written by hand: 4CAT has no Weibo data source yet. The first sync from
+// datasources/weibo_comments/search_weibo_comments.py replaces this block.)
+/**
+ * Map a Weibo comment to a flat item
+ *
+ * @param {Object} comment  Comment as captured, with `__import_meta`
+ * @returns {MappedItem}
+ */
+export function map_item(comment) {
+    if (!is_comment(comment)) {
+        const source = comment.__import_meta?.source_url ?? "";
+        throw new MapItemException(`Not a Weibo comment (source: ${source || 'unknown'})`);
+    }
+
+    const user = comment.user;
+    const post = commented_post(comment);
+    const is_reply = comment.rootidstr !== comment.idstr;
+    const unix_timestamp = weibo_unix_timestamp(comment.created_at);
+
+    return new MappedItem({
+        collected_from_url: normalize_url_encoding(comment.__import_meta?.source_platform_url ?? ""),
+        id: comment.idstr,
+        thread_id: post.id ?? new MissingMappedField(""),
+        post_id: post.id ?? new MissingMappedField(""),
+        post_url: post.id && post.author_id
+            ? `https://weibo.com/${post.author_id}/${post.id}` : new MissingMappedField(""),
+        author: user.screen_name,
+        author_id: user.idstr,
+        author_avatar_url: user.avatar_hd ?? user.profile_image_url ?? "",
+        body: typeof comment.text_raw === 'string' ? comment.text_raw : strip_tags(comment.text ?? ""),
+        timestamp: unix_timestamp !== null ? formatUtcTimestamp(unix_timestamp) : new MissingMappedField(""),
+        unix_timestamp: unix_timestamp ?? new MissingMappedField(""),
+        likes: comment.like_counts ?? comment.like_count ?? new MissingMappedField(""),
+        replies: is_reply ? "" : value_or_missing(comment, 'total_number', ""),
+        is_reply: is_reply ? "yes" : "no",
+        root_comment_id: comment.rootidstr,
+        reply_to_comment_id: comment.reply_comment?.idstr ?? "",
+        reply_to_author: comment.reply_comment?.user?.screen_name ?? "",
+        location: comment.source ? comment.source.replace(/^来自\s*/, '') : new MissingMappedField(""),
+        is_liked_by_post_author: comment.isLikedByMblogAuthor ? "yes" : "no",
+    });
+}
+
+/**
+ * The post a comment belongs to
+ *
+ * Comments do not name their post in a field of its own. Its ID and its
+ * author's ID are in `analysis_extra`, which reads like
+ * "author_uid:5781292544|mid:5353703604491306|rid:5246087537427886552".
+ *
+ * @param {Object} comment
+ * @returns {{id: string|undefined, author_id: string|undefined}}
+ */
+function commented_post(comment) {
+    const fields = Object.fromEntries(String(comment.analysis_extra ?? "")
+        .split("|")
+        .map(field => field.split(":"))
+        .filter(field => field.length === 2));
+    return {id: fields.mid, author_id: fields.author_uid};
+}
+
+/**
+ * Convert a Weibo date to a UNIX timestamp
+ *
+ * Weibo dates look like "Thu Oct 15 10:22:13 +0800 2026".
+ *
+ * @param {string} created_at
+ * @returns {number|null}  Seconds since the epoch, or null if not a Weibo date
+ */
+function weibo_unix_timestamp(created_at) {
+    const parts = /^\w{3} (\w{3}) (\d{1,2}) (\d{2}):(\d{2}):(\d{2}) ([+-])(\d{2})(\d{2}) (\d{4})$/.exec(created_at ?? "");
+    if (!parts) {
+        return null;
+    }
+
+    const [, month, day, hours, minutes, seconds, sign, offset_hours, offset_minutes, year] = parts;
+    const month_index = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'].indexOf(month);
+    if (month_index < 0) {
+        return null;
+    }
+
+    const as_if_utc = Date.UTC(+year, month_index, +day, +hours, +minutes, +seconds) / 1000;
+    const offset = (sign === '-' ? -1 : 1) * (+offset_hours * 3600 + +offset_minutes * 60);
+    return as_if_utc - offset;
+}
+// === end auto-generated ===