From 289f406b90c8e2fc988a742a3d4863cded201fcc Mon Sep 17 00:00:00 2001 From: dale-wahl <32108944+dale-wahl@users.noreply.github.com> Date: Tue, 20 Oct 2026 12:10:35 +0000 Subject: [PATCH 1/2] chore: sync map_item for weibo_comments from 4CAT b08c28092151098c189b122b0e3d8bc0758c34ca --- modules/weibo-comments.js | 170 +++++++++++++++++++++----------------- 1 file changed, 95 insertions(+), 75 deletions(-) diff --git a/modules/weibo-comments.js b/modules/weibo-comments.js index ccd0aee..1de09ef 100644 --- a/modules/weibo-comments.js +++ b/modules/weibo-comments.js @@ -74,89 +74,109 @@ function is_comment(item) { } // === auto-generated by 4cat map_item sync — BLOCK REPLACED AUTOMATICALLY === -// (written by hand: 4CAT has no Weibo data source yet. The first sync from -// datasources/weibo_comments/search_weibo_comments.py replaces this block.) -/** - * Map a Weibo comment to a flat item - * - * @param {Object} comment Comment as captured, with `__import_meta` - * @returns {MappedItem} - */ -export function map_item(comment) { - if (!is_comment(comment)) { - const source = comment.__import_meta?.source_url ?? ""; - throw new MapItemException(`Not a Weibo comment (source: ${source || 'unknown'})`); - } - - const user = comment.user; - const post = commented_post(comment); - const is_reply = comment.rootidstr !== comment.idstr; - const unix_timestamp = weibo_unix_timestamp(comment.created_at); +// (regenerated from datasources/weibo_comments/search_weibo_comments.py) +function weibo_is_comment(item) { + return typeof item === 'object' && item !== null + && typeof item.idstr === 'string' + && typeof item.rootidstr === 'string' + && typeof item.user === 'object' && item.user !== null + && typeof item.created_at === 'string'; +} - return new MappedItem({ - collected_from_url: normalize_url_encoding(comment.__import_meta?.source_platform_url ?? ""), - id: comment.idstr, - thread_id: post.id ?? new MissingMappedField(""), - post_id: post.id ?? new MissingMappedField(""), - post_url: post.id && post.author_id - ? `https://weibo.com/${post.author_id}/${post.id}` : new MissingMappedField(""), - author: user.screen_name, - author_id: user.idstr, - author_avatar_url: user.avatar_hd ?? user.profile_image_url ?? "", - body: typeof comment.text_raw === 'string' ? comment.text_raw : strip_tags(comment.text ?? ""), - timestamp: unix_timestamp !== null ? formatUtcTimestamp(unix_timestamp) : new MissingMappedField(""), - unix_timestamp: unix_timestamp ?? new MissingMappedField(""), - likes: comment.like_counts ?? comment.like_count ?? new MissingMappedField(""), - replies: is_reply ? "" : value_or_missing(comment, 'total_number', ""), - is_reply: is_reply ? "yes" : "no", - root_comment_id: comment.rootidstr, - reply_to_comment_id: comment.reply_comment?.idstr ?? "", - reply_to_author: comment.reply_comment?.user?.screen_name ?? "", - location: comment.source ? comment.source.replace(/^来自\s*/, '') : new MissingMappedField(""), - is_liked_by_post_author: comment.isLikedByMblogAuthor ? "yes" : "no", - }); +function weibo_commented_post(comment) { + const extra = py_get(comment, 'analysis_extra', ''); + const fields = {}; + if (extra) { + const parts = extra.split('|'); + for (const part of parts) { + const idx = part.indexOf(':'); + if (idx !== -1) { + const key = part.slice(0, idx); + const val = part.slice(idx + 1); + fields[key] = val; + } + } + } + return { id: fields['mid'] || null, author_id: fields['author_uid'] || null }; } -/** - * The post a comment belongs to - * - * Comments do not name their post in a field of its own. Its ID and its - * author's ID are in `analysis_extra`, which reads like - * "author_uid:5781292544|mid:5353703604491306|rid:5246087537427886552". - * - * @param {Object} comment - * @returns {{id: string|undefined, author_id: string|undefined}} - */ -function commented_post(comment) { - const fields = Object.fromEntries(String(comment.analysis_extra ?? "") - .split("|") - .map(field => field.split(":")) - .filter(field => field.length === 2)); - return {id: fields.mid, author_id: fields.author_uid}; +function weibo_first_set(...values) { + for (let i = 0; i < values.length; i++) { + const v = values[i]; + if (v !== null && v !== undefined) { + return v; + } + } + return values[values.length - 1]; } -/** - * Convert a Weibo date to a UNIX timestamp - * - * Weibo dates look like "Thu Oct 15 10:22:13 +0800 2026". - * - * @param {string} created_at - * @returns {number|null} Seconds since the epoch, or null if not a Weibo date - */ -function weibo_unix_timestamp(created_at) { - const parts = /^\w{3} (\w{3}) (\d{1,2}) (\d{2}):(\d{2}):(\d{2}) ([+-])(\d{2})(\d{2}) (\d{4})$/.exec(created_at ?? ""); - if (!parts) { - return null; +export function map_item(item) { + // Verify this is a Weibo comment + if (!weibo_is_comment(item)) { + const meta = py_get(item, '__import_meta', {}); + const src = py_get(meta, 'source_url', ''); + throw new MapItemException(`Not a Weibo comment (source: ${src || 'unknown'})`); } - const [, month, day, hours, minutes, seconds, sign, offset_hours, offset_minutes, year] = parts; - const month_index = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'].indexOf(month); - if (month_index < 0) { - return null; + const user = item['user']; + const post = weibo_commented_post(item); + const is_reply = item['rootidstr'] !== item['idstr']; + + // Parse created_at into a Unix timestamp (seconds) + let unix_timestamp = null; + const created_at = py_get(item, 'created_at'); + if (typeof created_at === 'string') { + const ms = Date.parse(created_at); + if (!Number.isNaN(ms)) { + unix_timestamp = Math.floor(ms / 1000); + } } - const as_if_utc = Date.UTC(+year, month_index, +day, +hours, +minutes, +seconds) / 1000; - const offset = (sign === '-' ? -1 : 1) * (+offset_hours * 3600 + +offset_minutes * 60); - return as_if_utc - offset; + const reply_to = py_get(item, 'reply_comment') || {}; + + return new MappedItem({ + collected_from_url: normalize_url_encoding( + py_get(py_get(item, '__import_meta', {}), 'source_platform_url', '') + ), + id: item['idstr'], + thread_id: post.id != null ? post.id : new MissingMappedField(''), + post_id: post.id != null ? post.id : new MissingMappedField(''), + post_url: post.id && post.author_id + ? `https://weibo.com/${post.author_id}/${post.id}` + : new MissingMappedField(''), + author: py_get(user, 'screen_name', ''), + author_id: user['idstr'], + author_avatar_url: weibo_first_set( + py_get(user, 'avatar_hd'), + py_get(user, 'profile_image_url'), + '' + ), + body: typeof py_get(item, 'text_raw') === 'string' + ? py_get(item, 'text_raw') + : strip_tags(py_get(item, 'text') || ''), + timestamp: unix_timestamp != null + ? formatUtcTimestamp(unix_timestamp) + : new MissingMappedField(''), + unix_timestamp: unix_timestamp != null + ? unix_timestamp + : new MissingMappedField(''), + likes: weibo_first_set( + py_get(item, 'like_counts'), + py_get(item, 'like_count'), + new MissingMappedField('') + ), + replies: is_reply ? '' : value_or_missing(item, 'total_number', ''), + is_reply: is_reply ? 'yes' : 'no', + root_comment_id: item['rootidstr'], + reply_to_comment_id: weibo_first_set(py_get(reply_to, 'idstr'), ''), + reply_to_author: weibo_first_set( + py_get(py_get(reply_to, 'user') || {}, 'screen_name'), + '' + ), + location: item['source'] + ? item['source'].replace(/^来自/, '').trimStart() + : new MissingMappedField(''), + is_liked_by_post_author: py_get(item, 'isLikedByMblogAuthor') ? 'yes' : 'no', + }); } // === end auto-generated === From fb895b16ab33f23ec9c04416b7306bcbbd40a116 Mon Sep 17 00:00:00 2001 From: Dale Wahl Date: Wed, 21 Oct 2026 17:27:16 +0200 Subject: [PATCH 2/2] weibo comments: ?? vs || (?? null/defined, || falsey) --- modules/weibo-comments.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/weibo-comments.js b/modules/weibo-comments.js index 2e6e89e..19f7ce6 100644 --- a/modules/weibo-comments.js +++ b/modules/weibo-comments.js @@ -97,7 +97,7 @@ function weibo_commented_post(comment) { } } } - return { id: fields['mid'] || null, author_id: fields['author_uid'] || null }; + return { id: fields['mid'] ?? null, author_id: fields['author_uid'] ?? null }; } function weibo_first_set(...values) {