diff --git a/modules/weibo.js b/modules/weibo.js index d38ab5c..9f29460 100644 --- a/modules/weibo.js +++ b/modules/weibo.js @@ -116,173 +116,157 @@ function is_shortened(post) { } // === auto-generated by 4cat map_item sync — BLOCK REPLACED AUTOMATICALLY === -// (written by hand: 4CAT has no Weibo data source yet. The first sync from -// datasources/weibo/search_weibo.py replaces this block.) -/** - * Map a Weibo post to a flat item - * - * @param {Object} post Post as captured, with `__import_meta` - * @returns {MappedItem} - */ -export function map_item(post) { - if (!is_post(post)) { - const source = post.__import_meta?.source_url ?? ""; - throw new MapItemException(`Not a Weibo post (source: ${source || 'unknown'})`); - } - - const user = post.user; - const reposted = post.retweeted_status ?? null; - const unix_timestamp = weibo_unix_timestamp(post.created_at); +// (regenerated from datasources/weibo/search_weibo.py) +function weibo_is_post(item) { + return typeof item === 'object' + && typeof item.mblogid === 'string' + && typeof item.user === 'object' && item.user !== null + && typeof item.created_at === 'string'; +} - return new MappedItem({ - collected_from_url: readable_url(post.__import_meta?.source_platform_url ?? ""), - id: post.idstr, - thread_id: post.idstr, - url: `https://weibo.com/${user.idstr}/${post.mblogid}`, - body: post_text(post), - body_is_shortened: is_shortened(post) ? "yes" : "no", - timestamp: unix_timestamp !== null ? formatUtcTimestamp(unix_timestamp) : new MissingMappedField(""), - unix_timestamp: unix_timestamp ?? new MissingMappedField(""), - author: user.screen_name, - author_id: user.idstr, - author_avatar_url: user.avatar_hd ?? user.profile_image_url ?? "", - author_verified: user.verified ? "yes" : "no", - author_followers: value_or_missing(user, 'followers_count', ""), - likes: post.attitudes_count, - comments: post.comments_count, - reposts: post.reposts_count, - hashtags: hashtags(post).join(","), - image_urls: image_urls(post).join(","), - video_url: video_url(post), - location: post.region_name ? post.region_name.replace(/^发布于\s*/, '') : new MissingMappedField(""), - posted_with: post.source ? strip_tags(post.source) : "", - is_promoted: post.isAd ? "yes" : "no", - is_repost: reposted ? "yes" : "no", - reposted_id: reposted ? reposted.idstr : "", - reposted_url: reposted && reposted.user ? `https://weibo.com/${reposted.user.idstr}/${reposted.mblogid}` : "", - reposted_author: reposted && reposted.user ? reposted.user.screen_name : "", - reposted_body: reposted ? post_text(reposted) : "", - reposted_body_is_shortened: reposted && is_shortened(reposted) ? "yes" : "no", - }); +function weibo_is_shortened(post) { + return post.isLongText === true && !post.longText; } -/** - * A post's text, in full when it was captured in full - * - * `text_raw` is plain text, `text` the same with HTML links. Super-topic - * pages send only `text`. - * - * @param {Object} post - * @returns {string} - */ -function post_text(post) { - if (post.longText && post.longText.content) { - return post.longText.content; +function weibo_first_set(...values) { + for (let v of values) { + if (v !== null && v !== undefined) return v; } + return values[values.length - 1]; +} - if (typeof post.text_raw === 'string') { - return post.text_raw; +function weibo_post_text(post) { + const longText = py_get(post, 'longText'); + if (typeof longText === 'object' && longText !== null && longText.content) { + return longText.content; } - - return strip_tags(post.text ?? ""); + const raw = py_get(post, 'text_raw'); + if (typeof raw === 'string') { + return raw; + } + return strip_tags(py_get(post, 'text') || ''); } -/** - * Hashtags in a post, without the # on either side - * - * Weibo hashtags are enclosed in #, as in #广交会#. `topic_struct` lists - * them when present. - * - * @param {Object} post - * @returns {string[]} - */ -function hashtags(post) { - if (Array.isArray(post.topic_struct) && post.topic_struct.length > 0) { - return post.topic_struct.map(topic => topic.topic_title).filter(Boolean); +function weibo_hashtags(post) { + const topics = py_get(post, 'topic_struct'); + if (Array.isArray(topics) && topics.length) { + return topics.map(t => py_get(t, 'topic_title')).filter(t => t); } - - return [...post_text(post).matchAll(/#([^#\n]+)#/g)].map(match => match[1]); + const text = weibo_post_text(post); + const matches = []; + const regex = /#([^#\n]+)#/g; + let m; + while ((m = regex.exec(text)) !== null) { + matches.push(m[1]); + } + return matches; } -/** - * Full-size image URLs, in the order the post shows them - * - * Images are in `pic_infos`, keyed by the IDs in `pic_ids`; posts that mix - * images and videos have them in `mix_media_info` instead. - * - * @param {Object} post - * @returns {string[]} - */ -function image_urls(post) { +function weibo_image_urls(post) { const pictures = []; - for (const pic_id of post.pic_ids ?? []) { - if (post.pic_infos && post.pic_infos[pic_id]) { - pictures.push(post.pic_infos[pic_id]); - } + const picInfos = py_get(post, 'pic_infos') || {}; + const picIds = py_get(post, 'pic_ids') || []; + for (const picId of picIds) { + if (picInfos[picId]) pictures.push(picInfos[picId]); } - - for (const media of post.mix_media_info?.items ?? []) { - if (media.type === 'pic' && media.data) { - pictures.push(media.data); - } + const mixItems = (py_get(post, 'mix_media_info') || {}).items || []; + for (const media of mixItems) { + if (media.type === 'pic' && media.data) pictures.push(media.data); } - - const urls = pictures - .map(picture => picture.largest?.url ?? picture.original?.url ?? picture.large?.url) - .filter(Boolean); - return [...new Set(urls)]; + const urls = []; + for (const picture of pictures) { + const url = weibo_first_set( + (picture.largest || {}).url, + (picture.original || {}).url, + (picture.large || {}).url, + null + ); + if (url && !urls.includes(url)) urls.push(url); + } + return urls; } -/** - * URL of a post's video, if it has one - * - * @param {Object} post - * @returns {string} - */ -function video_url(post) { - const pages = [post.page_info]; - for (const media of post.mix_media_info?.items ?? []) { - if (media.type === 'video') { - pages.push(media.data); - } +function weibo_video_url(post) { + const pages = [py_get(post, 'page_info')]; + const mixItems = (py_get(post, 'mix_media_info') || {}).items || []; + for (const media of mixItems) { + if (media.type === 'video') pages.push(media.data); } - for (const page of pages) { - const media_info = page?.object_type === 'video' ? page.media_info : null; - if (media_info) { - const url = media_info.mp4_720p_mp4 || media_info.mp4_hd_url || media_info.stream_url_hd - || media_info.stream_url; - if (url) { - return url; - } - } + if (typeof page !== 'object' || page === null) continue; + if (page.object_type !== 'video') continue; + const mediaInfo = page.media_info || {}; + const url = mediaInfo.mp4_720p_mp4 || mediaInfo.mp4_hd_url || mediaInfo.stream_url_hd || mediaInfo.stream_url; + if (url) return url; } - - return ""; + return ''; } -/** - * Convert a Weibo date to a UNIX timestamp - * - * Weibo dates look like "Thu Oct 15 10:22:13 +0800 2026". - * - * @param {string} created_at - * @returns {number|null} Seconds since the epoch, or null if not a Weibo date - */ -function weibo_unix_timestamp(created_at) { - const parts = /^\w{3} (\w{3}) (\d{1,2}) (\d{2}):(\d{2}):(\d{2}) ([+-])(\d{2})(\d{2}) (\d{4})$/.exec(created_at ?? ""); - if (!parts) { - return null; +export function map_item(item) { + // Validate that the item is a Weibo post + const importMeta = py_get(item, '__import_meta') || {}; + const sourceUrl = py_get(importMeta, 'source_url') || ''; + if (!weibo_is_post(item)) { + throw new MapItemException(`Not a Weibo post (source: ${sourceUrl || 'unknown'})`); } - const [, month, day, hours, minutes, seconds, sign, offset_hours, offset_minutes, year] = parts; - const month_index = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'].indexOf(month); - if (month_index < 0) { - return null; + const user = py_get(item, 'user') || {}; + const reposted = py_get(item, 'retweeted_status'); + const repostedUser = reposted ? py_get(reposted, 'user') : null; + + // Parse created_at into a Unix timestamp (seconds) + let unix_timestamp = null; + const createdAt = py_get(item, 'created_at'); + if (typeof createdAt === 'string') { + const parsed = Date.parse(createdAt); + if (!isNaN(parsed)) { + unix_timestamp = Math.floor(parsed / 1000); + } } - const as_if_utc = Date.UTC(+year, month_index, +day, +hours, +minutes, +seconds) / 1000; - const offset = (sign === '-' ? -1 : 1) * (+offset_hours * 3600 + +offset_minutes * 60); - return as_if_utc - offset; + return new MappedItem({ + collected_from_url: readable_url(py_get(importMeta, 'source_platform_url') || ''), + id: py_get(item, 'idstr'), + thread_id: py_get(item, 'idstr'), + url: `https://weibo.com/${py_get(user, 'idstr')}/${py_get(item, 'mblogid')}`, + body: weibo_post_text(item), + body_is_shortened: weibo_is_shortened(item) ? 'yes' : 'no', + timestamp: unix_timestamp !== null ? formatUtcTimestamp(unix_timestamp) : new MissingMappedField(''), + unix_timestamp: unix_timestamp !== null ? unix_timestamp : new MissingMappedField(''), + author: py_get(user, 'screen_name') || '', + author_id: py_get(user, 'idstr'), + author_avatar_url: weibo_first_set(py_get(user, 'avatar_hd'), py_get(user, 'profile_image_url'), ''), + author_verified: py_get(user, 'verified') ? 'yes' : 'no', + author_followers: value_or_missing(user, 'followers_count', ''), + likes: value_or_missing(item, 'attitudes_count', 0), + comments: value_or_missing(item, 'comments_count', 0), + reposts: value_or_missing(item, 'reposts_count', 0), + hashtags: weibo_hashtags(item).join(','), + image_urls: weibo_image_urls(item).join(','), + video_url: weibo_video_url(item), + location: (function() { + const region = py_get(item, 'region_name'); + if (!region) return new MissingMappedField(''); + let loc = region; + if (loc.startsWith('发布于')) loc = loc.slice('发布于'.length); + return loc.replace(/^\s+/, ''); + })(), + posted_with: (function() { + const src = py_get(item, 'source'); + return src ? strip_tags(src) : ''; + })(), + is_promoted: py_get(item, 'isAd') ? 'yes' : 'no', + is_repost: reposted ? 'yes' : 'no', + reposted_id: reposted ? (py_get(reposted, 'idstr') || '') : '', + reposted_url: (function() { + if (repostedUser && typeof repostedUser === 'object') { + return `https://weibo.com/${py_get(repostedUser, 'idstr')}/${py_get(reposted, 'mblogid')}`; + } + return ''; + })(), + reposted_author: repostedUser ? (py_get(repostedUser, 'screen_name') || '') : '', + reposted_body: reposted ? weibo_post_text(reposted) : '', + reposted_body_is_shortened: reposted && weibo_is_shortened(reposted) ? 'yes' : 'no', + }); } // === end auto-generated ===