Skip to content

Auto-translated map_item updates from 4CAT: weibo #106

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dale-wahl merged 3 commits into from
Oct 21, 2026
Merged

Auto-translated map_item updates from 4CAT: weibo #106

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
649b10c
chore: sync map_item for weibo from 4CAT b08c28092151098c189b122b0e3d…
dale-wahl Oct 20, 2026
ec6ef85
Merge remote-tracking branch 'origin/master' into auto/4cat-map-item-…
dale-wahl Oct 21, 2026
2f8ebaf
weibo: content exists
dale-wahl Oct 21, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
270 changes: 127 additions & 143 deletions modules/weibo.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,173 +116,157 @@ function is_shortened(post) {
}

// === auto-generated by 4cat map_item sync — BLOCK REPLACED AUTOMATICALLY ===
// (written by hand: 4CAT has no Weibo data source yet. The first sync from
// datasources/weibo/search_weibo.py replaces this block.)
/**
* Map a Weibo post to a flat item
*
* @param {Object} post Post as captured, with `__import_meta`
* @returns {MappedItem}
*/
export function map_item(post) {
if (!is_post(post)) {
const source = post.__import_meta?.source_url ?? "";
throw new MapItemException(`Not a Weibo post (source: ${source || 'unknown'})`);
}

const user = post.user;
const reposted = post.retweeted_status ?? null;
const unix_timestamp = weibo_unix_timestamp(post.created_at);
// (regenerated from datasources/weibo/search_weibo.py)
function weibo_is_post(item) {
return typeof item === 'object'
&& typeof item.mblogid === 'string'
&& typeof item.user === 'object' && item.user !== null
&& typeof item.created_at === 'string';
}

return new MappedItem({
collected_from_url: readable_url(post.__import_meta?.source_platform_url ?? ""),
id: post.idstr,
thread_id: post.idstr,
url: `https://weibo.com/${user.idstr}/${post.mblogid}`,
body: post_text(post),
body_is_shortened: is_shortened(post) ? "yes" : "no",
timestamp: unix_timestamp !== null ? formatUtcTimestamp(unix_timestamp) : new MissingMappedField(""),
unix_timestamp: unix_timestamp ?? new MissingMappedField(""),
author: user.screen_name,
author_id: user.idstr,
author_avatar_url: user.avatar_hd ?? user.profile_image_url ?? "",
author_verified: user.verified ? "yes" : "no",
author_followers: value_or_missing(user, 'followers_count', ""),
likes: post.attitudes_count,
comments: post.comments_count,
reposts: post.reposts_count,
hashtags: hashtags(post).join(","),
image_urls: image_urls(post).join(","),
video_url: video_url(post),
location: post.region_name ? post.region_name.replace(/^发布于\s*/, '') : new MissingMappedField(""),
posted_with: post.source ? strip_tags(post.source) : "",
is_promoted: post.isAd ? "yes" : "no",
is_repost: reposted ? "yes" : "no",
reposted_id: reposted ? reposted.idstr : "",
reposted_url: reposted && reposted.user ? `https://weibo.com/${reposted.user.idstr}/${reposted.mblogid}` : "",
reposted_author: reposted && reposted.user ? reposted.user.screen_name : "",
reposted_body: reposted ? post_text(reposted) : "",
reposted_body_is_shortened: reposted && is_shortened(reposted) ? "yes" : "no",
});
function weibo_is_shortened(post) {
return post.isLongText === true && !post.longText;
}

/**
* A post's text, in full when it was captured in full
*
* `text_raw` is plain text, `text` the same with HTML links. Super-topic
* pages send only `text`.
*
* @param {Object} post
* @returns {string}
*/
function post_text(post) {
if (post.longText && post.longText.content) {
return post.longText.content;
function weibo_first_set(...values) {
for (let v of values) {
if (v !== null && v !== undefined) return v;
}
return values[values.length - 1];
}

if (typeof post.text_raw === 'string') {
return post.text_raw;
function weibo_post_text(post) {
const longText = py_get(post, 'longText');
if (typeof longText === 'object' && longText !== null && longText.content) {
return longText.content;
}

return strip_tags(post.text ?? "");
const raw = py_get(post, 'text_raw');
if (typeof raw === 'string') {
return raw;
}
return strip_tags(py_get(post, 'text') || '');
}

/**
* Hashtags in a post, without the # on either side
*
* Weibo hashtags are enclosed in #, as in #广交会#. `topic_struct` lists
* them when present.
*
* @param {Object} post
* @returns {string[]}
*/
function hashtags(post) {
if (Array.isArray(post.topic_struct) && post.topic_struct.length > 0) {
return post.topic_struct.map(topic => topic.topic_title).filter(Boolean);
function weibo_hashtags(post) {
const topics = py_get(post, 'topic_struct');
if (Array.isArray(topics) && topics.length) {
return topics.map(t => py_get(t, 'topic_title')).filter(t => t);
}

return [...post_text(post).matchAll(/#([^#\n]+)#/g)].map(match => match[1]);
const text = weibo_post_text(post);
const matches = [];
const regex = /#([^#\n]+)#/g;
let m;
while ((m = regex.exec(text)) !== null) {
matches.push(m[1]);
}
return matches;
}

/**
* Full-size image URLs, in the order the post shows them
*
* Images are in `pic_infos`, keyed by the IDs in `pic_ids`; posts that mix
* images and videos have them in `mix_media_info` instead.
*
* @param {Object} post
* @returns {string[]}
*/
function image_urls(post) {
function weibo_image_urls(post) {
const pictures = [];
for (const pic_id of post.pic_ids ?? []) {
if (post.pic_infos && post.pic_infos[pic_id]) {
pictures.push(post.pic_infos[pic_id]);
}
const picInfos = py_get(post, 'pic_infos') || {};
const picIds = py_get(post, 'pic_ids') || [];
for (const picId of picIds) {
if (picInfos[picId]) pictures.push(picInfos[picId]);
}

for (const media of post.mix_media_info?.items ?? []) {
if (media.type === 'pic' && media.data) {
pictures.push(media.data);
}
const mixItems = (py_get(post, 'mix_media_info') || {}).items || [];
for (const media of mixItems) {
if (media.type === 'pic' && media.data) pictures.push(media.data);
}

const urls = pictures
.map(picture => picture.largest?.url ?? picture.original?.url ?? picture.large?.url)
.filter(Boolean);
return [...new Set(urls)];
const urls = [];
for (const picture of pictures) {
const url = weibo_first_set(
(picture.largest || {}).url,
(picture.original || {}).url,
(picture.large || {}).url,
null
);
if (url && !urls.includes(url)) urls.push(url);
}
return urls;
}

/**
* URL of a post's video, if it has one
*
* @param {Object} post
* @returns {string}
*/
function video_url(post) {
const pages = [post.page_info];
for (const media of post.mix_media_info?.items ?? []) {
if (media.type === 'video') {
pages.push(media.data);
}
function weibo_video_url(post) {
const pages = [py_get(post, 'page_info')];
const mixItems = (py_get(post, 'mix_media_info') || {}).items || [];
for (const media of mixItems) {
if (media.type === 'video') pages.push(media.data);
}

for (const page of pages) {
const media_info = page?.object_type === 'video' ? page.media_info : null;
if (media_info) {
const url = media_info.mp4_720p_mp4 || media_info.mp4_hd_url || media_info.stream_url_hd
|| media_info.stream_url;
if (url) {
return url;
}
}
if (typeof page !== 'object' || page === null) continue;
if (page.object_type !== 'video') continue;
const mediaInfo = page.media_info || {};
const url = mediaInfo.mp4_720p_mp4 || mediaInfo.mp4_hd_url || mediaInfo.stream_url_hd || mediaInfo.stream_url;
if (url) return url;
}

return "";
return '';
}

/**
* Convert a Weibo date to a UNIX timestamp
*
* Weibo dates look like "Thu Oct 15 10:22:13 +0800 2026".
*
* @param {string} created_at
* @returns {number|null} Seconds since the epoch, or null if not a Weibo date
*/
function weibo_unix_timestamp(created_at) {
const parts = /^\w{3} (\w{3}) (\d{1,2}) (\d{2}):(\d{2}):(\d{2}) ([+-])(\d{2})(\d{2}) (\d{4})$/.exec(created_at ?? "");
if (!parts) {
return null;
export function map_item(item) {
// Validate that the item is a Weibo post
const importMeta = py_get(item, '__import_meta') || {};
const sourceUrl = py_get(importMeta, 'source_url') || '';
if (!weibo_is_post(item)) {
throw new MapItemException(`Not a Weibo post (source: ${sourceUrl || 'unknown'})`);
}

const [, month, day, hours, minutes, seconds, sign, offset_hours, offset_minutes, year] = parts;
const month_index = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'].indexOf(month);
if (month_index < 0) {
return null;
const user = py_get(item, 'user') || {};
const reposted = py_get(item, 'retweeted_status');
const repostedUser = reposted ? py_get(reposted, 'user') : null;

// Parse created_at into a Unix timestamp (seconds)
let unix_timestamp = null;
const createdAt = py_get(item, 'created_at');
if (typeof createdAt === 'string') {
const parsed = Date.parse(createdAt);
if (!isNaN(parsed)) {
unix_timestamp = Math.floor(parsed / 1000);
}
}

const as_if_utc = Date.UTC(+year, month_index, +day, +hours, +minutes, +seconds) / 1000;
const offset = (sign === '-' ? -1 : 1) * (+offset_hours * 3600 + +offset_minutes * 60);
return as_if_utc - offset;
return new MappedItem({
collected_from_url: readable_url(py_get(importMeta, 'source_platform_url') || ''),
id: py_get(item, 'idstr'),
thread_id: py_get(item, 'idstr'),
url: `https://weibo.com/${py_get(user, 'idstr')}/${py_get(item, 'mblogid')}`,
body: weibo_post_text(item),
body_is_shortened: weibo_is_shortened(item) ? 'yes' : 'no',
timestamp: unix_timestamp !== null ? formatUtcTimestamp(unix_timestamp) : new MissingMappedField(''),
unix_timestamp: unix_timestamp !== null ? unix_timestamp : new MissingMappedField(''),
author: py_get(user, 'screen_name') || '',
author_id: py_get(user, 'idstr'),
author_avatar_url: weibo_first_set(py_get(user, 'avatar_hd'), py_get(user, 'profile_image_url'), ''),
author_verified: py_get(user, 'verified') ? 'yes' : 'no',
author_followers: value_or_missing(user, 'followers_count', ''),
likes: value_or_missing(item, 'attitudes_count', 0),
comments: value_or_missing(item, 'comments_count', 0),
reposts: value_or_missing(item, 'reposts_count', 0),
hashtags: weibo_hashtags(item).join(','),
image_urls: weibo_image_urls(item).join(','),
video_url: weibo_video_url(item),
location: (function() {
const region = py_get(item, 'region_name');
if (!region) return new MissingMappedField('');
let loc = region;
if (loc.startsWith('发布于')) loc = loc.slice('发布于'.length);
return loc.replace(/^\s+/, '');
})(),
posted_with: (function() {
const src = py_get(item, 'source');
return src ? strip_tags(src) : '';
})(),
is_promoted: py_get(item, 'isAd') ? 'yes' : 'no',
is_repost: reposted ? 'yes' : 'no',
reposted_id: reposted ? (py_get(reposted, 'idstr') || '') : '',
reposted_url: (function() {
if (repostedUser && typeof repostedUser === 'object') {
return `https://weibo.com/${py_get(repostedUser, 'idstr')}/${py_get(reposted, 'mblogid')}`;
}
return '';
})(),
reposted_author: repostedUser ? (py_get(repostedUser, 'screen_name') || '') : '',
reposted_body: reposted ? weibo_post_text(reposted) : '',
reposted_body_is_shortened: reposted && weibo_is_shortened(reposted) ? 'yes' : 'no',
});
}
// === end auto-generated ===
Loading