Skip to content

Auto-translated map_item updates from 4CAT: weibo #108

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
4cat-to-zeeschuimer-automation-pr wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

Auto-translated map_item updates from 4CAT: weibo #108

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
156 changes: 77 additions & 79 deletions modules/weibo.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,155 +118,153 @@ function is_shortened(post) {
// === auto-generated by 4cat map_item sync — BLOCK REPLACED AUTOMATICALLY ===
// (regenerated from datasources/weibo/search_weibo.py)
function weibo_is_post(item) {
return typeof item === 'object'
&& typeof item.mblogid === 'string'
&& typeof item.user === 'object' && item.user !== null
&& typeof item.created_at === 'string';
return typeof item === 'object' &&
typeof item.mblogid === 'string' &&
typeof item.user === 'object' && item.user !== null &&
typeof item.created_at === 'string';
}

function weibo_is_shortened(post) {
return post.isLongText === true && !post.longText;
}

function weibo_first_set(...values) {
for (let v of values) {
if (v !== null && v !== undefined) return v;
for (const v of values) {
if (v != null) {
return v;
}
}
return values[values.length - 1];
}

function weibo_post_text(post) {
const longText = py_get(post, 'longText');
if (typeof longText === 'object' && longText !== null && longText.content) {
return longText.content;
const long_text = post.longText;
if (typeof long_text === 'object' && long_text && typeof long_text.content === 'string') {
return long_text.content;
}
const raw = py_get(post, 'text_raw');
if (typeof raw === 'string') {
return raw;
if (typeof post.text_raw === 'string') {
return post.text_raw;
}
return strip_tags(py_get(post, 'text') || '');
return strip_tags(post.text || '');
}

function weibo_hashtags(post) {
const topics = py_get(post, 'topic_struct');
const topics = py_get(post, 'topic_struct', null);
if (Array.isArray(topics) && topics.length) {
return topics.map(t => py_get(t, 'topic_title')).filter(t => t);
return topics.map(t => t.topic_title).filter(Boolean);
}
const text = weibo_post_text(post);
const matches = [];
const regex = /#([^#\n]+)#/g;
let m;
while ((m = regex.exec(text)) !== null) {
matches.push(m[1]);
}
const matches = [...text.matchAll(/#([^#\n]+)#/g)].map(m => m[1]);
return matches;
}

function weibo_image_urls(post) {
const pictures = [];
const picInfos = py_get(post, 'pic_infos') || {};
const picIds = py_get(post, 'pic_ids') || [];
for (const picId of picIds) {
if (picInfos[picId]) pictures.push(picInfos[picId]);
const pic_infos = py_get(post, 'pic_infos', {});
const pic_ids = py_get(post, 'pic_ids', []);
for (const pic_id of pic_ids) {
if (pic_infos[pic_id]) {
pictures.push(pic_infos[pic_id]);
}
}
const mixItems = (py_get(post, 'mix_media_info') || {}).items || [];
for (const media of mixItems) {
if (media.type === 'pic' && media.data) pictures.push(media.data);
const mix_items = (py_get(post, 'mix_media_info', {})).items || [];
for (const media of mix_items) {
if (media.type === 'pic' && media.data) {
pictures.push(media.data);
}
}
const urls = [];
for (const picture of pictures) {
const url = weibo_first_set(
(picture.largest || {}).url,
(picture.original || {}).url,
(picture.large || {}).url,
picture.largest?.url,
picture.original?.url,
picture.large?.url,
null
);
if (url && !urls.includes(url)) urls.push(url);
if (url && !urls.includes(url)) {
urls.push(url);
}
}
return urls;
}

function weibo_video_url(post) {
const pages = [py_get(post, 'page_info')];
const mixItems = (py_get(post, 'mix_media_info') || {}).items || [];
for (const media of mixItems) {
if (media.type === 'video') pages.push(media.data);
const mix_items = (py_get(post, 'mix_media_info', {})).items || [];
for (const media of mix_items) {
if (media.type === 'video') {
pages.push(media.data);
}
}
for (const page of pages) {
if (typeof page !== 'object' || page === null) continue;
if (page.object_type !== 'video') continue;
const mediaInfo = page.media_info || {};
const url = mediaInfo.mp4_720p_mp4 || mediaInfo.mp4_hd_url || mediaInfo.stream_url_hd || mediaInfo.stream_url;
if (url) return url;
if (typeof page !== 'object' || page.object_type !== 'video') {
continue;
}
const media_info = page.media_info || {};
const url = media_info.mp4_720p_mp4 ||
media_info.mp4_hd_url ||
media_info.stream_url_hd ||
media_info.stream_url;
if (url) {
return url;
}
}
return '';
}

export function map_item(item) {
// Validate that the item is a Weibo post
const importMeta = py_get(item, '__import_meta') || {};
const sourceUrl = py_get(importMeta, 'source_url') || '';
// Verify this is a Weibo post
if (!weibo_is_post(item)) {
throw new MapItemException(`Not a Weibo post (source: ${sourceUrl || 'unknown'})`);
const meta = py_get(item, '__import_meta', {});
const source = py_get(meta, 'source_url', '');
throw new MapItemException(`Not a Weibo post (source: ${source || 'unknown'})`);
}

const user = py_get(item, 'user') || {};
const reposted = py_get(item, 'retweeted_status');
const repostedUser = reposted ? py_get(reposted, 'user') : null;
const user = item.user;
const reposted = py_get(item, 'retweeted_status', null);
const reposted_user = reposted && typeof reposted === 'object' ? reposted.user : null;

// Parse created_at into a Unix timestamp (seconds)
let unix_timestamp = null;
const createdAt = py_get(item, 'created_at');
if (typeof createdAt === 'string') {
const parsed = Date.parse(createdAt);
if (typeof item.created_at === 'string') {
const parsed = Date.parse(item.created_at);
if (!isNaN(parsed)) {
unix_timestamp = Math.floor(parsed / 1000);
}
}

const meta = py_get(item, '__import_meta', {});
const collected_from_url = readable_url(py_get(meta, 'source_platform_url', ''));

return new MappedItem({
collected_from_url: readable_url(py_get(importMeta, 'source_platform_url') || ''),
id: py_get(item, 'idstr'),
thread_id: py_get(item, 'idstr'),
url: `https://weibo.com/${py_get(user, 'idstr')}/${py_get(item, 'mblogid')}`,
collected_from_url: collected_from_url,
id: item.idstr,
thread_id: item.idstr,
url: `https://weibo.com/${user.idstr}/${item.mblogid}`,
body: weibo_post_text(item),
body_is_shortened: weibo_is_shortened(item) ? 'yes' : 'no',
timestamp: unix_timestamp !== null ? formatUtcTimestamp(unix_timestamp) : new MissingMappedField(''),
unix_timestamp: unix_timestamp !== null ? unix_timestamp : new MissingMappedField(''),
author: py_get(user, 'screen_name') || '',
author_id: py_get(user, 'idstr'),
author_avatar_url: weibo_first_set(py_get(user, 'avatar_hd'), py_get(user, 'profile_image_url'), ''),
author_verified: py_get(user, 'verified') ? 'yes' : 'no',
author: py_get(user, 'screen_name', ''),
author_id: user.idstr,
author_avatar_url: weibo_first_set(user.avatar_hd, user.profile_image_url, ''),
author_verified: user.verified ? 'yes' : 'no',
author_followers: value_or_missing(user, 'followers_count', ''),
likes: value_or_missing(item, 'attitudes_count', 0),
comments: value_or_missing(item, 'comments_count', 0),
reposts: value_or_missing(item, 'reposts_count', 0),
hashtags: weibo_hashtags(item).join(','),
image_urls: weibo_image_urls(item).join(','),
video_url: weibo_video_url(item),
location: (function() {
const region = py_get(item, 'region_name');
if (!region) return new MissingMappedField('');
let loc = region;
if (loc.startsWith('发布于')) loc = loc.slice('发布于'.length);
return loc.replace(/^\s+/, '');
})(),
posted_with: (function() {
const src = py_get(item, 'source');
return src ? strip_tags(src) : '';
})(),
is_promoted: py_get(item, 'isAd') ? 'yes' : 'no',
location: item.region_name ? item.region_name.replace(/^发布于/, '').trim() : new MissingMappedField(''),
posted_with: item.source ? strip_tags(item.source) : '',
is_promoted: item.isAd ? 'yes' : 'no',
is_repost: reposted ? 'yes' : 'no',
reposted_id: reposted ? (py_get(reposted, 'idstr') || '') : '',
reposted_url: (function() {
if (repostedUser && typeof repostedUser === 'object') {
return `https://weibo.com/${py_get(repostedUser, 'idstr')}/${py_get(reposted, 'mblogid')}`;
}
return '';
})(),
reposted_author: repostedUser ? (py_get(repostedUser, 'screen_name') || '') : '',
reposted_id: reposted ? py_get(reposted, 'idstr', '') : '',
reposted_url: reposted_user && typeof reposted_user === 'object' ? `https://weibo.com/${reposted_user.idstr}/${reposted.mblogid}` : '',
reposted_author: reposted_user && typeof reposted_user === 'object' ? py_get(reposted_user, 'screen_name', '') : '',
reposted_body: reposted ? weibo_post_text(reposted) : '',
reposted_body_is_shortened: reposted && weibo_is_shortened(reposted) ? 'yes' : 'no',
reposted_body_is_shortened: reposted && weibo_is_shortened(reposted) ? 'yes' : 'no'
});
}
// === end auto-generated ===
Loading