Skip to content

Auto-translated map_item updates from 4CAT: xiaohongshu #103

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dale-wahl merged 3 commits into from
Oct 21, 2026
Merged

Auto-translated map_item updates from 4CAT: xiaohongshu #103

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
df948ce
chore: sync map_item for xiaohongshu from 4CAT 451220dd9d7ca1e7111781…
dale-wahl Oct 13, 2026
ac5e7a9
Merge branch 'master' into auto/4cat-map-item-sync-xiaohongshu
dale-wahl Oct 21, 2026
805a1a3
rednote fix empty list return
dale-wahl Oct 21, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
172 changes: 102 additions & 70 deletions modules/rednote.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,18 +109,34 @@ export function capture(response, source_platform_url, source_url) {

// === auto-generated by 4cat map_item sync — BLOCK REPLACED AUTOMATICALLY ===
// (regenerated from datasources/xiaohongshu/search_rednote.py)
export function map_item(post) {
// Reject tile stub items
if (!('note_card' in post) && !('user' in post) && py_get(post, '_zs-origin') !== 'html' && !('note' in post)) {
const source = py_get(py_get(post, '__import_meta', {}), 'source_url', '');
throw new MapItemException(`Xiaohongshu tile stub without post content (source: ${source || 'unknown'})`);
}

if (py_get(post, '_zs-origin') === 'html') {
return map_item_from_html(post);
} else if ('note' in post) {
return map_item_from_json_embedded(post);
} else {
return map_item_from_json_api_explore(post);
}
}

function map_item_from_json_api_explore(post) {
const item = post.type !== 'video' ? post.note_card : post;
const item_id = post.id ?? post.note_id;
const item = py_get(post, 'type') !== 'video' ? py_get(post, 'note_card') : post;
const item_id = py_get(post, 'id', py_get(post, 'note_id'));

// Images handling
let images;
if (item.image_list) {
if (item && Array.isArray(item.image_list) && item.image_list.length) {
images = [];
for (const image of item.image_list) {
if (image.url_default) {
if ('url_default' in image && image.url_default) {
images.push(image.url_default);
} else if (image.info_list && image.info_list.length) {
} else if ('info_list' in image && image.info_list) {
let found = false;
for (const imgInfo of image.info_list) {
if (imgInfo.image_scene === 'WB_DFT') {
Expand All @@ -129,112 +145,128 @@ function map_item_from_json_api_explore(post) {
break;
}
}
if (!found) {
if (!found && image.info_list.length) {
images.push(image.info_list[0].url);
}
}
}
} else if (item.cover) {
} else if (item && item.cover) {
images = [item.cover.url_default];
} else {
images = new MissingMappedField("");
}

const xsec_bit = post.xsec_token ? `?xsec_token=${post.xsec_token}` : "";
const video_url = item.video?.media ? item.video.media.stream.h264[0].master_url : new MissingMappedField("");
const author = item.user.nickname ?? item.user.nick_name;
const timestamp = item.time ?? null;
const timestampStr = timestamp ? formatUtcTimestamp(timestamp / 1000) : new MissingMappedField("");
const hashtags = item.desc ? [...item.desc.matchAll(/#([^\s!@#$%^&*()_+{}:"|<>?\[\];',.\/`~]+)/g)].map(m => m[1]).join(",") : new MissingMappedField("");
const body = item.desc ?? new MissingMappedField("");
const image_urls = Array.isArray(images) ? images.join(",") : images;
const likes = item.interact_info?.liked_count ?? null;
const unix_ts = timestamp ? Math.floor(timestamp / 1000) : new MissingMappedField("");
const xsec_bit = py_get(post, 'xsec_token') ? `?xsec_token=${post.xsec_token}` : "";
let video_url;
if (item && item.video && item.video.media) {
video_url = getVideoUrl(item.video.media.stream || {});
} else {
video_url = new MissingMappedField("");
}

const author = (item.user.nickname !== undefined) ? item.user.nickname : item.user.nick_name;
const timestamp = py_get(item, 'time');

const hashtags = ('desc' in item)
? [...item.desc.matchAll(/#([^\s!@#$%^&*()_+{}:"|<>?\[\];',.\/`~]+)/g)].map(m => m[1]).join(',')
: new MissingMappedField("");

const body = ('desc' in item) ? item.desc : new MissingMappedField("");

const collected_from_url = readable_url(py_get(py_get(post, '__import_meta', {}), 'source_platform_url', ''));

return new MappedItem({
collected_from_url: readable_url(post.__import_meta?.source_platform_url ?? ""),
collected_from_url,
id: item_id,
thread_id: item_id,
url: `https://www.xiaohongshu.com/explore/${post.id}${xsec_bit}`,
title: item.display_title ?? "",
body: body,
hashtags: hashtags,
timestamp: timestampStr,
author: author,
title: py_get(item, 'display_title', ''),
body,
hashtags,
timestamp: timestamp ? formatUtcTimestamp(timestamp / 1000) : new MissingMappedField("") ,
author,
author_avatar_url: item.user.avatar,
image_urls: image_urls,
video_url: video_url,
likes: likes,
unix_timestamp: unix_ts,
image_urls: Array.isArray(images) ? images.join(',') : images,
video_url,
likes: item.interact_info.liked_count,
unix_timestamp: timestamp ? Math.floor(timestamp / 1000) : new MissingMappedField("")
});
}

function getVideoUrl(streams) {
const h264 = ['h264', 'EF4'];
const formats = [...h264, ...Object.keys(streams).filter(k => !h264.includes(k))];
for (const name of formats) {
const list = streams[name] || [];
for (const stream of list) {
if (stream.master_url) {
return stream.master_url;
}
}
}
return new MissingMappedField("");
}

function map_item_from_json_embedded(item) {
const note = item.note;
const image = note.imageList?.[0]?.urlDefault ?? new MissingMappedField("");
const image = note.imageList[0].urlDefault;
const xsec_bit = `?xsec_token=${note.xsecToken}`;
const timestamp = note.time ?? null;
const timestampStr = timestamp ? formatUtcTimestamp(timestamp / 1000) : new MissingMappedField("");
const hashtags = note.desc ? [...note.desc.matchAll(/#([^\s!@#$%^&*()_+{}:"|<>?\[\];',.\/`~]+)/g)].map(m => m[1]).join(",") : new MissingMappedField("");
const body = note.desc ?? new MissingMappedField("");
const author = note.user.nickname ?? note.user.nick_name;
const likes = note.interactInfo?.likedCount ??
note.interact_info?.liked_count ??
note.likes ??
new MissingMappedField("");
const unix_ts = timestamp ? Math.floor(timestamp / 1000) : new MissingMappedField("");
const timestamp = py_get(note, 'time');

let likes;
if ('interactInfo' in note) {
likes = note.interactInfo.likedCount;
} else if ('interact_info' in note) {
likes = note.interact_info.liked_count;
} else if ('likes' in note) {
likes = note.likes;
} else {
likes = new MissingMappedField("");
}

const author = (note.user.nickname !== undefined) ? note.user.nickname : note.user.nick_name;
const hashtags = ('desc' in note)
? [...note.desc.matchAll(/#([^\s!@#$%^&*()_+{}:"|<>?\[\];',.\/`~]+)/g)].map(m => m[1]).join(',')
: new MissingMappedField("");
const body = ('desc' in note) ? note.desc : new MissingMappedField("");

const collected_from_url = readable_url(py_get(py_get(item, '__import_meta', {}), 'source_platform_url', ''));

return new MappedItem({
collected_from_url: readable_url(item.__import_meta?.source_platform_url ?? ""),
collected_from_url,
id: item.id,
thread_id: item.id,
url: `https://www.xiaohongshu.com/explore/${item.id}${xsec_bit}`,
title: note.title ?? "",
body: body,
hashtags: hashtags,
timestamp: timestampStr,
author: author,
title: py_get(note, 'title', ''),
body,
hashtags,
timestamp: timestamp ? formatUtcTimestamp(timestamp / 1000) : new MissingMappedField("") ,
author,
author_avatar_url: note.user.avatar,
image_url: image,
video_url: new MissingMappedField(""),
likes: likes,
unix_timestamp: unix_ts,
video_url: new MissingMappedField("") ,
likes,
unix_timestamp: timestamp ? Math.floor(timestamp / 1000) : new MissingMappedField("")
});
}

function map_item_from_html(item) {
const collected_from_url = readable_url(py_get(py_get(item, '__import_meta', {}), 'source_platform_url', ''));
return new MappedItem({
collected_from_url: readable_url(item.__import_meta?.source_platform_url ?? ""),
collected_from_url,
id: item.id,
thread_id: item.id,
url: `https://www.xiaohongshu.com${item.url}`,
title: item.title,
body: new MissingMappedField(""),
hashtags: new MissingMappedField(""),
timestamp: new MissingMappedField(""),
body: new MissingMappedField("") ,
hashtags: new MissingMappedField("") ,
timestamp: new MissingMappedField("") ,
author: item.author_name,
author_avatar_url: item.author_avatar_url,
image_url: item.thumbnail_url,
video_url: new MissingMappedField(""),
video_url: new MissingMappedField("") ,
likes: item.likes,
unix_timestamp: new MissingMappedField(""),
unix_timestamp: new MissingMappedField("")
});
}

export function map_item(post) {
// Reject tile stub items – minimal thumbnail entries with no content
if (!post.note_card && !post.user && post['_zs-origin'] !== 'html' && !post.note) {
const source = post.__import_meta?.source_url ?? "";
throw new MapItemException(`Xiaohongshu tile stub without post content (source: ${source || 'unknown'})`);
}
if (post['_zs-origin'] === 'html') {
return map_item_from_html(post);
} else {
if (post.note) {
return map_item_from_json_embedded(post);
} else {
return map_item_from_json_api_explore(post);
}
}
}
// === end auto-generated ===
Loading