From 33c221620e2e604827cec600aacca979b2acf5e9 Mon Sep 17 00:00:00 2001 From: "Mark A. Greenwood" Date: Thu, 6 Jul 2023 15:24:13 +0100 Subject: [PATCH 1/3] first crack at supporting replies --- modules/twitter.js | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/modules/twitter.js b/modules/twitter.js index 1ab4086..e92ff71 100644 --- a/modules/twitter.js +++ b/modules/twitter.js @@ -15,6 +15,7 @@ zeeschuimer.register_module( && source_url.indexOf('UserTweets') < 0 && source_url.indexOf('Likes') < 0 && source_url.indexOf('SearchTimeline') < 0 + && source_url.indexOf('TweetDetail') < 0 // this one is not enabled because it is always loaded when viewing a user profile // even when not viewing the media tab // && source_url.indexOf('UserMedia') < 0 @@ -51,7 +52,7 @@ zeeschuimer.register_module( ) { for (let entry in child['entries']) { entry = child['entries'][entry]; - if('itemContent' in entry['content']) { + if('itemContent' in entry['content'] && entry['content']['itemContent']['itemType'] !== 'TimelineTimelineCursor') { // tweets are sometimes embedded directly in this object let tweet = entry['content']['itemContent']['tweet_results']['result'] if(!tweet || tweet['__typename'] === 'TweetUnavailable') { @@ -67,6 +68,25 @@ zeeschuimer.register_module( tweet['id'] = tweet['legacy']['id_str']; tweets.push(tweet); + } else if ('items' in entry['content']) { + for (let item in entry['content']['items']) { + item = entry['content']['items'][item]['item']; + if('itemContent' in item) { + let tweet = item['itemContent']['tweet_results']['result'] + if(!tweet || tweet['__typename'] === 'TweetUnavailable') { + // this sometimes happens + // no other data in the object, so just skip + continue; + } + + if('tweet' in tweet) { + // sometimes this is nested once more, for some reason + tweet = tweet['tweet']; + } + tweet['id'] = tweet['legacy']['id_str']; + tweets.push(tweet); + } + } } else { // in other cases this object only contains a reference to the full tweet, which is in turn // stored elsewhere in the parent object @@ -107,4 +127,4 @@ zeeschuimer.register_module( traverse(data); return tweets; } -); \ No newline at end of file +); From dbb2be9085751e5584712609422f72df74f156fe Mon Sep 17 00:00:00 2001 From: "Mark A. Greenwood" Date: Thu, 6 Jul 2023 15:42:00 +0100 Subject: [PATCH 2/3] refactor to avoid code duplication, and a fix for collecting tweets from the user profile page --- modules/twitter.js | 48 +++++++++++++++++++--------------------------- 1 file changed, 20 insertions(+), 28 deletions(-) diff --git a/modules/twitter.js b/modules/twitter.js index e92ff71..19fbdf3 100644 --- a/modules/twitter.js +++ b/modules/twitter.js @@ -32,6 +32,23 @@ zeeschuimer.register_module( return []; } + // we want to process tweets under multiple conditions so factor that out into a simple function + // so we can call it when we need it, rather than duplicating the code + let process = function (tweet) { + if(!tweet || tweet['__typename'] === 'TweetUnavailable') { + // this sometimes happens + // no other data in the object, so just skip + return; + } + + if('tweet' in tweet) { + // sometimes this is nested once more, for some reason + tweet = tweet['tweet']; + } + tweet['id'] = tweet['legacy']['id_str']; + tweets.push(tweet); + } + // find 'entries' in the API response // Twitter JSON objects are RPC-like objects that are interpreted // One of the 'instructions' is to add entries to the timeline, this is what we are interested in because what @@ -53,38 +70,13 @@ zeeschuimer.register_module( for (let entry in child['entries']) { entry = child['entries'][entry]; if('itemContent' in entry['content'] && entry['content']['itemContent']['itemType'] !== 'TimelineTimelineCursor') { - // tweets are sometimes embedded directly in this object - let tweet = entry['content']['itemContent']['tweet_results']['result'] - if(!tweet || tweet['__typename'] === 'TweetUnavailable') { - // this sometimes happens - // no other data in the object, so just skip - continue; - } - - if('tweet' in tweet) { - // sometimes this is nested once more, for some reason - tweet = tweet['tweet']; - } - tweet['id'] = tweet['legacy']['id_str']; - tweets.push(tweet); + process(entry['content']['itemContent']['tweet_results']['result']); } else if ('items' in entry['content']) { for (let item in entry['content']['items']) { item = entry['content']['items'][item]['item']; - if('itemContent' in item) { - let tweet = item['itemContent']['tweet_results']['result'] - if(!tweet || tweet['__typename'] === 'TweetUnavailable') { - // this sometimes happens - // no other data in the object, so just skip - continue; - } - - if('tweet' in tweet) { - // sometimes this is nested once more, for some reason - tweet = tweet['tweet']; - } - tweet['id'] = tweet['legacy']['id_str']; - tweets.push(tweet); + if('itemContent' in item && 'tweet_results' in item['itemContent']) { + process(item['itemContent']['tweet_results']['result']); } } } else { From 5f74aacd3aac85427a2bad016b0abcf2e70f8381 Mon Sep 17 00:00:00 2001 From: "Mark A. Greenwood" Date: Tue, 8 Aug 2023 16:01:34 +0100 Subject: [PATCH 3/3] now also tags promoted and related tweets --- modules/twitter.js | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/modules/twitter.js b/modules/twitter.js index 19fbdf3..e834378 100644 --- a/modules/twitter.js +++ b/modules/twitter.js @@ -34,7 +34,7 @@ zeeschuimer.register_module( // we want to process tweets under multiple conditions so factor that out into a simple function // so we can call it when we need it, rather than duplicating the code - let process = function (tweet) { + let process = function (tweet, promoted = false, related = false) { if(!tweet || tweet['__typename'] === 'TweetUnavailable') { // this sometimes happens // no other data in the object, so just skip @@ -46,6 +46,8 @@ zeeschuimer.register_module( tweet = tweet['tweet']; } tweet['id'] = tweet['legacy']['id_str']; + tweet['promoted'] = promoted; + tweet['related'] = related; tweets.push(tweet); } @@ -70,13 +72,18 @@ zeeschuimer.register_module( for (let entry in child['entries']) { entry = child['entries'][entry]; if('itemContent' in entry['content'] && entry['content']['itemContent']['itemType'] !== 'TimelineTimelineCursor') { - process(entry['content']['itemContent']['tweet_results']['result']); + process(entry['content']['itemContent']['tweet_results']['result'], + ('promotedMetadata' in entry['content']['itemContent']), + (entry['entryId'].indexOf('relatedtweets') >= 0)); } else if ('items' in entry['content']) { for (let item in entry['content']['items']) { + let entry_id = entry['content']['items'][item]['entryId']; item = entry['content']['items'][item]['item']; if('itemContent' in item && 'tweet_results' in item['itemContent']) { - process(item['itemContent']['tweet_results']['result']); + process(item['itemContent']['tweet_results']['result'], + ('promotedMetadata' in item['itemContent']), + (entry_id.indexOf('relatedtweets') >= 0)); } } } else { @@ -100,7 +107,9 @@ zeeschuimer.register_module( let tweet = { id: parseInt(tweet_id), legacy: data['globalObjects']['tweets'][tweet_id], - type: 'adaptive' + type: 'adaptive', + promoted: false, + related: false } // the user is also stored as a reference - so add the user data to the tweet