From 65018fcbedbb4f6e9d63e207e017efdcf05ee482 Mon Sep 17 00:00:00 2001 From: muskit <15199219+muskit@users.noreply.github.com> Date: Wed, 11 Jan 2023 22:50:47 -0800 Subject: [PATCH] store user date instead of timestamp in queue.txt --- README.md | 11 +++++------ src/catchup.py | 10 +++++----- src/ttweetqueue.py | 14 +++++++------- src/twapi.py | 9 +++++---- src/util.py | 15 +++++++++------ 5 files changed, 31 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 09caf62..dc6a662 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # NijiHolo EN/ID Bot -*Twitter bot that tracks interactions between Nijisanji EN/ID, hololive EN/ID, and holostars EN members.* +Twitter bot that tracks interactions between Nijisanji EN/ID, hololive EN/ID, and holostars EN members. ![The project banner](images/banner.png) @@ -10,10 +10,9 @@ The bot may run in these modes: * Catch-up (`c`): intended to run only once, scan all accounts for cross-company tweets and post them. Terminate when done posting all. - use `--auto-listen` to switch to listen mode when finished * Listen (`l`): listens for tweets from list, sharing it if it's cross-company -* Command-line (`cmd`): an interactive mode for manual control (drops into Python interpretor) +* Command-line (`cmd`): an interactive mode for manual control and debugging (drops into Python interpretor) -## Extra TODOs -* Fix main post's quote tweet formatting -* Fix mentions not showing when a standard TalentTweet is created +## TODOs +* Fix catch-up mode (`TWINT` is broken yet again *\*sigh\**) -**This project was created for the spirit of entertainment and in the name of unity.** +*Created for the spirit of entertainment and in the name of unity.* diff --git a/src/catchup.py b/src/catchup.py index bb44fe3..b258ca7 100644 --- a/src/catchup.py +++ b/src/catchup.py @@ -22,7 +22,7 @@ safe_to_post_tweets = True errored = False ## Returns the ID of all tweets (up to limit) from a user ID. -def get_user_tweets(id, since_timestamp=None, limit=None): +def get_user_tweets(id, since_date=None, limit=None): global safe_to_post_tweets qrt_count = 0 @@ -33,7 +33,7 @@ def get_user_tweets(id, since_timestamp=None, limit=None): c.Store_object = True c.Store_object_tweets_list = tweets c.Hide_output = True - c.Since = '' if since_timestamp == None else util.timestamp_to_tdate(since_timestamp) + c.Since = '' if since_date == None else f'{since_date} 00:00:00' user_str = f'@{util.get_username_local(id)}' print(f'Scraping tweets from {user_str} since {"forever ago" if c.Since == "" else c.Since}...') @@ -64,7 +64,7 @@ async def get_cross_talent_tweets(): for i, (talent_id, talent_username) in enumerate(talent_lists.talents.items()): print(f'[{i+1}/{len(talent_lists.talents)}] {talent_username}-----------------------------------') try: - tweets = get_user_tweets(talent_id, since_timestamp=queue.finished_user_timestamps.get(talent_id, None)) + tweets = get_user_tweets(talent_id, since_date=queue.finished_user_dates.get(talent_id, None)) for tweet in tweets: if tweet.id not in queue.ttweets_dict and tweet.id not in queue.finished_ttweets: ttweet = await tt.TalentTweet.create_from_twint_tweet(tweet) @@ -74,9 +74,9 @@ async def get_cross_talent_tweets(): print('Error occurred processing tweet data.') safe_to_post_tweets = False print(traceback.format_exc()) - queue.finished_user_timestamps[talent_id] = -1 + queue.finished_user_dates[talent_id] = '2000-01-01' else: - queue.finished_user_timestamps[talent_id] = util.get_current_timestamp() + queue.finished_user_dates[talent_id] = util.get_current_date() except: print('Unhandled error occurred while pulling tweets.') traceback.print_exc() diff --git a/src/ttweetqueue.py b/src/ttweetqueue.py index d9d0fb5..f297491 100644 --- a/src/ttweetqueue.py +++ b/src/ttweetqueue.py @@ -19,7 +19,7 @@ class TalentTweetQueue: self.finished_ttweets_path = f'{util.get_project_dir()}/finished_ttweets.txt' self.is_good = True self.__sorted = False - self.finished_user_timestamps = dict() + self.finished_user_dates = dict() self.ttweets_dict = dict() self.finished_ttweets = list() @@ -40,11 +40,11 @@ class TalentTweetQueue: if len(tokens) == 0: continue if tokens[0][0] != '#': - print(f'Stopped finding user timestamps at {line}') + print(f'Stopped finding user dates at {line}') # reached end of accounts list break if tokens[2] != '-1': - self.finished_user_timestamps[int(tokens[1])] = float(tokens[2]) + self.finished_user_dates[int(tokens[1])] = tokens[2] except: pass # ttweets try: @@ -56,7 +56,7 @@ class TalentTweetQueue: continue ttweet = tt.TalentTweet.deserialize(line) self.ttweets_dict[ttweet.tweet_id] = ttweet - print(f'Found {len(self.finished_user_timestamps)} scraped accounts and {len(self.ttweets_dict)} tweets in queue.') + print(f'Found {len(self.finished_user_dates)} scraped accounts and {len(self.ttweets_dict)} tweets in queue.') except: pass # finished ttweets try: @@ -108,9 +108,9 @@ class TalentTweetQueue: shutil.copyfile(self.queue_path, self.queue_backup_path) self.__sort_ttweets_dict() with open(self.queue_path, 'w') as f: - # write timestamps - for (id, timestamp) in self.finished_user_timestamps.items(): - f.write(f'# {id} {timestamp}\n') + # write dates + for (id, date) in self.finished_user_dates.items(): + f.write(f'# {id} {date}\n') f.write('\n') diff --git a/src/twapi.py b/src/twapi.py index 4c17cf6..50383ca 100644 --- a/src/twapi.py +++ b/src/twapi.py @@ -178,10 +178,10 @@ class TwAPI: async def post_ttweet(self, ttweet: tt.TalentTweet, is_catchup=False, dry_run=False): print(f'------{ttweet.tweet_id} ({util.get_username_local(ttweet.author_id)})------') - REPLY = '{0} replied to {1}\n' - QUOTE_TWEET = '{0} quote tweeted {1}\n' - TWEET = '{0} tweeted\n' - RETWEET = '{0} retweeted {1}\n' + REPLY = '{0} replied to {1}!\n' + QUOTE_TWEET = '{0} quote tweeted {1}!\n' + TWEET = '{0} tweeted!\n' + RETWEET = '{0} retweeted {1}!\n' def create_text(): author_username = f'@/{util.get_username_local(ttweet.author_id)}' @@ -225,6 +225,7 @@ class TwAPI: ttweet_url = util.ttweet_to_url(ttweet) if dry_run: # DRY-RUN: only print tweet + print('--------------- [DRY RUN] ---------------') print(text) print(f'QRT: {ttweet_url}') else: # NO DRY-RUN: post actual tweet diff --git a/src/util.py b/src/util.py index 1911f59..4f79740 100644 --- a/src/util.py +++ b/src/util.py @@ -2,7 +2,7 @@ import os import traceback -import datetime +from datetime import datetime import tweepy import pytz @@ -27,19 +27,22 @@ def get_queue_backup_path(): def clamp(n, smallest, largest): return max(smallest, min(n, largest)) -def datetime_to_tdate(date_time: datetime.datetime): +def datetime_to_tdate(date_time: datetime): return date_time.strftime("%Y-%m-%d") def tdate_to_datetime(tdate: str): - return datetime.datetime.strptime("%Y-%m-%d") + return datetime.strptime("%Y-%m-%d") def timestamp_to_tdate(timestamp=None): if timestamp==None: - timestamp = datetime.datetime.now().timestamp() - return datetime_to_tdate(datetime.datetime.fromtimestamp(timestamp, tz=pytz.utc)) + timestamp = datetime.now().timestamp() + return datetime_to_tdate(datetime.fromtimestamp(timestamp, tz=pytz.utc)) def get_current_timestamp(): - return datetime.datetime.now().timestamp() + return datetime.now().timestamp() + +def get_current_date(): + return datetime.today().strftime('%Y-%m-%d') def get_key_from_value(d, val): keys = [k for k, v in d.items() if v == val]