From 3ad7e42e739cf9ed171d928e10780cca7102990c Mon Sep 17 00:00:00 2001 From: muskit <15199219+muskit@users.noreply.github.com> Date: Wed, 30 Aug 2023 02:26:45 -0700 Subject: [PATCH] various adjustments --- src/account_pool.py | 2 +- src/catchup.py | 4 ++++ src/scraper.py | 10 +++++----- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/account_pool.py b/src/account_pool.py index 384bbd2..f51decf 100644 --- a/src/account_pool.py +++ b/src/account_pool.py @@ -4,7 +4,7 @@ from dotenv import dotenv_values class AccountPool: def __init__(self): self.__accounts: list[tuple[str, str]] = list() - self.__idx = -1 + self.__idx = 0 creds = dotenv_values() i = 0 while True: diff --git a/src/catchup.py b/src/catchup.py index f7cef26..ed4185c 100644 --- a/src/catchup.py +++ b/src/catchup.py @@ -25,11 +25,14 @@ async def get_cross_tweets_online(): global queue global scraper + safe_to_post_tweets = True + dbg_curr_user = '' # Begin getting tweets from online print('Pulling tweets from online!') try: for i, (talent_id, talent_username) in enumerate(talents.items()): print(f'[{i+1}/{len(talents)}] {talent_username}-----------------------------------') + dbg_curr_user = f'{talent_id}: {talent_username}' try: since_date = queue.finished_user_dates.get(talent_id, None) ttweets = scraper.get_cross_ttweets_from_user(talent_username, since_date=since_date) @@ -55,6 +58,7 @@ async def get_cross_tweets_online(): print('Unhandled error occurred while pulling tweets.') traceback.print_exc() with open("error_catchup.txt", "a") as f: + f.write(f'Error getting tweets from user {dbg_curr_user}\n') traceback.print_exc(file=f) safe_to_post_tweets = False else: diff --git a/src/scraper.py b/src/scraper.py index 2aba99c..2248f71 100644 --- a/src/scraper.py +++ b/src/scraper.py @@ -45,15 +45,15 @@ class Scraper: def login_wait(self, private=False): if private: - print(f"keeping pvt-accessible account ({self.__account.use_index(0)[0]}). sleeping for 2 minutes...") - sleep(120) + print(f"keeping pvt-accessible account ({self.__account.use_index(0)[0]}). sleeping for 4 minutes...") + sleep(240) print() l = self.try_login(0) else: l = self.try_login() if not l: - print("sleeping for 2 minutes...") - sleep(120) + print("sleeping for 4 minutes...") + sleep(240) print() self.try_login() @@ -81,7 +81,7 @@ class Scraper: return tweet def get_tweet(self, id: int, private_user=False): - print(f'{id}{" on private" if private_user else ""}') + # print(f'{id}{" on private" if private_user else ""}') if private_user: self.try_login(0) while True: