temporary patch for scraper lock outs

This commit is contained in:
muskit
2024-04-20 01:41:27 -07:00
parent bacc426a6d
commit 7fc86543e6
+8 -5
View File
@@ -15,8 +15,10 @@ from tweety_utils import *
from talenttweet import * from talenttweet import *
import talent_lists import talent_lists
# TODO: on RateLimit encounter, determine when it will probably
# unlock and wait just until then
class Scraper: class Scraper:
COOLDOWN = 16 # minutes
def __init__(self): def __init__(self):
Scraper.instance = self Scraper.instance = self
self.__account = AccountPool() self.__account = AccountPool()
@@ -48,16 +50,16 @@ class Scraper:
def login_wait(self, private=False): def login_wait(self, private=False):
if private: if private:
print( print(
f"keeping pvt-accessible account ({self.__account.use_index(0)[0]}). sleeping for 4 minutes..." f"keeping pvt-accessible account ({self.__account.use_index(0)[0]}). sleeping for {Scraper.COOLDOWN} minutes..."
) )
sleep(240) sleep(60*Scraper.COOLDOWN)
print() print()
l = self.try_login(0) l = self.try_login(0)
else: else:
l = self.try_login() l = self.try_login()
if not l: if not l:
print("sleeping for 4 minutes...") print(f"sleeping for {Scraper.COOLDOWN} minutes...")
sleep(240) sleep(60*Scraper.COOLDOWN)
print() print()
self.try_login() self.try_login()
@@ -191,6 +193,7 @@ class Scraper:
print(f"UnknownError occurred: {e.message.rstrip()}") print(f"UnknownError occurred: {e.message.rstrip()}")
print("treating like RateLimitReached...") print("treating like RateLimitReached...")
self.login_wait(uid in talent_lists.privated_accounts) self.login_wait(uid in talent_lists.privated_accounts)
sleep(5) # FIXME: temporary attempt to avoid scraper lock-up
tweets.sort(key=lambda t: t.id) tweets.sort(key=lambda t: t.id)
return tweets return tweets