temporary patch for scraper lock outs
This commit is contained in:
+8
-5
@@ -15,8 +15,10 @@ from tweety_utils import *
|
|||||||
from talenttweet import *
|
from talenttweet import *
|
||||||
import talent_lists
|
import talent_lists
|
||||||
|
|
||||||
|
# TODO: on RateLimit encounter, determine when it will probably
|
||||||
|
# unlock and wait just until then
|
||||||
class Scraper:
|
class Scraper:
|
||||||
|
COOLDOWN = 16 # minutes
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
Scraper.instance = self
|
Scraper.instance = self
|
||||||
self.__account = AccountPool()
|
self.__account = AccountPool()
|
||||||
@@ -48,16 +50,16 @@ class Scraper:
|
|||||||
def login_wait(self, private=False):
|
def login_wait(self, private=False):
|
||||||
if private:
|
if private:
|
||||||
print(
|
print(
|
||||||
f"keeping pvt-accessible account ({self.__account.use_index(0)[0]}). sleeping for 4 minutes..."
|
f"keeping pvt-accessible account ({self.__account.use_index(0)[0]}). sleeping for {Scraper.COOLDOWN} minutes..."
|
||||||
)
|
)
|
||||||
sleep(240)
|
sleep(60*Scraper.COOLDOWN)
|
||||||
print()
|
print()
|
||||||
l = self.try_login(0)
|
l = self.try_login(0)
|
||||||
else:
|
else:
|
||||||
l = self.try_login()
|
l = self.try_login()
|
||||||
if not l:
|
if not l:
|
||||||
print("sleeping for 4 minutes...")
|
print(f"sleeping for {Scraper.COOLDOWN} minutes...")
|
||||||
sleep(240)
|
sleep(60*Scraper.COOLDOWN)
|
||||||
print()
|
print()
|
||||||
self.try_login()
|
self.try_login()
|
||||||
|
|
||||||
@@ -191,6 +193,7 @@ class Scraper:
|
|||||||
print(f"UnknownError occurred: {e.message.rstrip()}")
|
print(f"UnknownError occurred: {e.message.rstrip()}")
|
||||||
print("treating like RateLimitReached...")
|
print("treating like RateLimitReached...")
|
||||||
self.login_wait(uid in talent_lists.privated_accounts)
|
self.login_wait(uid in talent_lists.privated_accounts)
|
||||||
|
sleep(5) # FIXME: temporary attempt to avoid scraper lock-up
|
||||||
|
|
||||||
tweets.sort(key=lambda t: t.id)
|
tweets.sort(key=lambda t: t.id)
|
||||||
return tweets
|
return tweets
|
||||||
|
|||||||
Reference in New Issue
Block a user