muskit/NijiHolo_EN_ID_Bot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

move working files into its own directory

Browse Source
This commit is contained in:
muskit
2024-01-25 16:29:01 -08:00
parent 22743c58ef
commit bfdcaf37fa
8 changed files with 280 additions and 188 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/catchup.py
+59 -44
View File
@@ -19,6 +19,7 @@ PROGRAM_ARGS = None
safe_to_post_tweets = True
scraper: Scraper
# Updates TTweetQueue
async def get_cross_tweets_online():
global safe_to_post_tweets
@@ -26,91 +27,101 @@ async def get_cross_tweets_online():
global scraper
safe_to_post_tweets = True
dbg_curr_user = ''
dbg_curr_user = ""
# Begin getting tweets from online
print('Pulling tweets from online!')
print("Pulling tweets from online!")
try:
for i, (talent_id, talent_username) in enumerate(talents.items()):
print(f'[{i+1}/{len(talents)}] {talent_username}-----------------------------------')
dbg_curr_user = f'{talent_id}: {talent_username}'
print(
f"[{i+1}/{len(talents)}] {talent_username}-----------------------------------"
)
dbg_curr_user = f"{talent_id}: {talent_username}"
try:
since_date = queue.finished_user_dates.get(talent_id, None)
ttweets = scraper.get_cross_ttweets_from_user(talent_username, since_date=since_date)
print(f'got {len(ttweets)} TalentTweets')
ttweets = scraper.get_cross_ttweets_from_user(
talent_username, since_date=since_date
)
print(f"got {len(ttweets)} TalentTweets")
for ttweet in ttweets:
if ttweet.tweet_id not in queue.finished_ttweets \
and ttweet.is_cross_company():
if (
ttweet.tweet_id not in queue.finished_ttweets
and ttweet.is_cross_company()
):
queue.add_ttweet(ttweet)
except KeyboardInterrupt as e:
raise e
except Exception as e:
print('Unhandled error occurred processing tweet data.')
print("Unhandled error occurred processing tweet data.")
safe_to_post_tweets = False
raise e
else:
queue.finished_user_dates[talent_id] = get_current_date()
queue.save_file()
except KeyboardInterrupt as e:
print('Interrupting tweet pulling... NOTE: remaining dates in queue file will not be updated!')
print(
"Interrupting tweet pulling... NOTE: remaining dates in queue file will not be updated!"
)
queue.save_file()
raise e
except:
print('Unhandled error occurred while pulling tweets.')
print("Unhandled error occurred while pulling tweets.")
traceback.print_exc()
with open("error_catchup.txt", "a") as f:
f.write(f'Error getting tweets from user {dbg_curr_user}\n')
with open(working_path(file="error_catchup.txt"), "a") as f:
f.write(f"Error getting tweets from user {dbg_curr_user}\n")
traceback.print_exc(file=f)
safe_to_post_tweets = False
else:
print('Successfully saved all tweets from online!')
print("Successfully saved all tweets from online!")
queue.save_file()
# return False = we posted at least one ttweet
# return True = we didn't post a single ttweet
async def process_queue() -> bool:
'''
"""
Go through the queue and post stored TalentTweets.
'''
"""
global scraper
global queue
queued_ttweets_count = queue.get_count()
WAIT_TIME = 60*15
WAIT_TIME = 60 * 15
ttweets_posted = 0
if queued_ttweets_count == 0:
print('Posting queue is empty!')
print("Posting queue is empty!")
return True
try:
while not queue.is_empty():
ttweet = queue.get_next_ttweet()
if ttweet.tweet_id in queue.finished_ttweets:
print('skipping finished tweet...')
print("skipping finished tweet...")
queue.good(ttweet.tweet_id)
continue
tweet_was_successful = await TwAPI.instance.post_ttweet(ttweet)
print('running queue.good()...')
print("running queue.good()...")
queue.good(ttweet.tweet_id)
if tweet_was_successful:
ttweets_posted += 1
print(f'({ttweets_posted}/{queued_ttweets_count}) done')
print(f"({ttweets_posted}/{queued_ttweets_count}) done")
if not queue.is_empty():
print(f'resting for {WAIT_TIME}s...')
await asyncio.sleep(WAIT_TIME-5)
print('5 second warning!')
print(f"resting for {WAIT_TIME}s...")
await asyncio.sleep(WAIT_TIME - 5)
print("5 second warning!")
await asyncio.sleep(5)
except Exception as e:
print('Unhandled error occurred while posting tweets from queue.')
print("Unhandled error occurred while posting tweets from queue.")
traceback.print_exc()
if ttweets_posted > 0:
return False
return True
# return True = no problems
# return False = issue occurred where we couldn't post all past tweets properly
async def run(PROGRAM_ARGS):
@@ -124,63 +135,67 @@ async def run(PROGRAM_ARGS):
# post tweets given in command line first
if PROGRAM_ARGS.post_id is not None and len(PROGRAM_ARGS.post_id) > 0:
PROGRAM_ARGS.post_id.sort()
print('Posting specified tweets first.')
print("Posting specified tweets first.")
for id in PROGRAM_ARGS.post_id:
try:
i = int(id)
except ValueError:
print(f'Invalid tweet {id}!')
print(f"Invalid tweet {id}!")
continue
posted = await TwAPI.instance.post_ttweet_by_id(i)
if posted:
queue.add_finished_tweet(i)
print('Successfully posted tweet. Sleeping for 5 minutes')
await asyncio.sleep(60*5)
print("Successfully posted tweet. Sleeping for 5 minutes")
await asyncio.sleep(60 * 5)
else:
print('Did not post tweet')
print('Done processing specified tweets')
print("Did not post tweet")
print("Done processing specified tweets")
PROGRAM_ARGS.post_id = None
# refresh stored queue first
if PROGRAM_ARGS.refresh_queue:
PROGRAM_ARGS.refresh_queue = False
print('Refreshing queue tweets...')
print("Refreshing queue tweets...")
for id in queue.ttweets_dict:
t = scraper.get_tweet(id, queue.ttweets_dict[id].author_id in privated_accounts)
t = scraper.get_tweet(
id, queue.ttweets_dict[id].author_id in privated_accounts
)
queue.ttweets_dict[id] = tt.TalentTweet.create_from_tweety(t)
queue.save_file()
async def queue_loop():
while True:
print(f'{queue.get_count()} cross-company tweets to announce.')
print(f"{queue.get_count()} cross-company tweets to announce.")
try:
if safe_to_post_tweets:
if await process_queue():
print("Finished processing queue")
return
else:
print('Posted no new tweets; we\'re caught up!')
print("Posted no new tweets; we're caught up!")
return
else:
print('Tweets were not retrieved cleanly. Not processing queue.')
print("Tweets were not retrieved cleanly. Not processing queue.")
return
except KeyboardInterrupt as e:
print('Interrupting queue processing...')
print("Interrupting queue processing...")
raise e
except:
print('Unhandled error occurred while running catch up in posting phase.')
print(
"Unhandled error occurred while running catch up in posting phase."
)
traceback.print_exc()
await get_cross_tweets_online()
try:
if PROGRAM_ARGS.straight_to_queue:
PROGRAM_ARGS.straight_to_queue = False
print('Processing queue first before fetching tweets...')
print("Processing queue first before fetching tweets...")
await queue_loop()
else:
await get_cross_tweets_online()
await queue_loop()
except KeyboardInterrupt:
print('Interrupt received. Ending catchup mode...')
print("Interrupt received. Ending catchup mode...")
return False