muskit/NijiHolo_EN_ID_Bot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

move working files into its own directory

Browse Source
This commit is contained in:
muskit
2024-01-25 16:29:01 -08:00
parent 22743c58ef
commit bfdcaf37fa
8 changed files with 280 additions and 188 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ttweetqueue.py
+48 -41
View File
@@ -9,15 +9,16 @@ import talenttweet as tt
# User timestamps line format:
# {user_id} {status_num} {UNIX_timestamp}
class TalentTweetQueue:
instance = None
def __init__(self):
TalentTweetQueue.instance = self
self.queue_path = util.get_queue_path()
self.queue_backup_path = util.get_queue_backup_path()
self.current_ttweet_path = f'{util.get_project_dir()}/_current_ttweet.txt'
self.finished_ttweets_path = f'{util.get_project_dir()}/finished_ttweets.txt'
self.queue_path = util.working_path(file="queue.txt")
self.queue_backup_path = util.working_path(file="_queue_backup.txt")
self.current_ttweet_path = util.working_path(file="_current_ttweet.txt")
self.finished_ttweets_path = util.working_path(file="finished_ttweets.txt")
self.is_good = True
self.__sorted = False
self.finished_user_dates: dict[int, str] = dict()
@@ -26,58 +27,62 @@ class TalentTweetQueue:
## file check, backup copy
if os.path.exists(self.queue_backup_path):
print('Found backup queue! We errored in the previous run.')
print("Found backup queue! We errored in the previous run.")
shutil.copyfile(self.queue_backup_path, self.queue_path)
elif os.path.exists(self.queue_path):
print('Creating backup queue...')
print("Creating backup queue...")
shutil.copyfile(self.queue_path, self.queue_backup_path)
## initialize structures
# user timestamps
try:
with open(self.queue_path, 'r') as f:
with open(self.queue_path, "r") as f:
for line in f:
tokens = line.split()
if len(tokens) == 0: continue
if len(tokens) == 0:
continue
if tokens[0][0] != '#':
print(f'Stopped finding user dates at {line}')
if tokens[0][0] != "#":
print(f"Stopped finding user dates at {line}")
# reached end of accounts list
break
if tokens[2] != '-1':
if tokens[2] != "-1":
self.finished_user_dates[int(tokens[1])] = tokens[2]
except: pass
except:
pass
# ttweets
try:
with open(self.queue_path, 'r') as f: # reset seek head
with open(self.queue_path, "r") as f: # reset seek head
# Get existing queued TalentTweets
for line in f:
tokens = line.split()
if len(tokens) == 0 or tokens[0][0] == '#':
if len(tokens) == 0 or tokens[0][0] == "#":
continue
ttweet = tt.TalentTweet.deserialize(line)
# print(f'{ttweet.tweet_id}:\n{ttweet}')
self.ttweets_dict[ttweet.tweet_id] = ttweet
print(f'Found {len(self.finished_user_dates)} scraped accounts and {len(self.ttweets_dict)} tweets in queue.')
print(
f"Found {len(self.finished_user_dates)} scraped accounts and {len(self.ttweets_dict)} tweets in queue."
)
except:
traceback.print_exc()
pass
# unfinished ttweet
if os.path.exists(self.current_ttweet_path):
with open(self.current_ttweet_path, 'r') as f:
with open(self.current_ttweet_path, "r") as f:
for line in f:
if len(line) > 0:
ttweet = tt.TalentTweet.deserialize(line)
if ttweet.tweet_id in self.ttweets_dict:
self.ttweets_dict[ttweet.tweet_id] = ttweet
print(f'adding unfinished tweet {ttweet.tweet_id}')
print(f"adding unfinished tweet {ttweet.tweet_id}")
# finished ttweets
try:
with open(self.finished_ttweets_path, 'r') as f:
with open(self.finished_ttweets_path, "r") as f:
for line in f:
self.finished_ttweets.add(int(line))
except: pass
except:
pass
def is_empty(self):
return self.get_count() <= 0
@@ -94,57 +99,59 @@ class TalentTweetQueue:
self.__sort_ttweets_dict()
key = list(self.ttweets_dict.keys())[0]
ttweet = self.ttweets_dict.pop(key)
with open(self.current_ttweet_path, 'w') as f:
with open(self.current_ttweet_path, "w") as f:
f.write(ttweet.serialize())
return ttweet
def get_count(self):
return len(self.ttweets_dict)
## Call when the TalentTweet retrieved from get_next_ttweet() was
# posted successfully.
def good(self, tweet_id: int):
try: os.remove(self.current_ttweet_path)
except: pass
try:
os.remove(self.current_ttweet_path)
except:
pass
self.add_finished_tweet(tweet_id)
self.save_file()
self.is_good = True
# overwrite queue.txt
def save_file(self, replace_backup=True):
print('saving queue...', end='')
print("saving queue...", end="")
if replace_backup:
print('overwriting backup...', end='')
print("overwriting backup...", end="")
shutil.copyfile(self.queue_path, self.queue_backup_path)
self.__sort_ttweets_dict()
with open(self.queue_path, 'w') as f:
with open(self.queue_path, "w") as f:
# write dates
for (id, date) in self.finished_user_dates.items():
f.write(f'# {id} {date}\n')
for id, date in self.finished_user_dates.items():
f.write(f"# {id} {date}\n")
f.write('\n')
f.write("\n")
# write sorted ttweets
for ttweet in self.ttweets_dict.values():
f.write(ttweet.serialize() + '\n')
print('done')
f.write(ttweet.serialize() + "\n")
print("done")
def add_finished_tweet(self, id):
self.finished_ttweets.add(id)
with open(self.finished_ttweets_path, 'a') as f:
f.write(f'{id}\n')
with open(self.finished_ttweets_path, "a") as f:
f.write(f"{id}\n")
def __sort_ttweets_dict(self):
if not self.__sorted:
self.ttweets_dict = dict(sorted(self.ttweets_dict.items()))
self.__sorted = True
# destructor
def __del__(self):
if self.is_good:
print('Ended in good state, deleting backup queue...')
print("Ended in good state, deleting backup queue...")
os.remove(self.queue_backup_path)
else:
print('Ended in bad state, keeping backup queue.')
print("Ended in bad state, keeping backup queue.")