diff --git a/.gitignore b/.gitignore index bf619a7..fa95045 100644 --- a/.gitignore +++ b/.gitignore @@ -144,4 +144,5 @@ cython_debug/ # project-specific run/ -*.json \ No newline at end of file +*.tw_session +.venv* \ No newline at end of file diff --git a/README.md b/README.md index fe6336c..f00833a 100644 --- a/README.md +++ b/README.md @@ -6,13 +6,20 @@ Twitter bot that tracks cross-company interactions between the non-JP branches o **This project was created to run [this account](https://twitter.com/NijiHolo_EN_ID).** ## Running -Install dependencies. -``` -pip install -r requirements.txt -``` -Setup the `.env` in the project root. Refer to the `.env` section for variables. +With the way packages are setup, **you must have Docker installed and running!!** -Run the program from project root (not in `src`). Refer to the following section for options. +Setup the `.env` in the project root. Refer to the [`.env`](#env) section for variables. + +Build and run the Docker container: +```bash +# to run attached (can CTRL+P,CTRL+Q to detach) +sh run.sh + +# ... or to run headless +sh run_detached.sh +``` + +If attached to a container prepared by Dockerfile, you can run the program from project root (not in `src`). Refer to the following section for options. ``` python src/main.py ``` diff --git a/lists/nijien.txt b/lists/nijien.txt index 4c3a72d..ed57bb6 100644 --- a/lists/nijien.txt +++ b/lists/nijien.txt @@ -26,11 +26,11 @@ 1465858739970273281 luca_kaneshiro # --- [Noctyx] --- -1490867613915828224　alban_knox -1491195742123397124　uki_violeta -1492604168145539072　Yugo_Asuma p -1493392149664219138　Fulgur_Ovid -1493394108014292993　sonny_brisko +1490867613915828224 alban_knox +1491195742123397124 uki_violeta +1492604168145539072 Yugo_Asuma p +1493392149664219138 Fulgur_Ovid +1493394108014292993 sonny_brisko # --- [ILUNA] --- 1545351225293426688 MariaMari0nette diff --git a/requirements.txt b/requirements.txt index 163d671..5282812 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ python-dotenv nest-asyncio pytz -git+https://github.com/muskit/tweety.git +git+https://github.com/mahrtayyab/tweety.git@e3d330280cb3b2e8f9d2bf2f20425c476f7671a5 tweepy tweet-capture diff --git a/requirements_dev.txt b/requirements_dev.txt new file mode 100644 index 0000000..b8178f8 --- /dev/null +++ b/requirements_dev.txt @@ -0,0 +1,7 @@ +python-dotenv +nest-asyncio +pytz +git+https://github.com/mahrtayyab/tweety.git@e3d330280cb3b2e8f9d2bf2f20425c476f7671a5 +tweepy +tweet-capture +opencv-python-headless \ No newline at end of file diff --git a/src/catchup.py b/src/catchup.py index be70804..77c2c5b 100644 --- a/src/catchup.py +++ b/src/catchup.py @@ -59,7 +59,7 @@ async def get_cross_tweets_online(): print(f"Queue has {queue.get_count()} tweets so far") except KeyboardInterrupt as e: print( - "Interrupting tweet pulling... NOTE: remaining dates in queue file will not be updated!" + "Interrupting tweet pulling. The remaining dates in queue file will not be updated!" ) queue.save_file() raise e @@ -144,7 +144,7 @@ async def run(PROGRAM_ARGS): print(f"Invalid tweet {id}!") continue - posted = await TwAPI.instance.post_ttweet_by_id(i) + posted = await TwAPI.instance.post_ttweet_by_id(i, PROGRAM_ARGS.dry_run) if posted: queue.add_finished_tweet(i) print("Successfully posted tweet. Sleeping for 5 minutes") diff --git a/src/listen.py b/src/listen.py index 9df17fb..202a21e 100644 --- a/src/listen.py +++ b/src/listen.py @@ -11,9 +11,9 @@ def run(PROGRAM_ARGS): while True: try: asyncio.run(catchup.run(PROGRAM_ARGS)) - print('Sleeping for 60 minutes...') - sleep(60*60) # run every hour + print("Sleeping for 60 minutes...") + sleep(60 * 30) # run every half-hour except KeyboardInterrupt: - print('Interrupt signal received. Exiting listen mode.') - print(f'errors encountered throughout session.') + print("Interrupt signal received. Exiting listen mode.") + print(f"errors encountered throughout session.") break diff --git a/src/main.py b/src/main.py index ab9c57e..076a844 100644 --- a/src/main.py +++ b/src/main.py @@ -14,27 +14,54 @@ from twapi import TwAPI PROGRAM_ARGS = None -MODES_HELP_STR = '''mode to run the bot at: +MODES_HELP_STR = """mode to run the bot at: scrape accounts in lists and post cross-company tweets if relevant -cmd drop into Python interpretor with access to initialized variables''' +cmd drop into Python interpretor with access to initialized variables""" + def init_argparse(): - p = argparse.ArgumentParser(description='Twitter bot that follows interactions between Nijisanji EN/ID and hololive EN/ID members.', formatter_class=RawTextHelpFormatter) - p.add_argument('mode', nargs='?', help=MODES_HELP_STR) - p.add_argument('--no-listen', action='store_true', help='Run one scraping-posting cycle without waiting to run again.') - p.add_argument('--refresh-queue', action='store_true', help='Refresh the details on each tweet currently in queue.') - p.add_argument('--straight-to-queue', action='store_true', help='Go through queue first before attempting to pull tweets.') - p.add_argument('--post-id', action='append', help='ID of a tweet to try and post right away. Specify multiple to post multiple tweets in a row.') + p = argparse.ArgumentParser( + description="Twitter bot that follows interactions between Nijisanji EN/ID and hololive EN/ID members.", + formatter_class=RawTextHelpFormatter, + ) + p.add_argument("mode", nargs="?", help=MODES_HELP_STR) + p.add_argument( + "--no-listen", + action="store_true", + help="Run one scraping-posting cycle without waiting to run again.", + ) + p.add_argument( + "--refresh-queue", + action="store_true", + help="Refresh the details on each tweet currently in queue.", + ) + p.add_argument( + "--straight-to-queue", + action="store_true", + help="Go through queue first before attempting to pull tweets.", + ) + p.add_argument( + "--dry-run", + action="store_true", + help="Don't actually post anything to Twitter; use to check outputs from console.", + ) + p.add_argument( + "--post-id", + action="append", + help="ID of a tweet to try and post right away. Specify multiple to post multiple tweets in a row.", + ) return p + def command_line(): # TODO (extra): implement command line mode for manually controlling the bot - print('Here\'s a Python interpretor.') + print("Here's a Python interpreter.") try: code.interact(local=globals()) except SystemExit: pass + async def async_main(): global PROGRAM_ARGS @@ -44,12 +71,13 @@ async def async_main(): else: listen.run(PROGRAM_ARGS) return - + mode = PROGRAM_ARGS.mode.lower() - if mode == 'cmd': + if mode == "cmd": command_line() else: - print('\nunknown mode. run with no arguments or -h for help and modes') + print("\nunknown mode. run with no arguments or -h for help and modes") + def init_data(): # Initialize shared API instance @@ -60,12 +88,13 @@ def init_data(): if PROGRAM_ARGS.mode: mode = PROGRAM_ARGS.mode.lower() - if mode != 'cmd': + if mode != "cmd": # Initialize queue files system ttq.TalentTweetQueue() else: ttq.TalentTweetQueue() + def main(): global PROGRAM_ARGS @@ -81,7 +110,7 @@ def main(): ## Asynchronous execution nest_asyncio.apply() asyncio.run(async_main()) - + if __name__ == "__main__": main() diff --git a/src/scraper.py b/src/scraper.py index 68fd414..df20f45 100644 --- a/src/scraper.py +++ b/src/scraper.py @@ -19,7 +19,7 @@ class Scraper: def __init__(self): Scraper.instance = self self.__account = AccountPool() - self.try_login() + self.try_login(0) def try_login(self, account_idx: int = None) -> bool: # decide on which account to use @@ -81,7 +81,7 @@ class Scraper: if tweet.is_reply and tweet.replied_to is None: # print(f'{tweet.author.username}/{tweet.id} is missing reply-to tweet! Recovering...') tweet.replied_to = self.get_tweet( - tweet.original_tweet["in_reply_to_status_id_str"] + tweet._original_tweet["in_reply_to_status_id_str"] ) return tweet @@ -159,7 +159,7 @@ class Scraper: search = self.app.search( f"from:{username}", filter_=SearchFilters.Latest(), cursor=cur ) - cur_page = search.tweets + cur_page = search.results print(f"obtained {len(cur_page)} tweets") if len(cur_page) == 0: @@ -168,7 +168,7 @@ class Scraper: for e in cur_page: if isinstance(e, Tweet): add_tweet(e) - elif isinstance(e, TweetThread): + elif isinstance(e, SelfThread): # FIXME: rework when replied_to is fixed (currently populates user_mentions) # latest tweet in thread = og author's reply for t in e: diff --git a/src/talenttweet.py b/src/talenttweet.py index 550c35f..88812a2 100644 --- a/src/talenttweet.py +++ b/src/talenttweet.py @@ -85,7 +85,7 @@ class TalentTweet: rt_mentions=rtm, ) - ## Creates a TalentTweet from a Tweety-library Tweet. + ## Creates a TalentTweet from a Tweety Tweet. @staticmethod def create_from_tweety(tweety: Tweet): if tweety.is_retweet: @@ -104,7 +104,7 @@ class TalentTweet: text=tweety.text, mrq=( {int(x.id) for x in tweety.user_mentions}, - int(tweety.original_tweet["in_reply_to_user_id_str"]) + int(tweety._original_tweet["in_reply_to_user_id_str"]) if tweety.is_reply else None, int(tweety.quoted_tweet.author.id) @@ -278,7 +278,7 @@ class TalentTweet: rt_username = ( util.get_username_with_company(self.rt_author_id) if self.rt_author_id != -1 - else None + else "someone" ) if rt_username == author_username: rt_username = "themselves" @@ -291,7 +291,7 @@ class TalentTweet: reply_username = ( util.get_username_with_company(self.reply_to) if self.reply_to != -1 - else None + else "someone" ) if reply_username == author_username: reply_username = "themselves" @@ -303,7 +303,7 @@ class TalentTweet: quoted_username = ( util.get_username_with_company(self.quote_tweeted) if self.quote_tweeted != -1 - else None + else "someone" ) if quoted_username == author_username: quoted_username = "themselves" diff --git a/src/twapi.py b/src/twapi.py index 368c328..a7adb6a 100644 --- a/src/twapi.py +++ b/src/twapi.py @@ -136,6 +136,8 @@ class TwAPI: # return True = successfully posted a single ttweet # return False = did not post ttweet (duplicate) async def post_ttweet(self, ttweet: tt.TalentTweet, dry_run=False): + import main + print( f"------{ttweet.tweet_id} ({util.get_username_local(ttweet.author_id)})------" ) @@ -145,11 +147,9 @@ class TwAPI: if dry_run: print("-------------------- DRY RUN --------------------") - print(ttweet) - if dry_run: + print(ttweet) return False - # NO DRY-RUN: actually post tweet # main tweet: text + screenshot try: print("creating main QRT w/ screenshot...") @@ -188,7 +188,7 @@ class TwAPI: raise e return True - async def post_ttweet_by_id(self, id: int): + async def post_ttweet_by_id(self, id: int, dry_run=False): from scraper import Scraper print(f"Manually posting tweet {id}") @@ -204,4 +204,4 @@ class TwAPI: return False print(f"Posting {ttweet.username}/{ttweet.tweet_id}...") - return await self.post_ttweet(ttweet) + return await self.post_ttweet(ttweet, dry_run) diff --git a/src/tweety_utils.py b/src/tweety_utils.py index ad21e05..02ba186 100644 --- a/src/tweety_utils.py +++ b/src/tweety_utils.py @@ -1,24 +1,26 @@ from tweety.types import * + def url(t: Tweet): - return f'https://twitter.com/{t.author.username}/status/{t.id}' + return f"https://twitter.com/{t.author.username}/status/{t.id}" -def print_tweets(tweets: list[Tweet | TweetThread]): - print(f'{len(tweets)} tweets:') - for t in tweets: - if isinstance(t, Tweet): - print(f'{t.date} : {url(t)} :', end=' ') - if t.is_retweet: - print(f'RT ({t.retweeted_tweet.author.username})', end=' ') +def print_tweets(tweets: list[Tweet | SelfThread]): + print(f"{len(tweets)} tweets:") + for t in tweets: + if isinstance(t, Tweet): + print(f"{t.date} : {url(t)} :", end=" ") - if t.is_reply: - print(f'is reply!', end=' ') - if t.replied_to is not None: - print(f'reply to {t.replied_to.author.username}', end=' ') + if t.is_retweet: + print(f"RT ({t.retweeted_tweet.author.username})", end=" ") - print("m=" + ",".join([x.username for x in t.user_mentions])) - elif isinstance(t, TweetThread): - print('-----------TTd----------') - print_tweets(t.tweets) - print('-----------end----------') \ No newline at end of file + if t.is_reply: + print(f"is reply!", end=" ") + if t.replied_to is not None: + print(f"reply to {t.replied_to.author.username}", end=" ") + + print("m=" + ",".join([x.username for x in t.user_mentions])) + elif isinstance(t, SelfThread): + print("-----------TTd----------") + print_tweets(t.tweets) + print("-----------end----------") diff --git a/src/util.py b/src/util.py index 14be2c6..6f8e8cb 100644 --- a/src/util.py +++ b/src/util.py @@ -26,7 +26,7 @@ def project_root(dir_path: tuple[str] = tuple(), file: str = None): def working_path(dir_path: tuple[str] = tuple(), file: str = None): - """Returns file path relative to the working ephemeral directory.""" + """Returns file path relative to the working ephemeral directory "run".""" dir_path = project_root(("run", *dir_path)) Path(dir_path).mkdir(parents=True, exist_ok=True)