Merge pull request #8 from muskit/cleanup

Cleanup

.gitignore

@@ -144,4 +144,5 @@ cython_debug/
 
 # project-specific
 run/
-*.json
+*.tw_session
+.venv*

README.md

@@ -6,13 +6,20 @@ Twitter bot that tracks cross-company interactions between the non-JP branches o
 **This project was created to run [this account](https://twitter.com/NijiHolo_EN_ID).**
 
 ## Running
-Install dependencies.
+With the way packages are setup, **you must have Docker installed and running!!**
-```
-pip install -r requirements.txt
-```
-Setup the `.env` in the project root. Refer to the `.env` section for variables.
 
-Run the program from project root (not in `src`). Refer to the following section for options.
+Setup the `.env` in the project root. Refer to the [`.env`](#env) section for variables.
+
+Build and run the Docker container:
+```bash
+# to run attached (can CTRL+P,CTRL+Q to detach)
+sh run.sh
+
+# ... or to run headless
+sh run_detached.sh
+```
+
+If attached to a container prepared by Dockerfile, you can run the program from project root (not in `src`). Refer to the following section for options.
 ```
 python src/main.py
 ```

lists/nijien.txt

@@ -26,11 +26,11 @@
 1465858739970273281 luca_kaneshiro
 
 # --- [Noctyx] ---
-1490867613915828224　alban_knox
+1490867613915828224 alban_knox
-1491195742123397124　uki_violeta
+1491195742123397124 uki_violeta
-1492604168145539072　Yugo_Asuma p
+1492604168145539072 Yugo_Asuma p
-1493392149664219138　Fulgur_Ovid
+1493392149664219138 Fulgur_Ovid
-1493394108014292993　sonny_brisko
+1493394108014292993 sonny_brisko
 
 # --- [ILUNA] ---
 1545351225293426688 MariaMari0nette

requirements.txt

@@ -1,6 +1,6 @@
 python-dotenv
 nest-asyncio
 pytz
-git+https://github.com/muskit/tweety.git
+git+https://github.com/mahrtayyab/tweety.git@e3d330280cb3b2e8f9d2bf2f20425c476f7671a5
 tweepy
 tweet-capture

requirements_dev.txt

@@ -0,0 +1,7 @@
+python-dotenv
+nest-asyncio
+pytz
+git+https://github.com/mahrtayyab/tweety.git@e3d330280cb3b2e8f9d2bf2f20425c476f7671a5
+tweepy
+tweet-capture
+opencv-python-headless

src/catchup.py

@@ -59,7 +59,7 @@ async def get_cross_tweets_online():
                 print(f"Queue has {queue.get_count()} tweets so far")
     except KeyboardInterrupt as e:
         print(
-            "Interrupting tweet pulling... NOTE: remaining dates in queue file will not be updated!"
+            "Interrupting tweet pulling. The remaining dates in queue file will not be updated!"
         )
         queue.save_file()
         raise e
@@ -144,7 +144,7 @@ async def run(PROGRAM_ARGS):
                 print(f"Invalid tweet {id}!")
                 continue
 
-            posted = await TwAPI.instance.post_ttweet_by_id(i)
+            posted = await TwAPI.instance.post_ttweet_by_id(i, PROGRAM_ARGS.dry_run)
             if posted:
                 queue.add_finished_tweet(i)
                 print("Successfully posted tweet. Sleeping for 5 minutes")

src/listen.py

@@ -11,9 +11,9 @@ def run(PROGRAM_ARGS):
     while True:
         try:
             asyncio.run(catchup.run(PROGRAM_ARGS))
-            print('Sleeping for 60 minutes...')
+            print("Sleeping for 60 minutes...")
-            sleep(60*60) # run every hour
+            sleep(60 * 30)  # run every half-hour
         except KeyboardInterrupt:
-            print('Interrupt signal received. Exiting listen mode.')
+            print("Interrupt signal received. Exiting listen mode.")
-            print(f'errors encountered throughout session.')
+            print(f"errors encountered throughout session.")
             break

src/main.py

@@ -14,27 +14,54 @@ from twapi import TwAPI
 
 PROGRAM_ARGS = None
 
-MODES_HELP_STR = '''mode to run the bot at:
+MODES_HELP_STR = """mode to run the bot at:
 <blank>      scrape accounts in lists and post cross-company tweets if relevant
-cmd          drop into Python interpretor with access to initialized variables'''
+cmd          drop into Python interpretor with access to initialized variables"""
+
 
 def init_argparse():
-    p = argparse.ArgumentParser(description='Twitter bot that follows interactions between Nijisanji EN/ID and hololive EN/ID members.', formatter_class=RawTextHelpFormatter)
+    p = argparse.ArgumentParser(
-    p.add_argument('mode', nargs='?', help=MODES_HELP_STR)
+        description="Twitter bot that follows interactions between Nijisanji EN/ID and hololive EN/ID members.",
-    p.add_argument('--no-listen', action='store_true', help='Run one scraping-posting cycle without waiting to run again.')
+        formatter_class=RawTextHelpFormatter,
-    p.add_argument('--refresh-queue', action='store_true', help='Refresh the details on each tweet currently in queue.')
+    )
-    p.add_argument('--straight-to-queue', action='store_true', help='Go through queue first before attempting to pull tweets.')
+    p.add_argument("mode", nargs="?", help=MODES_HELP_STR)
-    p.add_argument('--post-id', action='append', help='ID of a tweet to try and post right away. Specify multiple to post multiple tweets in a row.')
+    p.add_argument(
+        "--no-listen",
+        action="store_true",
+        help="Run one scraping-posting cycle without waiting to run again.",
+    )
+    p.add_argument(
+        "--refresh-queue",
+        action="store_true",
+        help="Refresh the details on each tweet currently in queue.",
+    )
+    p.add_argument(
+        "--straight-to-queue",
+        action="store_true",
+        help="Go through queue first before attempting to pull tweets.",
+    )
+    p.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="Don't actually post anything to Twitter; use to check outputs from console.",
+    )
+    p.add_argument(
+        "--post-id",
+        action="append",
+        help="ID of a tweet to try and post right away. Specify multiple to post multiple tweets in a row.",
+    )
     return p
 
+
 def command_line():
     # TODO (extra): implement command line mode for manually controlling the bot
-    print('Here\'s a Python interpretor.')
+    print("Here's a Python interpreter.")
     try:
         code.interact(local=globals())
     except SystemExit:
         pass
 
+
 async def async_main():
     global PROGRAM_ARGS
 
@@ -46,10 +73,11 @@ async def async_main():
         return
 
     mode = PROGRAM_ARGS.mode.lower()
-    if mode == 'cmd':
+    if mode == "cmd":
         command_line()
     else:
-        print('\nunknown mode. run with no arguments or -h for help and modes')
+        print("\nunknown mode. run with no arguments or -h for help and modes")
+
 
 def init_data():
     # Initialize shared API instance
@@ -60,12 +88,13 @@ def init_data():
 
     if PROGRAM_ARGS.mode:
         mode = PROGRAM_ARGS.mode.lower()
-        if mode != 'cmd':
+        if mode != "cmd":
             # Initialize queue files system
             ttq.TalentTweetQueue()
     else:
         ttq.TalentTweetQueue()
 
+
 def main():
     global PROGRAM_ARGS
 

src/scraper.py

@@ -19,7 +19,7 @@ class Scraper:
     def __init__(self):
         Scraper.instance = self
         self.__account = AccountPool()
-        self.try_login()
+        self.try_login(0)
 
     def try_login(self, account_idx: int = None) -> bool:
         # decide on which account to use
@@ -81,7 +81,7 @@ class Scraper:
         if tweet.is_reply and tweet.replied_to is None:
             # print(f'{tweet.author.username}/{tweet.id} is missing reply-to tweet! Recovering...')
             tweet.replied_to = self.get_tweet(
-                tweet.original_tweet["in_reply_to_status_id_str"]
+                tweet._original_tweet["in_reply_to_status_id_str"]
             )
         return tweet
 
@@ -159,7 +159,7 @@ class Scraper:
                 search = self.app.search(
                     f"from:{username}", filter_=SearchFilters.Latest(), cursor=cur
                 )
-                cur_page = search.tweets
+                cur_page = search.results
                 print(f"obtained {len(cur_page)} tweets")
 
                 if len(cur_page) == 0:
@@ -168,7 +168,7 @@ class Scraper:
                 for e in cur_page:
                     if isinstance(e, Tweet):
                         add_tweet(e)
-                    elif isinstance(e, TweetThread):
+                    elif isinstance(e, SelfThread):
                         # FIXME: rework when replied_to is fixed (currently populates user_mentions)
                         # latest tweet in thread = og author's reply
                         for t in e:

src/talenttweet.py

@@ -85,7 +85,7 @@ class TalentTweet:
             rt_mentions=rtm,
         )
 
-    ## Creates a TalentTweet from a Tweety-library Tweet.
+    ## Creates a TalentTweet from a Tweety Tweet.
     @staticmethod
     def create_from_tweety(tweety: Tweet):
         if tweety.is_retweet:
@@ -104,7 +104,7 @@ class TalentTweet:
             text=tweety.text,
             mrq=(
                 {int(x.id) for x in tweety.user_mentions},
-                int(tweety.original_tweet["in_reply_to_user_id_str"])
+                int(tweety._original_tweet["in_reply_to_user_id_str"])
                 if tweety.is_reply
                 else None,
                 int(tweety.quoted_tweet.author.id)
@@ -278,7 +278,7 @@ class TalentTweet:
             rt_username = (
                 util.get_username_with_company(self.rt_author_id)
                 if self.rt_author_id != -1
-                else None
+                else "someone"
             )
             if rt_username == author_username:
                 rt_username = "themselves"
@@ -291,7 +291,7 @@ class TalentTweet:
             reply_username = (
                 util.get_username_with_company(self.reply_to)
                 if self.reply_to != -1
-                else None
+                else "someone"
             )
             if reply_username == author_username:
                 reply_username = "themselves"
@@ -303,7 +303,7 @@ class TalentTweet:
             quoted_username = (
                 util.get_username_with_company(self.quote_tweeted)
                 if self.quote_tweeted != -1
-                else None
+                else "someone"
             )
             if quoted_username == author_username:
                 quoted_username = "themselves"

src/twapi.py

@@ -136,6 +136,8 @@ class TwAPI:
     # return True = successfully posted a single ttweet
     # return False = did not post ttweet (duplicate)
     async def post_ttweet(self, ttweet: tt.TalentTweet, dry_run=False):
+        import main
+
         print(
             f"------{ttweet.tweet_id} ({util.get_username_local(ttweet.author_id)})------"
         )
@@ -146,10 +148,8 @@ class TwAPI:
         if dry_run:
             print("-------------------- DRY RUN --------------------")
             print(ttweet)
-        if dry_run:
             return False
 
-        # NO DRY-RUN: actually post tweet
         # main tweet: text + screenshot
         try:
             print("creating main QRT w/ screenshot...")
@@ -188,7 +188,7 @@ class TwAPI:
                     raise e
         return True
 
-    async def post_ttweet_by_id(self, id: int):
+    async def post_ttweet_by_id(self, id: int, dry_run=False):
         from scraper import Scraper
 
         print(f"Manually posting tweet {id}")
@@ -204,4 +204,4 @@ class TwAPI:
             return False
 
         print(f"Posting {ttweet.username}/{ttweet.tweet_id}...")
-        return await self.post_ttweet(ttweet)
+        return await self.post_ttweet(ttweet, dry_run)

src/tweety_utils.py

@@ -1,24 +1,26 @@
 from tweety.types import *
 
-def url(t: Tweet):
-	return f'https://twitter.com/{t.author.username}/status/{t.id}'
 
-def print_tweets(tweets: list[Tweet | TweetThread]):
+def url(t: Tweet):
-	print(f'{len(tweets)} tweets:')
+    return f"https://twitter.com/{t.author.username}/status/{t.id}"
+
+
+def print_tweets(tweets: list[Tweet | SelfThread]):
+    print(f"{len(tweets)} tweets:")
     for t in tweets:
         if isinstance(t, Tweet):
-			print(f'{t.date} : {url(t)} :', end=' ')
+            print(f"{t.date} : {url(t)} :", end=" ")
 
             if t.is_retweet:
-				print(f'RT ({t.retweeted_tweet.author.username})', end=' ')
+                print(f"RT ({t.retweeted_tweet.author.username})", end=" ")
 
             if t.is_reply:
-				print(f'is reply!', end=' ')
+                print(f"is reply!", end=" ")
             if t.replied_to is not None:
-				print(f'reply to {t.replied_to.author.username}', end=' ')
+                print(f"reply to {t.replied_to.author.username}", end=" ")
 
             print("m=" + ",".join([x.username for x in t.user_mentions]))
-		elif isinstance(t, TweetThread):
+        elif isinstance(t, SelfThread):
-			print('-----------TTd----------')
+            print("-----------TTd----------")
             print_tweets(t.tweets)
-			print('-----------end----------')
+            print("-----------end----------")

src/util.py

@@ -26,7 +26,7 @@ def project_root(dir_path: tuple[str] = tuple(), file: str = None):
 
 
 def working_path(dir_path: tuple[str] = tuple(), file: str = None):
-    """Returns file path relative to the working ephemeral directory."""
+    """Returns file path relative to the working ephemeral directory "run"."""
     dir_path = project_root(("run", *dir_path))
     Path(dir_path).mkdir(parents=True, exist_ok=True)