adapt scraping code to mainline library version
This commit is contained in:
+2
-2
@@ -19,7 +19,7 @@ class Scraper:
|
|||||||
def __init__(self):
|
def __init__(self):
|
||||||
Scraper.instance = self
|
Scraper.instance = self
|
||||||
self.__account = AccountPool()
|
self.__account = AccountPool()
|
||||||
self.try_login()
|
self.try_login(0)
|
||||||
|
|
||||||
def try_login(self, account_idx: int = None) -> bool:
|
def try_login(self, account_idx: int = None) -> bool:
|
||||||
# decide on which account to use
|
# decide on which account to use
|
||||||
@@ -159,7 +159,7 @@ class Scraper:
|
|||||||
search = self.app.search(
|
search = self.app.search(
|
||||||
f"from:{username}", filter_=SearchFilters.Latest(), cursor=cur
|
f"from:{username}", filter_=SearchFilters.Latest(), cursor=cur
|
||||||
)
|
)
|
||||||
cur_page = search.tweets
|
cur_page = search.results
|
||||||
print(f"obtained {len(cur_page)} tweets")
|
print(f"obtained {len(cur_page)} tweets")
|
||||||
|
|
||||||
if len(cur_page) == 0:
|
if len(cur_page) == 0:
|
||||||
|
|||||||
+13
-11
@@ -1,24 +1,26 @@
|
|||||||
from tweety.types import *
|
from tweety.types import *
|
||||||
|
|
||||||
def url(t: Tweet):
|
|
||||||
return f'https://twitter.com/{t.author.username}/status/{t.id}'
|
|
||||||
|
|
||||||
def print_tweets(tweets: list[Tweet | TweetThread]):
|
def url(t: Tweet):
|
||||||
print(f'{len(tweets)} tweets:')
|
return f"https://twitter.com/{t.author.username}/status/{t.id}"
|
||||||
|
|
||||||
|
|
||||||
|
def print_tweets(tweets: list[Tweet | SelfThread]):
|
||||||
|
print(f"{len(tweets)} tweets:")
|
||||||
for t in tweets:
|
for t in tweets:
|
||||||
if isinstance(t, Tweet):
|
if isinstance(t, Tweet):
|
||||||
print(f'{t.date} : {url(t)} :', end=' ')
|
print(f"{t.date} : {url(t)} :", end=" ")
|
||||||
|
|
||||||
if t.is_retweet:
|
if t.is_retweet:
|
||||||
print(f'RT ({t.retweeted_tweet.author.username})', end=' ')
|
print(f"RT ({t.retweeted_tweet.author.username})", end=" ")
|
||||||
|
|
||||||
if t.is_reply:
|
if t.is_reply:
|
||||||
print(f'is reply!', end=' ')
|
print(f"is reply!", end=" ")
|
||||||
if t.replied_to is not None:
|
if t.replied_to is not None:
|
||||||
print(f'reply to {t.replied_to.author.username}', end=' ')
|
print(f"reply to {t.replied_to.author.username}", end=" ")
|
||||||
|
|
||||||
print("m=" + ",".join([x.username for x in t.user_mentions]))
|
print("m=" + ",".join([x.username for x in t.user_mentions]))
|
||||||
elif isinstance(t, TweetThread):
|
elif isinstance(t, SelfThread):
|
||||||
print('-----------TTd----------')
|
print("-----------TTd----------")
|
||||||
print_tweets(t.tweets)
|
print_tweets(t.tweets)
|
||||||
print('-----------end----------')
|
print("-----------end----------")
|
||||||
|
|||||||
+1
-1
@@ -26,7 +26,7 @@ def project_root(dir_path: tuple[str] = tuple(), file: str = None):
|
|||||||
|
|
||||||
|
|
||||||
def working_path(dir_path: tuple[str] = tuple(), file: str = None):
|
def working_path(dir_path: tuple[str] = tuple(), file: str = None):
|
||||||
"""Returns file path relative to the working ephemeral directory."""
|
"""Returns file path relative to the working ephemeral directory "run"."""
|
||||||
dir_path = project_root(("run", *dir_path))
|
dir_path = project_root(("run", *dir_path))
|
||||||
Path(dir_path).mkdir(parents=True, exist_ok=True)
|
Path(dir_path).mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user