Can't pull more than 100 tweets
Closed this issue · 2 comments
I am trying to collect users' liked tweets and I can only gather the most recent 100, I am very new to web scraping so I am struggling a little bit figuring out what to do to fix this error. If it isn't possible to gather more than 100 at a time, is there a way to gather them in batches (meaning I would get the first most recent hundred then the second most recent hundred and so on and so forth)? I've put my code below if anyone could help!
`from twikit import Client
import json
import pandas as pd
client=Client('en-US')
client.login(
auth_info_1='username,'
password='password'
)
client.save_cookies('cookies.json')
client.load_cookies(path='cookies.json')
user=client.get_user_by_screen_name('username')
tweets=user.get_tweets('Likes',count=500)
tweets_to_store=[]
for tweet in tweets:
tweets_to_store.append({
'created_at':tweet.created_at,
'favorite_count':tweet.favorite_count,
'full_text':tweet.full_text
})
df=pd.DataFrame(tweets_to_store)
df.to_csv('liked_tweets.csv', index=False)`
@KirenCO123
You can retrieve more tweets by calling next
method. Below is an example of the code.
from twikit import Client
import json
import os
import pandas as pd
client=Client('en-US')
if os.path.exists('cookies.json'):
client.load_cookies(path='cookies.json')
else:
client.login(
auth_info_1='username',
auth_info_2='email'
password='password'
)
client.save_cookies('cookies.json')
user = client.get_user_by_screen_name('username')
limit = 500
tweets_ = []
tweets = user.get_tweets('Likes', count=500)
while len(tweets_to_store) < limit and len(tweets) > 0:
tweets_ += tweets
try:
tweets = tweets.next() # Call next method to retrieve more tweets
except:
break
tweets_to_store=[]
for tweet in tweets:
tweets_to_store.append({
'created_at': tweet.created_at,
'favorite_count': tweet.favorite_count,
'full_text': tweet.full_text
})
df=pd.DataFrame(tweets_to_store)
df.to_csv('liked_tweets.csv', index=False)
Thank you so much! Here is the code that ended up working for me in the end :)
`from twikit import Client
import json
import os
import pandas as pd
client = Client('en-US')
if os.path.exists('cookies.json'):
client.load_cookies(path='cookies.json')
else:
client.login(
auth_info_1='your_auth_info_1',
auth_info_2='your_auth_info_2',
password='your_password'
)
user = client.get_user_by_screen_name('username')username
limit = 500
tweets_to_store = []
tweets = user.get_tweets('Likes', count=500)
while len(tweets_to_store) < limit and len(tweets) > 0:
for tweet in tweets:
tweets_to_store.append({
'created_at': tweet.created_at,
'favorite_count': tweet.favorite_count,
'full_text': tweet.full_text
})
if len(tweets_to_store) >= limit:
break
try:
tweets = tweets.next()
except:
break
df = pd.DataFrame(tweets_to_store)
df.to_csv('tweets.csv', index=False)
`