- Fetch data -
Asynchronously
- Parse data - with
multiprocessing
examples
from bs4 import BeautifulSoup
from pAsynCrawler import AsynCrawler, flattener
def parser_0(response_text):
soup = BeautifulSoup(response_text)
menus = soup.select('ul > li > span > a')
datas = tuple(x.text for x in menus)
urls = tuple(x.attrs['href'] for x in menus)
return (datas, urls)
def parser_0(response_text):
soup = BeautifulSoup(response_text)
menus = soup.select('ul > li > a')
datas = tuple(x.text for x in menus)
urls = tuple(x.attrs['href'] for x in menus)
return (datas, urls)
if __name__ == '__main__':
ac = AsynCrawler(asy_fetch=20, mp_parse=8)
datas_1, urls_1 = ac.fetch_and_parse(parser_0, ['https://www.example.com'])
datas_2, urls_2 = ac.fetch_and_parse(parser_1, flattener(urls_1))