Google搜索算法原理与代码实现.docxVIP

Google搜索算法原理与代码实现

一、搜索引擎核心原理

1.网页爬取(Crawling)

python

importrequestsfrombs4importBeautifulSoupfromurllib.parseimporturljoin,urlparseimporttimefromcollectionsimportdeque

classWebCrawler:

def__init__(self,delay=1):

self.visited=set()

self.to_crawl=deque()

self.delay=delay

self.index={}

defcrawl(self,start_url,max_pages=100):

self.to_crawl.append(start_url)

whileself.to_crawlandlen(self.visited)max_pages:

url=self.to_crawl.popleft()

ifurlinself.visited:

continue

try:

#遵守robots.txt和延迟

time.sleep(self.delay)

response=requests.get(url,timeout=5)

soup=BeautifulSoup(response.content,html.parser)

#提取文本内容

text=self.extract_text(soup)

self.index_page(url,text)

#提取并处理链接

links=self.extract_links(soup,url)

forlinkinlinks:

iflinknotinself.visited:

self.to_crawl.append(link)

self.visited.add(url)

exceptExceptionase:

print(fErrorcrawling{url}:{e})

defextract_text(self,soup):

#移除脚本和样式

forscriptinsoup([script,style]):

script.decompose()

#获取文本

text=soup.get_text()

lines=(line.strip()forlineintext.splitlines())

chunks=(phrase.strip()forlineinlinesforphraseinline.split())

text=.join(chunkforchunkinchunksifchunk)

returntext

defextract_links(self,soup,base_url):

links=[]

forlinkinsoup.find_all(a,href=True):

absolute_url=urljoin(base_url,link[href])