{"id":12762,"date":"2023-11-16T19:37:24","date_gmt":"2023-11-16T10:37:24","guid":{"rendered":"http:\/\/www.tyosuke20xx.com\/blog\/?p=12762"},"modified":"2023-11-16T19:37:26","modified_gmt":"2023-11-16T10:37:26","slug":"python-webcrawler","status":"publish","type":"post","link":"http:\/\/www.tyosuke20xx.com\/blog\/?p=12762","title":{"rendered":"Python Webcrawler"},"content":{"rendered":"\n

import requests
from bs4 import BeautifulSoup

def crawl(url, max_depth=2):
if max_depth < 0:
return

try:
response = requests.get(url)
content = response.content
soup = BeautifulSoup(content, ‘html.parser’)

links = set()

for link in soup.find_all(‘a’):
href = link.get(‘href’)
if href and href.startswith(‘http’):
links.add(href)

print(f”Found {len(links)} links at {url}”)

for link in links:
crawl(link, max_depth – 1)

except requests.RequestException as e:
print(f”Error during requests to {url} : {str(e)}”)

# \u4f7f\u7528\u4f8b
start_url = “https:\/\/b.hatena.ne.jp<\/strong>\/<\/em><\/a>” # \u30b9\u30bf\u30fc\u30c8\u3059\u308bURL
crawl(start_url, max_depth=2) # \u6df1\u30552\u3067\u30af\u30ed\u30fc\u30eb<\/p>\n\n\n\n

<\/a><\/a><\/p>\n","protected":false},"excerpt":{"rendered":"

import requestsfrom bs4 import BeautifulSoup def crawl(url, max_depth=2):if max_depth < 0:return try:respon … “Python Webcrawler” \u306e<\/span>\u7d9a\u304d\u3092\u8aad\u3080<\/a><\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"closed","sticky":false,"template":"","format":"standard","meta":{"_monsterinsights_skip_tracking":false,"_monsterinsights_sitenote_active":false,"_monsterinsights_sitenote_note":"","_monsterinsights_sitenote_category":0,"_uf_show_specific_survey":0,"_uf_disable_surveys":false,"footnotes":""},"categories":[4,65,6],"tags":[39,17],"class_list":["post-12762","post","type-post","status-publish","format-standard","hentry","category-programming","category-python","category-web","tag-python","tag-web"],"aioseo_notices":[],"jetpack_featured_media_url":"","_links":{"self":[{"href":"http:\/\/www.tyosuke20xx.com\/blog\/index.php?rest_route=\/wp\/v2\/posts\/12762","targetHints":{"allow":["GET"]}}],"collection":[{"href":"http:\/\/www.tyosuke20xx.com\/blog\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"http:\/\/www.tyosuke20xx.com\/blog\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"http:\/\/www.tyosuke20xx.com\/blog\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"http:\/\/www.tyosuke20xx.com\/blog\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=12762"}],"version-history":[{"count":1,"href":"http:\/\/www.tyosuke20xx.com\/blog\/index.php?rest_route=\/wp\/v2\/posts\/12762\/revisions"}],"predecessor-version":[{"id":12763,"href":"http:\/\/www.tyosuke20xx.com\/blog\/index.php?rest_route=\/wp\/v2\/posts\/12762\/revisions\/12763"}],"wp:attachment":[{"href":"http:\/\/www.tyosuke20xx.com\/blog\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=12762"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"http:\/\/www.tyosuke20xx.com\/blog\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=12762"},{"taxonomy":"post_tag","embeddable":true,"href":"http:\/\/www.tyosuke20xx.com\/blog\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=12762"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}