import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin, urlparse
import sqlite3
import threading
import queue
import tkinter as tk
from tkinter import ttk, messagebox
# データベース設定
DB_NAME = "gui_search_engine.db"
conn = sqlite3.connect(DB_NAME, check_same_thread=False)
cursor = conn.cursor()
# テーブル作成
cursor.execute("""
CREATE TABLE IF NOT EXISTS pages (
id INTEGER PRIMARY KEY AUTOINCREMENT,
url TEXT UNIQUE,
title TEXT,
description TEXT,
content TEXT
)
""")
conn.commit()
# グローバル変数
visited = set()
visited_lock = threading.Lock()
task_queue = queue.Queue()
MAX_THREADS = 5
# ページをデータベースに保存
def save_page_to_db(url, title, description, content):
try:
cursor.execute("INSERT INTO pages (url, title, description, content) VALUES (?, ?, ?, ?)",
(url, title, description, content))
conn.commit()
except sqlite3.IntegrityError:
pass # URL重複時は無視
# URL正規化
def normalize_url(base, link):
return urljoin(base, link).split('#')[0]
# クローラー
def crawl(url, domain, status_label):
with visited_lock:
if url in visited:
return
visited.add(url)
try:
response = requests.get(url, timeout=10)
soup = BeautifulSoup(response.content, "html.parser")
# メタデータ収集
title = soup.title.string if soup.title else "No Title"
description_tag = soup.find("meta", attrs={"name": "description"})
description = description_tag["content"] if description_tag else "No Description"
content = soup.get_text()
# データベースに保存
save_page_to_db(url, title, description, content)
# ステータス更新
status_label.config(text=f"Crawling: {url}")
# 次のURLを収集
for link in soup.find_all('a', href=True):
full_url = normalize_url(url, link['href'])
if urlparse(full_url).netloc == domain:
task_queue.put(full_url)
except Exception as e:
print(f"Error crawling {url}: {e}")
# 並列クローリング
def start_crawling(start_url, domain, status_label):
visited.clear()
task_queue.put(start_url)
def worker():
while not task_queue.empty():
url = task_queue.get()
crawl(url, domain, status_label)
status_label.config(text="Crawling Complete")
threads = []
for _ in range(MAX_THREADS):
thread = threading.Thread(target=worker)
threads.append(thread)
thread.start()
for thread in threads:
thread.join()
# 検索機能
def search(query, results_box):
cursor.execute("SELECT url, title, description FROM pages WHERE content LIKE ?", (f"%{query}%",))
results = cursor.fetchall()
results_box.delete(*results_box.get_children()) # 結果をクリア
if not results:
messagebox.showinfo("検索結果", "該当する結果はありませんでした。")
else:
for url, title, description in results:
results_box.insert("", "end", values=(title, url, description[:100]))
# GUI設計
def create_gui():
root = tk.Tk()
root.title("クローラー型検索エンジン")
# フレーム構成
frame_crawl = ttk.Frame(root, padding=10)
frame_crawl.grid(row=0, column=0, sticky="ew")
frame_search = ttk.Frame(root, padding=10)
frame_search.grid(row=1, column=0, sticky="nsew")
# クローリングセクション
ttk.Label(frame_crawl, text="スタートURL:").grid(row=0, column=0, padx=5, pady=5, sticky="w")
url_entry = ttk.Entry(frame_crawl, width=50)
url_entry.grid(row=0, column=1, padx=5, pady=5, sticky="w")
status_label = ttk.Label(frame_crawl, text="Status: Ready", foreground="blue")
status_label.grid(row=1, column=0, columnspan=2, padx=5, pady=5, sticky="w")
def on_crawl():
start_url = url_entry.get().strip()
if not start_url:
messagebox.showerror("Error", "スタートURLを入力してください。")
return
# スキーム補完
if not start_url.startswith(("http://", "https://")):
start_url = "https://" + start_url
# ドメインを取得
domain = urlparse(start_url).netloc
status_label.config(text="Crawling in Progress...")
threading.Thread(target=start_crawling, args=(start_url, domain, status_label)).start()
ttk.Button(frame_crawl, text="クローリング開始", command=on_crawl).grid(row=0, column=2, padx=5, pady=5)
# 検索セクション
ttk.Label(frame_search, text="検索クエリ:").grid(row=0, column=0, padx=5, pady=5, sticky="w")
query_entry = ttk.Entry(frame_search, width=30)
query_entry.grid(row=0, column=1, padx=5, pady=5, sticky="w")
results_box = ttk.Treeview(frame_search, columns=("Title", "URL", "Description"), show="headings")
results_box.heading("Title", text="タイトル")
results_box.heading("URL", text="URL")
results_box.heading("Description", text="説明")
results_box.grid(row=1, column=0, columnspan=3, padx=5, pady=5, sticky="nsew")
def on_search():
query = query_entry.get().strip()
if not query:
messagebox.showerror("Error", "検索クエリを入力してください。")
return
search(query, results_box)
ttk.Button(frame_search, text="検索", command=on_search).grid(row=0, column=2, padx=5, pady=5)
# ウィンドウサイズ調整
root.columnconfigure(0, weight=1)
frame_search.rowconfigure(1, weight=1)
root.mainloop()
# GUI起動
if __name__ == "__main__":
create_gui()
# データベース接続を閉じる
conn.close()
Decided to set a calendar reminder to revisit, and a stop at echohollowmarket extended that revisit list, calendar entries for content are a level of commitment I rarely make but when I do they signal a higher regard than a simple bookmark and this site has earned that calendar tier of relationship from me today.
Worth recognising the absence of the usual blog tropes here, and a look at radiantfieldcollective continued that fresh quality, sites that avoid the standard moves of the medium read as more original even when the content is on familiar topics and this one has clearly chosen its own path through the conventional terrain skilfully.
Thanks for the readable length, I finished it without checking how much was left, and a stop at stylishcartcenter kept me reading the same way, when I stop noticing the length of a piece because the content is engaging enough to sustain attention without willpower the writer has done their job well today.
A welcome contrast to the loud takes that have dominated my feed lately, and a look at amberridgegoods extended that calm voice, content that arrives without yelling has become unusual in the modern attention economy and this site is one of the few places I have found that consistently delivers without raising its voice.
Liked everything about the experience, from the opening through to the closing notes, and a stop at mysticthreadstore extended that into more pages, finding a site where the editorial vision shows through every choice rather than feeling random is an increasingly rare experience and one I am glad to have today during this particular reading session.
Reading this in a quiet coffee shop matched the calm energy of the writing, and a stop at frostedenclavegoods extended that environmental match, content that has its own ambient quality which can match or clash with surroundings is content with a personality and this site has the kind of personality that suits calm reading.
Took me back a step or two on an assumption I had been making, and a stop at frostbaycollective pushed that reconsideration further, writing that gently corrects the reader without being aggressive about it is a rare diplomatic skill and the team here clearly knows how to land critical points without turning readers off.
Genuine pleasure to read, and that is not something I say often after a casual click through, and a quick visit to elitetrendcenter kept the same feeling going across the rest of the site, finding writing that actually feels good to spend time with rather than just functional is increasingly rare on the open web.
Now planning to come back when I have the right kind of attention to read carefully, and a stop at mysticoakmarket reinforced that plan, choosing the right moment to read certain content is a quiet form of respect for the work and this site is generating those careful planning behaviours from me consistently as a reader.
Definitely returning here, that is decided, and a look at crystalfernstore only made the case stronger, this is one of those rare websites that rewards regular visits rather than feeling stale after the first read which is something I cannot say about most of the places I bookmark today across all my topics.
The tone stayed consistent across the whole post which is harder than it looks for longer pieces, and a look at stylishgoodsarena continued the same voice, this kind of editorial consistency is a sign of either a single careful writer or a tightly run team and either is impressive today across the broader media environment.
Liked how the post handled an objection I was forming as I read, and a stop at buyloopshop similarly anticipated where my thinking was going next, the rare writer who can predict reader concerns and address them in advance is doing something most online content fails to do despite that being basic editorial work.
Just want to acknowledge that the writing here is doing something right, and a quick visit to velvetharborgoods confirmed the same standards run across the broader site, recognising good work is something I try to do when I find it because the alternative is silence and silence rewards mediocrity.
Now sitting with the thoughts the post triggered rather than rushing on to the next thing, and a stop at swifttrendhub extended that reflective pause, content that earns time for thought after closing the tab is content of higher value than the merely interesting and this site has clearly produced that lasting effect today.
Looking back on this reading session it stands as one of the better ones recently, and a look at silverhorizonstore extended that ranking, the informal ranking of reading sessions against each other is something I do mentally and this session ranks high largely because of this site and a couple of related pages here.
Well structured and easy to read, that combination is rarer than people think, and a stop at freshcartcorner confirmed the same standard runs across the rest of the site, definitely the kind of place I will be coming back to when this topic comes up in conversation later again over the weeks ahead.
Really liked the calm tone running through the post, no shouting and no urgency forced into the writing, and a look at premiumpickzone kept that quiet confidence going, the kind of voice that makes the reader feel respected rather than yelled at which is depressingly common across most modern blog content these days.
Reading this on a long flight and finding it the best thing I read across hours of trying, and a stop at dreamwovenbazaar kept the streak going, when content beats long flight reading you know it has substance because flight reading is a hard test of a piece given the alternatives available everywhere.
Worth recognising that the post did not pretend to be the final word on the topic, and a stop at copperwindessentials continued that humility, content that admits its own scope and limits is more trustworthy than content that overreaches and this site has clearly developed the editorial maturity to know what it can and cannot claim well.