タグ: python

import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin, urlparse
import sqlite3
import threading
import queue
import tkinter as tk
from tkinter import ttk, messagebox

# データベース設定
DB_NAME = "gui_search_engine.db"
conn = sqlite3.connect(DB_NAME, check_same_thread=False)
cursor = conn.cursor()

# テーブル作成
cursor.execute("""
CREATE TABLE IF NOT EXISTS pages (
    id INTEGER PRIMARY KEY AUTOINCREMENT,
    url TEXT UNIQUE,
    title TEXT,
    description TEXT,
    content TEXT
)
""")
conn.commit()

# グローバル変数
visited = set()
visited_lock = threading.Lock()
task_queue = queue.Queue()
MAX_THREADS = 5

# ページをデータベースに保存
def save_page_to_db(url, title, description, content):
    try:
        cursor.execute("INSERT INTO pages (url, title, description, content) VALUES (?, ?, ?, ?)",
                       (url, title, description, content))
        conn.commit()
    except sqlite3.IntegrityError:
        pass  # URL重複時は無視

# URL正規化
def normalize_url(base, link):
    return urljoin(base, link).split('#')[0]

# クローラー
def crawl(url, domain, status_label):
    with visited_lock:
        if url in visited:
            return
        visited.add(url)

    try:
        response = requests.get(url, timeout=10)
        soup = BeautifulSoup(response.content, "html.parser")

        # メタデータ収集
        title = soup.title.string if soup.title else "No Title"
        description_tag = soup.find("meta", attrs={"name": "description"})
        description = description_tag["content"] if description_tag else "No Description"
        content = soup.get_text()

        # データベースに保存
        save_page_to_db(url, title, description, content)

        # ステータス更新
        status_label.config(text=f"Crawling: {url}")

        # 次のURLを収集
        for link in soup.find_all('a', href=True):
            full_url = normalize_url(url, link['href'])
            if urlparse(full_url).netloc == domain:
                task_queue.put(full_url)
    except Exception as e:
        print(f"Error crawling {url}: {e}")

# 並列クローリング
def start_crawling(start_url, domain, status_label):
    visited.clear()
    task_queue.put(start_url)

    def worker():
        while not task_queue.empty():
            url = task_queue.get()
            crawl(url, domain, status_label)
        status_label.config(text="Crawling Complete")

    threads = []
    for _ in range(MAX_THREADS):
        thread = threading.Thread(target=worker)
        threads.append(thread)
        thread.start()
    for thread in threads:
        thread.join()

# 検索機能
def search(query, results_box):
    cursor.execute("SELECT url, title, description FROM pages WHERE content LIKE ?", (f"%{query}%",))
    results = cursor.fetchall()
    results_box.delete(*results_box.get_children())  # 結果をクリア

    if not results:
        messagebox.showinfo("検索結果", "該当する結果はありませんでした。")
    else:
        for url, title, description in results:
            results_box.insert("", "end", values=(title, url, description[:100]))

# GUI設計
def create_gui():
    root = tk.Tk()
    root.title("クローラー型検索エンジン")

    # フレーム構成
    frame_crawl = ttk.Frame(root, padding=10)
    frame_crawl.grid(row=0, column=0, sticky="ew")
    frame_search = ttk.Frame(root, padding=10)
    frame_search.grid(row=1, column=0, sticky="nsew")

    # クローリングセクション
    ttk.Label(frame_crawl, text="スタートURL:").grid(row=0, column=0, padx=5, pady=5, sticky="w")
    url_entry = ttk.Entry(frame_crawl, width=50)
    url_entry.grid(row=0, column=1, padx=5, pady=5, sticky="w")

    status_label = ttk.Label(frame_crawl, text="Status: Ready", foreground="blue")
    status_label.grid(row=1, column=0, columnspan=2, padx=5, pady=5, sticky="w")

    def on_crawl():
        start_url = url_entry.get().strip()
        if not start_url:
            messagebox.showerror("Error", "スタートURLを入力してください。")
            return

        # スキーム補完
        if not start_url.startswith(("http://", "https://")):
            start_url = "https://" + start_url

        # ドメインを取得
        domain = urlparse(start_url).netloc
        status_label.config(text="Crawling in Progress...")
        threading.Thread(target=start_crawling, args=(start_url, domain, status_label)).start()

    ttk.Button(frame_crawl, text="クローリング開始", command=on_crawl).grid(row=0, column=2, padx=5, pady=5)

    # 検索セクション
    ttk.Label(frame_search, text="検索クエリ:").grid(row=0, column=0, padx=5, pady=5, sticky="w")
    query_entry = ttk.Entry(frame_search, width=30)
    query_entry.grid(row=0, column=1, padx=5, pady=5, sticky="w")

    results_box = ttk.Treeview(frame_search, columns=("Title", "URL", "Description"), show="headings")
    results_box.heading("Title", text="タイトル")
    results_box.heading("URL", text="URL")
    results_box.heading("Description", text="説明")
    results_box.grid(row=1, column=0, columnspan=3, padx=5, pady=5, sticky="nsew")

    def on_search():
        query = query_entry.get().strip()
        if not query:
            messagebox.showerror("Error", "検索クエリを入力してください。")
            return
        search(query, results_box)

    ttk.Button(frame_search, text="検索", command=on_search).grid(row=0, column=2, padx=5, pady=5)

    # ウィンドウサイズ調整
    root.columnconfigure(0, weight=1)
    frame_search.rowconfigure(1, weight=1)

    root.mainloop()

# GUI起動
if __name__ == "__main__":
    create_gui()

    # データベース接続を閉じる
    conn.close()

import torch
import torch.nn as nn
import torch.optim as optim
from torchvision import datasets, transforms
from torch.utils.data import DataLoader
from torchvision.utils import save_image  # 画像保存のための関数
import os  # ファイルの保存先を指定するためのライブラリ

# VAEのエンコーダーとデコーダー
class VAE(nn.Module):
    def __init__(self, input_dim=784, hidden_dim=400, latent_dim=20):
        super(VAE, self).__init__()
        self.fc1 = nn.Linear(input_dim, hidden_dim)
        self.fc21 = nn.Linear(hidden_dim, latent_dim)
        self.fc22 = nn.Linear(hidden_dim, latent_dim)
        self.fc3 = nn.Linear(latent_dim, hidden_dim)
        self.fc4 = nn.Linear(hidden_dim, input_dim)

    def encode(self, x):
        h1 = torch.relu(self.fc1(x))
        return self.fc21(h1), self.fc22(h1)

    def reparameterize(self, mu, logvar):
        std = torch.exp(0.5 * logvar)
        eps = torch.randn_like(std)
        return mu + eps * std

    def decode(self, z):
        h3 = torch.relu(self.fc3(z))
        return torch.sigmoid(self.fc4(h3))

    def forward(self, x):
        mu, logvar = self.encode(x.view(-1, 784))
        z = self.reparameterize(mu, logvar)
        return self.decode(z), mu, logvar

# VAEの損失関数
def loss_function(recon_x, x, mu, logvar):
    BCE = nn.functional.binary_cross_entropy(recon_x, x.view(-1, 784), reduction='sum')
    KLD = -0.5 * torch.sum(1 + logvar - mu.pow(2) - logvar.exp())
    return BCE + KLD

# モデルの定義とデータローダーの準備
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
vae = VAE().to(device)
optimizer = optim.Adam(vae.parameters(), lr=1e-3)

transform = transforms.ToTensor()
train_dataset = datasets.MNIST('.', train=True, download=True, transform=transform)
train_loader = DataLoader(train_dataset, batch_size=128, shuffle=True)

# トレーニングループ
vae.train()
for epoch in range(10):  # 10エポックで学習
    train_loss = 0
    for batch_idx, (data, _) in enumerate(train_loader):
        data = data.to(device)
        optimizer.zero_grad()
        recon_batch, mu, logvar = vae(data)
        loss = loss_function(recon_batch, data, mu, logvar)
        loss.backward()
        train_loss += loss.item()
        optimizer.step()
    
    print(f'Epoch {epoch + 1}, Loss: {train_loss / len(train_loader.dataset)}')

# 学習したモデルで画像生成
vae.eval()
with torch.no_grad():
    z = torch.randn(64, 20).to(device)
    sample = vae.decode(z).cpu()
    sample = sample.view(64, 1, 28, 28)  # 28x28の画像サイズに変換 (MNISTデータのフォーマット)
    
    # 保存先ディレクトリを指定
    os.makedirs('generated_images', exist_ok=True)
    save_image(sample, 'generated_images/sample.png')

    print("画像生成完了: 'generated_images/sample.png' に保存されました")