scraper_functions #1

Merged
texhno merged 6 commits from scraper_functions into master 2024-02-06 01:24:59 +00:00
3 changed files with 15 additions and 3 deletions

View File

@ -1,10 +1,11 @@
import ollama import ollama
import scraper_functions as sf
def processmsg(msg, rcpt): def processmsg(msg, rcpt):
if msg.startswith("!"): if "youtube.com/watch" in msg:
return command(msg, rcpt)
elif "youtube.com/watch" in msg:
return msg.replace("youtube.com", "iv.datura.network") return msg.replace("youtube.com", "iv.datura.network")
elif msg.startswith("!wiki"):
return sf.query_external_website("https://en.wikipedia.org/wiki/", msg.split(" ")[1])
elif "good bot" in msg: elif "good bot" in msg:
return "^_^" return "^_^"

View File

@ -1,2 +1,4 @@
slixmpp slixmpp
ollama ollama
requests
beautifulsoup4

9
scraper_functions.py Normal file
View File

@ -0,0 +1,9 @@
import requests
from bs4 import BeautifulSoup
def query_external_website(base_url, query):
page = requests.get(base_url + query)
soup = BeautifulSoup(page.content, "html.parser")
title = soup.find("span", class_="mw-page-title-main").text
content = soup.find(id="mw-content-text").select("p")[2].text
return "\nTITLE:\n" + title + "\n\nCONTENT:\n" + content + "\n\nFULL LINK:\n" + base_url + query