diff options
| -rw-r--r-- | src/main.py | 8 | ||||
| -rw-r--r-- | src/scrape_lexicanum.py | 21 |
2 files changed, 24 insertions, 5 deletions
diff --git a/src/main.py b/src/main.py index a50e233..c531faa 100644 --- a/src/main.py +++ b/src/main.py @@ -34,7 +34,8 @@ async def help(ctx): @bot.command(name="Explain", aliases=["explain", "What's", "what's", "whats"]) async def explain(ctx, *args): config["site"] = "https://ageofsigmar.lexicanum.com" - + api_root = "/mediawiki" + include_image = True include_links = True @@ -49,6 +50,9 @@ async def explain(ctx, *args): config["site"] = "https://wh40k.lexicanum.com" case "!aos": config["site"] = "https://ageofsigmar.lexicanum.com" + case "!experimental-wod": + config["site"] = "https://whitewolf.fandom.com" + api_root = "" case "!noimage": include_image = False case "!nolinks": @@ -58,7 +62,7 @@ async def explain(ctx, *args): query = " ".join([x.replace('"', "") for x in args]) try: - search_content = scrape_lexicanum.get_search_response(config, query) + search_content = scrape_lexicanum.get_search_response(config, api_root, query) page_header, page_content, page_img_link = scrape_lexicanum.get_page_content(config, search_content[0]) embed = discord.Embed( diff --git a/src/scrape_lexicanum.py b/src/scrape_lexicanum.py index f4cb071..d55f58b 100644 --- a/src/scrape_lexicanum.py +++ b/src/scrape_lexicanum.py @@ -1,6 +1,7 @@ from bs4 import BeautifulSoup from markdownify import markdownify from requests.exceptions import HTTPError +from urllib.parse import urlparse import requests import textwrap @@ -16,9 +17,9 @@ def decompose(node, field): for unwanted in node.select(field): unwanted.decompose() -def get_search_response(config, query): +def get_search_response(config, api_root, query): try: - response = requests.get(config['site'] + '/mediawiki/api.php?action=opensearch&format=json&formatversion=2&search=' + query + '&namespace=0&limit=6&suggest=true', timeout=7) + response = requests.get(config['site'] + api_root + '/api.php?action=opensearch&format=json&formatversion=2&search=' + query + '&namespace=0&limit=6&suggest=true', timeout=7) except requests.exceptions.Timeout: raise WikiError("Request timed out.") except requests.exceptions.RequestException as e: @@ -55,7 +56,7 @@ def get_page_content(config, page_name): soup = BeautifulSoup(response.content, "html.parser") - parser_output = soup.find(id="mw-content-text").contents[0] + parser_output = soup.select_one(".mw-parser-output") decompose(parser_output, "sup") decompose(parser_output, "div.portal") @@ -75,8 +76,19 @@ def get_page_content(config, page_name): fix_link(config, image.contents[0], "src") image_link = image.contents[0]["src"] + try: + result = urlparse(image_link) + + if not result.netloc: + fix_link(config, image.contents[0], "data-src") + image_link = image.contents[0]["data-src"] + except ValueError: + fix_link(config, image.contents[0], "data-src") + image_link = image.contents[0]["data-src"] + decompose(parser_output, "br") decompose(parser_output, "table") + decompose(parser_output, "aside") for p in parser_output.find_all("p"): if not p.get_text(strip=True): @@ -87,6 +99,9 @@ def get_page_content(config, page_name): fix_link(config, a, "href") header_md = markdownify(str(soup.find(id="firstHeading").contents[0])) + if not header_md: + header_md = markdownify(str(soup.select_one(".mw-page-title-main"))) + content_md = textwrap.shorten(markdownify(str(content)), width=1000, placeholder="...") return header_md, content_md, image_link |