diff options
Diffstat (limited to 'src/scrape_lexicanum.py')
| -rw-r--r-- | src/scrape_lexicanum.py | 21 |
1 files changed, 18 insertions, 3 deletions
diff --git a/src/scrape_lexicanum.py b/src/scrape_lexicanum.py index f4cb071..d55f58b 100644 --- a/src/scrape_lexicanum.py +++ b/src/scrape_lexicanum.py @@ -1,6 +1,7 @@ from bs4 import BeautifulSoup from markdownify import markdownify from requests.exceptions import HTTPError +from urllib.parse import urlparse import requests import textwrap @@ -16,9 +17,9 @@ def decompose(node, field): for unwanted in node.select(field): unwanted.decompose() -def get_search_response(config, query): +def get_search_response(config, api_root, query): try: - response = requests.get(config['site'] + '/mediawiki/api.php?action=opensearch&format=json&formatversion=2&search=' + query + '&namespace=0&limit=6&suggest=true', timeout=7) + response = requests.get(config['site'] + api_root + '/api.php?action=opensearch&format=json&formatversion=2&search=' + query + '&namespace=0&limit=6&suggest=true', timeout=7) except requests.exceptions.Timeout: raise WikiError("Request timed out.") except requests.exceptions.RequestException as e: @@ -55,7 +56,7 @@ def get_page_content(config, page_name): soup = BeautifulSoup(response.content, "html.parser") - parser_output = soup.find(id="mw-content-text").contents[0] + parser_output = soup.select_one(".mw-parser-output") decompose(parser_output, "sup") decompose(parser_output, "div.portal") @@ -75,8 +76,19 @@ def get_page_content(config, page_name): fix_link(config, image.contents[0], "src") image_link = image.contents[0]["src"] + try: + result = urlparse(image_link) + + if not result.netloc: + fix_link(config, image.contents[0], "data-src") + image_link = image.contents[0]["data-src"] + except ValueError: + fix_link(config, image.contents[0], "data-src") + image_link = image.contents[0]["data-src"] + decompose(parser_output, "br") decompose(parser_output, "table") + decompose(parser_output, "aside") for p in parser_output.find_all("p"): if not p.get_text(strip=True): @@ -87,6 +99,9 @@ def get_page_content(config, page_name): fix_link(config, a, "href") header_md = markdownify(str(soup.find(id="firstHeading").contents[0])) + if not header_md: + header_md = markdownify(str(soup.select_one(".mw-page-title-main"))) + content_md = textwrap.shorten(markdownify(str(content)), width=1000, placeholder="...") return header_md, content_md, image_link |