Abbot – “404-finder v1.0.6” by Serious Business

Code

<br /> # Description: Check all the links on a page to make sure they all resolve<br /> #<br /> # Package URL: /packages/aseriousbiz/404-finder<br /> #<br /> # Usage:<br /> # This skill is best used as a scheduled skill.<br /> #<br /> # @abbot schedule 404-finder</p> <p>from bs4 import BeautifulSoup<br /> from urllib.parse import urlparse<br /> import requests</p> <p># Let’s pretend to be Chrome in order to check Twitter<br /> bot_headers = {‘user-agent’: ‘Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36’}</p> <p>def check_link(url):<br /> if url.startswith(“mailto:”) or url.startswith(“tel:”):<br /> return<br /> try:<br /> r = requests.head(url, headers=bot_headers)<br /> if r.status_code in [404, 500, 502, 503, 504]:<br /> bot.reply(“:warning: {} returned status **{}**”.format(url, r.status_code))</p> <p> except:<br /> bot.reply(“:octagonal-sign: Could not reach `{}` while crawling `{}`”.format(url, args))</p> <p>def check_links(url):<br /> “””Check any hyperlinks in the data and report any issues to chat.”””<br /> try:<br /> r = requests.get(url, headers=bot_headers)<br /> r.raise_for_status()</p> <p> soup = BeautifulSoup(r.text, ‘html.parser’)<br /> links = soup.find_all(‘a’, href=True)<br /> url_parts = urlparse(url)<br /> for link in links:<br /> target = link[‘href’]<br /> if target[0] == “/” or target[0] == “#”:<br /> target = “{}://{}{}”.format(url_parts.scheme, url_parts.netloc, link[‘href’])<br /> check_link(target)<br /> bot.reply(“Finished checking `{}` for 404s”.format(url))<br /> except ConnectionError:<br /> bot.reply(“Error checking for 404s. Could not reach `{}`.”.format(url))</p> <p>if len(bot.arguments) > 0:<br /> check_links(bot.arguments)<br /> else:<br /> bot.reply(“You didn’t say anything”)