Quick and dirty search for 404s among Markdown links

search_for_404s.py

# Run like so at repository root directory:
#
# python3 search_for_404s.py > output &
# tail -f output

import glob
import re
import requests
import sys

regex_voodoo = '((http|https)://)[-a-zA-Z0-9@:%._\\+~#?&//=]{2,256}\\.[a-z]{2,6}\\b([-a-zA-Z0-9@:%._\\+~#?&//=]*)'
unique_pages = set()

for md in glob.glob('**/*.md', recursive=True):
    with open(md) as f:
        lines = f.readlines()

    for line in lines:
        m = re.search(regex_voodoo, line)

        if m:
            unique_pages.add(m.group())

for page in unique_pages:
    print(page)

    if requests.get(page).status_code == 404:
        print('MISSING')

    sys.stdout.flush()