Kosmodular grid image scraper

main.py

# Quick and dirty script to save external images used on kosmodulargrid directly into the repository.
# This helps prevent images disappearing from the site as their URLs go bad.
# The outputted JSON does need some manual cleanup - do so at your own risk!
# This is free and unencumbered software released into the public domain.


# Clone the repository into the same directory as this script.
import requests
import json
import urllib
import os

with open("kosmodulargrid/public/data/modules.json") as f:
    modules_file = f.read()
    f.close()
    
new_modules_file = ""
modules = json.loads(modules_file)

print(f"There are {len(modules)} modules to download.")
for i, item in enumerate(modules):
    link = item["imageLink"]
    id = item["id"]
    #get file extension from url
    path = urllib.parse.urlparse(link).path
    ext = os.path.splitext(path)[1]
    try:
        urllib.request.urlretrieve(item["imageLink"], f"kosmodulargrid/public/data/images/{id}{ext}")
    except Exception: # lazy, i know
        print(item["imageLink"])
        print(f"Download failed for {i}, entry not changed.")
        new_modules_file += str(item)
        continue

    # update module entry with new url and append to file
    new_item = item
    new_item["imageLink"] = f"https://raw.githubusercontent.com/Sonosus/kosmodulargrid/main/public/data/images/{id}{ext}"
    new_modules_file += str(new_item)
    print(f"Saved {i} of {len(modules)}")

# some find and replace magic is required to clean up this json file but its nothing too tragic
with open("kosmodulargrid/public/data/modules.json", "w") as f:
    f.write(new_modules_file)
    f.close()