jklymak

Hash-Based Baseline Image Storage for Matplotlib

Problem: Matplotlib's baseline image tests generate 507MB .git directory (40MB current baselines + 467MB history). With 2,330 baseline images and ~106 baseline-touching commits per year, this continues to grow, making FreeType updates painful and new contributor clones slow.

Solution: Store perceptual hashes (~200KB total) instead of images in the main repo. Download actual images on-demand only for test failures.

---

Core Architecture

jklymak

import numpy as np
import dbdreader

dbd=dbdreader.DBD("realtime_raw/02870227.scd", cacheDir='cac/')

# print what parameters are available:
for i,p in enumerate(dbd.parameterNames):
    tm, v = dbd.get(p)
    print(f"{i:02d}: {p} {len(v)}")

jklymak

import smtplib
import yaml

from email.message import EmailMessage
from email.utils import make_msgid
from email.mime.base import MIMEBase
from email import encoders
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from email.mime.application import MIMEApplication

jklymak

import numpy as np
import matplotlib.pyplot as plt
import seawater
%matplotlib widget
import pooch
import xarray as xr
import pandas

"""
eg 

jklymak

import xarray as xr
# https://oceandata.sci.gsfc.nasa.gov/ob/getfile/AQUA_MODIS.20230606T215500.L2.SST.NRT.nc

with xr.open_dataset('/Users/jklymak/Downloads/AQUA_MODIS.20230606T215500.L2.OC.NRT.nc',group='geophysical_data') as ds, xr.open_dataset('/Users/jklymak/Downloads/AQUA_MODIS.20230606T215500.L2.OC.NRT.nc',group='navigation_data') as nav:
    print(nav)
    ds['lon'] = (('number_of_lines', 'pixels_per_line'), nav['longitude'].values)
    ds['lat'] = (('number_of_lines', 'pixels_per_line'), nav['latitude'].values)
    for j in range(0, 2030):
        if np.any(~np.isfinite(ds.lat[j, :])):
            print(j)

jklymak

# output from the MITgcm using xmitgcm

import xarray as xr
import numpy as np
import xgcm

with xr.open_dataset(fname, decode_times=False) as ds:
    grid = xgcm.Grid(ds, coords={'Z':{'center':'Z', 'outer':'Zp1'}}, periodic=False)
    ds = preprocess(ds)

jklymak

#!/bin/bash
#SBATCH --account=def-jklymak
#SBATCH --mail-user=jklymak@gmail.com
#SBATCH --mail-type=ALL
#SBATCH --ntasks-per-node=1
#SBATCH --time=0-00:15
#SBATCH --mem=64G

cd ${SLURM_SUBMIT_DIR}

jklymak

#!/bin/bash -l
for todo in OneHill100lowU10N10Amp305f141B059Rough
do
    day=86400
    jobid=$(sbatch -J $todo --export=start=0,stop=$((day*6 + 180)),dt=90 runModel.sh)
    #echo $jobid
    jobid1=$(sbatch  -J $todo  --dependency=afterok:${jobid##* } --export=start=$((day*6)),stop=$((day*12 + 180)),dt=90 runModel.sh)
    jobid2=$(sbatch  -J $todo --dependency=afterok:${jobid1##* } --export=start=$((day*12)),stop=$((day*18 + 180)),dt=90 runModel.sh)
    jobid3=$(sbatch  -J $todo --dependency=afterok:${jobid2##* } --export=start=$((day*18)),stop=$((day*24 + 180)),dt=90 runModel.sh)

jklymak

#MITgcm checkin runs

Sometimes we want to store versions of config files for a model run. This uses runname to make a new branch, and to make a commit on the main branch. You need to set the model config up to be tracked, of course.

_log.info('doing this via git!!')

jklymak

import sys
import os
import subprocess

os.system('sq')
out=subprocess.check_output('squeue -h -t RUNNING -u jklymak --Format="Name:50"', 
                            shell=True, text=True)
out = out.splitlines()
# print(out)
for dd in out: