memory optimized so it would run on my laptop. Also sorts by sound not pitch.

video.py

from moviepy.editor import *
import numpy as np
clip = VideoFileClip("Steamed Hams.mpeg")
import sounddevice as sd
import soundfile as sf
from queue import Queue
from collections import OrderedDict
import pprint
import pdb

pp = pprint.PrettyPrinter(indent=4)
print('started')
print('sarted')

def freq(x):
    # pdb.set_trace()
    # b = x.mean(1)
    # d = (np.fft.rfft(b)**2)
    # w = d[1:].argmax() + 1
    return np.linalg.norm(x)

normal_audioarr = [ clip.audio.get_frame(t/clip.audio.fps) for t in range(0,int(clip.duration*clip.audio.fps))]
#normal_audioarr = [ clip.audio.get_frame(t/clip.audio.fps) for t in range(0,int(5*clip.audio.fps))]
video_clips = [clip.get_frame(t/clip.fps) for t in range(0,int(clip.fps*clip.duration))]
chunked =  enumerate(np.array_split(np.array(normal_audioarr), int(clip.duration) * 15))
chunked_vid = dict(enumerate(np.array_split(video_clips, int(clip.duration) * 15)))
#weird_audio = sorted(chunked, key=lambda x: abs(x[1]).sum())
weird_audio = sorted(chunked, key=lambda x: freq(x[1]))
weird_audio_dict = OrderedDict(weird_audio)
weird_vid = [chunked_vid[i] for i in weird_audio_dict.keys()]

# pdb.set_trace()
# l2 = np.zeros((sum(map(lambda x: x.shape[0], weird_vid)), *weird_vid[0].shape[1:]),dtype = 'uint8')
# index = 0
# for chunk in weird_vid:
#     length = chunk.shape[0]
#     l2[index:index+length] = chunk
#     index += length



print('concating audio')
data = list(map(lambda x: x[1], weird_audio))

l = np.array(data[0])
for i in range(1,len(data)):
    l = np.append(l, data[i], 0)
print(l.shape)
print(len(l))
data = l
#data = normal_audioarr



#aclip = AudioClip(make_frame=make_frame, duration=clip.duration)
#aclip.
fs = 44100
print("playing")
# NOTE, we will probably have to do it like 4 frames at a time so that it sounds like not static

sf.write('myfile.wav', data, fs)
print('concating video')
l2 = np.zeros((sum(map(lambda x: x.shape[0], weird_vid)), *weird_vid[0].shape[1:]),dtype = 'uint8')
index = 0
for chunk in weird_vid:
    length = chunk.shape[0]
    l2[index:index+length] = chunk
    index += length
# l_1 = np.array(weird_vid[0])    
# for i in range(1,len(weird_vid)):
#     l = np.append(l, weird_vid[i], 0)

print(l2.shape)
# pdb.set_trace()
del weird_audio
del weird_vid
del chunked
del chunked_vid
del weird_audio_dict
del video_clips
del normal_audioarr
del l
del data

global it
it = 0
def make_frame(t):
    global it
    it += 1
    # if it >= 4846:
    #     return l2[5]
    try:
        return l2[it]
    except IndexError:
        return l2[5]
vid = VideoClip(make_frame, duration = clip.duration)
# audio = AudioFileClip('myfile.wav')
# vid.set_audio(audio)
vid.write_videofile('out.mp4',fps=clip.fps)


print('done')
#b has the suond data assoc with its frame
# sort b, then give each of b its frame using get_frame maybe?
#then make an audio clip using audioarrayclip which doesent exist yet
#then profit
#audioclip.preview()

the "freq(x)" function now is actually just a norm of the wave form, but I was too lazy to change function name. Can be substituted for any other way you want to sort it.