Skip to content

Instantly share code, notes, and snippets.

@michalfratczak
Created October 16, 2024 10:03
Show Gist options
  • Save michalfratczak/9f9d795bbd2ec907880991862bcfa92d to your computer and use it in GitHub Desktop.
Save michalfratczak/9f9d795bbd2ec907880991862bcfa92d to your computer and use it in GitHub Desktop.
execute parallel commands in python
def parallel_exec(commands_arr):
processes = set()
max_processes = 8
for cmd in commands_arr:
print(cmd)
processes.add(subprocess.Popen( cmd ) )
while len(processes) >= max_processes:
time.sleep(1)
diffP = [p for p in processes if p.poll() is not None]
processes.difference_update(diffP)
# wait for unfinished processes
while len(processes):
time.sleep(.5)
diffP = [p for p in processes if p.poll() is not None]
processes.difference_update(diffP)
print('DONE')
sys.exit(0)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment