AWS tools

aws_unpaginated.sh

#! /bin/bash

set -eo pipefail

AWS_PARAMS=("$@")
NEXT_TOKEN=""
OUTPUT="{}"


deep_merge() {  
    jq -s '
def meld(a; b):
  a as $a | b as $b
  | if ($a|type) == "object" and ($b|type) == "object"
    then reduce ([$a,$b]|add|keys_unsorted[]) as $k ({}; 
      .[$k] = meld( $a[$k]; $b[$k]) )
    elif ($a|type) == "array" and ($b|type) == "array"
    then $a+$b
    elif $b == null then $a
    else $b
    end;

meld(.[0]; .[1])
    ' "$1" "$2"
}

while [ "$NEXT_TOKEN" != "null" ]; do
    MY_PARAMS=("${AWS_PARAMS[@]}")
    if [ -n "$NEXT_TOKEN" ]; then
        MY_PARAMS+=(--next-token "$NEXT_TOKEN")
    fi
    RESPONSE=$(aws "${MY_PARAMS[@]}" --output json)
    OUTPUT=$(deep_merge <(echo "$OUTPUT") <(echo "$RESPONSE"))
    NEXT_TOKEN=$(echo "$RESPONSE" | jq -r '.nextToken // "null"')
done

echo "$OUTPUT" | jq 'del(.nextToken)'

aws_unpaginated.sh --> a script that will proxy AWS CLI calls to automatically loop through paginated response until the end of the list