fust download with r2 rclone and huggingface parallel

fastdl.sh

curl https://rclone.org/install.sh | sudo bash
mkdir ~/.config/rclone
echo "
[r2]
type = s3
provider = Cloudflare
access_key_id = $a
secret_access_key = $b
region = auto
endpoint = https://$c.r2.cloudflarestorage.com
acl = private
" > ~/.config/rclone/rclone.conf

pip install hf_transfer 'huggingface_hub[hf_transfer]'
export HF_HUB_ENABLE_HF_TRANSFER=1
huggingface-cli login --token $d

started=$SECONDS
for digit in $(seq 0 4); do
huggingface-cli download --max-workers=1 --include="*${digit}-of-*.safetensors" --local-dir ~/dsv3 deepseek-ai/DeepSeek-V3 | cat &
rclone copy --include "*$((digit + 5))-of-*" --transfers=1 --multi-thread-streams=32 r2:hfbuk/hff/deepseek-v3-base/ ~/dsv3/  --size-only --ignore-errors --ignore-checksum --log-level=INFO | cat &
sleep 3
done

wait

# remaining files besides .safetensors
rclone copy  --transfers=32 r2:hfbuk/hff/deepseek-v3-base/ ~/dsv3/  --size-only --ignore-errors --ignore-checksum --log-level=INFO | cat &

elapsed=$((SECONDS - started))
echo "took $elapsed seconds"