Skip to content

Instantly share code, notes, and snippets.

@Mazyod

Mazyod/minio-transfer.html

Last active October 4, 2025 06:33
Show Gist options
  • Save Mazyod/cdf03e6155b677e363ad2aa0e25811d3 to your computer and use it in GitHub Desktop.
Save Mazyod/cdf03e6155b677e363ad2aa0e25811d3 to your computer and use it in GitHub Desktop.
Download ZIP
MinIO object transfer in a single, self-contained HTML page. Load objects from one server, pick the objects to transfer, and push them to the other side.
Raw
minio-transfer.html
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>Transfer Console — SRC ➜ DST (S3→S3, No Auth)</title>
<style>
:root { --fg:#0f172a; --muted:#475569; --bg:#f8fafc; --card:#ffffff; --ok:#16a34a; --warn:#f59e0b; --err:#dc2626; --accent:#2563eb; }
* { box-sizing: border-box; }
body { margin:0; font-family: ui-sans-serif, system-ui, -apple-system, Segoe UI, Roboto, Helvetica, Arial, "Apple Color Emoji", "Segoe UI Emoji"; color:var(--fg); background:var(--bg); }
header { padding:16px 20px; background:var(--card); border-bottom:1px solid #e5e7eb; position:sticky; top:0; z-index:10; }
h1 { font-size:18px; margin:0 0 8px; }
main { display:grid; grid-template-columns: 360px 1fr; gap:16px; padding:16px; }
section, .panel { background:var(--card); border:1px solid #e5e7eb; border-radius:12px; padding:12px; }
.grid { display:grid; gap:8px; }
label { font-size:12px; color:var(--muted); }
input[type="text"], input[type="number"] { width:100%; padding:8px; border:1px solid #cbd5e1; border-radius:8px; background:#fff; }
button { appearance:none; border:1px solid #cbd5e1; background:#fff; padding:8px 12px; border-radius:8px; cursor:pointer; }
button.primary { background:var(--accent); border-color:var(--accent); color:#fff; }
button.ghost { background:transparent; }
button:disabled { opacity:0.5; cursor:not-allowed; }
.row { display:flex; gap:8px; align-items:center; }
.col { display:flex; flex-direction:column; gap:8px; }
.muted { color:var(--muted); }
.ok { color:var(--ok); }
.warn { color:var(--warn); }
.err { color:var(--err); }
table { width:100%; border-collapse:collapse; }
th, td { padding:8px; border-bottom:1px solid #f1f5f9; text-align:left; font-size:13px; }
thead th { position:sticky; top:0; background:var(--card); z-index:1; }
.scroll { max-height:420px; overflow:auto; }
.progress { height:8px; background:#e5e7eb; border-radius:999px; overflow:hidden; }
.progress > span { display:block; height:100%; background:linear-gradient(90deg, var(--accent), #60a5fa); width:0%; }
code { background:#f1f5f9; padding:2px 6px; border-radius:6px; }
.tag { background:#eef2ff; color:#3730a3; border:1px solid #c7d2fe; padding:2px 6px; border-radius:999px; font-size:12px; }
footer { padding:16px; text-align:center; color:var(--muted); }
</style>
<style id="layout-fixes">
/* Layout fixes: keep columns separate; no overlay between panes */
main { align-items: start; }
main > .col { position: relative; min-width: 0; }
section, .panel { position: relative; }
</style>
</head>
<body>
<header>
<h1>Transfer Console — SRC ➜ DST (S3→S3, No Auth)</h1>
<div class="muted">Copy large objects by Range GET from SRC and Multipart Upload to DST. Resumable, browser‑only.</div>
</header>
<main>
<!-- LEFT: Configuration & SRC browser -->
<div class="col" style="gap:16px;">
<section class="grid">
<div class="row" style="justify-content:space-between; align-items:flex-end;">
<div class="col" style="flex:1;">
<label>SRC Endpoint (S3 API)</label>
<input id="srcEndpoint" type="text" placeholder="https://src-minio.internal.example" />
</div>
</div>
<div class="row">
<div class="col" style="flex:1;">
<label>SRC Bucket</label>
<input id="srcBucket" type="text" placeholder="open-llms" />
</div>
<div class="col" style="flex:1;">
<label>SRC Prefix (optional)</label>
<input id="srcPrefix" type="text" placeholder="models/" />
</div>
</div>
<div class="row" style="justify-content:space-between; align-items:flex-end;">
<div class="col" style="flex:1;">
<label>DST Endpoint (S3 API)</label>
<input id="dstEndpoint" type="text" placeholder="https://dst-minio.internal.example" />
</div>
</div>
<div class="row">
<div class="col" style="flex:1;">
<label>DST Bucket</label>
<input id="dstBucket" type="text" placeholder="ingest-quarantine" />
</div>
<div class="col" style="flex:1;">
<label>DST Prefix</label>
<input id="dstPrefix" type="text" placeholder="from-src/" />
</div>
</div>
<div class="row">
<div class="col" style="flex:1;">
<label>Part Size (MiB)</label>
<input id="partSizeMiB" type="number" min="5" step="1" value="128" />
</div>
<div class="col" style="flex:1;">
<label>Concurrency (parts/file)</label>
<input id="concurrency" type="number" min="1" max="8" step="1" value="3" />
</div>
</div>
<div class="row" style="gap:12px;">
<button id="applyCfg" class="primary">Apply Config</button>
<span class="muted">Or edit the constants in the code block below (// 🔧 SET ME)</span>
</div>
</section>
</div>
<!-- RIGHT: Jobs -->
<div class="col" style="gap:16px;">
<section>
<div class="row" style="justify-content:space-between; align-items:center; margin-bottom:8px;">
<div class="row" style="gap:8px; align-items:center;">
<strong>SRC Objects</strong>
<span class="tag" id="prefixTag"></span>
</div>
<div class="row" style="gap:8px;">
<input id="filterPrefix" type="text" placeholder="prefix filter (optional)" style="width:220px;" />
<button id="btnList" class="primary">List</button>
<button id="btnMore">Load more</button>
<button id="btnQueue" disabled>Queue Selected ➜</button>
</div>
</div>
<div class="scroll">
<table>
<thead>
<tr><th style="width:28px;"><input type="checkbox" id="chkAll"></th><th>Key</th><th style="width:140px;">Size</th></tr>
</thead>
<tbody id="srcTbody"></tbody>
</table>
</div>
</section>
<section class="row" style="justify-content:space-between; align-items:center;">
<strong>Job Queue</strong>
<div class="row" style="gap:8px;">
<button id="pauseAll">Pause All</button>
<button id="resumeAll" class="primary">Resume All</button>
<button id="clearDone">Clear Done</button>
</div>
</section>
<section class="scroll">
<table>
<thead>
<tr>
<th>File</th>
<th style="width:180px;">Progress</th>
<th style="width:160px;">Status</th>
<th style="width:200px;">Actions</th>
</tr>
</thead>
<tbody id="jobsTbody"></tbody>
</table>
</section>
<section class="panel">
<div id="log" style="font-family:ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, monospace; font-size:12px; white-space:pre-wrap; line-height:1.4; max-height:160px; overflow:auto; background:#0b1020; color:#d1e7ff; padding:8px; border-radius:8px;">
</div>
</section>
</div>
</main>
<footer>“Many a little makes a mickle.” — Browser copies parts; MinIO stitches them.</footer>
<script type="module">
/***************************
* 🔧 SET ME — DEFAULTS
* You can also edit these at runtime using the form and "Apply Config".
***************************/
const CONFIG = {
SRC_ENDPOINT: 'https://src-minio.internal.example', // e.g., https://src-minio.internal.example
SRC_BUCKET: 'open-llms', // e.g., open-llms
SRC_PREFIX: '', // optional list prefix
DST_ENDPOINT:'https://dst-minio.internal.example',// e.g., https://dst-minio.internal.example
DST_BUCKET: 'ingest-quarantine', // e.g., ingest-quarantine
DST_PREFIX: 'from-src/', // e.g., from-src/
DEFAULT_PART_SIZE_MIB: 128, // ≥ 5 MiB; auto-raised to meet 10k part cap
DEFAULT_CONCURRENCY: 3, // parts in flight per file (keep modest for RAM)
MAX_ACTIVE_JOBS: 1, // process one file at a time for simplicity
};
/***************************
* Helpers
***************************/
const MiB = 1024*1024;
const $ = (id)=>document.getElementById(id);
const fmtBytes = (n)=>{
const u=["B","KB","MB","GB","TB","PB"]; let i=0; let x=n;
while(x>=1024 && i<u.length-1){x/=1024;i++;}
return `${x.toFixed(x<10&&i>0?2:0)} ${u[i]}`;
};
const encodeS3Key = (k)=>k.split('/').map(encodeURIComponent).join('/');
const s3url = (endpoint,bucket,key)=>`${endpoint.replace(/\/$/,'')}/${encodeURIComponent(bucket)}/${encodeS3Key(key)}`;
const xmlText = (xml, tag)=> xml.getElementsByTagName(tag)[0]?.textContent ?? '';
const sleep = (ms)=>new Promise(r=>setTimeout(r,ms));
function pickPartSize(totalBytes, desiredMiB){
const minPart = 5*MiB;
let part = Math.max(minPart, desiredMiB*MiB);
const maxParts = 10000;
if (Math.ceil(totalBytes/part) > maxParts){
part = Math.ceil(totalBytes / maxParts);
// round up to nearest 5 MiB
part = Math.ceil(part / (5*MiB)) * (5*MiB);
}
return part;
}
/***************************
* S3 (anonymous) operations
***************************/
async function s3ListV2({endpoint,bucket,prefix,maxKeys=1000,continuationToken=null}){
const params = new URLSearchParams({ 'list-type':'2', 'max-keys': String(maxKeys) });
if(prefix) params.set('prefix', prefix);
if(continuationToken) params.set('continuation-token', continuationToken);
const url = `${endpoint.replace(/\/$/,'')}/${encodeURIComponent(bucket)}?${params.toString()}`;
const res = await fetch(url, { method:'GET' });
if(!res.ok) throw new Error(`ListObjectsV2 failed: ${res.status}`);
const text = await res.text();
const xml = new DOMParser().parseFromString(text, 'application/xml');
const isTrunc = xmlText(xml,'IsTruncated') === 'true';
const token = xmlText(xml,'NextContinuationToken') || null;
const items = [...xml.getElementsByTagName('Contents')].map(n=>({
key: n.getElementsByTagName('Key')[0].textContent,
size: Number(n.getElementsByTagName('Size')[0].textContent)
}));
return {items, isTrunc, token};
}
async function s3Head({endpoint,bucket,key}){
const res = await fetch(s3url(endpoint,bucket,key), { method:'HEAD' });
if(!res.ok) throw new Error(`HEAD failed: ${res.status}`);
const len = Number(res.headers.get('Content-Length'));
return { size:len };
}
async function s3RangeBlob({endpoint,bucket,key,start,end}){
const url = s3url(endpoint,bucket,key);
const res = await fetch(url, { method:'GET', headers: { 'Range': `bytes=${start}-${end}` } });
if(res.status!==206 && res.status!==200) throw new Error(`Range GET failed: ${res.status}`);
// Use Blob to keep memory manageable; browser sets Content-Length on PUT
const buf = await res.arrayBuffer();
return new Blob([buf]);
}
async function s3InitMultipart({endpoint,bucket,key}){
const url = `${s3url(endpoint,bucket,key)}?uploads`;
const res = await fetch(url, { method:'POST' });
if(!res.ok) throw new Error(`Init multipart failed: ${res.status}`);
const xml = new DOMParser().parseFromString(await res.text(), 'application/xml');
const uploadId = xmlText(xml,'UploadId');
if(!uploadId) throw new Error('No UploadId in response');
return uploadId;
}
async function s3UploadPart({endpoint,bucket,key,uploadId,partNumber,body}){
const url = `${s3url(endpoint,bucket,key)}?partNumber=${partNumber}&uploadId=${encodeURIComponent(uploadId)}`;
const res = await fetch(url, { method:'PUT', body });
if(!res.ok) throw new Error(`UploadPart #${partNumber} failed: ${res.status}`);
const etag = res.headers.get('ETag');
if(!etag) console.warn('No ETag header returned (MinIO may still accept).');
return etag;
}
async function s3ListParts({endpoint,bucket,key,uploadId}){
const url = `${s3url(endpoint,bucket,key)}?uploadId=${encodeURIComponent(uploadId)}`;
const res = await fetch(url, { method:'GET' });
if(!res.ok) throw new Error(`ListParts failed: ${res.status}`);
const xml = new DOMParser().parseFromString(await res.text(), 'application/xml');
const parts = [...xml.getElementsByTagName('Part')].map(p=>({
partNumber: Number(p.getElementsByTagName('PartNumber')[0].textContent),
etag: p.getElementsByTagName('ETag')[0].textContent
}));
return parts;
}
async function s3CompleteMultipart({endpoint,bucket,key,uploadId,parts}){
const doc = [`<CompleteMultipartUpload>`];
parts.sort((a,b)=>a.partNumber-b.partNumber).forEach(p=>{
doc.push(`<Part><PartNumber>${p.partNumber}</PartNumber><ETag>${p.etag}</ETag></Part>`);
});
doc.push(`</CompleteMultipartUpload>`);
const body = new Blob([doc.join('')], { type:'application/xml' });
const url = `${s3url(endpoint,bucket,key)}?uploadId=${encodeURIComponent(uploadId)}`;
const res = await fetch(url, { method:'POST', body, headers: { 'Content-Type':'application/xml' } });
if(!res.ok) throw new Error(`CompleteMultipart failed: ${res.status}`);
return true;
}
async function s3AbortMultipart({endpoint,bucket,key,uploadId}){
const url = `${s3url(endpoint,bucket,key)}?uploadId=${encodeURIComponent(uploadId)}`;
const res = await fetch(url, { method:'DELETE' });
if(!res.ok) throw new Error(`AbortMultipart failed: ${res.status}`);
return true;
}
/***************************
* Persistence (localStorage)
***************************/
const LS_KEY = 'transferConsoleJobs.v1';
function loadJobsState(){ try{ return JSON.parse(localStorage.getItem(LS_KEY)||'{}'); }catch{ return {}; } }
function saveJobsState(obj){ localStorage.setItem(LS_KEY, JSON.stringify(obj)); }
/***************************
* Job model
***************************/
class Job {
constructor({srcKey, size, destKey}){
this.srcKey = srcKey;
this.destKey = destKey;
this.size = size;
this.status = 'pending'; // pending|running|paused|done|error|canceled
this.uploadId = null;
this.partSize = pickPartSize(size, STATE.concurrencyMiB);
this.completed = {}; // partNumber -> ETag
this.totalParts = Math.ceil(size / this.partSize);
this.nextPartHint = 1;
this.retries = {}; // partNumber -> tries
}
toState(){
return {
srcKey:this.srcKey, destKey:this.destKey, size:this.size,
status:this.status, uploadId:this.uploadId, partSize:this.partSize,
completed:this.completed, totalParts:this.totalParts,
nextPartHint:this.nextPartHint
};
}
static fromState(s){
const j = new Job({srcKey:s.srcKey, size:s.size, destKey:s.destKey});
j.status = s.status; j.uploadId = s.uploadId; j.partSize = s.partSize;
j.completed = s.completed||{}; j.totalParts = s.totalParts; j.nextPartHint = s.nextPartHint||1;
return j;
}
}
/***************************
* Global state & UI wiring
***************************/
const STATE = {
cfg:{...CONFIG},
srcList:[],
srcToken:null,
selected:new Set(),
jobs:[], // Job[]
running:false,
concurrency: CONFIG.DEFAULT_CONCURRENCY,
concurrencyMiB: CONFIG.DEFAULT_PART_SIZE_MIB,
};
function logln(msg){ const el=$('log'); el.textContent += `\n${new Date().toLocaleTimeString()} ${msg}`; el.scrollTop = el.scrollHeight; }
function hydrateForm(){
$('srcEndpoint').value = STATE.cfg.SRC_ENDPOINT;
$('srcBucket').value = STATE.cfg.SRC_BUCKET;
$('srcPrefix').value = STATE.cfg.SRC_PREFIX;
$('dstEndpoint').value= STATE.cfg.DST_ENDPOINT;
$('dstBucket').value = STATE.cfg.DST_BUCKET;
$('dstPrefix').value = STATE.cfg.DST_PREFIX;
$('partSizeMiB').value = STATE.concurrencyMiB;
$('concurrency').value = STATE.concurrency;
$('prefixTag').textContent = STATE.cfg.SRC_PREFIX || '(no prefix)';
}
function applyCfg(){
STATE.cfg.SRC_ENDPOINT = $('srcEndpoint').value.trim();
STATE.cfg.SRC_BUCKET = $('srcBucket').value.trim();
STATE.cfg.SRC_PREFIX = $('srcPrefix').value.trim();
STATE.cfg.DST_ENDPOINT = $('dstEndpoint').value.trim();
STATE.cfg.DST_BUCKET = $('dstBucket').value.trim();
STATE.cfg.DST_PREFIX = $('dstPrefix').value.trim();
STATE.concurrencyMiB = Math.max(5, Number($('partSizeMiB').value));
STATE.concurrency = Math.max(1, Number($('concurrency').value));
$('prefixTag').textContent = STATE.cfg.SRC_PREFIX || '(no prefix)';
logln('Applied configuration.');
}
/***************************
* SRC listing UI
***************************/
function renderSRC(){
const tbody=$('srcTbody'); tbody.innerHTML='';
STATE.srcList.forEach((it,idx)=>{
const tr=document.createElement('tr');
const td0=document.createElement('td'); const td1=document.createElement('td'); const td2=document.createElement('td');
const cb=document.createElement('input'); cb.type='checkbox'; cb.checked=STATE.selected.has(it.key);
cb.addEventListener('change',()=>{ if(cb.checked) STATE.selected.add(it.key); else STATE.selected.delete(it.key); updateQueueButton(); });
td0.appendChild(cb);
td1.textContent=it.key;
td2.textContent=fmtBytes(it.size);
tr.append(td0,td1,td2); tbody.appendChild(tr);
});
$('chkAll').checked = STATE.srcList.length>0 && STATE.srcList.every(it=>STATE.selected.has(it.key));
}
function updateQueueButton(){
$('btnQueue').disabled = STATE.selected.size===0;
}
$('chkAll').addEventListener('change', (e)=>{
if(e.target.checked){ STATE.srcList.forEach(it=>STATE.selected.add(it.key)); }
else { STATE.selected.clear(); }
renderSRC(); updateQueueButton();
});
$('btnList').addEventListener('click', async ()=>{
STATE.selected.clear(); updateQueueButton();
const prefix = $('filterPrefix').value.trim() || STATE.cfg.SRC_PREFIX;
logln(`Listing SRC: bucket=${STATE.cfg.SRC_BUCKET}, prefix="${prefix}"...`);
try{
const {items,isTrunc,token} = await s3ListV2({endpoint:STATE.cfg.SRC_ENDPOINT, bucket:STATE.cfg.SRC_BUCKET, prefix});
STATE.srcList = items; STATE.srcToken = isTrunc? token : null;
renderSRC();
logln(`Listed ${items.length} objects${isTrunc? ' (truncated, use Load more)':''}.`);
}catch(err){ logln(`List error: ${err.message}`); }
});
$('btnMore').addEventListener('click', async ()=>{
if(!STATE.srcToken) { logln('No more pages.'); return; }
const prefix = $('filterPrefix').value.trim() || STATE.cfg.SRC_PREFIX;
try{
const {items,isTrunc,token} = await s3ListV2({endpoint:STATE.cfg.SRC_ENDPOINT, bucket:STATE.cfg.SRC_BUCKET, prefix, continuationToken:STATE.srcToken});
STATE.srcList.push(...items); STATE.srcToken = isTrunc? token : null;
renderSRC();
logln(`Loaded +${items.length} objects.`);
}catch(err){ logln(`List error: ${err.message}`); }
});
$('btnQueue').addEventListener('click', ()=>{
const destPrefix = STATE.cfg.DST_PREFIX || '';
const selectedKeys = [...STATE.selected.values()];
selectedKeys.forEach(k=>{
const item = STATE.srcList.find(it=>it.key===k); if(!item) return;
const destKey = `${destPrefix}${k}`.replace(/\/+/, '/');
const job = new Job({srcKey:k, size:item.size, destKey});
STATE.jobs.push(job);
});
STATE.selected.clear(); renderSRC(); updateQueueButton();
persistJobs();
renderJobs();
maybeRunScheduler();
});
/***************************
* Jobs UI & scheduler
***************************/
function persistJobs(){
const obj = { cfg:STATE.cfg, jobs: STATE.jobs.map(j=>j.toState()) };
saveJobsState(obj);
}
function loadPersisted(){
const obj = loadJobsState(); if(!obj.jobs) return;
STATE.cfg = {...STATE.cfg, ...(obj.cfg||{})};
STATE.jobs = obj.jobs.map(Job.fromState);
}
function renderJobs(){
const tbody = $('jobsTbody'); tbody.innerHTML='';
STATE.jobs.forEach((j,idx)=>{
const tr=document.createElement('tr');
const td0=document.createElement('td');
td0.innerHTML = `<div style="display:flex;flex-direction:column;gap:4px;"><div>${j.srcKey}</div><div class="muted" style="font-size:12px;">${fmtBytes(j.size)} ➜ <code>${j.destKey}</code></div></div>`;
const td1=document.createElement('td');
const pct = Math.round((Object.keys(j.completed).length / j.totalParts) * 100);
td1.innerHTML = `<div class="progress"><span style="width:${pct}%"></span></div><div class="muted" style="font-size:12px;margin-top:4px;">${Object.keys(j.completed).length}/${j.totalParts} parts</div>`;
const td2=document.createElement('td');
td2.innerHTML = `<span class="tag">${j.status}</span>`;
const td3=document.createElement('td');
const row=document.createElement('div'); row.className='row';
const b1=document.createElement('button'); b1.textContent='Pause'; b1.onclick=()=>pauseJob(j);
const b2=document.createElement('button'); b2.textContent='Resume'; b2.className='primary'; b2.onclick=()=>resumeJob(j);
const b3=document.createElement('button'); b3.textContent='Cancel'; b3.className='ghost'; b3.onclick=()=>cancelJob(j);
row.append(b1,b2,b3); td3.appendChild(row);
tr.append(td0,td1,td2,td3); tbody.appendChild(tr);
});
}
$('pauseAll').addEventListener('click', ()=>{ STATE.jobs.forEach(j=>{ if(j.status==='running') j.status='paused'; }); persistJobs(); renderJobs(); });
$('resumeAll').addEventListener('click', ()=>{ STATE.jobs.forEach(j=>{ if(j.status==='paused') j.status='pending'; }); persistJobs(); renderJobs(); maybeRunScheduler(); });
$('clearDone').addEventListener('click', ()=>{ STATE.jobs = STATE.jobs.filter(j=>j.status!=='done' && j.status!=='canceled'); persistJobs(); renderJobs(); });
async function pauseJob(j){ if(j.status==='running'){ j.status='paused'; logln(`Paused: ${j.srcKey}`); persistJobs(); renderJobs(); } }
async function resumeJob(j){ if(j.status==='paused' || j.status==='error' || j.status==='pending'){ j.status='pending'; logln(`Queued: ${j.srcKey}`); persistJobs(); renderJobs(); maybeRunScheduler(); } }
async function cancelJob(j){
if(j.uploadId){
try{ await s3AbortMultipart({ endpoint:STATE.cfg.DST_ENDPOINT, bucket:STATE.cfg.DST_BUCKET, key:j.destKey, uploadId:j.uploadId }); logln(`Aborted uploadId=${j.uploadId}`); }
catch(e){ logln(`Abort failed (ignored): ${e.message}`); }
}
j.status='canceled'; persistJobs(); renderJobs();
}
function maybeRunScheduler(){
if(STATE.running) return;
const active = STATE.jobs.filter(j=>j.status==='running').length;
const slots = STATE.cfg.MAX_ACTIVE_JOBS - active;
if(slots<=0) return;
const next = STATE.jobs.find(j=>j.status==='pending');
if(next){ runJob(next); }
}
async function runJob(j){
STATE.running = true; j.status='running'; persistJobs(); renderJobs();
logln(`▶ Start: ${j.srcKey} (${fmtBytes(j.size)}), partSize=${fmtBytes(j.partSize)}, totalParts=${j.totalParts}`);
// ensure uploadId
if(!j.uploadId){
try{ j.uploadId = await s3InitMultipart({ endpoint:STATE.cfg.DST_ENDPOINT, bucket:STATE.cfg.DST_BUCKET, key:j.destKey }); }
catch(e){ j.status='error'; logln(`Init failed: ${e.message}`); persistJobs(); renderJobs(); STATE.running=false; maybeRunScheduler(); return; }
}
// reconcile already uploaded parts
try{
const remote = await s3ListParts({ endpoint:STATE.cfg.DST_ENDPOINT, bucket:STATE.cfg.DST_BUCKET, key:j.destKey, uploadId:j.uploadId });
remote.forEach(p=>{ j.completed[p.partNumber] = p.etag; });
}catch(e){ logln(`ListParts warn: ${e.message}`); }
// work queue of remaining part numbers
const remaining = [];
for(let n=1;n<=j.totalParts;n++) if(!j.completed[n]) remaining.push(n);
let inFlight = 0; let idx=0; let stop=false;
const pump = async ()=>{
while(!stop && idx < remaining.length){
if(j.status!== 'running') { await sleep(200); continue; }
if(inFlight >= STATE.concurrency){ await sleep(50); continue; }
const partNumber = remaining[idx++]; inFlight++;
uploadOnePart(j, partNumber).then(()=>{
inFlight--; persistJobs(); renderJobs();
}).catch(err=>{
inFlight--; j.status='error'; logln(`Part ${partNumber} error: ${err.message}`);
});
}
// wait for all inflight
while(inFlight>0) await sleep(50);
};
await pump();
if(j.status==='running'){
// complete
try{
const parts = Object.keys(j.completed).map(n=>({ partNumber:Number(n), etag:j.completed[n] }));
if(parts.length!==j.totalParts) throw new Error(`Missing parts: have ${parts.length} of ${j.totalParts}`);
await s3CompleteMultipart({ endpoint:STATE.cfg.DST_ENDPOINT, bucket:STATE.cfg.DST_BUCKET, key:j.destKey, uploadId:j.uploadId, parts });
j.status='done'; logln(`✅ Completed: ${j.destKey}`);
}catch(e){ j.status='error'; logln(`Complete failed: ${e.message}`); }
}
persistJobs(); renderJobs(); STATE.running=false; maybeRunScheduler();
}
async function uploadOnePart(j, partNumber){
// compute byte range
const start = (partNumber-1)*j.partSize;
const end = Math.min(j.size-1, start + j.partSize - 1);
const dest = `${STATE.cfg.DST_PREFIX||''}${j.srcKey}`.replace(/\/+/, '/'); // informational
logln(`↔ part ${partNumber}/${j.totalParts}: bytes ${start}-${end}`);
// backoff retries
let attempt=0, maxTry=5; let lastErr=null;
while(attempt<maxTry){
attempt++;
try{
const blob = await s3RangeBlob({ endpoint:STATE.cfg.SRC_ENDPOINT, bucket:STATE.cfg.SRC_BUCKET, key:j.srcKey, start, end });
const etag = await s3UploadPart({ endpoint:STATE.cfg.DST_ENDPOINT, bucket:STATE.cfg.DST_BUCKET, key:j.destKey, uploadId:j.uploadId, partNumber, body:blob });
if(etag) j.completed[partNumber]=etag; else j.completed[partNumber]='"unknown-etag"';
return;
}catch(e){ lastErr=e; logln(` retry part ${partNumber} #${attempt}: ${e.message}`); await sleep( 1000 * Math.pow(2, attempt-1) ); }
}
throw lastErr || new Error('Unknown upload error');
}
/***************************
* Startup
***************************/
loadPersisted();
hydrateForm();
applyCfg();
renderJobs();
$('applyCfg').addEventListener('click', ()=>{ applyCfg(); persistJobs(); });
</script>
</body>
</html>
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment