Upload and download from an nginx fileserver

dl.ps1

#!/bin/pwsh
#Downloads matching items from the given url to the current directory
. $PSScriptRoot/l.ps1

$auth   = Get-Credential
$from   = 'https://some-web.site:1337/directory/root/'
$filter = (
	'things we',
	'might*want',
	'to',
	'pull$'
) -join '|'

$down = { Param($cd, $auth, $job)
	$params['Uri']        = $job.url
	$params['Credential'] = $auth

	Set-Location -LiteralPath $cd
	if ($job.url[-1] -ne '/') {
		Set-Location -LiteralPath $job.dir
		Invoke-WebRequest -OutFile $job.name @params | Out-Null
		return
	}

	$dir = $job.dir
	if ($dir) {
		New-Item -Type Directory -Path $dir
	}
	else {
		$dir = './'
	}
	
	#simply `autoindex_format json` in nginx config, edit if you need custom parsing
	Invoke-WebRequest @params `
	| ConvertFrom-Json `
	| %{ $_ } `
	| %{
		$name = $_.Name
		$url  = $job.url + [System.Uri]::EscapeDataString($name)
		switch ($_.Type) {
			'directory' { [PSCustomObject]@{url="$url/";dir="$dir$name/"} }
			'file'      { [PSCustomObject]@{url=$url;dir=$dir;name=$name} }
			Default     { if ($job.dir) { throw $_ } }
		}
	}
}

Start-Job `
	-ScriptBlock          $down `
	-InitializationScript $load `
	-ArgumentList         '.',$auth,([PSCustomObject]@{url=$from;dir=''}) `
| Receive-Job -Wait -AutoRemoveJob `
| ?{
	$name = $_.name
	if (!$name) {
		$name = $_.dir -replace '^.*/(?=[^/]+($|/))',''
	}
	$name -match $filter
} `
| Batch
Load 5 'down' $auth $down

l.ps1

#!/bin/pwsh
#Common functions handling the setup and parallel running
$ErrorActionPreference = 'Stop'

$wait = Get-Job
if ($wait.Count) {
	'Waiting for pre-existing jobs to complete'
	$wait | Receive-Job -Wait -AutoRemoveJob
}

if ($PSVersionTable.PSVersion.Major -gt 5) {
	$PSStyle.Progress.View = 'Classic'
}

$load = {
	$ErrorActionPreference = 'Stop'
	$params = @{}
	if ($PSVersionTable.PSVersion.Major -gt 5) {
		$params['SkipCertificateCheck'] = $True
	}
	elseif (![System.Net.ServicePointManager]::ServerCertificateValidationCallback) {
		class ServerCertificateValidationCallback {
			static [bool] Ignore(
				[System.Object]                                                 $obj, 
				[System.Security.Cryptography.X509Certificates.X509Certificate] $certificate, 
				[System.Security.Cryptography.X509Certificates.X509Chain]       $chain, 
				[System.Net.Security.SslPolicyErrors]                           $errors
			) { return $True }
		}
		[System.Net.ServicePointManager]::ServerCertificateValidationCallback += [System.Delegate]::CreateDelegate(
			[System.Net.Security.RemoteCertificateValidationCallback],
			[ServerCertificateValidationCallback],
			'Ignore'
		)
	}
}

Function Done { Param($colour)
	Begin {
		$path = [System.Collections.Queue]::new()
	}
	Process {
		if ($_) {
			$path.Enqueue($_)
		}
	}
	End {
		while ($path.Count -gt 1) {
			$path.Dequeue() | Write-Host -NoNewline
			'/' | Write-Host -NoNewline -ForegroundColor DarkYellow
		}
		$path | Write-Host -ForegroundColor $colour
	}
}

$item = 0
Function Run { Param($load)
	$job = $Input | %{ $_ }
	$count = $job.Count
	if (!$count) {
		Write-Progress -Id 1 -Activity 'Processing'
		return
	}
	
	$name = ''
	$Script:item = ($item + 1) % $count
	$job  = $job[$item]
	$name = $running[$job.Id]
	if ($name.name) {
		$name = $name.dir + $name.name
	}
	elseif ($name.dir) {
		$name = $name.dir
	}
	else {
		$name = 'Initial'
	}
	
	if (($job = $job.ChildJobs.Progress).Count) {
		Write-Progress `
			-Id               1 `
			-Activity         $name `
			-Status           $job[-1].StatusDescription `
			-PercentComplete  $job[-1].PercentComplete `
			-CurrentOperation "$($count - 1) other files $($load)loading currently"
	}
	else {
		Write-Progress `
			-Id       1 `
			-Activity $name `
			-CurrentOperation "$count concurrent $($load)loads"
	}
}

$running = @{}
Filter Complete {
	$job = $running[$_.Id]
	$running.Remove($_.Id)

	if ($job.name) {
		($job.dir,$job.name -join '/') -split '/' | Done 'DarkGreen'
	}
	else {
		$job.dir -split '/' | Done 'DarkYellow'
	}
	
	$_ `
	| Receive-Job -Wait -AutoRemoveJob `
	| %{
		if ($_ -is [String]) {
			$_ | Out-Host
		}
		else {
			$_
		}
	} `
	| Batch
}

Filter Fail {
	$running[$_.Id] `
	| %{
		$_.dir -split '/'
		$_.name
	} `
	| Done 'Red'
	$running.Remove($_.Id)
	try {
		$_ | Receive-Job -Wait -AutoRemoveJob
	}
	catch {
		$_ | Out-Host
	}
}

$queue = [System.Collections.Stack]::new()
Function Batch {
	$Input `
	| ?{ $_.url } `
	| Sort-Object -Property (
		@{Expression={!$_.Name}; Descending=$True},
		@{Expression='url'; Descending=$True}
	) `
	| %{ $queue.Push($_) }
}

Function Load { Param([int]$parallel, [string]$name, [Object[]]$extra, [ScriptBlock]$callback)
	$params = &{
		$PWD.ProviderPath
		$extra
		$NULL
	}
	while ($running.Count -or $queue.Count) {
		while (
			($running.Count -ge $parallel) -or
			($running.Count -and !$queue.Count)
		) {
			Get-Job -State Running   | Run $name
			Get-Job -State Completed | Complete
			Get-Job -State Failed    | Fail
			Start-Sleep 1
		}
		
		while (
			$queue.Count -and
			$running.Count -lt $parallel
		) {
			$job = $queue.Pop()
			$params[-1] = $job
			$running[(Start-Job `
				-InitializationScript $load `
				-ScriptBlock          $callback `
				-ArgumentList         $params `
			).Id] = $job
		}
	}
}

ul.ps1

#!/bin/pwsh
#Uploads the current directory 
. $PSScriptRoot/l.ps1

$auth = Get-Credential
$to   = 'https://some-web.site:1337/'

$up = { Param($cd, $auth, $job)
	$params['Uri']        = $job.url
	$params['Credential'] = $auth
	$params['Method']     = 'Post'

	Set-Location -LiteralPath $cd
	if ($job.url[-1] -ne '/') {
		Set-Location -LiteralPath $job.dir
		#my upload script that accepts POST data in nginx returns with the
		# -url you can access your file
		# -hash of the received body stream
		# -size of the received body stream
		# -a completed message
		#and requires no headers just the url to save at and the file data via POST
		$url,$hash,$size,$msg = (Invoke-WebRequest -InFile $job.name @params) -split "`n"
		return "`t$hash`t$(($size / 1MB).ToString() -replace '(?<=\.\d{2}).+$','')mB"
	}
	
	Invoke-WebRequest @params | Out-Null
	Get-ChildItem -LiteralPath $job.dir `
	| %{
		$name = $_.Name
		$url  = $job.url + [System.Uri]::EscapeDataString($name)
		switch ($_.GetType().Name) {
			'DirectoryInfo' { [PSCustomObject]@{url="$url/";dir=$job.dir + "$name/"} }
			'FileInfo'      { [PSCustomObject]@{url=$url;dir=$job.dir;name=$name} }
			Default         { throw $_ }
		}
	}
}

#check if where we want to save our data already has an item
#NB this script is already aware that saved uploads become
#available at the '/files/_upload' prefix location for my usecase
$name = [System.Uri]::EscapeDataString((Get-Item '.').Name) + '/'
Start-Job `
	-InitializationScript $load `
	-ArgumentList         $auth,$to,$name `
	-ScriptBlock          { Param($auth, $to, $name)
		try {
			$url = $to + 'files/_upload/' + $name
			Invoke-WebRequest -Uri $url -Credential $auth @params | Out-Null
			throw "Already exists: $url"
		}
		catch {
			if ($_.Exception.Response.StatusCode -ne 404) {
				throw $_.Exception
			}
		}
		Invoke-WebRequest `
			-Uri        "$to$name" `
			-Credential $auth `
			-Method     Post `
			@params
	} `
| Receive-Job -Wait -AutoRemoveJob `
| Out-Null

([PSCustomObject]@{url="$to$name"; dir='./'}) `
| Batch
Load 5 'up' $auth $up