Created
May 3, 2019 05:39
-
-
Save SP3269/e2952bd50500f8eb419f1a20b0b1b570 to your computer and use it in GitHub Desktop.
This PowerShell script is using Google BigQuery REST API to run a query and creates a result set using returned schema for field names and types
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
$project = "your-gcp-project" | |
$accesstoken = gcloud auth print-access-token # Lazy way. Can also get access token using service account and 2LO - ref. https://gist.github.com/SP3269/da43b00692de5b3f591b3068d76df577 | |
# Using Google BigQuery REST API per https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs/query | |
# Example query using public dataset available to anyone in BQ | |
$query = 'SELECT * FROM `bigquery-public-data.san_francisco_trees.street_trees` WHERE plant_date >= "1969-01-01 00:00:00 UTC" AND plant_date < "1970-01-01 00:00:00 UTC"' | |
$body = @{ | |
useLegacySql = "false" | |
query = $query | |
} | |
$request = @{ | |
Uri = "https://www.googleapis.com/bigquery/v2/projects/$project/queries" | |
Method = "POST" | |
Headers = @{Authorization = "Bearer $accesstoken"; "Content-Type" = "application/json"; Accept = "application/json"} | |
Body = $body | ConvertTo-Json | |
} | |
$res = Invoke-RestMethod @request -Verbose | |
# Column names in $res.schema.fields.name | |
# Row values are in $res.rows[0].f.v | |
# TODO: types. Types can be STRING, FLOAT - need conversion of value before Add-Member | |
$epoch = Get-Date "1970-01-01T00:00:00Z" | |
$fieldcount = $res.schema.fields.count | |
$columnnames = $res.schema.fields.name | |
$columntypes = $res.schema.fields.type | |
$rowcount = $res.rows.count | |
$data = for ($j=0; $j -lt $rowcount; $j++) { | |
$row = $res.rows[$j].f.v | |
$entry = New-Object PSCustomObject | |
for ($i=0; $i -lt $fieldcount; $i++) { | |
$fieldvalue = $row[$i] | |
if ($null -eq $fieldvalue) { | |
$entry | Add-Member -NotePropertyMembers @{$columnnames[$i]=$null} | |
} | |
else { | |
switch ($columntypes[$i]) { | |
"FLOAT" { $entry | Add-Member -NotePropertyMembers @{$columnnames[$i]=[float]$fieldvalue} } | |
"INTEGER" { $entry | Add-Member -NotePropertyMembers @{$columnnames[$i]=[int]$fieldvalue} } | |
"TIMESTAMP" { $entry | Add-Member -NotePropertyMembers @{$columnnames[$i]=$epoch.AddSeconds($fieldvalue)} } | |
Default { $entry | Add-Member -NotePropertyMembers @{$columnnames[$i]=$fieldvalue} } # String as the default | |
} | |
} | |
} | |
$entry | |
} | |
$data |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment