Last active
April 1, 2025 07:59
-
-
Save marcoslot/f97454d10d4619440cbc8fb2c50f68b0 to your computer and use it in GitHub Desktop.
ESRI pagination in Crunchy Data Warehouse
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
select load_esri('esri', 'https://map.oshawa.ca/arcgis/rest/services/Operational/OpenData/MapServer/1/query?where=1%3D1&orderByFields=OBJECTID&OutFields=*&f=pjson'); |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
CREATE OR REPLACE FUNCTION load_esri(table_name text, url text) | |
RETURNS int | |
LANGUAGE plpgsql | |
AS $function$ | |
DECLARE | |
max_result_size bigint := 1000; | |
result_offset bigint := 0; | |
rows_inserted bigint; | |
command text; | |
BEGIN | |
command := format($$create table %I () with (definition_from = %L, format = 'gdal')$$, table_name, url); | |
raise notice '%', command; | |
execute command; | |
loop | |
command := format($$copy %I from %L with (format 'gdal')$$, table_name, url || '&resultRecordCount='|| max_result_size ||'&resultOffset=' || result_offset); | |
raise notice '%', command; | |
execute command; | |
get diagnostics rows_inserted = ROW_COUNT; | |
if rows_inserted < max_result_size then | |
exit; | |
end if; | |
result_offset := result_offset + max_result_size; | |
end loop; | |
END; | |
$function$; |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment