Skip to content

Instantly share code, notes, and snippets.

@marcoslot
Last active April 1, 2025 07:59
Show Gist options
  • Save marcoslot/f97454d10d4619440cbc8fb2c50f68b0 to your computer and use it in GitHub Desktop.
Save marcoslot/f97454d10d4619440cbc8fb2c50f68b0 to your computer and use it in GitHub Desktop.
ESRI pagination in Crunchy Data Warehouse
select load_esri('esri', 'https://map.oshawa.ca/arcgis/rest/services/Operational/OpenData/MapServer/1/query?where=1%3D1&orderByFields=OBJECTID&OutFields=*&f=pjson');
CREATE OR REPLACE FUNCTION load_esri(table_name text, url text)
RETURNS int
LANGUAGE plpgsql
AS $function$
DECLARE
max_result_size bigint := 1000;
result_offset bigint := 0;
rows_inserted bigint;
command text;
BEGIN
command := format($$create table %I () with (definition_from = %L, format = 'gdal')$$, table_name, url);
raise notice '%', command;
execute command;
loop
command := format($$copy %I from %L with (format 'gdal')$$, table_name, url || '&resultRecordCount='|| max_result_size ||'&resultOffset=' || result_offset);
raise notice '%', command;
execute command;
get diagnostics rows_inserted = ROW_COUNT;
if rows_inserted < max_result_size then
exit;
end if;
result_offset := result_offset + max_result_size;
end loop;
END;
$function$;
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment