Get data for UK geometries (vector data) and lookups for fast geocoding from ONS, OS, Government Data sources

get_uk_open_data.sh

# get the middle output are shapefile (MSOA)
curl --header 'Host: borders.ukdataservice.ac.uk' --user-agent 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:133.0) Gecko/20100101 Firefox/133.0' --header 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' --header 'Accept-Language: en-US,en;q=0.5' --referer 'https://statistics.ukdataservice.ac.uk/' --header 'Upgrade-Insecure-Requests: 1' --header 'Sec-Fetch-Dest: document' --header 'Sec-Fetch-Mode: navigate' --header 'Sec-Fetch-Site: same-site' --header 'Sec-Fetch-User: ?1' 'https://borders.ukdataservice.ac.uk/ukborders/easy_download/prebuilt/shape/infuse_msoa_lyr_2011.zip' --output 'infuse_msoa_lyr_2011.zip'

# postcode to lsoa msoa
# https://geoportal.statistics.gov.uk/datasets/15469f4d3a484c37a41b441212667c22/about
curl --header 'Host: www.arcgis.com' --user-agent 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:133.0) Gecko/20100101 Firefox/133.0' --header 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' --header 'Accept-Language: en-US,en;q=0.5' --header 'Upgrade-Insecure-Requests: 1' --header 'Sec-Fetch-Dest: document' --header 'Sec-Fetch-Mode: navigate' --header 'Sec-Fetch-Site: cross-site' --header 'Sec-Fetch-User: ?1' 'https://www.arcgis.com/sharing/rest/content/items/15469f4d3a484c37a41b441212667c22/data' --output 'PCD_OA_LSOA_MSOA_LAD_MAY20_UK_LU.zip'

curl --header 'Host: www.arcgis.com' --user-agent 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:133.0) Gecko/20100101 Firefox/133.0' --header 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' --header 'Accept-Language: en-US,en;q=0.5' --header 'Upgrade-Insecure-Requests: 1' --header 'Sec-Fetch-Dest: document' --header 'Sec-Fetch-Mode: navigate' --header 'Sec-Fetch-Site: cross-site' --header 'Sec-Fetch-User: ?1' 'https://www.arcgis.com/sharing/rest/content/items/3770c5e8b0c24f1dbe6d2fc6b46a0b18/data' --output 'PCD_OA21_LSOA21_MSOA21_LAD_AUG23_UK_LU.zip'

# LSOA
# https://www.data.gov.uk/dataset/c481f2d3-91fc-4767-ae10-2efdf6d58996/lower-layer-super-output-areas-lsoas
# Get n-triples (<subject> <predicate> <object>)
curl --header 'Host: pmd3-production-grafter-dluhc.publishmydata.com' --user-agent 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:133.0) Gecko/20100101 Firefox/133.0' --header 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' --header 'Accept-Language: en-US,en;q=0.5' --referer 'https://opendatacommunities.org/' --header 'Upgrade-Insecure-Requests: 1' --header 'Sec-Fetch-Dest: document' --header 'Sec-Fetch-Mode: navigate' --header 'Sec-Fetch-Site: cross-site' --header 'Sec-Fetch-User: ?1' 'https://pmd3-production-grafter-dluhc.publishmydata.com/v1/pipelines/download/job/8bdb7fcf-502f-44f1-8a7a-64a1a1337944' --output 'download7266234069698100910.nt'

# Get OS CodePoint Open
# https://www.ordnancesurvey.co.uk/products/code-point-open
curl --header 'Host: omseprd1stdstordownload.blob.core.windows.net' --user-agent 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:133.0) Gecko/20100101 Firefox/133.0' --header 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' --header 'Accept-Language: en-US,en;q=0.5' --header 'Upgrade-Insecure-Requests: 1' --header 'Sec-Fetch-Dest: document' --header 'Sec-Fetch-Mode: navigate' --header 'Sec-Fetch-Site: cross-site' --header 'Sec-Fetch-User: ?1' 'https://omseprd1stdstordownload.blob.core.windows.net/downloads/CodePointOpen/2024-11/allGB/CSV/codepo_gb.zip?sv=2023-08-03&spr=https&se=2024-12-21T21%3A47%3A37Z&sr=b&sp=r&sig=C7fgLtAyyyjaJbzHtM6%2FXGyFEj3BlxA3DmdFn4hVY40%3D' --output 'codepo_gb.zip'

# LOCAL AUTHORITY DISTRICT geojson shapes!
curl --header 'Host: stg-arcgisazurecdataprod1.az.arcgis.com' --user-agent 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:133.0) Gecko/20100101 Firefox/133.0' --header 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' --header 'Accept-Language: en-US,en;q=0.5' --referer 'https://geoportal.statistics.gov.uk/' --header 'Upgrade-Insecure-Requests: 1' --header 'Sec-Fetch-Dest: document' --header 'Sec-Fetch-Mode: navigate' --header 'Sec-Fetch-Site: cross-site' --header 'Sec-Fetch-User: ?1' 'https://stg-arcgisazurecdataprod1.az.arcgis.com/exportfiles-1559-23740/Local_Authority_Districts_May_2024_Boundaries_UK_BFC_-6788913184658251542.geojson?sv=2018-03-28&sr=b&sig=tFgN6lAvWRPNHlic2lB%2F1QmAjVLPNmBycgWs6whvEU8%3D&se=2024-12-21T23%3A04%3A53Z&sp=r' --output 'Local_Authority_Districts_May_2024_Boundaries_UK_BFC_-6788913184658251542.geojson'

# GET MSOA Data shapes
# https://geoportal.statistics.gov.uk/datasets/ons::middle-layer-super-output-areas-december-2021-boundaries-ew-bgc-v3-2/about
curl --header 'Host: stg-arcgisazurecdataprod1.az.arcgis.com' --user-agent 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:133.0) Gecko/20100101 Firefox/133.0' --header 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' --header 'Accept-Language: en-US,en;q=0.5' --referer 'https://geoportal.statistics.gov.uk/' --header 'Upgrade-Insecure-Requests: 1' --header 'Sec-Fetch-Dest: document' --header 'Sec-Fetch-Mode: navigate' --header 'Sec-Fetch-Site: cross-site' 'https://stg-arcgisazurecdataprod1.az.arcgis.com/exportfiles-1559-23723/Middle_layer_Super_Output_Areas_December_2021_Boundaries_EW_BGC_V3_4916445166053426.geojson?sv=2018-03-28&sr=b&sig=0c6rbrJs9KfRGUvB12qtOUqbzUgeXaOX45zv2YzILZg%3D&se=2024-12-21T23%3A20%3A03Z&sp=r' --output 'Middle_layer_Super_Output_Areas_December_2021_Boundaries_EW_BGC_V3_4916445166053426.geojson'

# GET LSOA Data shapes 2021
# https://geoportal.statistics.gov.uk/datasets/ons::lower-layer-super-output-areas-december-2021-boundaries-ew-bsc-v4-2/about
curl --header 'Host: stg-arcgisazurecdataprod1.az.arcgis.com' --user-agent 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:133.0) Gecko/20100101 Firefox/133.0' --header 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' --header 'Accept-Language: en-US,en;q=0.5' --referer 'https://geoportal.statistics.gov.uk/' --header 'Upgrade-Insecure-Requests: 1' --header 'Sec-Fetch-Dest: document' --header 'Sec-Fetch-Mode: navigate' --header 'Sec-Fetch-Site: cross-site' --header 'Sec-Fetch-User: ?1' 'https://stg-arcgisazurecdataprod1.az.arcgis.com/exportfiles-1559-23706/Lower_layer_Super_Output_Areas_December_2021_Boundaries_EW_BSC_V4_-4299016806856585929.geojson?sv=2018-03-28&sr=b&sig=Kn%2Bga26iwdH8ybXcttpWawxxq9p2BQaldvdIxxy6v7o%3D&se=2024-12-21T23%3A24%3A47Z&sp=r' --output 'Lower_layer_Super_Output_Areas_December_2021_Boundaries_EW_BSC_V4_-4299016806856585929.geojson'