From ef770bfdf76e7aa2a07e6f635e9e51070cb24202 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Motiejus=20Jak=C5=A1tys?= Date: Sun, 16 May 2021 12:51:57 +0300 Subject: [PATCH] also import GDR50 and GDR250 replaces *.shp import with *.gdb import for GDB10. --- IV/Makefile | 44 +++++++++++++++++++++-------------------- IV/aggregate-rivers.sql | 12 +++++------ IV/gdr2pgsql | 34 +++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 27 deletions(-) create mode 100755 IV/gdr2pgsql diff --git a/IV/Makefile b/IV/Makefile index 551494f..bcb73d9 100644 --- a/IV/Makefile +++ b/IV/Makefile @@ -74,6 +74,9 @@ test-rivers: .faux_test-rivers ## Rivers tests (slow) .PHONY: slides slides: $(SLIDES) +.PHONY: refresh-rivers +refresh-rivers: rivers.sql rivers-50.sql rivers-250.sql ## Refresh river data from national datasets + ########################### # The report, quick version ########################### @@ -209,9 +212,13 @@ salvis-wm-overlaid-250k-zoom_1SELECT = wm_visuals where name='salvis' salvis-wm-overlaid-250k-zoom_2SELECT = wm_visuals where name='salvis-wm-220' salvis-wm-overlaid-250k-zoom_1COLOR = orange -.faux_db: db init.sql rivers.sql +.faux_db_pre: db init.sql bash db start - bash db -f init.sql -f rivers.sql + bash db -f init.sql + touch $@ + +.faux_db: .faux_db_pre rivers.sql + bash db -f rivers.sql touch $@ .faux_test: test.sql wm.sql .faux_db @@ -273,8 +280,8 @@ mj-msc-gray.pdf: mj-msc.pdf clean: ## Clean the current working directory -bash db stop -rm -r .faux_test .faux_aggregate-rivers .faux_test-rivers .faux_visuals \ - .faux_db version.inc.tex vars.inc.tex version.aux version.fdb_latexmk \ - _minted-mj-msc \ + .faux_db .faux_db_pre version.inc.tex vars.inc.tex version.aux \ + version.fdb_latexmk _minted-mj-msc .tmp \ $(shell git ls-files -o mj-msc*) \ $(addsuffix .pdf,$(FIGURES)) \ $(addsuffix .pdf,$(RIVERS)) \ @@ -287,7 +294,8 @@ clean-tables: ## Remove tables created during unit or rivers tests .PHONY: help help: ## Print this help message - @awk -F':.*?## ' '/^[a-z0-9.-]*: *.*## */{printf "%-18s %s\n",$$1,$$2}' $(MAKEFILE_LIST) + @awk -F':.*?## ' '/^[a-z0-9.-]*: *.*## */{printf "%-18s %s\n",$$1,$$2}' \ + $(MAKEFILE_LIST) .PHONY: wc wc: mj-msc.pdf ## Character and page count @@ -296,21 +304,15 @@ wc: mj-msc.pdf ## Character and page count tr -d '[:space:]' | wc -c | \ awk '{printf("Chars: %d, pages: %.1f\n", $$1, $$1/1500)}' -.PHONY: refresh-rivers -refresh-rivers: aggregate-rivers.sql .faux_db ## Refresh rivers.sql from GDB10LT - @if [ ! -f "$(GDB10LT)" ]; then \ - echo "ERROR: GDB10LT-static-*.zip not found. Do GDB10LT=<...>"; \ +define rivers_template +$(1): aggregate-rivers.sql gdr2pgsql .faux_db_pre + @if [ ! -f "$$($(2))" ]; then \ + echo "ERROR: $(2)-static-*.zip not found. Run env $(2)=<...>"; \ exit 1; \ fi - mkdir -p .tmp/shp; unzip -d .tmp/shp "$(GDB10LT)" 'HIDRO_L.*' - shp2pgsql -s 3857 -d ".tmp/shp/HIDRO_L.shp" | \ - awk '!/^INSERT/{print}; /^INSERT/&&/$(RIVERFILTER)/{print;next}' | \ - bash ./db - bash db -f $< - (\ - echo '-- Generated at $(shell TZ=UTC date +"%FT%TZ") on $(shell whoami)@$(shell hostname -f)'; \ - echo '-- Rivers: $(RIVERFILTER)'; \ - docker exec wm-mj pg_dump --clean -Uosm osm -t wm_rivers | tr -d '\r' \ - ) > rivers.sql.tmp - mv rivers.sql.tmp rivers.sql - rm -fr .tmp/shp + ./gdr2pgsql "$$($(2))" "$(3)" "$(RIVERFILTER)" "$(1)" +endef + +$(eval $(call rivers_template,rivers.sql,GDB10LT,wm_rivers)) +$(eval $(call rivers_template,rivers-50.sql,GDR50LT,wm_rivers_50)) +$(eval $(call rivers_template,rivers-250.sql,GDR250LT,wm_rivers_250)) diff --git a/IV/aggregate-rivers.sql b/IV/aggregate-rivers.sql index 4153fc3..d305445 100644 --- a/IV/aggregate-rivers.sql +++ b/IV/aggregate-rivers.sql @@ -1,7 +1,7 @@ /* Aggregates rivers by name and proximity. */ drop function if exists aggregate_rivers; create function aggregate_rivers() returns table( - id bigint, + id integer, name text, way geometry ) as $$ @@ -35,15 +35,15 @@ $$ language plpgsql; drop index if exists wm_rivers_tmp_id; drop index if exists wm_rivers_tmp_gix; drop table if exists wm_rivers_tmp; -create temporary table wm_rivers_tmp (id bigint, name text, way geometry); +create temporary table wm_rivers_tmp (id serial, name text, way geometry); create index wm_rivers_tmp_id on wm_rivers_tmp(id); create index wm_rivers_tmp_gix on wm_rivers_tmp using gist(way) include(name); -insert into wm_rivers_tmp - select p.gid as id, p.vardas as name, p.geom as way from hidro_l p; +insert into wm_rivers_tmp (name, way) + select p.vardas as name, p.shape as way from :srctable p; -drop table if exists wm_rivers; -create table wm_rivers as ( +drop table if exists :dsttable; +create table :dsttable as ( select * from aggregate_rivers() where st_length(way) >= 50000 ); drop table wm_rivers_tmp; diff --git a/IV/gdr2pgsql b/IV/gdr2pgsql new file mode 100755 index 0000000..2a3a7df --- /dev/null +++ b/IV/gdr2pgsql @@ -0,0 +1,34 @@ +#!/bin/bash +set -euo pipefail + +dbzip=$(realpath -s "$1") +tbl=$2 +filter=$3 +outfile=$4 + +if [[ "$dbzip" =~ " " ]]; then + echo "ERROR: $dbzip contains spaces" + exit 1 +fi + +set -x + +gdbname=$(unzip -Z1 "$dbzip" | awk -Fgdb '/.gdb/{print $1"gdb";exit}') + +now=$(TZ=UTC date +"%FT%TZ") +here=$(whoami)@$(hostname -f) + +mkdir -p ".tmp" +ogr2ogr -skipfailures -f PGDump /dev/stdout -t_srs epsg:3857 \ + "/vsizip/$dbzip/$gdbname" -nln "src_$tbl" hidro_l | \ + awk "!/^INSERT/{print}; /^INSERT/&&/${filter}/{print;next}" | \ + bash ./db | \ + grep -v '^INSERT 0 1' + +bash db -f aggregate-rivers.sql -v "srctable=src_$tbl" -v "dsttable=$tbl" +( + echo "-- Generated at $now on $here"; + echo "-- Rivers: $filter"; + docker exec wm-mj pg_dump --clean -Uosm osm -t "$tbl" | tr -d '\r' +) > ".tmp/$outfile" +mv ".tmp/$outfile" "$outfile"