also import GDR50 and GDR250

replaces *.shp import with *.gdb import for GDB10.

Makefile

@@ -74,6 +74,9 @@ test-rivers: .faux_test-rivers ## Rivers tests (slow)
 .PHONY: slides
 slides: $(SLIDES)
 
+.PHONY: refresh-rivers
+refresh-rivers: rivers.sql rivers-50.sql rivers-250.sql ## Refresh river data from national datasets
+
 ###########################
 # The report, quick version
 ###########################
@@ -209,9 +212,13 @@ salvis-wm-overlaid-250k-zoom_1SELECT = wm_visuals where name='salvis'
 salvis-wm-overlaid-250k-zoom_2SELECT = wm_visuals where name='salvis-wm-220'
 salvis-wm-overlaid-250k-zoom_1COLOR = orange
 
-.faux_db: db init.sql rivers.sql
+.faux_db_pre: db init.sql
 	bash db start
-	bash db -f init.sql -f rivers.sql
+	bash db -f init.sql
+	touch $@
+
+.faux_db: .faux_db_pre rivers.sql
+	bash db -f rivers.sql
 	touch $@
 
 .faux_test: test.sql wm.sql .faux_db
@@ -273,8 +280,8 @@ mj-msc-gray.pdf: mj-msc.pdf
 clean: ## Clean the current working directory
 	-bash db stop
 	-rm -r .faux_test .faux_aggregate-rivers .faux_test-rivers .faux_visuals \
-		.faux_db version.inc.tex vars.inc.tex version.aux version.fdb_latexmk \
+		.faux_db .faux_db_pre version.inc.tex vars.inc.tex version.aux \
-		_minted-mj-msc \
+		version.fdb_latexmk _minted-mj-msc .tmp \
 		$(shell git ls-files -o mj-msc*) \
 		$(addsuffix .pdf,$(FIGURES)) \
 		$(addsuffix .pdf,$(RIVERS)) \
@@ -287,7 +294,8 @@ clean-tables: ## Remove tables created during unit or rivers tests
 
 .PHONY: help
 help: ## Print this help message
-	@awk -F':.*?## ' '/^[a-z0-9.-]*: *.*## */{printf "%-18s %s\n",$$1,$$2}' $(MAKEFILE_LIST)
+	@awk -F':.*?## ' '/^[a-z0-9.-]*: *.*## */{printf "%-18s %s\n",$$1,$$2}' \
+		$(MAKEFILE_LIST)
 
 .PHONY: wc
 wc: mj-msc.pdf ## Character and page count
@@ -296,21 +304,15 @@ wc: mj-msc.pdf ## Character and page count
 		tr -d '[:space:]' | wc -c | \
 		awk '{printf("Chars: %d, pages: %.1f\n", $$1, $$1/1500)}'
 
-.PHONY: refresh-rivers
+define rivers_template
-refresh-rivers: aggregate-rivers.sql .faux_db ## Refresh rivers.sql from GDB10LT
+$(1): aggregate-rivers.sql gdr2pgsql .faux_db_pre
-	@if [ ! -f "$(GDB10LT)" ]; then \
+	@if [ ! -f "$$($(2))" ]; then \
-		echo "ERROR: GDB10LT-static-*.zip not found. Do GDB10LT=<...>"; \
+		echo "ERROR: $(2)-static-*.zip not found. Run env $(2)=<...>"; \
 		exit 1; \
 	fi
-	mkdir -p .tmp/shp; unzip -d .tmp/shp "$(GDB10LT)" 'HIDRO_L.*'
+	./gdr2pgsql "$$($(2))" "$(3)" "$(RIVERFILTER)" "$(1)"
-	shp2pgsql -s 3857 -d ".tmp/shp/HIDRO_L.shp" | \
+endef
-		awk '!/^INSERT/{print}; /^INSERT/&&/$(RIVERFILTER)/{print;next}' | \
+
-		bash ./db
+$(eval $(call rivers_template,rivers.sql,GDB10LT,wm_rivers))
-	bash db -f $<
+$(eval $(call rivers_template,rivers-50.sql,GDR50LT,wm_rivers_50))
-	(\
+$(eval $(call rivers_template,rivers-250.sql,GDR250LT,wm_rivers_250))
-		echo '-- Generated at $(shell TZ=UTC date +"%FT%TZ") on $(shell whoami)@$(shell hostname -f)'; \
-		echo '-- Rivers: $(RIVERFILTER)'; \
-		docker exec wm-mj pg_dump --clean -Uosm osm -t wm_rivers | tr -d '\r' \
-	) > rivers.sql.tmp
-	mv rivers.sql.tmp rivers.sql
-	rm -fr .tmp/shp

aggregate-rivers.sql

@@ -1,7 +1,7 @@
 /* Aggregates rivers by name and proximity. */
 drop function if exists aggregate_rivers;
 create function aggregate_rivers() returns table(
-  id bigint,
+  id integer,
   name text,
   way geometry
 ) as $$
@@ -35,15 +35,15 @@ $$ language plpgsql;
 drop index if exists wm_rivers_tmp_id;
 drop index if exists wm_rivers_tmp_gix;
 drop table if exists wm_rivers_tmp;
-create temporary table wm_rivers_tmp (id bigint, name text, way geometry);
+create temporary table wm_rivers_tmp (id serial, name text, way geometry);
 create index wm_rivers_tmp_id on wm_rivers_tmp(id);
 create index wm_rivers_tmp_gix on wm_rivers_tmp using gist(way) include(name);
 
-insert into wm_rivers_tmp
+insert into wm_rivers_tmp (name, way)
-  select p.gid as id, p.vardas as name, p.geom as way from hidro_l p;
+  select p.vardas as name, p.shape as way from :srctable p;
 
-drop table if exists wm_rivers;
+drop table if exists :dsttable;
-create table wm_rivers as (
+create table :dsttable as (
   select * from aggregate_rivers() where st_length(way) >= 50000
 );
 drop table wm_rivers_tmp;

gdr2pgsql

@@ -0,0 +1,34 @@
+#!/bin/bash
+set -euo pipefail
+
+dbzip=$(realpath -s "$1")
+tbl=$2
+filter=$3
+outfile=$4
+
+if [[ "$dbzip" =~ " " ]]; then
+    echo "ERROR: $dbzip contains spaces"
+    exit 1
+fi
+
+set -x
+
+gdbname=$(unzip -Z1 "$dbzip" | awk -Fgdb '/.gdb/{print $1"gdb";exit}')
+
+now=$(TZ=UTC date +"%FT%TZ")
+here=$(whoami)@$(hostname -f)
+
+mkdir -p ".tmp"
+ogr2ogr -skipfailures -f PGDump /dev/stdout -t_srs epsg:3857 \
+    "/vsizip/$dbzip/$gdbname" -nln "src_$tbl" hidro_l | \
+        awk "!/^INSERT/{print}; /^INSERT/&&/${filter}/{print;next}" | \
+        bash ./db | \
+        grep -v '^INSERT 0 1'
+
+bash db -f aggregate-rivers.sql -v "srctable=src_$tbl" -v "dsttable=$tbl"
+(
+    echo "-- Generated at $now on $here";
+    echo "-- Rivers: $filter";
+    docker exec wm-mj pg_dump --clean -Uosm osm -t "$tbl" | tr -d '\r'
+) > ".tmp/$outfile"
+mv ".tmp/$outfile" "$outfile"