simultaneous unit + integration tests

IV/Makefile

@@ -1,10 +1,11 @@
 SOURCE ?= lithuania-latest.osm.pbf
-WHERE ?= name like '%'
+WHERE ?= name='Visinčia' OR name='Šalčia' OR name='Nemunas' OR name='Žeimena' OR name='Lakaja'
-#WHERE ?= name='Visinčia' OR name='Šalčia' OR name='Nemunas' OR name='Žeimena' OR name='Lakaja'
+#WHERE ?= name like '%'
 SLIDES = slides-2021-03-29.pdf
 
-NON_ARCHIVABLES = notes.txt referatui.txt slides-2021-03-29.txt
+################################################################
-ARCHIVABLES = $(filter-out $(NON_ARCHIVABLES),$(shell git ls-files .))
+# This has to be before first use due to how macros are expanded
+################################################################
 
 FIGURES = test-figures \
 					fig8-definition-of-a-bend \
@@ -15,19 +16,43 @@ FIGURES = test-figures \
 					fig6-self-crossing-before \
 					fig6-self-crossing-after
 
+#################################
+# The thesis, publishable version
+#################################
+
+mj-msc-full.pdf: mj-msc.pdf version.inc.tex $(ARCHIVABLES) ## Thesis for publishing
+	cp $< .tmp-$@
+	for f in $^; do \
+		if [ "$$f" = "$<" ]; then continue; fi; \
+		pdfattach .tmp-$@ $$f .tmp2-$@; \
+		mv .tmp2-$@ .tmp-$@; \
+	done
+	mv .tmp-$@ $@
+
+###############################
+# Auxiliary targets for humans
+###############################
+
 .PHONY: test
-test: .faux_test
+test: .faux_test ## Unit tests (fast)
 
 .PHONY: test-integration
-test-integration: .faux_filter-rivers
+test-integration: .faux_test-integration ## Integration tests (slow)
-	./db -f tests-integration.sql
 
 .PHONY: slides
 slides: $(SLIDES)
 
+###########################
+# The report, quick version
+###########################
+
 mj-msc.pdf: mj-msc.tex version.inc.tex vars.inc.tex bib.bib $(addsuffix .pdf,$(FIGURES))
 	latexmk -shell-escape -g -pdf $<
 
+############################
+# Report's test dependencies
+############################
+
 define FIG_template
 $(1).pdf: layer2img.py Makefile .faux_test
 	python ./layer2img.py --outfile=$(1).pdf \
@@ -72,17 +97,38 @@ fig6-self-crossing-before_3SELECT = wm_visuals where name='fig6-newline'
 fig6-self-crossing-after_WIDTHDIV = 4
 fig6-self-crossing-after_1SELECT = wm_debug where name='fig6' AND stage='dcrossings' AND gen=1
 
-.faux_test: tests.sql wm.sql .faux_db
+
-	./db -f tests.sql
+.faux_test-integration: tests-integration.sql wm.sql .faux_aggregate-rivers
+	./db -f $<
 	touch $@
 
-.faux_db:
+.faux_test: tests.sql wm.sql .faux_db
+	./db -f $<
+	touch $@
+
+.faux_db: db init.sql
 	./db start
+	./db -f init.sql
 	touch $@
 
 $(SOURCE):
 	wget http://download.geofabrik.de/europe/$@
 
+.faux_aggregate-rivers: aggregate-rivers.sql .faux_import-osm Makefile
+	./db -v where="$(WHERE)" -f $<
+	touch $@
+
+.faux_import-osm: $(SOURCE) .faux_db
+	PGPASSWORD=osm osm2pgsql -c --multi-geometry -H 127.0.0.1 -d osm -U osm $<
+	touch $@
+
+################################
+# Report's non-test dependencies
+################################
+
+NON_ARCHIVABLES = notes.txt referatui.txt slides-2021-03-29.txt
+ARCHIVABLES = $(filter-out $(NON_ARCHIVABLES),$(shell git ls-files .))
+
 REF = $(shell git describe --abbrev=12 --always --dirty)
 version.inc.tex: Makefile $(shell git rev-parse --git-dir 2>/dev/null)
 	TZ=UTC date '+\gdef\VCDescribe{%F ($(REF))}%' > $@
@@ -90,6 +136,16 @@ version.inc.tex: Makefile $(shell git rev-parse --git-dir 2>/dev/null)
 vars.inc.tex: vars.awk wm.sql Makefile
 	awk -f $< wm.sql
 
+#####################
+# Almost never needed
+#####################
+
+slides-2021-03-29.pdf: slides-2021-03-29.txt
+	pandoc -t beamer -i $< -o $@
+
+dump-debug_wm.sql.xz:
+	docker exec -ti wm-mj pg_dump -Uosm osm -t debug_wm | xz -v > $@
+
 mj-msc-gray.pdf: mj-msc.pdf
 	gs \
 		-sOutputFile=$@ \
@@ -101,19 +157,10 @@ mj-msc-gray.pdf: mj-msc.pdf
 		-dBATCH \
 		$<
 
-mj-msc-full.pdf: mj-msc.pdf version.inc.tex $(ARCHIVABLES)
-	cp $< .tmp-$@
-	for f in $^; do \
-		if [ "$$f" = "$<" ]; then continue; fi; \
-		pdfattach .tmp-$@ $$f .tmp2-$@; \
-		mv .tmp2-$@ .tmp-$@; \
-	done
-	mv .tmp-$@ $@
-
 .PHONY: clean
-clean:
+clean: ## Clean the current working directory
 	-./db stop
-	-rm -r .faux_test .faux_filter-rivers .faux_import-osm .faux_db \
+	-rm -r .faux_test .faux_aggregate-rivers .faux_import-osm .faux_db \
 		version.inc.tex vars.inc.tex version.aux version.fdb_latexmk \
 		_minted-mj-msc \
 		$(shell git ls-files -o mj-msc*) \
@@ -121,22 +168,12 @@ clean:
 		$(SLIDES)
 
 .PHONY: clean-tables
-clean-tables:
+clean-tables: ## Remove tables created during unit or integration tests
 	for t in $$(./db -c '\dt' | awk '/\ywm_\w+\y/{print $$3}'); do \
 		./db -c "drop table $$t"; \
 	done
 	-rm .faux_test
 
-.faux_filter-rivers: aggregate-rivers.sql .faux_import-osm Makefile
+.PHONY: help
-	./db -v where="$(WHERE)" -f $<
+help: ## Print this help message
-	touch $@
+	@awk -F':.*?## ' '/^[a-z0-9.-]*: *.*## */{printf "%-18s %s\n",$$1,$$2}' $(MAKEFILE_LIST)
-
-.faux_import-osm: $(SOURCE) .faux_db
-	PGPASSWORD=osm osm2pgsql -c --multi-geometry -H 127.0.0.1 -d osm -U osm $<
-	touch $@
-
-slides-2021-03-29.pdf: slides-2021-03-29.txt
-	pandoc -t beamer -i $< -o $@
-
-dump-debug_wm.sql.xz:
-	docker exec -ti wm-mj pg_dump -Uosm osm -t debug_wm | xz -v > $@

IV/aggregate-rivers.sql

@@ -6,35 +6,35 @@ declare
   cc record;
   changed boolean;
 begin
-  while (select count(1) from aggregate_rivers_tmp) > 0 loop
+  while (select count(1) from wm_rivers_tmp) > 0 loop
-    select * from aggregate_rivers_tmp limit 1 into c;
+    select * from wm_rivers_tmp limit 1 into c;
-    delete from aggregate_rivers_tmp a where a.osm_id = c.osm_id;
+    delete from wm_rivers_tmp a where a.osm_id = c.osm_id;
     changed = true;
     while changed loop
       changed = false;
-      for cc in (select * from aggregate_rivers_tmp a where a.name = c.name and st_dwithin(a.way, c.way, 500)) loop
+      for cc in (select * from wm_rivers_tmp a where a.name = c.name and st_dwithin(a.way, c.way, 500)) loop
         c.way = st_linemerge(st_union(c.way, cc.way));
-        delete from aggregate_rivers_tmp a where a.osm_id = cc.osm_id;
+        delete from wm_rivers_tmp a where a.osm_id = cc.osm_id;
         changed = true;
       end loop;
     end loop; -- while changed
     return query select c.osm_id, c.name, c.way;
-  end loop; -- count(1) from aggregate_rivers_tmp > 0
+  end loop; -- count(1) from wm_rivers_tmp > 0
   return;
 end
 $$ language plpgsql;
 
-drop index if exists aggregate_rivers_tmp_id;
+drop index if exists wm_rivers_tmp_id;
-drop index if exists aggregate_rivers_tmp_gix;
+drop index if exists wm_rivers_tmp_gix;
-drop table if exists aggregate_rivers_tmp;
+drop table if exists wm_rivers_tmp;
-create temporary table aggregate_rivers_tmp (osm_id bigint, name text, way geometry);
+create temporary table wm_rivers_tmp (osm_id bigint, name text, way geometry);
-create index aggregate_rivers_tmp_id on aggregate_rivers_tmp(osm_id);
+create index wm_rivers_tmp_id on wm_rivers_tmp(osm_id);
-create index aggregate_rivers_tmp_gix on aggregate_rivers_tmp using gist(way) include(name);
+create index wm_rivers_tmp_gix on wm_rivers_tmp using gist(way) include(name);
 
-insert into aggregate_rivers_tmp
+insert into wm_rivers_tmp
   select p.osm_id, p.name, p.way from planet_osm_line p
     where waterway in ('river', 'stream', 'canal') and :where;
 
-drop table if exists agg_rivers;
+drop table if exists wm_rivers;
-create table agg_rivers as (select * from aggregate_rivers() where st_length(way) >= 50000);
+create table wm_rivers as (select * from aggregate_rivers() where st_length(way) >= 50000);
-drop table aggregate_rivers_tmp;
+drop table wm_rivers_tmp;

IV/init.sql

@@ -0,0 +1,13 @@
+-- This file initializes tables for unit and integration tests.
+-- ST_SimplifyWM, when dbgname is non-empty, expects `wm_debug` table to be
+-- created.
+
+-- to preview this somewhat conveniently in QGIS:
+-- stage || '_' || name || ' gen:' || coalesce(gen, 'Ø') || ' nbend:' || lpad(nbend, 4, '0')
+drop table if exists wm_debug;
+create table wm_debug(stage text, name text, gen bigint, nbend bigint, way geometry, props jsonb);
+
+-- Run ST_SimplifyWM in debug mode, so `wm_debug` is populated. That table
+-- is used for geometric assertions later in the file.
+drop table if exists wm_demo;
+create table wm_demo (name text, i bigint, way geometry);

IV/mj-msc.tex

@@ -121,7 +121,9 @@ different trade-offs.
 A number of cartographic line generalization algorithms have been researched.
 The "classical" ones are {\DP} and {\VW}.
 
-\subsection{{\DP} and {\VW}}
+\subsection{Available algorithms}
+
+\subsubsection{{\DP}, {\VW} and Chaikin's}
 
 {\DP} \cite{douglas1973algorithms} and {\VW} \cite{visvalingam1993line} are
 "classical" line generalization computer graphics algorithms. They are
@@ -144,13 +146,8 @@ algorithm \cite{chaikin1974algorithm} via
 \href{https://postgis.net/docs/ST_ChaikinSmoothing.html}{PostGIS
 ChaikinSmoothing}.
 
-Even though {\DP} and {\VW} are simple to understand and computationally
-efficient, they have serious deficiencies for cartographic natural line
-generalization.
 
-<TODO: expand on deficiencies>
+\subsubsection{Modern approaches}
-
-\subsection{Modern approaches}
 
 Due to their simplicity and ubiquity, {\DP} and {\VW} have been established as
 go-to algorithms for line generalization. During recent years, alternatives
@@ -184,7 +181,7 @@ open-source tools is an important foundation for future cartographic
 experimentation and development, thus it it benefits the cartographic society
 as a whole.
 
-<TODO: expand on each paper>
+\subsection{Problematic with generalization of rivers}
 
 \section{Methodology}
 \label{sec:methodology}

IV/tests-integration.sql

@@ -1,18 +1,13 @@
 \i wm.sql
 
-drop table if exists wm_debug;
-create table wm_debug(stage text, name text, gen bigint, nbend bigint, way geometry, props jsonb);
-
 do $$
 declare
   npoints bigint;
   secs bigint;
 begin
-  select * from ST_SimplifyWM_Estimate((select st_union(way) from agg_rivers)) into npoints, secs;
+  select * from ST_SimplifyWM_Estimate((select st_union(way) from wm_rivers)) into npoints, secs;
   raise notice 'Total points: %', npoints;
   raise notice 'Expected duration: %s (+-%s)', ceil(secs), floor(secs*.5);
 end $$ language plpgsql;
 
-drop table if exists wm_demo;
+insert into wm_demo (name, way) select name, ST_SimplifyWM(way, name) from wm_rivers;
-create table wm_demo (name text, i bigint, way geometry);
-insert into wm_demo (name, way) select name, ST_SimplifyWM(way, name) from agg_rivers;

IV/tests.sql

@@ -26,11 +26,6 @@ begin
 end
 $$ language plpgsql;
 
--- to preview this somewhat conveniently in QGIS:
--- stage || '_' || name || ' gen:' || coalesce(gen, 'Ø') || ' nbend:' || lpad(nbend, 4, '0')
-drop table if exists wm_debug;
-create table wm_debug(stage text, name text, gen bigint, nbend bigint, way geometry, props jsonb);
-
 drop table if exists wm_figures;
 create table wm_figures (name text, way geometry);
 -- add fig8.gpkg to postgis:
@@ -56,10 +51,6 @@ insert into wm_figures (name, way) values ('multi-island',ST_GeomFromText('MULTI
 insert into wm_figures (name, way) values ('selfcrossing-1',ST_GeomFromText('LINESTRING(-27 180,-20 166,-21 142,-18 136,55 136,55 136,71 145,44 165,37 146,22 145,14 164,11 164,3 146,-12 146,-13 176,-18 184)'));
 insert into wm_figures (name, way) values ('selfcrossing-1-rev',ST_Reverse(ST_Translate((select way from wm_figures where name='selfcrossing-1'), 0, 60)));
 
--- Run ST_SimplifyWM in debug mode, so `wm_debug` is populated. That table
--- is used for geometric assertions later in the file.
-drop table if exists wm_demo;
-create table wm_demo (name text, i bigint, way geometry);
 insert into wm_demo (name, way) select name, ST_SimplifyWM(way, name) from wm_figures;
 
 -- wm_visuals holds visual aids for the paper.