motiejus/stud
1
Fork 0
Code Releases Activity

simultaneous unit + integration tests

Browse Source
This commit is contained in:
Motiejus Jakštys 2021-04-17 16:16:27 +03:00
parent 395c52d4f4
commit cc82f3d1bb
6 changed files with 108 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

107
IV/Makefile
View File

@ -1,10 +1,11 @@
SOURCE ?= lithuania-latest.osm.pbf
WHERE ?= name like '%'
#WHERE ?= name='Visinčia' OR name='Šalčia' OR name='Nemunas' OR name='Žeimena' OR name='Lakaja'
WHERE ?= name='Visinčia' OR name='Šalčia' OR name='Nemunas' OR name='Žeimena' OR name='Lakaja'
#WHERE ?= name like '%'
SLIDES = slides-2021-03-29.pdf
NON_ARCHIVABLES = notes.txt referatui.txt slides-2021-03-29.txt
ARCHIVABLES = $(filter-out $(NON_ARCHIVABLES),$(shell git ls-files .))
################################################################
# This has to be before first use due to how macros are expanded
################################################################
FIGURES = test-figures \
fig8-definition-of-a-bend \
@ -15,19 +16,43 @@ FIGURES = test-figures \
fig6-self-crossing-before \
fig6-self-crossing-after
#################################
# The thesis, publishable version
#################################
mj-msc-full.pdf: mj-msc.pdf version.inc.tex $(ARCHIVABLES) ## Thesis for publishing
cp $< .tmp-$@
for f in $^; do \
if [ "$$f" = "$<" ]; then continue; fi; \
pdfattach .tmp-$@ $$f .tmp2-$@; \
mv .tmp2-$@ .tmp-$@; \
done
mv .tmp-$@ $@
###############################
# Auxiliary targets for humans
###############################
.PHONY: test
test: .faux_test
test: .faux_test ## Unit tests (fast)
.PHONY: test-integration
test-integration: .faux_filter-rivers
./db -f tests-integration.sql
test-integration: .faux_test-integration ## Integration tests (slow)
.PHONY: slides
slides: $(SLIDES)
###########################
# The report, quick version
###########################
mj-msc.pdf: mj-msc.tex version.inc.tex vars.inc.tex bib.bib $(addsuffix .pdf,$(FIGURES))
latexmk -shell-escape -g -pdf $<
############################
# Report's test dependencies
############################
define FIG_template
$(1).pdf: layer2img.py Makefile .faux_test
python ./layer2img.py --outfile=$(1).pdf \
@ -72,17 +97,38 @@ fig6-self-crossing-before_3SELECT = wm_visuals where name='fig6-newline'
fig6-self-crossing-after_WIDTHDIV = 4
fig6-self-crossing-after_1SELECT = wm_debug where name='fig6' AND stage='dcrossings' AND gen=1
.faux_test: tests.sql wm.sql .faux_db
./db -f tests.sql
.faux_test-integration: tests-integration.sql wm.sql .faux_aggregate-rivers
./db -f $<
touch $@
.faux_db:
.faux_test: tests.sql wm.sql .faux_db
./db -f $<
touch $@
.faux_db: db init.sql
./db start
./db -f init.sql
touch $@
$(SOURCE):
wget http://download.geofabrik.de/europe/$@
.faux_aggregate-rivers: aggregate-rivers.sql .faux_import-osm Makefile
./db -v where="$(WHERE)" -f $<
touch $@
.faux_import-osm: $(SOURCE) .faux_db
PGPASSWORD=osm osm2pgsql -c --multi-geometry -H 127.0.0.1 -d osm -U osm $<
touch $@
################################
# Report's non-test dependencies
################################
NON_ARCHIVABLES = notes.txt referatui.txt slides-2021-03-29.txt
ARCHIVABLES = $(filter-out $(NON_ARCHIVABLES),$(shell git ls-files .))
REF = $(shell git describe --abbrev=12 --always --dirty)
version.inc.tex: Makefile $(shell git rev-parse --git-dir 2>/dev/null)
TZ=UTC date '+\gdef\VCDescribe{%F ($(REF))}%' > $@
@ -90,6 +136,16 @@ version.inc.tex: Makefile $(shell git rev-parse --git-dir 2>/dev/null)
vars.inc.tex: vars.awk wm.sql Makefile
awk -f $< wm.sql
#####################
# Almost never needed
#####################
slides-2021-03-29.pdf: slides-2021-03-29.txt
pandoc -t beamer -i $< -o $@
dump-debug_wm.sql.xz:
docker exec -ti wm-mj pg_dump -Uosm osm -t debug_wm | xz -v > $@
mj-msc-gray.pdf: mj-msc.pdf
gs \
-sOutputFile=$@ \
@ -101,19 +157,10 @@ mj-msc-gray.pdf: mj-msc.pdf
-dBATCH \
$<
mj-msc-full.pdf: mj-msc.pdf version.inc.tex $(ARCHIVABLES)
cp $< .tmp-$@
for f in $^; do \
if [ "$$f" = "$<" ]; then continue; fi; \
pdfattach .tmp-$@ $$f .tmp2-$@; \
mv .tmp2-$@ .tmp-$@; \
done
mv .tmp-$@ $@
.PHONY: clean
clean:
clean: ## Clean the current working directory
-./db stop
-rm -r .faux_test .faux_filter-rivers .faux_import-osm .faux_db \
-rm -r .faux_test .faux_aggregate-rivers .faux_import-osm .faux_db \
version.inc.tex vars.inc.tex version.aux version.fdb_latexmk \
_minted-mj-msc \
$(shell git ls-files -o mj-msc*) \
@ -121,22 +168,12 @@ clean:
$(SLIDES)
.PHONY: clean-tables
clean-tables:
clean-tables: ## Remove tables created during unit or integration tests
for t in $$(./db -c '\dt' | awk '/\ywm_\w+\y/{print $$3}'); do \
./db -c "drop table $$t"; \
done
-rm .faux_test
.faux_filter-rivers: aggregate-rivers.sql .faux_import-osm Makefile
./db -v where="$(WHERE)" -f $<
touch $@
.faux_import-osm: $(SOURCE) .faux_db
PGPASSWORD=osm osm2pgsql -c --multi-geometry -H 127.0.0.1 -d osm -U osm $<
touch $@
slides-2021-03-29.pdf: slides-2021-03-29.txt
pandoc -t beamer -i $< -o $@
dump-debug_wm.sql.xz:
docker exec -ti wm-mj pg_dump -Uosm osm -t debug_wm | xz -v > $@
.PHONY: help
help: ## Print this help message
@awk -F':.*?## ' '/^[a-z0-9.-]*: *.*## */{printf "%-18s %s\n",$$1,$$2}' $(MAKEFILE_LIST)

32
IV/aggregate-rivers.sql
View File

@ -6,35 +6,35 @@ declare
cc record;
changed boolean;
begin
while (select count(1) from aggregate_rivers_tmp) > 0 loop
select * from aggregate_rivers_tmp limit 1 into c;
delete from aggregate_rivers_tmp a where a.osm_id = c.osm_id;
while (select count(1) from wm_rivers_tmp) > 0 loop
select * from wm_rivers_tmp limit 1 into c;
delete from wm_rivers_tmp a where a.osm_id = c.osm_id;
changed = true;
while changed loop
changed = false;
for cc in (select * from aggregate_rivers_tmp a where a.name = c.name and st_dwithin(a.way, c.way, 500)) loop
for cc in (select * from wm_rivers_tmp a where a.name = c.name and st_dwithin(a.way, c.way, 500)) loop
c.way = st_linemerge(st_union(c.way, cc.way));
delete from aggregate_rivers_tmp a where a.osm_id = cc.osm_id;
delete from wm_rivers_tmp a where a.osm_id = cc.osm_id;
changed = true;
end loop;
end loop; -- while changed
return query select c.osm_id, c.name, c.way;
end loop; -- count(1) from aggregate_rivers_tmp > 0
end loop; -- count(1) from wm_rivers_tmp > 0
return;
end
$$ language plpgsql;
drop index if exists aggregate_rivers_tmp_id;
drop index if exists aggregate_rivers_tmp_gix;
drop table if exists aggregate_rivers_tmp;
create temporary table aggregate_rivers_tmp (osm_id bigint, name text, way geometry);
create index aggregate_rivers_tmp_id on aggregate_rivers_tmp(osm_id);
create index aggregate_rivers_tmp_gix on aggregate_rivers_tmp using gist(way) include(name);
drop index if exists wm_rivers_tmp_id;
drop index if exists wm_rivers_tmp_gix;
drop table if exists wm_rivers_tmp;
create temporary table wm_rivers_tmp (osm_id bigint, name text, way geometry);
create index wm_rivers_tmp_id on wm_rivers_tmp(osm_id);
create index wm_rivers_tmp_gix on wm_rivers_tmp using gist(way) include(name);
insert into aggregate_rivers_tmp
insert into wm_rivers_tmp
select p.osm_id, p.name, p.way from planet_osm_line p
where waterway in ('river', 'stream', 'canal') and :where;
drop table if exists agg_rivers;
create table agg_rivers as (select * from aggregate_rivers() where st_length(way) >= 50000);
drop table aggregate_rivers_tmp;
drop table if exists wm_rivers;
create table wm_rivers as (select * from aggregate_rivers() where st_length(way) >= 50000);
drop table wm_rivers_tmp;

13
IV/init.sql Normal file
View File

@ -0,0 +1,13 @@
-- This file initializes tables for unit and integration tests.
-- ST_SimplifyWM, when dbgname is non-empty, expects `wm_debug` table to be
-- created.
-- to preview this somewhat conveniently in QGIS:
-- stage || '_' || name || ' gen:' || coalesce(gen, 'Ø') || ' nbend:' || lpad(nbend, 4, '0')
drop table if exists wm_debug;
create table wm_debug(stage text, name text, gen bigint, nbend bigint, way geometry, props jsonb);
-- Run ST_SimplifyWM in debug mode, so `wm_debug` is populated. That table
-- is used for geometric assertions later in the file.
drop table if exists wm_demo;
create table wm_demo (name text, i bigint, way geometry);

13
IV/mj-msc.tex
View File

@ -121,7 +121,9 @@ different trade-offs.
A number of cartographic line generalization algorithms have been researched.
The "classical" ones are {\DP} and {\VW}.
\subsection{{\DP} and {\VW}}
\subsection{Available algorithms}
\subsubsection{{\DP}, {\VW} and Chaikin's}
{\DP} \cite{douglas1973algorithms} and {\VW} \cite{visvalingam1993line} are
"classical" line generalization computer graphics algorithms. They are
@ -144,13 +146,8 @@ algorithm \cite{chaikin1974algorithm} via
\href{https://postgis.net/docs/ST_ChaikinSmoothing.html}{PostGIS
ChaikinSmoothing}.
Even though {\DP} and {\VW} are simple to understand and computationally
efficient, they have serious deficiencies for cartographic natural line
generalization.
<TODO: expand on deficiencies>
\subsection{Modern approaches}
\subsubsection{Modern approaches}
Due to their simplicity and ubiquity, {\DP} and {\VW} have been established as
go-to algorithms for line generalization. During recent years, alternatives
@ -184,7 +181,7 @@ open-source tools is an important foundation for future cartographic
experimentation and development, thus it it benefits the cartographic society
as a whole.
<TODO: expand on each paper>
\subsection{Problematic with generalization of rivers}
\section{Methodology}
\label{sec:methodology}

9
IV/tests-integration.sql
View File

@ -1,18 +1,13 @@
\i wm.sql
drop table if exists wm_debug;
create table wm_debug(stage text, name text, gen bigint, nbend bigint, way geometry, props jsonb);
do $$
declare
npoints bigint;
secs bigint;
begin
select * from ST_SimplifyWM_Estimate((select st_union(way) from agg_rivers)) into npoints, secs;
select * from ST_SimplifyWM_Estimate((select st_union(way) from wm_rivers)) into npoints, secs;
raise notice 'Total points: %', npoints;
raise notice 'Expected duration: %s (+-%s)', ceil(secs), floor(secs*.5);
end $$ language plpgsql;
drop table if exists wm_demo;
create table wm_demo (name text, i bigint, way geometry);
insert into wm_demo (name, way) select name, ST_SimplifyWM(way, name) from agg_rivers;
insert into wm_demo (name, way) select name, ST_SimplifyWM(way, name) from wm_rivers;

9
IV/tests.sql
View File

@ -26,11 +26,6 @@ begin
end
$$ language plpgsql;
-- to preview this somewhat conveniently in QGIS:
-- stage || '_' || name || ' gen:' || coalesce(gen, 'Ø') || ' nbend:' || lpad(nbend, 4, '0')
drop table if exists wm_debug;
create table wm_debug(stage text, name text, gen bigint, nbend bigint, way geometry, props jsonb);
drop table if exists wm_figures;
create table wm_figures (name text, way geometry);
-- add fig8.gpkg to postgis:
@ -56,10 +51,6 @@ insert into wm_figures (name, way) values ('multi-island',ST_GeomFromText('MULTI
insert into wm_figures (name, way) values ('selfcrossing-1',ST_GeomFromText('LINESTRING(-27 180,-20 166,-21 142,-18 136,55 136,55 136,71 145,44 165,37 146,22 145,14 164,11 164,3 146,-12 146,-13 176,-18 184)'));
insert into wm_figures (name, way) values ('selfcrossing-1-rev',ST_Reverse(ST_Translate((select way from wm_figures where name='selfcrossing-1'), 0, 60)));
-- Run ST_SimplifyWM in debug mode, so `wm_debug` is populated. That table
-- is used for geometric assertions later in the file.
drop table if exists wm_demo;
create table wm_demo (name text, i bigint, way geometry);
insert into wm_demo (name, way) select name, ST_SimplifyWM(way, name) from wm_figures;
-- wm_visuals holds visual aids for the paper.