From 56a5fc8a4d63931ff65a7fbc2a0ada604e048961 Mon Sep 17 00:00:00 2001 From: Jochen Topf Date: Sun, 8 Feb 2026 11:28:48 +0100 Subject: [PATCH] Add option for diff expire The "expire" configuration for a geometry column in the table definition now has an option "diff_expire", which is set to "false" by default but can be set to "true" to get "diff expire", which means expire will be based on the symmetrical difference between old and new geometry. Note that "diff expire" will only be used for ways and relations that change based on their members. If a way or relation is changed itself, we always do a complete expire. This is needed, because we don't know what changes there are (the tags or attributes could have changed, or even the geometry type). Internally this is implemented by not calculating expire tiles first for the "delete" of the old geometry and then for the "add" of the new geometry, but by writing old and new geometries for each object in a cache and then, after we have collected all geometries, calculating the symmetrical differences between those geometries. --- src/expire-config.hpp | 3 + src/flex-lua-table.cpp | 9 + src/flex-table-column.cpp | 140 ++++++++++- src/flex-table-column.hpp | 45 +++- src/flex-write.cpp | 13 +- src/flex-write.hpp | 7 +- src/output-flex.cpp | 52 +++-- src/output-flex.hpp | 10 + tests/bdd/flex/expire-diff.feature | 361 +++++++++++++++++++++++++++++ tests/bdd/flex/lua-expire.feature | 42 ++++ 10 files changed, 644 insertions(+), 38 deletions(-) create mode 100644 tests/bdd/flex/expire-diff.feature diff --git a/src/expire-config.hpp b/src/expire-config.hpp index c4ae83b49..e6acf043c 100644 --- a/src/expire-config.hpp +++ b/src/expire-config.hpp @@ -43,6 +43,9 @@ struct expire_config_t /// Expire mode. expire_mode mode = expire_mode::full_area; + /// Do expire based on symmetric difference of old and new geometry + bool diff_expire = false; + }; // struct expire_config_t #endif // OSM2PGSQL_EXPIRE_CONFIG_HPP diff --git a/src/flex-lua-table.cpp b/src/flex-lua-table.cpp index abb55ed1f..b88c58261 100644 --- a/src/flex-lua-table.cpp +++ b/src/flex-lua-table.cpp @@ -310,6 +310,15 @@ void parse_and_set_expire_options(lua_State *lua_state, } lua_pop(lua_state, 1); // "buffer" + lua_getfield(lua_state, -1, "diff_expire"); + if (lua_isboolean(lua_state, -1)) { + config.diff_expire = lua_toboolean(lua_state, -1); + } else if (!lua_isnil(lua_state, -1)) { + throw std::runtime_error{ + "Optional expire field 'diff_expire' must contain a boolean."}; + } + lua_pop(lua_state, 1); // "diff_expire" + // Actually add the expire only if we are in append mode. if (append_mode) { column->add_expire(config); diff --git a/src/flex-table-column.cpp b/src/flex-table-column.cpp index 205c44f09..90911eaf6 100644 --- a/src/flex-table-column.cpp +++ b/src/flex-table-column.cpp @@ -10,6 +10,8 @@ #include "flex-table-column.hpp" #include "format.hpp" +#include "geom-boost-adaptor.hpp" +#include "overloaded.hpp" #include "pgsql-capabilities.hpp" #include "projection.hpp" #include "util.hpp" @@ -208,18 +210,146 @@ void flex_table_column_t::add_expire(expire_config_t const &config) m_expires.push_back(config); } +namespace { + +/** + * This expires all geometries in "geoms" by themselves. Used when we don't + * need diff expire. + */ +void separate_expire(std::vector const &geoms, + expire_config_t const &expire_config, + expire_tiles_t &expire_tiles, + std::vector *expire_outputs) +{ + assert(expire_outputs); + + for (auto const &geom : geoms) { + expire_tiles.from_geometry(geom, expire_config); + } + expire_tiles.commit_tiles(&expire_outputs->at(expire_config.expire_output)); +} + +/** + * When doing diff expire, we need to calculate the symmetric difference + * between old and new geometries. The difference is done by type, so points + * are compared with points, linestrings with linestrings, etc. This function + * separates out the input geometries by the three fundamental types. + */ +// NOLINTBEGIN(cppcoreguidelines-rvalue-reference-param-not-moved) +template +void classify_geometries(T input_geoms, geom::multipoint_t *points, + geom::multilinestring_t *linestrings, + geom::multipolygon_t *polygons) +{ + assert(points); + assert(linestrings); + assert(polygons); + + for (auto &&geom : *input_geoms) { + visit(overloaded{ + [&](geom::nullgeom_t && /*input*/) {}, + [&](geom::point_t &&input) { points->add_geometry(input); }, + [&](geom::linestring_t &&input) { + linestrings->add_geometry(std::move(input)); + }, + [&](geom::polygon_t &&input) { + polygons->add_geometry(std::move(input)); + }, + [&](geom::multipoint_t &&input) { + for (auto &&point : input) { + points->add_geometry(point); + } + }, + [&](geom::multilinestring_t &&input) { + for (auto &&linestring : input) { + linestrings->add_geometry(std::move(linestring)); + } + }, + [&](geom::multipolygon_t &&input) { + for (auto &&polygon : input) { + polygons->add_geometry(std::move(polygon)); + } + }, + [&](geom::collection_t &&input) { + classify_geometries(&input, points, linestrings, + polygons); + }}, + std::move(geom)); + } +} +// NOLINTEND(cppcoreguidelines-rvalue-reference-param-not-moved) + +template +void diff_and_expire(geom::multigeometry_t const &old_geoms, + geom::multigeometry_t const &new_geoms, + expire_config_t const &expire_config, + expire_tiles_t &expire_tiles) +{ + std::vector diffs; + boost::geometry::sym_difference(old_geoms, new_geoms, diffs); + for (auto const &geom : diffs) { + expire_tiles.from_geometry(geom, expire_config); + } +} + +void diff_expire(std::vector *geoms_old, + std::vector *geoms_new, + expire_config_t const &expire_config, + expire_tiles_t &expire_tiles, + std::vector *expire_outputs) +{ + assert(geoms_old); + assert(geoms_new); + assert(expire_outputs); + + geom::multipoint_t old_points; + geom::multilinestring_t old_linestrings; + geom::multipolygon_t old_polygons; + + classify_geometries(geoms_old, &old_points, &old_linestrings, + &old_polygons); + + geom::multipoint_t new_points; + geom::multilinestring_t new_linestrings; + geom::multipolygon_t new_polygons; + + classify_geometries(geoms_new, &new_points, &new_linestrings, + &new_polygons); + + diff_and_expire(old_points, new_points, expire_config, expire_tiles); + diff_and_expire(old_linestrings, new_linestrings, expire_config, + expire_tiles); + diff_and_expire(old_polygons, new_polygons, expire_config, expire_tiles); + + expire_tiles.commit_tiles(&expire_outputs->at(expire_config.expire_output)); +} + +} // anonymous namespace + void flex_table_column_t::do_expire( - geom::geometry_t const &geom, std::vector *expire, - std::vector *expire_outputs) const + std::vector *geoms_old, + std::vector *geoms_new, + std::vector *expire, + std::vector *expire_outputs, bool enable_diff_expire) const { + assert(geoms_old); + assert(geoms_new); assert(expire); assert(expire_outputs); for (auto const &expire_config : m_expires) { assert(expire_config.expire_output < expire->size()); auto &expire_tiles = expire->at(expire_config.expire_output); - expire_tiles.from_geometry(geom, expire_config); - expire_tiles.commit_tiles( - &expire_outputs->at(expire_config.expire_output)); + + if (!expire_config.diff_expire || !enable_diff_expire || + geoms_old->empty() || geoms_new->empty()) { + separate_expire(*geoms_old, expire_config, expire_tiles, + expire_outputs); + separate_expire(*geoms_new, expire_config, expire_tiles, + expire_outputs); + } else { + diff_expire(geoms_old, geoms_new, expire_config, expire_tiles, + expire_outputs); + } } } diff --git a/src/flex-table-column.hpp b/src/flex-table-column.hpp index 558ce29d0..cb9d367ad 100644 --- a/src/flex-table-column.hpp +++ b/src/flex-table-column.hpp @@ -18,6 +18,7 @@ #include #include #include +#include #include enum class table_column_type : uint8_t @@ -54,6 +55,8 @@ enum class table_column_type : uint8_t id_num }; +class geometry_cache_t; + /** * A column in a flex_table_t. */ @@ -134,9 +137,11 @@ class flex_table_column_t return m_expires; } - void do_expire(geom::geometry_t const &geom, + void do_expire(std::vector *geoms_old, + std::vector *geoms_new, std::vector *expire, - std::vector *expire_outputs) const; + std::vector *expire_outputs, + bool enable_diff_expire) const; private: std::vector m_expires; @@ -170,6 +175,40 @@ class flex_table_column_t /// Column will be created but not filled by osm2pgsql. bool m_create_only = false; -}; +}; // class flex_table_column_t + +/** + * While processing an OSM object, this cache is used to hold all old and all + * new geometries stored in all geometry columns with expire config in a table. + * Later those geometries are used to calculate the expire. + */ +class geometry_cache_t +{ +public: + template + void add_old(flex_table_column_t const *column, GEOM &&geom) + { + m_geometries[column].first.push_back(std::forward(geom)); + } + + template + void add_new(flex_table_column_t const *column, GEOM &&geom) + { + m_geometries[column].second.push_back(std::forward(geom)); + } + + auto begin() noexcept { return m_geometries.begin(); } + + auto end() noexcept { return m_geometries.end(); } + + void clear() { m_geometries.clear(); } + +private: + std::unordered_map< + flex_table_column_t const *, + std::pair, std::vector>> + m_geometries; + +}; // class geometry_cache_t #endif // OSM2PGSQL_FLEX_TABLE_COLUMN_HPP diff --git a/src/flex-write.cpp b/src/flex-write.cpp index f507eafe5..3e0883859 100644 --- a/src/flex-write.cpp +++ b/src/flex-write.cpp @@ -8,6 +8,7 @@ */ #include "flex-lua-geom.hpp" +#include "flex-table-column.hpp" #include "flex-write.hpp" #include "geom-functions.hpp" #include "json-writer.hpp" @@ -257,11 +258,9 @@ bool is_compatible(geom::geometry_t const &geom, } // anonymous namespace -void flex_write_column(lua_State *lua_state, +void flex_write_column(lua_State *lua_state, geometry_cache_t *geom_cache, db_copy_mgr_t *copy_mgr, - flex_table_column_t const &column, - std::vector *expire, - std::vector *expire_outputs) + flex_table_column_t const &column) { lua_getfield(lua_state, -1, column.name().c_str()); int const ltype = lua_type(lua_state, -1); @@ -446,13 +445,13 @@ void flex_write_column(lua_State *lua_state, type == table_column_type::multilinestring || type == table_column_type::multipolygon); if (geom->srid() == column.srid()) { - column.do_expire(*geom, expire, expire_outputs); copy_mgr->add_hex_geom(geom_to_ewkb(*geom, wrap_multi)); + geom_cache->add_new(&column, *geom); } else { auto const &proj = get_projection(column.srid()); - auto const tgeom = geom::transform(*geom, proj); - column.do_expire(tgeom, expire, expire_outputs); + auto tgeom = geom::transform(*geom, proj); copy_mgr->add_hex_geom(geom_to_ewkb(tgeom, wrap_multi)); + geom_cache->add_new(&column, std::move(tgeom)); } } else { write_null(copy_mgr, column); diff --git a/src/flex-write.hpp b/src/flex-write.hpp index bef1b45c0..dc2cb79cc 100644 --- a/src/flex-write.hpp +++ b/src/flex-write.hpp @@ -19,6 +19,7 @@ #include class expire_tiles_t; +class geometry_cache_t; class not_null_exception_t : public std::runtime_error { @@ -34,10 +35,8 @@ class not_null_exception_t : public std::runtime_error flex_table_column_t const *m_column; }; // class not_null_exception_t -void flex_write_column(lua_State *lua_state, +void flex_write_column(lua_State *lua_state, geometry_cache_t *geom_cache, db_copy_mgr_t *copy_mgr, - flex_table_column_t const &column, - std::vector *expire, - std::vector *expire_outputs); + flex_table_column_t const &column); #endif // OSM2PGSQL_FLEX_WRITE_HPP diff --git a/src/output-flex.cpp b/src/output-flex.cpp index b8a41fbed..379e111d6 100644 --- a/src/output-flex.cpp +++ b/src/output-flex.cpp @@ -810,8 +810,8 @@ int output_flex_t::table_insert() } else if (column.type() == table_column_type::id_num) { copy_mgr->add_column(id); } else { - flex_write_column(lua_state(), copy_mgr, column, - &m_expire_tiles, m_expire_outputs.get()); + flex_write_column(lua_state(), &m_geometry_cache, copy_mgr, + column); } } table_connection.increment_insert_counter(); @@ -927,6 +927,7 @@ void output_flex_t::pending_way(osmid_t id) if (func) { get_mutex_and_call_lua_function(func, m_way_cache.get()); } + expire_geoms_from_cache(true); } void output_flex_t::select_relation_members() @@ -997,6 +998,7 @@ void output_flex_t::pending_relation(osmid_t id) select_relation_members(); delete_from_tables(osmium::item_type::relation, id); process_relation(); + expire_geoms_from_cache(true); } void output_flex_t::pending_relation_stage1c(osmid_t id) @@ -1121,13 +1123,13 @@ void output_flex_t::node_add(osmium::Node const &node) auto const &func = node.tags().empty() ? m_process_untagged_node : m_process_node; - if (!func) { - return; + if (func) { + m_context_node = &node; + get_mutex_and_call_lua_function(func, node); + m_context_node = nullptr; } - m_context_node = &node; - get_mutex_and_call_lua_function(func, node); - m_context_node = nullptr; + expire_geoms_from_cache(); } void output_flex_t::way_add(osmium::Way *way) @@ -1137,12 +1139,12 @@ void output_flex_t::way_add(osmium::Way *way) auto const &func = way->tags().empty() ? m_process_untagged_way : m_process_way; - if (!func) { - return; + if (func) { + m_way_cache.init(way); + get_mutex_and_call_lua_function(func, m_way_cache.get()); } - m_way_cache.init(way); - get_mutex_and_call_lua_function(func, m_way_cache.get()); + expire_geoms_from_cache(); } void output_flex_t::relation_add(osmium::Relation const &relation) @@ -1150,13 +1152,13 @@ void output_flex_t::relation_add(osmium::Relation const &relation) auto const &func = relation.tags().empty() ? m_process_untagged_relation : m_process_relation; - if (!func) { - return; + if (func) { + m_relation_cache.init(relation); + select_relation_members(); + get_mutex_and_call_lua_function(func, relation); } - m_relation_cache.init(relation); - select_relation_members(); - get_mutex_and_call_lua_function(func, relation); + expire_geoms_from_cache(); } void output_flex_t::delete_from_table(table_connection_t *table_connection, @@ -1175,9 +1177,8 @@ void output_flex_t::delete_from_table(table_connection_t *table_connection, for (auto const &column : table_connection->table().columns()) { if (column.has_expire()) { for (int i = 0; i < num_tuples; ++i) { - auto const geom = ewkb_to_geom(result.get(i, col)); - column.do_expire(geom, &m_expire_tiles, - m_expire_outputs.get()); + m_geometry_cache.add_old( + &column, ewkb_to_geom(result.get(i, col))); } ++col; } @@ -1206,6 +1207,7 @@ void output_flex_t::node_delete(osmium::Node const &node) } node_delete(node.id()); + expire_geoms_from_cache(); } void output_flex_t::way_delete(osmium::Way *way) @@ -1217,6 +1219,7 @@ void output_flex_t::way_delete(osmium::Way *way) } way_delete(way->id()); + expire_geoms_from_cache(); } void output_flex_t::relation_delete(osmium::Relation const &rel) @@ -1227,6 +1230,7 @@ void output_flex_t::relation_delete(osmium::Relation const &rel) } relation_delete(rel.id()); + expire_geoms_from_cache(); } /* Delete is easy, just remove all traces of this object. We don't need to @@ -1488,6 +1492,16 @@ void output_flex_t::init_lua(std::string const &filename, lua_remove(lua_state(), 1); // global "osm2pgsql" } +void output_flex_t::expire_geoms_from_cache(bool enable_diff_expire) +{ + for (auto &[column, geoms] : m_geometry_cache) { + column->do_expire(&geoms.first, &geoms.second, &m_expire_tiles, + m_expire_outputs.get(), enable_diff_expire); + } + + m_geometry_cache.clear(); +} + idlist_t const &output_flex_t::get_marked_node_ids() { if (m_stage2_node_ids->empty()) { diff --git a/src/output-flex.hpp b/src/output-flex.hpp index 7bbb6bef3..791aa0f1a 100644 --- a/src/output-flex.hpp +++ b/src/output-flex.hpp @@ -215,6 +215,13 @@ class output_flex_t : public output_t void delete_from_tables(osmium::item_type type, osmid_t osm_id); + /** + * Actually do expire from the geometries in the cache. Diff expire is + * only enabled in stage 1c, because we are only sure then that only the + * geometry changed and nothing else. + */ + void expire_geoms_from_cache(bool enable_diff_expire = false); + lua_State *lua_state() noexcept { return m_lua_state.get(); } void create_id_cache(flex_table_t const &table) @@ -307,6 +314,9 @@ class output_flex_t : public output_t // accessed while protected using the lua_mutex. std::shared_ptr m_lua_state; + // Caches for old and new geometries from a single OSM object + geometry_cache_t m_geometry_cache; + std::vector m_expire_tiles; way_cache_t m_way_cache; diff --git a/tests/bdd/flex/expire-diff.feature b/tests/bdd/flex/expire-diff.feature new file mode 100644 index 000000000..28f2d6254 --- /dev/null +++ b/tests/bdd/flex/expire-diff.feature @@ -0,0 +1,361 @@ +Feature: Diff expire + + Scenario: non-diff expire way node changes with diff_expire disabled + Given the OSM data + """ + n1 v1 x0 y0 + n2 v1 x2 y0 + n3 v1 x2 y1 + n4 v1 x4 y1 + w1 v1 Thighway=primary Nn1,n2,n3,n4 + """ + And the lua style + """ + local eo = osm2pgsql.define_expire_output({ + table = 'osm2pgsql_test_expire', + maxzoom = 8, + }) + + local the_table = osm2pgsql.define_way_table('osm2pgsql_test', { + { column = 'geom', type = 'linestring', expire = { + { output = eo, diff_expire = false } + } + }, + }) + + function osm2pgsql.process_way(object) + the_table:insert{ + geom = object:as_linestring() + } + end + """ + When running osm2pgsql flex with parameters + | --slim | -c | + Then table osm2pgsql_test contains exactly + | way_id | geom!geo | + | 1 | 0 0,222638.98158654713 0,222638.98158654713 111325.14285463623,445277.96317309426 111325.14285463623 | + Then table osm2pgsql_test_expire has 0 rows + + Given the OSM data + """ + n2 v2 x0 y1 + """ + When running osm2pgsql flex with parameters + | --slim | -a | + Then table osm2pgsql_test contains exactly + | way_id | geom!geo | + | 1 | 0 0,0 111325.14285463623,222638.98158654713 111325.14285463623,445277.96317309426 111325.14285463623 | + Then table osm2pgsql_test_expire contains exactly + | zoom | x | y | + | 8 | 127 | 127 | + | 8 | 128 | 127 | + | 8 | 129 | 127 | + | 8 | 130 | 127 | + | 8 | 127 | 128 | + | 8 | 128 | 128 | + | 8 | 129 | 128 | + + Scenario: non-diff expire if way changes + Given the OSM data + """ + n1 v1 x0 y0 + n2 v1 x2 y0 + n3 v1 x2 y1 + n4 v1 x4 y1 + w1 v1 Thighway=primary Nn1,n2,n3,n4 + """ + And the lua style + """ + local eo = osm2pgsql.define_expire_output({ + table = 'osm2pgsql_test_expire', + maxzoom = 8, + }) + + local the_table = osm2pgsql.define_way_table('osm2pgsql_test', { + { column = 'geom', type = 'linestring', expire = { + { output = eo, diff_expire = true } + } + }, + }) + + function osm2pgsql.process_way(object) + the_table:insert{ + geom = object:as_linestring() + } + end + """ + When running osm2pgsql flex with parameters + | --slim | -c | + Then table osm2pgsql_test contains exactly + | way_id | geom!geo | + | 1 | 0 0,222638.98158654713 0,222638.98158654713 111325.14285463623,445277.96317309426 111325.14285463623 | + Then table osm2pgsql_test_expire has 0 rows + + Given the OSM data + """ + w1 v2 Thighway=secondary Nn1,n2,n3,n4 + """ + When running osm2pgsql flex with parameters + | --slim | -a | + Then table osm2pgsql_test contains exactly + | way_id | geom!geo | + | 1 | 0 0,222638.98158654713 0,222638.98158654713 111325.14285463623,445277.96317309426 111325.14285463623 | + Then table osm2pgsql_test_expire contains exactly + | zoom | x | y | + | 8 | 127 | 127 | + | 8 | 128 | 127 | + | 8 | 129 | 127 | + | 8 | 130 | 127 | + | 8 | 127 | 128 | + | 8 | 128 | 128 | + | 8 | 129 | 128 | + + Scenario: diff expire way node changes with diff_expire enabled + Given the OSM data + """ + n1 v1 x0 y0 + n2 v1 x2 y0 + n3 v1 x2 y1 + n4 v1 x4 y1 + w1 v1 Thighway=primary Nn1,n2,n3,n4 + """ + And the lua style + """ + local eo = osm2pgsql.define_expire_output({ + table = 'osm2pgsql_test_expire', + maxzoom = 8, + }) + + local the_table = osm2pgsql.define_way_table('osm2pgsql_test', { + { column = 'geom', type = 'linestring', expire = { + { output = eo, diff_expire = true } + } + }, + }) + + function osm2pgsql.process_way(object) + the_table:insert{ + geom = object:as_linestring() + } + end + """ + When running osm2pgsql flex with parameters + | --slim | -c | + Then table osm2pgsql_test contains exactly + | way_id | geom!geo | + | 1 | 0 0,222638.98158654713 0,222638.98158654713 111325.14285463623,445277.96317309426 111325.14285463623 | + Then table osm2pgsql_test_expire has 0 rows + + Given the OSM data + """ + n2 v2 x0 y1 + """ + When running osm2pgsql flex with parameters + | --slim | -a | + Then table osm2pgsql_test contains exactly + | way_id | geom!geo | + | 1 | 0 0,0 111325.14285463623,222638.98158654713 111325.14285463623,445277.96317309426 111325.14285463623 | + Then table osm2pgsql_test_expire contains exactly + | zoom | x | y | + | 8 | 127 | 127 | + | 8 | 128 | 127 | + | 8 | 129 | 127 | + | 8 | 127 | 128 | + | 8 | 128 | 128 | + | 8 | 129 | 128 | + + Scenario: non-diff expire of relation when way changes with diff_expire disabled + Given the OSM data + """ + n11 v1 x0 y0 + n12 v1 x1 y0 + n13 v1 x1 y1 + n14 v1 x0 y1 + n15 v1 x2 y2 + n16 v1 x3 y2 + n17 v1 x3 y3 + n18 v1 x2 y3 + w20 v1 Nn11,n12,n13,n14,n11 + w21 v1 Nn15,n16,n17,n18,n15 + r30 v1 Ttype=multipolygon Mw20@,w21@ + """ + And the lua style + """ + local eo = osm2pgsql.define_expire_output({ + table = 'osm2pgsql_test_expire', + maxzoom = 8, + }) + + local the_table = osm2pgsql.define_relation_table('osm2pgsql_test', { + { column = 'geom', type = 'polygon', expire = { + { output = eo, diff_expire = false } + } + }, + }) + + function osm2pgsql.process_relation(object) + for geom in object:as_multipolygon():geometries() do + the_table:insert{ + geom = geom + } + end + end + """ + When running osm2pgsql flex with parameters + | --slim | -c | + Then table osm2pgsql_test contains exactly + | relation_id | geom!geo | + | 30 | (0 0,111319.49079327357 0,111319.49079327357 111325.14285463623,0 111325.14285463623,0 0) | + | 30 | (222638.98158654713 222684.20848178727,333958.4723798207 222684.20848178727,333958.4723798207 334111.17136656796,222638.98158654713 334111.17136656796,222638.98158654713 222684.20848178727) | + Then table osm2pgsql_test_expire has 0 rows + + Given the OSM data + """ + w21 v2 Nn15,n16,n17,n15 + """ + When running osm2pgsql flex with parameters + | --slim | -a | + Then table osm2pgsql_test contains exactly + | relation_id | geom!geo | + | 30 | (0 0,111319.49079327357 0,111319.49079327357 111325.14285463623,0 111325.14285463623,0 0) | + | 30 | (222638.98158654713 222684.20848178727,333958.4723798207 222684.20848178727,333958.4723798207 334111.17136656796,222638.98158654713 222684.20848178727) | + Then table osm2pgsql_test_expire contains exactly + | zoom | x | y | + | 8 | 127 | 127 | + | 8 | 128 | 127 | + | 8 | 127 | 128 | + | 8 | 128 | 128 | + | 8 | 129 | 125 | + | 8 | 130 | 125 | + | 8 | 129 | 126 | + | 8 | 130 | 126 | + + Scenario: non-diff expire of relation when relation changes + Given the OSM data + """ + n11 v1 x0 y0 + n12 v1 x1 y0 + n13 v1 x1 y1 + n14 v1 x0 y1 + n15 v1 x2 y2 + n16 v1 x3 y2 + n17 v1 x3 y3 + n18 v1 x2 y3 + w20 v1 Nn11,n12,n13,n14,n11 + w21 v1 Nn15,n16,n17,n18,n15 + r30 v1 Ttype=multipolygon Mw20@,w21@ + """ + And the lua style + """ + local eo = osm2pgsql.define_expire_output({ + table = 'osm2pgsql_test_expire', + maxzoom = 8, + }) + + local the_table = osm2pgsql.define_relation_table('osm2pgsql_test', { + { column = 'geom', type = 'polygon', expire = { + { output = eo, diff_expire = true } + } + }, + }) + + function osm2pgsql.process_relation(object) + for geom in object:as_multipolygon():geometries() do + the_table:insert{ + geom = geom + } + end + end + """ + When running osm2pgsql flex with parameters + | --slim | -c | + Then table osm2pgsql_test contains exactly + | relation_id | geom!geo | + | 30 | (0 0,111319.49079327357 0,111319.49079327357 111325.14285463623,0 111325.14285463623,0 0) | + | 30 | (222638.98158654713 222684.20848178727,333958.4723798207 222684.20848178727,333958.4723798207 334111.17136656796,222638.98158654713 334111.17136656796,222638.98158654713 222684.20848178727) | + Then table osm2pgsql_test_expire has 0 rows + + Given the OSM data + """ + r30 v2 Ttype=multipolygon,landuse=forest Mw20@,w21@ + """ + When running osm2pgsql flex with parameters + | --slim | -a | + Then table osm2pgsql_test contains exactly + | relation_id | geom!geo | + | 30 | (0 0,111319.49079327357 0,111319.49079327357 111325.14285463623,0 111325.14285463623,0 0) | + | 30 | (222638.98158654713 222684.20848178727,333958.4723798207 222684.20848178727,333958.4723798207 334111.17136656796,222638.98158654713 334111.17136656796,222638.98158654713 222684.20848178727) | + Then table osm2pgsql_test_expire contains exactly + | zoom | x | y | + | 8 | 127 | 127 | + | 8 | 128 | 127 | + | 8 | 127 | 128 | + | 8 | 128 | 128 | + | 8 | 129 | 125 | + | 8 | 130 | 125 | + | 8 | 129 | 126 | + | 8 | 130 | 126 | + + Scenario: non-diff expire of relation when way changes with diff_expire enabled + Given the OSM data + """ + n11 v1 x0 y0 + n12 v1 x1 y0 + n13 v1 x1 y1 + n14 v1 x0 y1 + n15 v1 x2 y2 + n16 v1 x3 y2 + n17 v1 x3 y3 + n18 v1 x2 y3 + w20 v1 Nn11,n12,n13,n14,n11 + w21 v1 Nn15,n16,n17,n18,n15 + r30 Ttype=multipolygon Mw20@,w21@ + """ + And the lua style + """ + local eo = osm2pgsql.define_expire_output({ + table = 'osm2pgsql_test_expire', + maxzoom = 8, + }) + + local the_table = osm2pgsql.define_relation_table('osm2pgsql_test', { + { column = 'geom', type = 'polygon', expire = { + { output = eo, diff_expire = true } + } + }, + }) + + function osm2pgsql.process_relation(object) + for geom in object:as_multipolygon():geometries() do + the_table:insert{ + geom = geom + } + end + end + """ + When running osm2pgsql flex with parameters + | --slim | -c | + Then table osm2pgsql_test has 2 rows + Then table osm2pgsql_test contains exactly + | relation_id | geom!geo | + | 30 | (0 0,111319.49079327357 0,111319.49079327357 111325.14285463623,0 111325.14285463623,0 0) | + | 30 | (222638.98158654713 222684.20848178727,333958.4723798207 222684.20848178727,333958.4723798207 334111.17136656796,222638.98158654713 334111.17136656796,222638.98158654713 222684.20848178727) | + Then table osm2pgsql_test_expire has 0 rows + + Given the OSM data + """ + w21 v2 Nn15,n16,n17,n15 + """ + When running osm2pgsql flex with parameters + | --slim | -a | + Then table osm2pgsql_test contains exactly + | relation_id | geom!geo | + | 30 | (0 0,111319.49079327357 0,111319.49079327357 111325.14285463623,0 111325.14285463623,0 0) | + | 30 | (222638.98158654713 222684.20848178727,333958.4723798207 222684.20848178727,333958.4723798207 334111.17136656796,222638.98158654713 222684.20848178727) | + Then table osm2pgsql_test_expire contains exactly + | zoom | x | y | + | 8 | 129 | 125 | + | 8 | 130 | 125 | + | 8 | 129 | 126 | + | 8 | 130 | 126 | + diff --git a/tests/bdd/flex/lua-expire.feature b/tests/bdd/flex/lua-expire.feature index fbff960ba..de4f25be3 100644 --- a/tests/bdd/flex/lua-expire.feature +++ b/tests/bdd/flex/lua-expire.feature @@ -220,6 +220,48 @@ Feature: Expire configuration in Lua file When running osm2pgsql flex Then table bar has 1562 rows + Scenario: Expire with diff_expire that's not a boolean fails + Given the input file 'liechtenstein-2013-08-03.osm.pbf' + And the lua style + """ + local eo = osm2pgsql.define_expire_output({ + filename = 'bar', + maxzoom = 12 + }) + osm2pgsql.define_node_table('bar', { + { column = 'some', + type = 'geometry', + expire = { + { output = eo, diff_expire = 'foo' } + }} + }) + """ + When running osm2pgsql flex + Then execution fails + And the error output contains + """ + Optional expire field 'diff_expire' must contain a boolean. + """ + + Scenario: Expire with diff_expire that's a boolean is okay + Given the input file 'liechtenstein-2013-08-03.osm.pbf' + And the lua style + """ + local eo = osm2pgsql.define_expire_output({ + filename = 'bar', + maxzoom = 12 + }) + osm2pgsql.define_node_table('bar', { + { column = 'some', + type = 'geometry', + expire = { + { output = eo, diff_expire = true } + }} + }) + """ + When running osm2pgsql flex + Then execution is successful + Scenario: Expire into table is okay Given the input file 'liechtenstein-2013-08-03.osm.pbf' And the lua style