Libosmium  2.22.0
Fast and flexible C++ library for working with OpenStreetMap data
Loading...
Searching...
No Matches
collector.hpp
Go to the documentation of this file.
1#ifndef OSMIUM_RELATIONS_COLLECTOR_HPP
2#define OSMIUM_RELATIONS_COLLECTOR_HPP
3
4/*
5
6This file is part of Osmium (https://osmcode.org/libosmium).
7
8Copyright 2013-2025 Jochen Topf <jochen@topf.org> and others (see README).
9
10Boost Software License - Version 1.0 - August 17th, 2003
11
12Permission is hereby granted, free of charge, to any person or organization
13obtaining a copy of the software and accompanying documentation covered by
14this license (the "Software") to use, reproduce, display, distribute,
15execute, and transmit the Software, and to prepare derivative works of the
16Software, and to permit third-parties to whom the Software is furnished to
17do so, all subject to the following:
18
19The copyright notices in the Software and this entire statement, including
20the above license grant, this restriction and the following disclaimer,
21must be included in all copies of the Software, in whole or in part, and
22all derivative works of the Software, unless such copies or derivative
23works are solely in the form of machine-executable object code generated by
24a source language processor.
25
26THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
29SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
30FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
31ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
32DEALINGS IN THE SOFTWARE.
33
34*/
35
36#include <osmium/handler.hpp>
40#include <osmium/osm/object.hpp>
42#include <osmium/osm/types.hpp>
43#include <osmium/relations/detail/member_meta.hpp>
44#include <osmium/relations/detail/relation_meta.hpp>
46#include <osmium/visitor.hpp>
47
48#include <algorithm>
49#include <array>
50#include <cassert>
51#include <cstddef>
52#include <cstdint>
53#include <functional>
54#include <iomanip>
55#include <iostream>
56#include <utility>
57#include <vector>
58
59namespace osmium {
60
61 class Node;
62 class Way;
63
67 namespace relations {
68
97 template <typename TCollector, bool TNodes, bool TWays, bool TRelations>
98 class Collector {
99
104
105 TCollector& m_collector;
106
107 public:
108
109 explicit HandlerPass1(TCollector& collector) noexcept :
110 m_collector(collector) {
111 }
112
114 if (m_collector.keep_relation(relation)) {
115 m_collector.add_relation(relation);
116 }
117 }
118
119 }; // class HandlerPass1
120
121 public:
122
127
129 TCollector& m_collector;
130
131 public:
132
133 explicit HandlerPass2(TCollector& collector) noexcept :
134 m_collector(collector) {
135 }
136
137 void node(const osmium::Node& node) {
138 if (TNodes) {
139 m_check_order.node(node);
140 if (!m_collector.find_and_add_object(node)) {
141 m_collector.node_not_in_any_relation(node);
142 }
143 }
144 }
145
146 void way(const osmium::Way& way) {
147 if (TWays) {
148 m_check_order.way(way);
149 if (!m_collector.find_and_add_object(way)) {
150 m_collector.way_not_in_any_relation(way);
151 }
152 }
153 }
154
156 if (TRelations) {
157 m_check_order.relation(relation);
158 if (!m_collector.find_and_add_object(relation)) {
159 m_collector.relation_not_in_any_relation(relation);
160 }
161 }
162 }
163
164 void flush() {
165 m_collector.flush();
166 }
167
168 }; // class HandlerPass2
169
170 private:
171
172 HandlerPass2 m_handler_pass2;
173
174 // All relations we are interested in will be kept in this buffer
175 osmium::memory::Buffer m_relations_buffer;
176
177 // All members we are interested in will be kept in this buffer
178 osmium::memory::Buffer m_members_buffer;
179
181 std::vector<RelationMeta> m_relations;
182
187 using mm_vector_type = std::vector<MemberMeta>;
188 using mm_iterator = mm_vector_type::iterator;
189 std::array<mm_vector_type, 3> m_member_meta;
190
192
193 using callback_func_type = std::function<void(osmium::memory::Buffer&&)>;
195
196 enum {
197 initial_buffer_size = 1024UL * 1024UL
198 };
199
201 auto& mmv = member_meta(type);
202 return make_range(std::equal_range(mmv.begin(), mmv.end(), MemberMeta(id)));
203 }
204
205 public:
206
211 m_handler_pass2(*static_cast<TCollector*>(this)),
212 m_relations_buffer(initial_buffer_size, osmium::memory::Buffer::auto_grow::yes),
213 m_members_buffer(initial_buffer_size, osmium::memory::Buffer::auto_grow::yes) {
214 }
215
216 protected:
217
218 std::vector<MemberMeta>& member_meta(const item_type type) {
219 return m_member_meta[static_cast<uint16_t>(type) - 1];
220 }
221
225
226 const std::vector<RelationMeta>& relations() const {
227 return m_relations;
228 }
229
239 bool keep_relation(const osmium::Relation& /*relation*/) const {
240 return true;
241 }
242
253 bool keep_member(const RelationMeta& /*relation_meta*/, const osmium::RelationMember& /*member*/) const {
254 return true;
255 }
256
265 }
266
275 }
276
285 }
286
298 void flush() {
299 }
300
301 const osmium::Relation& get_relation(size_t offset) const {
302 assert(m_relations_buffer.committed() > offset);
303 return m_relations_buffer.get<osmium::Relation>(offset);
304 }
305
309 const osmium::Relation& get_relation(const RelationMeta& relation_meta) const {
310 return get_relation(relation_meta.relation_offset());
311 }
312
316 const osmium::Relation& get_relation(const MemberMeta& member_meta) const {
317 return get_relation(m_relations[member_meta.relation_pos()]);
318 }
319
320 osmium::OSMObject& get_member(size_t offset) const {
321 assert(m_members_buffer.committed() > offset);
322 return m_members_buffer.get<osmium::OSMObject>(offset);
323 }
324
325 private:
326
336 const size_t offset = m_relations_buffer.committed();
338
339 RelationMeta relation_meta{offset};
340
341 int n = 0;
342 for (auto& member : m_relations_buffer.get<osmium::Relation>(offset).members()) {
343 if (static_cast<TCollector*>(this)->keep_member(relation_meta, member)) {
344 member_meta(member.type()).emplace_back(member.ref(), m_relations.size(), n);
345 relation_meta.increment_need_members();
346 } else {
347 member.set_ref(0); // set member id to zero to indicate we are not interested
348 }
349 ++n;
350 }
351
352 assert(offset == m_relations_buffer.committed());
353 if (relation_meta.has_all_members()) {
354 m_relations_buffer.rollback();
355 } else {
356 m_relations_buffer.commit();
357 m_relations.push_back(relation_meta);
358 }
359 }
360
366 std::sort(m_member_meta[0].begin(), m_member_meta[0].end());
367 std::sort(m_member_meta[1].begin(), m_member_meta[1].end());
368 std::sort(m_member_meta[2].begin(), m_member_meta[2].end());
369 }
370
372 return std::count_if(range.begin(), range.end(), [](const MemberMeta& mm) {
373 return !mm.removed();
374 });
375 }
376
385 auto range = find_member_meta(object.type(), object.id());
386
387 if (count_not_removed(range) == 0) {
388 // nothing found
389 return false;
390 }
391
392 {
393 members_buffer().add_item(object);
394 const size_t member_offset = members_buffer().commit();
395
396 for (auto& member : range) {
397 member.set_buffer_offset(member_offset);
398 }
399 }
400
401 for (auto& member : range) {
402 if (member.removed()) {
403 break;
404 }
405 assert(member.member_id() == object.id());
406 assert(member.relation_pos() < m_relations.size());
407 RelationMeta& relation_meta = m_relations[member.relation_pos()];
408 assert(member.member_pos() < get_relation(relation_meta).members().size());
409 relation_meta.got_one_member();
410 if (relation_meta.has_all_members()) {
411 const size_t relation_offset = member.relation_pos();
412 static_cast<TCollector*>(this)->complete_relation(relation_meta);
413 clear_member_metas(relation_meta);
414 m_relations[relation_offset] = RelationMeta{};
416 }
417 }
418
419 return true;
420 }
421
422 void clear_member_metas(const RelationMeta& relation_meta) {
423 const osmium::Relation& relation = get_relation(relation_meta);
424 for (const auto& member : relation.members()) {
425 if (member.ref() != 0) {
426 const auto range = find_member_meta(member.type(), member.ref());
427 assert(!range.empty());
428
429 // if this is the last time this object was needed
430 // then mark it as removed
431 if (count_not_removed(range) == 1) {
432 get_member(range.begin()->buffer_offset()).set_removed(true);
433 }
434
435 for (auto& member_meta : range) {
436 if (!member_meta.removed() && relation.id() == get_relation(member_meta).id()) {
437 member_meta.remove();
438 break;
439 }
440 }
441 }
442 }
443 }
444
445 public:
446
447 uint64_t used_memory() const {
448 const uint64_t nmembers = m_member_meta[0].capacity() + m_member_meta[1].capacity() + m_member_meta[2].capacity();
449 const uint64_t members = nmembers * sizeof(MemberMeta);
450 const uint64_t relations_size = m_relations.capacity() * sizeof(RelationMeta);
451 const uint64_t relations_buffer_capacity = m_relations_buffer.capacity();
452 const uint64_t members_buffer_capacity = m_members_buffer.capacity();
453
454 std::cerr << " nR = m_relations.capacity() ........... = " << std::setw(12) << m_relations.capacity() << "\n";
455 std::cerr << " nMN = m_member_meta[NODE].capacity() ... = " << std::setw(12) << m_member_meta[0].capacity() << "\n";
456 std::cerr << " nMW = m_member_meta[WAY].capacity() .... = " << std::setw(12) << m_member_meta[1].capacity() << "\n";
457 std::cerr << " nMR = m_member_meta[RELATION].capacity() = " << std::setw(12) << m_member_meta[2].capacity() << "\n";
458 std::cerr << " nM = m_member_meta[*].capacity() ...... = " << std::setw(12) << nmembers << "\n";
459
460 std::cerr << " sRM = sizeof(RelationMeta) ............. = " << std::setw(12) << sizeof(RelationMeta) << "\n";
461 std::cerr << " sMM = sizeof(MemberMeta) ............... = " << std::setw(12) << sizeof(MemberMeta) << "\n\n";
462
463 std::cerr << " nR * sRM ............................... = " << std::setw(12) << relations_size << "\n";
464 std::cerr << " nM * sMM ............................... = " << std::setw(12) << members << "\n";
465 std::cerr << " relations_buffer_capacity .............. = " << std::setw(12) << relations_buffer_capacity << "\n";
466 std::cerr << " members_buffer_capacity ................ = " << std::setw(12) << members_buffer_capacity << "\n";
467
468 const uint64_t total = relations_size + members + relations_buffer_capacity + members_buffer_capacity;
469
470 std::cerr << " total .................................. = " << std::setw(12) << total << "\n";
471 std::cerr << " =======================================================\n";
472
473 return relations_buffer_capacity + members_buffer_capacity + relations_size + members;
474 }
475
479 HandlerPass2& handler(const callback_func_type& callback = nullptr) {
481 return m_handler_pass2;
482 }
483
484 osmium::memory::Buffer& members_buffer() {
485 return m_members_buffer;
486 }
487
500 const auto range = find_member_meta(type, id);
501 assert(!range.empty());
502 return range.begin()->is_available();
503 }
504
515 const auto range = find_member_meta(type, id);
516 assert(!range.empty());
517 assert(range.begin()->is_available());
518 return range.begin()->buffer_offset();
519 }
520
538 const auto range = find_member_meta(type, id);
539 assert(!range.empty());
540 if (range.begin()->is_available()) {
541 return std::make_pair(true, range.begin()->buffer_offset());
542 }
543 return std::make_pair(false, 0);
544 }
545
546 template <typename TIter>
547 void read_relations(TIter begin, TIter end) {
548 HandlerPass1 handler_pass1{*static_cast<TCollector*>(this)};
549 osmium::apply(begin, end, handler_pass1);
551 }
552
553 template <typename TSource>
554 void read_relations(TSource& source) {
555 using std::begin;
556 using std::end;
557 read_relations(begin(source), end(source));
558 source.close();
559 }
560
561 void moving_in_buffer(size_t old_offset, size_t new_offset) {
562 const osmium::OSMObject& object = m_members_buffer.get<osmium::OSMObject>(old_offset);
563 auto range = find_member_meta(object.type(), object.id());
564 for (auto& member : range) {
565 assert(member.buffer_offset() == old_offset);
566 member.set_buffer_offset(new_offset);
567 }
568 }
569
578 if (m_count_complete > 10000) { // XXX
579// const size_t size_before = m_members_buffer.committed();
580 m_members_buffer.purge_removed(this);
581/*
582 const size_t size_after = m_members_buffer.committed();
583 double percent = static_cast<double>(size_before - size_after);
584 percent /= size_before;
585 percent *= 100;
586 std::cerr << "PURGE (size before=" << size_before << " after=" << size_after << " purged=" << (size_before - size_after) << " / " << static_cast<int>(percent) << "%)\n";
587*/
589 }
590 }
591
600 std::vector<const osmium::Relation*> get_incomplete_relations() const {
601 std::vector<const osmium::Relation*> incomplete_relations;
602 for (const auto& relation_meta : m_relations) {
603 if (!relation_meta.has_all_members()) {
604 incomplete_relations.push_back(&get_relation(relation_meta));
605 }
606 }
607 return incomplete_relations;
608 }
609
610 }; // class Collector
611
612 } // namespace relations
613
614} // namespace osmium
615
616#endif // OSMIUM_RELATIONS_COLLECTOR_HPP
Definition node.hpp:48
Definition object.hpp:64
Definition relation.hpp:56
Definition relation.hpp:161
RelationMemberList & members()
Get a reference to the member list.
Definition relation.hpp:179
Definition way.hpp:72
Definition check_order.hpp:87
Definition handler.hpp:71
void set_removed(const bool removed) noexcept
Definition item.hpp:179
TCollector & m_collector
Definition collector.hpp:105
HandlerPass1(TCollector &collector) noexcept
Definition collector.hpp:109
void relation(const osmium::Relation &relation)
Definition collector.hpp:113
void flush()
Definition collector.hpp:164
osmium::handler::CheckOrder m_check_order
Definition collector.hpp:128
HandlerPass2(TCollector &collector) noexcept
Definition collector.hpp:133
void way(const osmium::Way &way)
Definition collector.hpp:146
TCollector & m_collector
Definition collector.hpp:129
void relation(const osmium::Relation &relation)
Definition collector.hpp:155
void node(const osmium::Node &node)
Definition collector.hpp:137
osmium::memory::Buffer & members_buffer()
Definition collector.hpp:484
const osmium::Relation & get_relation(size_t offset) const
Definition collector.hpp:301
void clear_member_metas(const RelationMeta &relation_meta)
Definition collector.hpp:422
Collector()
Definition collector.hpp:210
HandlerPass2 & handler(const callback_func_type &callback=nullptr)
Definition collector.hpp:479
void sort_member_meta()
Definition collector.hpp:365
void add_relation(const osmium::Relation &relation)
Definition collector.hpp:335
const std::vector< RelationMeta > & relations() const
Definition collector.hpp:226
osmium::OSMObject & get_member(size_t offset) const
Definition collector.hpp:320
std::vector< MemberMeta > & member_meta(const item_type type)
Definition collector.hpp:218
size_t get_offset(osmium::item_type type, osmium::object_id_type id)
Definition collector.hpp:514
uint64_t used_memory() const
Definition collector.hpp:447
std::vector< RelationMeta > m_relations
Definition collector.hpp:181
void way_not_in_any_relation(const osmium::Way &)
Definition collector.hpp:274
void flush()
Definition collector.hpp:298
callback_func_type callback()
Definition collector.hpp:222
void relation_not_in_any_relation(const osmium::Relation &)
Definition collector.hpp:284
const osmium::Relation & get_relation(const MemberMeta &member_meta) const
Definition collector.hpp:316
bool is_available(osmium::item_type type, osmium::object_id_type id)
Definition collector.hpp:499
bool find_and_add_object(const osmium::OSMObject &object)
Definition collector.hpp:384
std::pair< bool, size_t > get_availability_and_offset(osmium::item_type type, osmium::object_id_type id)
Definition collector.hpp:537
bool keep_member(const RelationMeta &, const osmium::RelationMember &) const
Definition collector.hpp:253
std::vector< const osmium::Relation * > get_incomplete_relations() const
Definition collector.hpp:600
const osmium::Relation & get_relation(const RelationMeta &relation_meta) const
Definition collector.hpp:309
bool keep_relation(const osmium::Relation &) const
Definition collector.hpp:239
std::function< void(osmium::memory::Buffer &&)> callback_func_type
Definition collector.hpp:193
void read_relations(TIter begin, TIter end)
Definition collector.hpp:547
static iterator_range< mm_iterator >::iterator::difference_type count_not_removed(const iterator_range< mm_iterator > &range)
Definition collector.hpp:371
void moving_in_buffer(size_t old_offset, size_t new_offset)
Definition collector.hpp:561
void node_not_in_any_relation(const osmium::Node &)
Definition collector.hpp:264
void possibly_purge_removed_members()
Definition collector.hpp:576
std::array< mm_vector_type, 3 > m_member_meta
Definition collector.hpp:189
iterator_range< mm_iterator > find_member_meta(osmium::item_type type, osmium::object_id_type id)
Definition collector.hpp:200
void read_relations(TSource &source)
Definition collector.hpp:554
Definition osm_object_builder.hpp:66
Code related to the assembly of OSM relations.
Definition multipolygon_collector.hpp:53
Namespace for everything in the Osmium library.
Definition assembler.hpp:53
void apply(TIterator it, TIterator end, THandlers &&... handlers)
Definition visitor.hpp:326
item_type
Definition item_type.hpp:45
@ relation
Definition item_type.hpp:50
iterator_range< It > make_range(P &&p) noexcept
Definition iterator.hpp:68
int64_t object_id_type
Type for OSM object (node, way, or relation) IDs.
Definition types.hpp:45
Definition iterator.hpp:42
It begin() const noexcept
Definition iterator.hpp:50
It end() const noexcept
Definition iterator.hpp:54