[Tile-serving] [openstreetmap/osm2pgsql] pgsql middle ram leaks (#731)

Paul Norman notifications at github.com
Sat Mar 25 19:33:49 UTC 2017 

with `./osm2pgsql --slim -d osm2pgsql_test -S ../default.style ~/osm/planet/vancouver_canada.osm.pbf --cache 1`

```
Direct leak of 26809952 byte(s) in 128894 object(s) allocated from:
    #0 0x7f1d8d7aad28 in malloc (/usr/lib/x86_64-linux-gnu/libasan.so.3+0xc1d28)
    #1 0x7f1d8d0ac10c in PQmakeEmptyPGresult (/usr/lib/x86_64-linux-gnu/libpq.so.5+0x1110c)
    #2 0x7f1d8d0b7607  (/usr/lib/x86_64-linux-gnu/libpq.so.5+0x1c607)
    #3 0x7f1d8d0ae1d7 in PQgetResult (/usr/lib/x86_64-linux-gnu/libpq.so.5+0x131d7)
    #4 0x7f1d8d0ae52c  (/usr/lib/x86_64-linux-gnu/libpq.so.5+0x1352c)
    #5 0x5645eeb83888 in pgsql_execPrepared(pg_conn*, char const*, int, char const* const*, ExecStatusType) /home/pnorman/osm/osm2pgsql/pgsql.cpp:121
    #6 0x5645eeb3758a in middle_pgsql_t::local_nodes_get_list(osmium::WayNodeList*) const /home/pnorman/osm/osm2pgsql/middle-pgsql.cpp:323
    #7 0x5645eeb37ad6 in middle_pgsql_t::nodes_get_list(osmium::WayNodeList*) const /home/pnorman/osm/osm2pgsql/middle-pgsql.cpp:359
    #8 0x5645eeb74d6f in output_pgsql_t::pending_way(long, int) /home/pnorman/osm/osm2pgsql/output-pgsql.cpp:167
    #9 0x5645eea84e7e in do_jobs /home/pnorman/osm/osm2pgsql/osmdata.cpp:194
    #10 0x5645eea9c1fe in void std::_Bind_simple<void (*(std::reference_wrapper<std::vector<std::shared_ptr<output_t>, std::allocator<std::shared_ptr<output_t> > > const>, std::reference_wrapper<std::stack<pending_job_t, std::deque<pending_job_t, std::allocator<pending_job_t> > > >, std::reference_wrapper<unsigned long>, std::refe
rence_wrapper<std::mutex>, bool, bool))(std::vector<std::shared_ptr<output_t>, std::allocator<std::shared_ptr<output_t> > > const&, std::stack<pending_job_t, std::deque<pending_job_t, std::allocator<pending_job_t> > >&, unsigned long&, std::mutex&, int, bool)>::_M_invoke<0ul, 1ul, 2ul, 3ul, 4ul, 5ul>(std::_Index_tuple<0ul, 1ul, 2u
l, 3ul, 4ul, 5ul>) (/home/pnorman/osm/osm2pgsql/build/osm2pgsql+0x27a1fe)
    #11 0x5645eea9b75a in std::_Bind_simple<void (*(std::reference_wrapper<std::vector<std::shared_ptr<output_t>, std::allocator<std::shared_ptr<output_t> > > const>, std::reference_wrapper<std::stack<pending_job_t, std::deque<pending_job_t, std::allocator<pending_job_t> > > >, std::reference_wrapper<unsigned long>, std::reference
_wrapper<std::mutex>, bool, bool))(std::vector<std::shared_ptr<output_t>, std::allocator<std::shared_ptr<output_t> > > const&, std::stack<pending_job_t, std::deque<pending_job_t, std::allocator<pending_job_t> > >&, unsigned long&, std::mutex&, int, bool)>::operator()() (/home/pnorman/osm/osm2pgsql/build/osm2pgsql+0x27975a)
    #12 0x5645eea9ab33 in std::__future_base::_Task_setter<std::unique_ptr<std::__future_base::_Result<void>, std::__future_base::_Result_base::_Deleter>, std::_Bind_simple<void (*(std::reference_wrapper<std::vector<std::shared_ptr<output_t>, std::allocator<std::shared_ptr<output_t> > > const>, std::reference_wrapper<std::stack<pe
nding_job_t, std::deque<pending_job_t, std::allocator<pending_job_t> > > >, std::reference_wrapper<unsigned long>, std::reference_wrapper<std::mutex>, bool, bool))(std::vector<std::shared_ptr<output_t>, std::allocator<std::shared_ptr<output_t> > > const&, std::stack<pending_job_t, std::deque<pending_job_t, std::allocator<pending_j
ob_t> > >&, unsigned long&, std::mutex&, int, bool)>, void>::operator()() const (/home/pnorman/osm/osm2pgsql/build/osm2pgsql+0x278b33)
    #13 0x5645eea9a268 in std::_Function_handler<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> (), std::__future_base::_Task_setter<std::unique_ptr<std::__future_base::_Result<void>, std::__future_base::_Result_base::_Deleter>, std::_Bind_simple<void (*(std::reference_wrapper<std::ve
ctor<std::shared_ptr<output_t>, std::allocator<std::shared_ptr<output_t> > > const>, std::reference_wrapper<std::stack<pending_job_t, std::deque<pending_job_t, std::allocator<pending_job_t> > > >, std::reference_wrapper<unsigned long>, std::reference_wrapper<std::mutex>, bool, bool))(std::vector<std::shared_ptr<output_t>, std::all
ocator<std::shared_ptr<output_t> > > const&, std::stack<pending_job_t, std::deque<pending_job_t, std::allocator<pending_job_t> > >&, unsigned long&, std::mutex&, int, bool)>, void> >::_M_invoke(std::_Any_data const&) (/home/pnorman/osm/osm2pgsql/build/osm2pgsql+0x278268)
    #14 0x5645eea898d2 in std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>::operator()() const /usr/include/c++/6/functional:2127
    #15 0x5645eea8876e in std::__future_base::_State_baseV2::_M_do_set(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*) /usr/include/c++/6/future:533
    #16 0x5645eea8f030 in void std::__invoke_impl<void, void (std::__future_base::_State_baseV2::*)(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*), std::__future_base::_State_baseV2*, std::function<std::unique_ptr<std::__future_base::_Result_base, std::__fut
ure_base::_Result_base::_Deleter> ()>*, bool*>(std::__invoke_memfun_deref, void (std::__future_base::_State_baseV2::*&&)(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*), std::__future_base::_State_baseV2*&&, std::function<std::unique_ptr<std::__future_base::_
Result_base, std::__future_base::_Result_base::_Deleter> ()>*&&, bool*&&) /usr/include/c++/6/functional:227
    #17 0x5645eea8bfab in std::result_of<void (std::__future_base::_State_baseV2::*&&(std::__future_base::_State_baseV2*&&, std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*&&, bool*&&))(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base
::_Result_base::_Deleter> ()>*, bool*)>::type std::__invoke<void (std::__future_base::_State_baseV2::*)(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*), std::__future_base::_State_baseV2*, std::function<std::unique_ptr<std::__future_base::_Result_base, std::_
_future_base::_Result_base::_Deleter> ()>*, bool*>(void (std::__future_base::_State_baseV2::*&&)(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*), std::__future_base::_State_baseV2*&&, std::function<std::unique_ptr<std::__future_base::_Result_base, std::__futu
re_base::_Result_base::_Deleter> ()>*&&, bool*&&) /usr/include/c++/6/functional:251
    #18 0x5645eea894e7 in void std::call_once<void (std::__future_base::_State_baseV2::*)(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*), std::__future_base::_State_baseV2*, std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::
_Result_base::_Deleter> ()>*, bool*>(std::once_flag&, void (std::__future_base::_State_baseV2::*&&)(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*), std::__future_base::_State_baseV2*&&, std::function<std::unique_ptr<std::__future_base::_Result_base, std::__f
uture_base::_Result_base::_Deleter> ()>*&&, bool*&&)::{lambda()#1}::operator()() const /usr/include/c++/6/mutex:602
    #19 0x5645eea8954a in void std::call_once<void (std::__future_base::_State_baseV2::*)(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*), std::__future_base::_State_baseV2*, std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::
_Result_base::_Deleter> ()>*, bool*>(std::once_flag&, void (std::__future_base::_State_baseV2::*&&)(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*), std::__future_base::_State_baseV2*&&, std::function<std::unique_ptr<std::__future_base::_Result_base, std::__f
uture_base::_Result_base::_Deleter> ()>*&&, bool*&&)::{lambda()#2}::operator()() const /usr/include/c++/6/mutex:607
    #20 0x5645eea8955b in void std::call_once<void (std::__future_base::_State_baseV2::*)(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*), std::__future_base::_State_baseV2*, std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::
_Result_base::_Deleter> ()>*, bool*>(std::once_flag&, void (std::__future_base::_State_baseV2::*&&)(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*), std::__future_base::_State_baseV2*&&, std::function<std::unique_ptr<std::__future_base::_Result_base, std::__f
uture_base::_Result_base::_Deleter> ()>*&&, bool*&&)::{lambda()#2}::_FUN() /usr/include/c++/6/mutex:607
    #21 0x7f1d8cc72738 in __pthread_once_slow (/lib/x86_64-linux-gnu/libpthread.so.0+0xe738)
```

```
Direct leak of 2496208 byte(s) in 12001 object(s) allocated from:
    #0 0x7f1d8d7aad28 in malloc (/usr/lib/x86_64-linux-gnu/libasan.so.3+0xc1d28)
    #1 0x7f1d8d0ac10c in PQmakeEmptyPGresult (/usr/lib/x86_64-linux-gnu/libpq.so.5+0x1110c)
    #2 0x7f1d8d0b7607  (/usr/lib/x86_64-linux-gnu/libpq.so.5+0x1c607)
    #3 0x7f1d8d0ae1d7 in PQgetResult (/usr/lib/x86_64-linux-gnu/libpq.so.5+0x131d7)
    #4 0x7f1d8d0ae52c  (/usr/lib/x86_64-linux-gnu/libpq.so.5+0x1352c)
    #5 0x5645eeb83888 in pgsql_execPrepared(pg_conn*, char const*, int, char const* const*, ExecStatusType) /home/pnorman/osm/osm2pgsql/pgsql.cpp:121
    #6 0x5645eeb3758a in middle_pgsql_t::local_nodes_get_list(osmium::WayNodeList*) const /home/pnorman/osm/osm2pgsql/middle-pgsql.cpp:323
    #7 0x5645eeb37ad6 in middle_pgsql_t::nodes_get_list(osmium::WayNodeList*) const /home/pnorman/osm/osm2pgsql/middle-pgsql.cpp:359
    #8 0x5645eeb75c88 in output_pgsql_t::way_add(osmium::Way*) /home/pnorman/osm/osm2pgsql/output-pgsql.cpp:301
    #9 0x5645eea83e35 in osmdata_t::way_add(osmium::Way*) /home/pnorman/osm/osm2pgsql/osmdata.cpp:54
    #10 0x5645eeac299f in parse_osmium_t::way(osmium::Way&) /home/pnorman/osm/osm2pgsql/parse-osmium.cpp:168
    #11 0x5645eeb15977 in void osmium::detail::apply_item_impl<parse_osmium_t&, osmium::memory::Item>(osmium::memory::Item&, parse_osmium_t&) /home/pnorman/osm/osm2pgsql/contrib/libosmium/osmium/visitor.hpp:64
    #12 0x5645eeb0fa8d in void osmium::apply_item<osmium::memory::Item, parse_osmium_t&>(osmium::memory::Item&, parse_osmium_t&) /home/pnorman/osm/osm2pgsql/contrib/libosmium/osmium/visitor.hpp:206
    #13 0x5645eeb0734c in void osmium::apply<osmium::io::InputIterator<osmium::io::Reader, osmium::memory::Item>, parse_osmium_t&>(osmium::io::InputIterator<osmium::io::Reader, osmium::memory::Item>, osmium::io::InputIterator<osmium::io::Reader, osmium::memory::Item>, parse_osmium_t&) /home/pnorman/osm/osm2pgsql/contrib/libosmium/
osmium/visitor.hpp:220
    #14 0x5645eeafd920 in void osmium::apply<osmium::io::Reader, parse_osmium_t&>(osmium::io::Reader&, parse_osmium_t&) /home/pnorman/osm/osm2pgsql/contrib/libosmium/osmium/visitor.hpp:227
    #15 0x5645eeac2507 in parse_osmium_t::stream_file(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) /home/pnorman/osm/osm2pgsql/parse-osmium.cpp:128
    #16 0x5645eea67d37 in main /home/pnorman/osm/osm2pgsql/osm2pgsql.cpp:86
    #17 0x7f1d8b3662b0 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x202b0)
```

```
Direct leak of 7712 byte(s) in 24 object(s) allocated from:
    #0 0x7f1d8d740f30 in strdup (/usr/lib/x86_64-linux-gnu/libasan.so.3+0x57f30)
    #1 0x5645eeb3c07b in set_prefix_and_tbls /home/pnorman/osm/osm2pgsql/middle-pgsql.cpp:927
    #2 0x5645eeb3c242 in middle_pgsql_t::connect(middle_pgsql_t::table_desc&) /home/pnorman/osm/osm2pgsql/middle-pgsql.cpp:937
    #3 0x5645eeb3e233 in middle_pgsql_t::get_instance() const /home/pnorman/osm/osm2pgsql/middle-pgsql.cpp:1187
    #4 0x5645eea8528d in pending_threaded_processor /home/pnorman/osm/osm2pgsql/osmdata.cpp:222
    #5 0x5645eea86e26 in osmdata_t::stop() /home/pnorman/osm/osm2pgsql/osmdata.cpp:395
    #6 0x5645eea67e05 in main /home/pnorman/osm/osm2pgsql/osm2pgsql.cpp:97
    #7 0x7f1d8b3662b0 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x202b0)
```

Lots more leaks from middle-pgsql.cpp:927 with different functions calling it.

`SUMMARY: AddressSanitizer: 483618738 byte(s) leaked in 440293 allocation(s).`


-- 
You are receiving this because you are subscribed to this thread.
Reply to this email directly or view it on GitHub:
https://github.com/openstreetmap/osm2pgsql/issues/731
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.openstreetmap.org/pipermail/tile-serving/attachments/20170325/035b9d4a/attachment-0001.html>

More information about the Tile-serving mailing list