[Tile-serving] [openstreetmap/osm2pgsql] Something wrong with indexes during import (#1436)

[Andrey Novikov]

```
local srid = 3857

local prefix = 'osm_'

local tables = {}

tables.points = osm2pgsql.define_node_table(prefix .. 'points', {
    { column = 'tags', type = 'hstore' },
    { column = 'names', type = 'hstore' },
    { column = 'geom', type = 'point', projection = srid },
})

tables.lines = osm2pgsql.define_way_table(prefix .. 'lines', {
    { column = 'tags', type = 'hstore' },
    { column = 'names', type = 'hstore' },
    { column = 'geom', type = 'linestring', projection = srid },
})

tables.boundaries = osm2pgsql.define_way_table(prefix .. 'boundaries', {
    { column = 'admin_level', type = 'int', not_null = true },
    { column = 'maritime', type = 'text' },
    { column = 'geom', type = 'linestring', projection = srid },
})

tables.polygons = osm2pgsql.define_area_table(prefix .. 'polygons', {
    { column = 'tags', type = 'hstore' },
    { column = 'names', type = 'hstore' },
    { column = 'label_node', type = 'int8' },
    { column = 'geom', type = 'geometry', projection = srid },
    { column = 'area', type = 'area' },
})

tables.routes = osm2pgsql.define_relation_table(prefix .. 'routes', {
    { column = 'type', type = 'text' },
    { column = 'tags', type = 'hstore' },
    { column = 'geom', type = 'multilinestring', projection = srid },
})

local way2boundary = {}

local delete_keys = {
    -- cleaned to save DB space (addr:housenumber specially treated)
    'addr:*',

    -- everything else here is from flex examples, deleted to save space
}

local clean_tags = osm2pgsql.make_clean_tags_func(delete_keys)

function has_area_tags(tags)
    -- exactly as in flex examples, deleted to save space
end

function osm2pgsql.process_node(object)
    local addr_housenumber = object.tags['addr:housenumber']

    if clean_tags(object.tags) and not addr_housenumber then
        return
    end

    object.tags['addr:housenumber'] = addr_housenumber
    object.tags.layer = layer(object.tags.layer)
    local names = names(object.tags)

    tables.points:add_row({
        tags = object.tags,
        names = names
    })
end

function osm2pgsql.process_way(object)
    local boundaries = way2boundary[object.id]
    if boundaries or object.tags.boundary == 'administrative' then
        local admin_level = tonumber(object:grab_tag('admin_level')) or 12
        local maritime = object:grab_tag('maritime')

        if boundaries then
            for _, boundary in pairs(boundaries) do
                if boundary.admin_level < admin_level then
                    admin_level = boundary.admin_level
                end
                if boundary.maritime and not maritime then
                    maritime = boundary.maritime
                end
            end
        end

        if admin_level < 5 then
            tables.boundaries:add_row({
                admin_level = admin_level,
                maritime = maritime,
                geom = { create = 'line' }
            })
        end
    end

    local addr_housenumber = object.tags['addr:housenumber']
    local addr_interpolation = object.tags['addr:interpolation']

    if clean_tags(object.tags) and not addr_housenumber and not addr_interpolation then
        return
    end

    object.tags['addr:housenumber'] = addr_housenumber
    object.tags['addr:interpolation'] = addr_interpolation
    object.tags.layer = layer(object.tags.layer)
    local names = names(object.tags)

    if object.is_closed and has_area_tags(object.tags) then
        tables.polygons:add_row({
            tags = object.tags,
            names = names,
            geom = { create = 'area' }
        })
    else
        tables.lines:add_row({
            tags = object.tags,
            names = names
        })
    end
end

function osm2pgsql.select_relation_members(relation)
    if (relation.tags.type == 'boundary' or (relation.tags.type == 'multipolygon' and relation.tags.boundary)) and relation.tags.boundary == 'administrative' and relation.tags.admin_level then
        local admin_level = tonumber(relation.tags.admin_level) or 12
        if admin_level < 5 then
            return { ways = osm2pgsql.way_member_ids(relation) }
        end
    end
end

function osm2pgsql.process_relation(object)
    local addr_housenumber = object.tags['addr:housenumber']
    local addr_interpolation = object.tags['addr:interpolation']

    if clean_tags(object.tags) and not addr_housenumber and not addr_interpolation then
        return
    end

    object.tags['addr:housenumber'] = addr_housenumber
    object.tags['addr:interpolation'] = addr_interpolation
    object.tags.layer = layer(object.tags.layer)
    local names = names(object.tags)

    local type = object:grab_tag('type')

    if type == 'route' then
        tables.routes:add_row({
            type = object:grab_tag('route'),
            tags = object.tags,
            geom = { create = 'line' }
        })
        return
    end

    if type == 'boundary' or (type == 'multipolygon' and object.tags.boundary) then
        if object.tags.boundary == 'administrative' then
            local admin_level = tonumber(object.tags.admin_level) or 12
            if admin_level < 5 then
                local boundary = {
                    admin_level = admin_level,
                    maritime = object.tags.maritime
                }

                -- Go through all the members and store relation ids and refs so they
                -- can be found by the way id.
                for _, member in ipairs(object.members) do
                    if member.type == 'w' then
                        if not way2boundary[member.ref] then
                            way2boundary[member.ref] = {}
                        end
                        way2boundary[member.ref][object.id] = boundary
                    end
                end
            end
            if not object.tags.place then
                return
            end
            object.tags.boundary = nil
        end

        tables.polygons:add_row({
            tags = object.tags,
            names = names,
            label_node = relation_label(object.members, object.tags.boundary == 'administrative'),
            geom = { create = 'area' }
        })
        return
    end

    if type == 'multipolygon' then
        tables.polygons:add_row({
            tags = object.tags,
            names = names,
            label_node = relation_label(object.members, false),
            geom = { create = 'area' }
        })
    end
end

--- Normalizes layer tags
--- @param v string The layer tag value
--- @return number An integer for the layer tag
function layer(v)
    return v and string.find(v, "^-?%d+$") and tonumber(v) < 100 and tonumber(v) > -100 and v or nil
end

--- Put all name:* tags in their own substructure
--- @param tags table Object tags
--- @return table Extracted names
function names(tags)
    local object_names = {}
    for k, v in pairs(tags) do
        if k == 'name' then
            object_names[''] = v
            tags.name = nil
        elseif osm2pgsql.has_prefix(k, "name:") then
            -- extract language
            local lang = k:sub(6, -1)
            tags[k] = nil
            if lang == 'en' or lang == 'de' or lang == 'ru' then
                object_names[lang] = v
            end
        end
    end
    return object_names
end

--- Get label node reference for relation
--- @param members table Relation members
--- @param is_admin boolean Relation is administrative boundary
--- @return number node id or nil
function relation_label(members, is_admin)
    local label_ref
    for _, member in ipairs(members) do
        if member.type == 'n' then
            if member.role == 'label' then
                label_ref = member.ref
            end
            if is_admin and member.role == 'admin_centre' and not label_ref then
                label_ref = member.ref
            end
        end
    end
    return label_ref
end
```

-- 
You are receiving this because you are subscribed to this thread.
Reply to this email directly or view it on GitHub:
https://github.com/openstreetmap/osm2pgsql/issues/1436#issuecomment-807104717
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.openstreetmap.org/pipermail/tile-serving/attachments/20210325/55670504/attachment-0001.htm>