diff --git a/3party/osrm/BUILD_NOTE.txt b/3party/osrm/BUILD_NOTE.txt deleted file mode 100644 index f59e65d96bd..00000000000 --- a/3party/osrm/BUILD_NOTE.txt +++ /dev/null @@ -1,8 +0,0 @@ -If you youse luabind library with a version lesser than 0.9.2, -you may have troubles compiling OSRM code with a boost >=1.57 -(more details here: https://github.com/luabind/luabind/issues/27 ). - -You can fix it either by using latest luabind from source https://github.com/rpavlik/luabind -or https://github.com/DennisOSRM/luabind, or by applying the following patch -to /usr/include/luabind/object.hpp: -https://github.com/alex85k/luabind/commit/2340928e00f4d606255dd5d570e1e142353a5fdd diff --git a/3party/osrm/README.txt b/3party/osrm/README.txt new file mode 100644 index 00000000000..fa350702626 --- /dev/null +++ b/3party/osrm/README.txt @@ -0,0 +1,54 @@ +To build the Mapsme plugin for osrm-backend: +============================================ + +0. Build the regular omim project in the directory ${OMIM_BUILD_PATH}. + +1. Ignore all cmake files except those in omim/3party/osrm/osrm-backend, i.e. + run + + > cd osrm-backend-build + > cmake ${OMIM_ROOT}/3party/osrm/osrm-backend/ -DOMIM_BUILD_PATH=${OMIM_BUILD_PATH} + +2. Build the usual OSRM tools. + + > make -k -j4 osrm-extract osrm-prepare osrm-routed + +3. Run the OSRM pipeline on an *.osm.pbf file, see project OSRM docs or use + omim/tools/unix/generate_planet_routing.sh to do it automatically. + +4. Run the routing daemon (osrm-routed). + +5. The mapsme API that current version of the repository expects is + + GET /{service}?loc={latlon1}&loc={latlon2} + + where {latlon1} and {latlon2} are {coordinates} of source and destination points, respectively, in the usual OSRM format. + + For example, if osrm-routed runs on localhost on the default port 5000, use + + > curl "http://127.0.0.1:5000/mapsme?loc=55,37&loc=55,40" + + The expected response is (note the omim-mercator coordinates) + {"used_mwms":[[36.140419,60.524949,"Russia_Kursk Oblast"],[37.872379,67.58203,"Russia_Moscow Oblast_East"],[37.557854,67.355247,"Russia_Moscow"]]} + + + + +LUABIND +======= + +If you youse luabind library with a version lesser than 0.9.2, +you may have troubles compiling OSRM code with a boost >=1.57 +(more details here: https://github.com/luabind/luabind/issues/27 ). + +You can fix it either by using latest luabind from source https://github.com/rpavlik/luabind +or https://github.com/DennisOSRM/luabind, or by applying the following patch +to /usr/include/luabind/object.hpp: +https://github.com/alex85k/luabind/commit/2340928e00f4d606255dd5d570e1e142353a5fdd + + +On macOS, luabind is no longer supported by Homebrew so you may need to + + > brew edit luabind + +and comment out the "disable!" line. diff --git a/3party/osrm/osrm-backend/CMakeLists.txt b/3party/osrm/osrm-backend/CMakeLists.txt index 87dec16335d..c0a0968249d 100755 --- a/3party/osrm/osrm-backend/CMakeLists.txt +++ b/3party/osrm/osrm-backend/CMakeLists.txt @@ -92,7 +92,6 @@ file(GLOB AlgorithmTestsGlob unit_tests/algorithms/*.cpp) file(GLOB MapsMeSources mapsme/*.cpp) file(GLOB MapsMeHeaders mapsme/*.h) -file(GLOB MapsMeGenerator "${OMIM_PATH}/storage/country.cpp" "${OMIM_PATH}/storage/country_decl.cpp" "${OMIM_PATH}/storage/country_info.cpp") set( OSRMSources @@ -101,9 +100,16 @@ set( ${DatastructureGlob} ${AlgorithmGlob} ${HttpGlob} - ${MapsMeGenerator} ) +# Hacky but hey, all this time we've been compiling plugins by including them from a cpp-file bypassing cmake. +# This can't be worse, can it? +# Problem is our 2015 version of OSRM must be built with C++98, and MapsMePlugin compiles +# files from omim/ (because this CMakeLists.txt is inherited from the OSRM project and is +# not integrated in the omim cmake hierarchy). So MapsMePlugin must be built with C++11 or higher. +# That's why we cannot move this line to the beginning of the file but everything works when it's here. +set(CMAKE_CXX_STANDARD 17) + add_library(COORDINATE OBJECT ${CoordinateGlob}) add_library(GITDESCRIPTION OBJECT util/git_sha.cpp) add_library(OSRM ${OSRMSources} $ $ $ $ $ $ $ $ $) diff --git a/3party/osrm/osrm-backend/mapsme/converter.cpp b/3party/osrm/osrm-backend/mapsme/converter.cpp index aae3a05833c..d3e0fd87890 100644 --- a/3party/osrm/osrm-backend/mapsme/converter.cpp +++ b/3party/osrm/osrm-backend/mapsme/converter.cpp @@ -3,6 +3,8 @@ #include "../server/data_structures/internal_datafacade.hpp" #include +#include +#include #include "../../../../base/bits.hpp" #include "../../../../base/logging.hpp" @@ -18,6 +20,17 @@ #include "../../../succinct/rs_bit_vector.hpp" #include "../../../succinct/mapper.hpp" +#define ROUTING_MATRIX_FILE_TAG "mercedes" +#define ROUTING_EDGEDATA_FILE_TAG "daewoo" +#define ROUTING_EDGEID_FILE_TAG "infinity" +#define ROUTING_SHORTCUTS_FILE_TAG "skoda" +#define ROUTING_CROSS_CONTEXT_TAG "chrysler" + +#define ROUTING_FTSEG_FILE_TAG "ftseg" +#define ROUTING_NODEIND_TO_FTSEGIND_FILE_TAG "node2ftseg" + +using namespace std; + namespace { template @@ -74,8 +87,6 @@ string EdgeDataToString(EdgeOsrmT const & d) return ss.str(); } - - void GenerateRoutingIndex(const std::string & fPath) { ServerPaths server_paths; @@ -274,7 +285,7 @@ void GenerateRoutingIndex(const std::string & fPath) container.Open(path); typedef routing::OsrmDataFacade DataFacadeT; DataFacadeT facadeNew; - facadeNew.Load(container); + // facadeNew.Load(container); uint64_t edgesCount = facadeNew.GetNumberOfEdges() - copiedEdges + ignoredEdges; std::cout << "Check node count " << facade.GetNumberOfNodes() << " == " << facadeNew.GetNumberOfNodes() << "..." << std::endl; @@ -403,5 +414,4 @@ void GenerateRoutingIndex(const std::string & fPath) } PrintStatus(!error); } - -} +} // namespace mapsme diff --git a/3party/osrm/osrm-backend/mapsme/converter.hpp b/3party/osrm/osrm-backend/mapsme/converter.hpp index 25ab53aaf57..b7b097a33a5 100644 --- a/3party/osrm/osrm-backend/mapsme/converter.hpp +++ b/3party/osrm/osrm-backend/mapsme/converter.hpp @@ -6,8 +6,5 @@ namespace mapsme { - - void GenerateRoutingIndex(const std::string & fPath); - } diff --git a/3party/osrm/osrm-backend/plugins/MapsMePlugin.hpp b/3party/osrm/osrm-backend/plugins/MapsMePlugin.hpp index 1e6d805df39..2e4be0b9704 100644 --- a/3party/osrm/osrm-backend/plugins/MapsMePlugin.hpp +++ b/3party/osrm/osrm-backend/plugins/MapsMePlugin.hpp @@ -1,39 +1,42 @@ #pragma once -#include "../../../../base/string_utils.hpp" -#include "../../../../coding/files_container.hpp" -#include "../../../../coding/read_write_utils.hpp" -#include "../../../../defines.hpp" -#include "../../../../geometry/mercator.hpp" -#include "../../../../geometry/region2d.hpp" -#include "../../../../storage/country_decl.hpp" -#include "../../../../storage/country_polygon.hpp" -#include "plugin_base.hpp" - -#include "../algorithms/object_encoder.hpp" -#include "../data_structures/search_engine.hpp" -#include "../data_structures/edge_based_node_data.hpp" -#include "../descriptors/descriptor_base.hpp" -#include "../descriptors/gpx_descriptor.hpp" -#include "../descriptors/json_descriptor.hpp" -#include "../util/integer_range.hpp" -#include "../util/json_renderer.hpp" -#include "../util/make_unique.hpp" -#include "../util/simple_logger.hpp" +#include "storage/country_decl.hpp" + +#include "coding/files_container.hpp" +#include "coding/read_write_utils.hpp" + +#include "geometry/region2d.hpp" + +#include "base/string_utils.hpp" + +#include "defines.hpp" #include #include #include -#include #include +#include +#include #include -using TMapRepr = pair; +#include "3party/osrm/osrm-backend/algorithms/object_encoder.hpp" +#include "3party/osrm/osrm-backend/data_structures/edge_based_node_data.hpp" +#include "3party/osrm/osrm-backend/data_structures/search_engine.hpp" +#include "3party/osrm/osrm-backend/descriptors/descriptor_base.hpp" +#include "3party/osrm/osrm-backend/descriptors/gpx_descriptor.hpp" +#include "3party/osrm/osrm-backend/descriptors/json_descriptor.hpp" +#include "3party/osrm/osrm-backend/plugins/plugin_base.hpp" +#include "3party/osrm/osrm-backend/util/integer_range.hpp" +#include "3party/osrm/osrm-backend/util/json_renderer.hpp" +#include "3party/osrm/osrm-backend/util/make_unique.hpp" +#include "3party/osrm/osrm-backend/util/simple_logger.hpp" + +using TMapRepr = std::pair; class UsedMwmChecker { public: - static size_t constexpr kInvalidIndex = numeric_limits::max(); + static size_t constexpr kInvalidIndex = std::numeric_limits::max(); UsedMwmChecker() : m_lastUsedMwm(kInvalidIndex) {} @@ -49,7 +52,7 @@ class UsedMwmChecker m_lastMwmPoints.push_back(pt); } - vector const & GetUsedMwms() + std::vector const & GetUsedMwms() { // Get point from the last mwm. CommitUsedPoints(); @@ -79,9 +82,9 @@ class UsedMwmChecker m_lastMwmPoints.clear(); } - vector m_usedMwms; + std::vector m_usedMwms; size_t m_lastUsedMwm; - vector m_lastMwmPoints; + std::vector m_lastMwmPoints; }; template class MapsMePlugin final : public BasePlugin @@ -94,7 +97,7 @@ template class MapsMePlugin final : public BasePlugin public: size_t m_res; - GetByPoint(std::vector> const ®ions, m2::PointD const &pt) + GetByPoint(std::vector> const & regions, m2::PointD const &pt) : m_pt(pt), m_regions(regions), m_res(-1) { } @@ -103,13 +106,12 @@ template class MapsMePlugin final : public BasePlugin /// @return false If point is in country. bool operator()(size_t id) { - auto it = - find_if(m_regions[id].begin(), m_regions[id].end(), [&](m2::RegionD const ®ion) - { return region.Contains(m_pt);}); - if (it == m_regions[id].end()) - return true; - m_res = id; - return false; + auto it = std::find_if(m_regions[id].begin(), m_regions[id].end(), + [&](m2::RegionD const & region) { return region.Contains(m_pt); }); + if (it == m_regions[id].end()) + return true; + m_res = id; + return false; } }; @@ -133,21 +135,26 @@ template class MapsMePlugin final : public BasePlugin uint32_t const count = ReadVarUint(src); for (size_t j = 0; j < count; ++j) { - vector points; + std::vector points; serial::LoadOuterPath(src, serial::GeometryCodingParams(), points); - m_regions[i].emplace_back(move(m2::RegionD(points.begin(), points.end()))); + m_regions[i].emplace_back(std::move(m2::RegionD(points.begin(), points.end()))); } } m_searchEngine = osrm::make_unique>(facade); } - template void ForEachCountry(m2::PointD const &pt, ToDo &toDo) const + template + void ForEachCountry(m2::PointD const & pt, ToDo & toDo) const { - for (size_t i = 0; i < m_countries.size(); ++i) - if (m_countries[i].m_rect.IsPointInside(pt)) - if (!toDo(i)) - return; + for (size_t i = 0; i < m_countries.size(); ++i) + { + if (m_countries[i].m_rect.IsPointInside(pt)) + { + if (!toDo(i)) + return; + } + } } virtual ~MapsMePlugin() {} @@ -196,12 +203,12 @@ template class MapsMePlugin final : public BasePlugin return phantom_pair.first.component_id != 0; }; - const bool every_phantom_is_in_tiny_cc = + bool const every_phantom_is_in_tiny_cc = std::all_of(std::begin(phantom_node_pair_list), std::end(phantom_node_pair_list), check_component_id_is_tiny); // are all phantoms from a tiny cc? - const auto component_id = phantom_node_pair_list.front().first.component_id; + auto const component_id = phantom_node_pair_list.front().first.component_id; auto check_component_id_is_equal = [component_id](const phantom_node_pair &phantom_pair) { @@ -238,7 +245,7 @@ template class MapsMePlugin final : public BasePlugin osrm::for_each_pair(phantom_node_pair_list, build_phantom_pairs); - vector uturns; + std::vector uturns; m_searchEngine->shortest_path(raw_route.segment_end_coordinates, uturns, raw_route); if (INVALID_EDGE_WEIGHT == raw_route.shortest_path_length) { @@ -258,7 +265,7 @@ template class MapsMePlugin final : public BasePlugin if (data.m_segments.empty()) continue; auto const & seg = data.m_segments.front(); - m2::PointD pt = MercatorBounds::FromLatLon(seg.lat1, seg.lon1); + m2::PointD pt = FromLatLon(seg.lat1, seg.lon1); GetByPoint doGet(m_regions, pt); ForEachCountry(pt, doGet); usedChecker.AddPoint(doGet.m_res, pt); @@ -280,6 +287,17 @@ template class MapsMePlugin final : public BasePlugin } private: + // Reimplementation of coding/mercator.hpp due to namespace clash with OSRM's mercator struct. + double ClampY(double d) { return base::Clamp(d, -180.0, 180.0); } + double LonToX(double lon) { return lon; } + double LatToY(double lat) + { + double const sinx = std::sin(base::DegToRad(base::Clamp(lat, -86.0, 86.0))); + double const res = base::RadToDeg(0.5 * log((1.0 + sinx) / (1.0 - sinx))); + return ClampY(res); + } + m2::PointD FromLatLon(double lat, double lon) { return m2::PointD(LonToX(lon), LatToY(lat)); } + std::unique_ptr> m_searchEngine; std::vector m_countries; std::vector> m_regions; diff --git a/3party/osrm/osrm-backend/plugins/WayIdPlugin.hpp b/3party/osrm/osrm-backend/plugins/WayIdPlugin.hpp index 3a1ea081570..90b55d711d9 100644 --- a/3party/osrm/osrm-backend/plugins/WayIdPlugin.hpp +++ b/3party/osrm/osrm-backend/plugins/WayIdPlugin.hpp @@ -19,6 +19,7 @@ #include #include #include +#include #include template @@ -121,7 +122,7 @@ class WayIdPlugin final : public BasePlugin osrm::for_each_pair(phantom_node_pair_list, build_phantom_pairs); - vector uturns; + std::vector uturns; m_searchEngine->shortest_path(raw_route.segment_end_coordinates, uturns, raw_route); if (INVALID_EDGE_WEIGHT == raw_route.shortest_path_length) { @@ -130,7 +131,7 @@ class WayIdPlugin final : public BasePlugin } // Get ids of ways used in path. - set wayIds; + std::set wayIds; for (auto i : osrm::irange(0, raw_route.unpacked_path_segments.size())) { @@ -147,7 +148,7 @@ class WayIdPlugin final : public BasePlugin // Format answer. osrm::json::Array json_array; json_array.values.assign(wayIds.begin(), wayIds.end()); - reply.values["way_ids"] = move(json_array); + reply.values["way_ids"] = std::move(json_array); return 200; } diff --git a/routing/cross_routing_context.hpp b/routing/cross_routing_context.hpp new file mode 100644 index 00000000000..7f7ea099b87 --- /dev/null +++ b/routing/cross_routing_context.hpp @@ -0,0 +1,174 @@ +#pragma once + +#include "coding/files_container.hpp" + +#include "geometry/latlon.hpp" +#include "geometry/point2d.hpp" +#include "geometry/rect2d.hpp" +#include "geometry/tree4d.hpp" + +#include +#include +#include +#include +#include + +namespace routing +{ +using TWrittenNodeId = uint32_t; +using TWrittenEdgeWeight = uint32_t; + +TWrittenEdgeWeight constexpr kInvalidContextEdgeNodeId = std::numeric_limits::max(); +TWrittenEdgeWeight constexpr kInvalidContextEdgeWeight = std::numeric_limits::max(); +size_t constexpr kInvalidAdjacencyIndex = std::numeric_limits::max(); +double constexpr kMwmCrossingNodeEqualityMeters = 80; + +struct IngoingCrossNode +{ + ms::LatLon m_point; + TWrittenNodeId m_nodeId; + size_t m_adjacencyIndex; + + IngoingCrossNode() + : m_point(ms::LatLon::Zero()) + , m_nodeId(kInvalidContextEdgeNodeId) + , m_adjacencyIndex(kInvalidAdjacencyIndex) + { + } + + IngoingCrossNode(TWrittenNodeId nodeId, ms::LatLon const & point, size_t const adjacencyIndex) + : m_point(point), m_nodeId(nodeId), m_adjacencyIndex(adjacencyIndex) + { + } + + bool IsValid() const { return m_nodeId != kInvalidContextEdgeNodeId; } + + void Save(Writer & w) const; + + size_t Load(Reader const & r, size_t pos, size_t adjacencyIndex); + + m2::RectD const GetLimitRect() const { return m2::RectD(m_point.m_lat, m_point.m_lon, m_point.m_lat, m_point.m_lon); } +}; + +struct OutgoingCrossNode +{ + ms::LatLon m_point; + TWrittenNodeId m_nodeId; + unsigned char m_outgoingIndex; + size_t m_adjacencyIndex; + + OutgoingCrossNode() + : m_point(ms::LatLon::Zero()) + , m_nodeId(kInvalidContextEdgeNodeId) + , m_outgoingIndex(0) + , m_adjacencyIndex(kInvalidAdjacencyIndex) + { + } + + OutgoingCrossNode(TWrittenNodeId nodeId, size_t const index, ms::LatLon const & point, + size_t const adjacencyIndex) + : m_point(point) + , m_nodeId(nodeId) + , m_outgoingIndex(static_cast(index)) + , m_adjacencyIndex(adjacencyIndex) + { + } + + bool IsValid() const { return m_nodeId != kInvalidContextEdgeNodeId; } + + void Save(Writer & w) const; + + size_t Load(Reader const & r, size_t pos, size_t adjacencyIndex); + + m2::RectD const GetLimitRect() const { return m2::RectD(m_point.m_lat, m_point.m_lon, m_point.m_lat, m_point.m_lon); } +}; + +using IngoingEdgeIteratorT = std::vector::const_iterator; +using OutgoingEdgeIteratorT = std::vector::const_iterator; + +/// Reader class from cross context section in mwm.routing file +class CrossRoutingContextReader +{ + std::vector m_outgoingNodes; + std::vector m_neighborMwmList; + std::vector m_adjacencyMatrix; + m4::Tree m_ingoingIndex; + m4::Tree m_outgoingIndex; + +public: + void Load(Reader const & r); + + const std::string & GetOutgoingMwmName(OutgoingCrossNode const & outgoingNode) const; + + TWrittenEdgeWeight GetAdjacencyCost(IngoingCrossNode const & ingoing, + OutgoingCrossNode const & outgoing) const; + + std::vector const & GetNeighboringMwmList() const { return m_neighborMwmList; } + + m2::RectD GetMwmCrossingNodeEqualityRect(ms::LatLon const & point) const; + + template + bool ForEachIngoingNodeNearPoint(ms::LatLon const & point, Fn && fn) const + { + bool found = false; + m_ingoingIndex.ForEachInRect(GetMwmCrossingNodeEqualityRect(point), + [&found, &fn](IngoingCrossNode const & node) { + fn(node); + found = true; + }); + return found; + } + + template + bool ForEachOutgoingNodeNearPoint(ms::LatLon const & point, Fn && fn) const + { + bool found = false; + m_outgoingIndex.ForEachInRect(GetMwmCrossingNodeEqualityRect(point), + [&found, &fn](OutgoingCrossNode const & node) { + fn(node); + found = true; + }); + return found; + } + + template + void ForEachIngoingNode(TFunctor f) const + { + m_ingoingIndex.ForEach(f); + } + + template + void ForEachOutgoingNode(TFunctor f) const + { + for_each(m_outgoingNodes.cbegin(), m_outgoingNodes.cend(), f); + } +}; + +/// Helper class to generate cross context section in mwm.routing file +class CrossRoutingContextWriter +{ + std::vector m_ingoingNodes; + std::vector m_outgoingNodes; + std::vector m_adjacencyMatrix; + std::vector m_neighborMwmList; + + size_t GetIndexInAdjMatrix(IngoingEdgeIteratorT ingoing, OutgoingEdgeIteratorT outgoing) const; + +public: + void Save(Writer & w) const; + + void AddIngoingNode(TWrittenNodeId const nodeId, ms::LatLon const & point); + + void AddOutgoingNode(TWrittenNodeId const nodeId, std::string const & targetMwm, + ms::LatLon const & point); + + void ReserveAdjacencyMatrix(); + + void SetAdjacencyCost(IngoingEdgeIteratorT ingoing, OutgoingEdgeIteratorT outgoin, + TWrittenEdgeWeight value); + + std::pair GetIngoingIterators() const; + + std::pair GetOutgoingIterators() const; +}; +} // namespace routing diff --git a/routing/osrm_data_facade.hpp b/routing/osrm_data_facade.hpp new file mode 100644 index 00000000000..20d01c7eb71 --- /dev/null +++ b/routing/osrm_data_facade.hpp @@ -0,0 +1,313 @@ +#pragma once + +#include "routing/cross_routing_context.hpp" + +#include "coding/files_container.hpp" +#include "coding/read_write_utils.hpp" + +#include "base/bits.hpp" + +#include "defines.hpp" + +#include "3party/succinct/elias_fano.hpp" +#include "3party/succinct/elias_fano_compressed_list.hpp" +#include "3party/succinct/gamma_vector.hpp" +#include "3party/succinct/rs_bit_vector.hpp" +#include "3party/succinct/mapper.hpp" + +#if defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wunused-local-typedef" +#endif + +// TODO (ldragunov) exclude osrm specific headers from here! They causes "coordinate" problem +#include "3party/osrm/osrm-backend/server/data_structures/datafacade_base.hpp" +#include "3party/osrm/osrm-backend/data_structures/travel_mode.hpp" + +#if defined(__clang__) +#pragma clang diagnostic pop +#endif + +#include +#include + +namespace routing +{ + +template class OsrmRawDataFacade : public BaseDataFacade +{ + template void ClearContainer(T & t) + { + T().swap(t); + } + +protected: + succinct::elias_fano_compressed_list m_edgeData; + succinct::rs_bit_vector m_shortcuts; + succinct::elias_fano m_matrix; + succinct::elias_fano_compressed_list m_edgeId; + + uint32_t m_numberOfNodes = 0; + +public: + //OsrmRawDataFacade(): m_numberOfNodes(0) {} + + void LoadRawData(char const * pRawEdgeData, char const * pRawEdgeIds, char const * pRawEdgeShortcuts, char const * pRawFanoMatrix) + { + ClearRawData(); + + ASSERT(pRawEdgeData, ()); + succinct::mapper::map(m_edgeData, pRawEdgeData); + + ASSERT(pRawEdgeIds, ()); + succinct::mapper::map(m_edgeId, pRawEdgeIds); + + ASSERT(pRawEdgeShortcuts, ()); + succinct::mapper::map(m_shortcuts, pRawEdgeShortcuts); + + ASSERT(pRawFanoMatrix, ()); + m_numberOfNodes = *reinterpret_cast(pRawFanoMatrix); + succinct::mapper::map(m_matrix, pRawFanoMatrix + sizeof(m_numberOfNodes)); + } + + void ClearRawData() + { + ClearContainer(m_edgeData); + ClearContainer(m_edgeId); + ClearContainer(m_shortcuts); + ClearContainer(m_matrix); + } + + unsigned GetNumberOfNodes() const override + { + return m_numberOfNodes; + } + + unsigned GetNumberOfEdges() const override + { + return static_cast(m_edgeData.size()); + } + + unsigned GetOutDegree(const NodeID n) const override + { + return EndEdges(n) - BeginEdges(n); + } + + NodeID GetTarget(const EdgeID e) const override + { + return (m_matrix.select(e) / 2) % GetNumberOfNodes(); + } + + EdgeDataT GetEdgeData(const EdgeID e, NodeID node) const override + { + EdgeDataT res; + + res.shortcut = m_shortcuts[e]; + res.id = res.shortcut ? (node - static_cast(bits::ZigZagDecode(m_edgeId[static_cast(m_shortcuts.rank(e))]))) : 0; + res.backward = (m_matrix.select(e) % 2 == 1); + res.forward = !res.backward; + res.distance = static_cast(m_edgeData[e]); + + return res; + } + + EdgeDataT & GetEdgeData(const EdgeID) const override + { + static EdgeDataT res; + ASSERT(false, ("Maps me routing facade does not supports this edge unpacking method")); + return res; + } + + //! TODO: Make proper travelmode getter when we add it to routing file + TravelMode GetTravelModeForEdgeID(const unsigned) const override + { + return TRAVEL_MODE_DEFAULT; + } + + EdgeID BeginEdges(const NodeID n) const override + { + uint64_t idx = 2 * n * (uint64_t)GetNumberOfNodes(); + return n == 0 ? 0 : static_cast(m_matrix.rank(std::min(idx, m_matrix.size()))); + } + + EdgeID EndEdges(const NodeID n) const override + { + uint64_t const idx = 2 * (n + 1) * (uint64_t)GetNumberOfNodes(); + return static_cast(m_matrix.rank(std::min(idx, m_matrix.size()))); + } + + EdgeRange GetAdjacentEdgeRange(const NodeID node) const override + { + return osrm::irange(BeginEdges(node), EndEdges(node)); + } + + // searches for a specific edge + EdgeID FindEdge(const NodeID from, const NodeID to) const override + { + EdgeID smallest_edge = SPECIAL_EDGEID; + EdgeWeight smallest_weight = INVALID_EDGE_WEIGHT; + for (auto edge : GetAdjacentEdgeRange(from)) + { + const NodeID target = GetTarget(edge); + const EdgeWeight weight = GetEdgeData(edge).distance; + if (target == to && weight < smallest_weight) + { + smallest_edge = edge; + smallest_weight = weight; + } + } + return smallest_edge; + } + + EdgeID FindEdgeInEitherDirection(const NodeID, const NodeID) const override + { + return (EdgeID)0; + } + + EdgeID FindEdgeIndicateIfReverse(const NodeID, const NodeID, bool &) const override + { + return (EdgeID)0; + } + + // node and edge information access + FixedPointCoordinate GetCoordinateOfNode(const unsigned) const override + { + return FixedPointCoordinate(); + } + + bool EdgeIsCompressed(const unsigned) const override + { + return false; + } + + unsigned GetGeometryIndexForEdgeID(const unsigned) const override + { + return false; + } + + void GetUncompressedGeometry(const unsigned, std::vector &) const override + { + } + + TurnInstruction GetTurnInstructionForEdgeID(const unsigned) const override + { + return TurnInstruction::NoTurn; + } + + bool LocateClosestEndPointForCoordinate(const FixedPointCoordinate &, + FixedPointCoordinate &, + const unsigned = 18) override + { + return false; + } + + /*bool FindPhantomNodeForCoordinate(const FixedPointCoordinate &input_coordinate, + PhantomNode &resulting_phantom_node, + const unsigned zoom_level) override + { + return false; + }*/ + + /*bool IncrementalFindPhantomNodeForCoordinate(const FixedPointCoordinate &input_coordinate, + std::vector &resulting_phantom_node_vector, + const unsigned zoom_level, + const unsigned number_of_results) override + { + return false; + }*/ + + bool IncrementalFindPhantomNodeForCoordinate(const FixedPointCoordinate &, + std::vector &, + const unsigned) override + { + return false; + } + + bool IncrementalFindPhantomNodeForCoordinate(const FixedPointCoordinate &, + PhantomNode &) override + { + return false; + } + + bool IncrementalFindPhantomNodeForCoordinateWithMaxDistance( + const FixedPointCoordinate &, + std::vector> &, + const double, + const unsigned, + const unsigned) override + { + return false; + } + + unsigned GetCheckSum() const override + { + return 0; + } + + unsigned GetNameIndexFromEdgeID(const unsigned) const override + { + return -1; + } + + //void GetName(const unsigned name_id, std::string &result) const override + //{ + //} + + std::string get_name_for_id(const unsigned) const override + { + return std::string(); + } + + std::string GetTimestamp() const override + { + return std::string(); + } +}; + + +template class OsrmDataFacade : public OsrmRawDataFacade +{ + typedef OsrmRawDataFacade super; + + FilesMappingContainer::Handle m_handleEdgeData; + FilesMappingContainer::Handle m_handleEdgeId; + FilesMappingContainer::Handle m_handleEdgeIdFano; + FilesMappingContainer::Handle m_handleShortcuts; + FilesMappingContainer::Handle m_handleFanoMatrix; + + using OsrmRawDataFacade::LoadRawData; + using OsrmRawDataFacade::ClearRawData; + +public: + + // void Load(FilesMappingContainer const & container) + // { + // Clear(); + + // // Map huge data first, as we hope it will reduce fragmentation of the program address space. + // m_handleFanoMatrix.Assign(container.Map(ROUTING_MATRIX_FILE_TAG)); + // ASSERT(m_handleFanoMatrix.IsValid(), ()); + + // m_handleEdgeData.Assign(container.Map(ROUTING_EDGEDATA_FILE_TAG)); + // ASSERT(m_handleEdgeData.IsValid(), ()); + + // m_handleEdgeId.Assign(container.Map(ROUTING_EDGEID_FILE_TAG)); + // ASSERT(m_handleEdgeId.IsValid(), ()); + + // m_handleShortcuts.Assign(container.Map(ROUTING_SHORTCUTS_FILE_TAG)); + // ASSERT(m_handleShortcuts.IsValid(), ()); + + // LoadRawData(m_handleEdgeData.GetData(), m_handleEdgeId.GetData(), m_handleShortcuts.GetData(), m_handleFanoMatrix.GetData()); + // } + + void Clear() + { + ClearRawData(); + m_handleEdgeData.Unmap(); + m_handleEdgeId.Unmap(); + m_handleShortcuts.Unmap(); + m_handleFanoMatrix.Unmap(); + } +}; + +} diff --git a/tools/unix/generate_planet_routing.sh b/tools/unix/generate_planet_routing.sh new file mode 100755 index 00000000000..151b1764d35 --- /dev/null +++ b/tools/unix/generate_planet_routing.sh @@ -0,0 +1,205 @@ +#!/bin/bash +############################################ +# Builds routing indices for given regions # +############################################ + +set -e # Exit when any of commands fail +set -o pipefail # Capture all errors in command chains +set -u # Fail on undefined variables +#set -x # Echo every script line + +if [ $# -lt 1 ]; then + echo '' + echo "Usage:" + echo " $0 pbf" + echo " $0 prepare" + echo " $0 mwm" + echo " $0 online" + echo " $0 serve" + echo " $0 stop" + echo '' + exit 1 +fi + +fail() { + [ $# -gt 0 ] && echo "$@" >&2 + exit 1 +} + +OMIM_PATH="${OMIM_PATH:-$(cd "$(dirname "$0")/../.."; pwd)}" +DATA_PATH="${DATA_PATH:-$OMIM_PATH/data}" +BORDERS_PATH="${BORDERS_PATH:-$DATA_PATH/borders}" +TARGET="${TARGET:-$DATA_PATH}" +[ ! -d "$TARGET" ] && fail "$TARGET should be a writable folder" +INTDIR="${INTDIR:-$TARGET/intermediate_data}" +mkdir -p "$INTDIR" +NUM_PROCESSES=${NUM_PROCESSES:-8} +KEEP_INTDIR=${KEEP_INTDIR-} +LOG_PATH=${LOG_PATH:-.} +echo "[$(date +%Y/%m/%d\ %H:%M:%S)] $0 $1" + +if [ "$1" == "pbf" ]; then + PLANET="${PLANET:-$HOME/planet/planet-latest.o5m}" + OSMCTOOLS="${OSMCTOOLS:-$HOME/osmctools}" + [ ! -d "$OSMCTOOLS" ] && OSMCTOOLS="$INTDIR" + [ ! -x "$OSMCTOOLS/osmconvert" ] && cc -x c -O3 "$OMIM_PATH/tools/osmctools/osmconvert.c" -o "$OSMCTOOLS/osmconvert" -lz + + TMPBORDERS="$INTDIR/tmpborders" + [ -d "$TMPBORDERS" ] && rm -r "$TMPBORDERS" + mkdir "$TMPBORDERS" + if [ -z "${REGIONS-}" ]; then + cp "$BORDERS_PATH"/*.poly "$TMPBORDERS" + else + echo "$REGIONS" | xargs -I % cp "%" "$TMPBORDERS" + fi + [ -z "$(ls "$TMPBORDERS" | grep '\.poly$')" ] && fail "No regions to create routing files for" + export OSMCTOOLS + export PLANET + export INTDIR + find "$TMPBORDERS" -maxdepth 1 -name '*.poly' -print0 | xargs -0 -P $NUM_PROCESSES -I % \ + sh -c '"$OSMCTOOLS/osmconvert" "$PLANET" --hash-memory=2000 -B="%" --complete-ways --out-pbf -o="$INTDIR/$(basename "%" .poly).pbf"' + rm -r "$TMPBORDERS" + +elif [ "$1" == "prepare" ]; then + [ -z "$(ls "$INTDIR" | grep '\.pbf$')" ] && fail "Please build PBF files first" + OSRM_PATH="${OSRM_PATH:-$OMIM_PATH/3party/osrm/osrm-backend}" + OSRM_BUILD_PATH="${OSRM_BUILD_PATH:-$OMIM_PATH/../osrm-backend-release}" + [ ! -x "$OSRM_BUILD_PATH/osrm-extract" ] && fail "Please compile OSRM binaries to $OSRM_BUILD_PATH" + + OSRM_THREADS=${OSRM_THREADS:-15} + OSRM_MEMORY=${OSRM_MEMORY:-50} + EXTRACT_CFG="$INTDIR/extractor.ini" + PREPARE_CFG="$INTDIR/contractor.ini" + echo "threads = $OSRM_THREADS" > "$EXTRACT_CFG" + echo "memory = $OSRM_MEMORY" > "$PREPARE_CFG" + echo "threads = $OSRM_THREADS" >> "$PREPARE_CFG" + PROFILE="${PROFILE:-$OSRM_PATH/profiles/car.lua}" + [ $# -gt 1 ] && PROFILE="$2" + [ ! -r "$PROFILE" ] && fail "Lua profile $PROFILE is not found" + export STXXLCFG="$HOME/.stxxl" + [ ! -f "$STXXLCFG" ] && fail "For routing, you need ~/.stxxl file. Run this: echo 'disk=$HOME/stxxl_disk1,400G,syscall' > $STXXLCFG" + + for PBF in "$INTDIR"/*.pbf; do + OSRM_FILE="${PBF%.*}.osrm" + RESTRICTIONS_FILE="$OSRM_FILE.restrictions" + LOG="$LOG_PATH/$(basename "$PBF" .pbf).log" + rm -f "$OSRM_FILE" + "$OSRM_BUILD_PATH/osrm-extract" --config "$EXTRACT_CFG" --profile "$PROFILE" "$PBF" >> "$LOG" 2>&1 || true + "$OSRM_BUILD_PATH/osrm-prepare" --config "$PREPARE_CFG" --profile "$PROFILE" "$OSRM_FILE" -r "$RESTRICTIONS_FILE" >> "$LOG" 2>&1 || true + "$OSRM_BUILD_PATH/osrm-mapsme" -i "$OSRM_FILE" >> "$LOG" 2>&1 || true + if [ -s "$OSRM_FILE" ]; then + [ -z "$KEEP_INTDIR" ] && rm -f "$PBF" + ONE_OSRM_READY=1 + else + echo "Failed to create $OSRM_FILE" >> "$LOG" + fi + done + [ -z "${ONE_OSRM_READY-}" ] && fail "No osrm files were prepared" + +elif [ "$1" == "mwm" ]; then + [ -z "$(ls "$INTDIR" | grep '\.osrm$')" ] && fail "Please build OSRM files first" + source "$(dirname "$0")/find_generator_tool.sh" + + if [ ! -d "$TARGET/borders" -o -z "$(ls "$TARGET/borders" | grep '\.poly$')" ]; then + # copy polygons to a temporary directory + POLY_DIR="$TARGET/borders" + mkdir -p "$POLY_DIR" + if [ -z "${REGIONS-}" ]; then + cp "$BORDERS_PATH"/*.poly "$POLY_DIR" + else + echo "$REGIONS" | xargs -I % cp "%" "$POLY_DIR" + fi + fi + + # Xargs has 255 chars limit for exec string, so we use short variable names. + export G="$GENERATOR_TOOL" + export K="--make_routing --make_cross_section --make_routing_index --generate_traffic_keys" + export I="--intermediate_data_path=$INTDIR" + export TARGET + export LOG_PATH + export DATA_PATH + set +e + find "$INTDIR" -maxdepth 1 -name '*.osrm' -print0 | xargs -0 -P $NUM_PROCESSES -I % \ + sh -c 'O="%"; B="$(basename "$O" .osrm)"; "$G" $K "$I" --osrm_file_name="$O" --data_path="$TARGET" --user_resource_path="$DATA_PATH" --output="$B" 2>> "$LOG_PATH/$B.log"' + set -e + + if [ -n "${POLY_DIR-}" ]; then + # delete temporary polygons + rm "$POLY_DIR"/*.poly + if [ -z "$(ls -A "$POLY_DIR")" ]; then + rmdir "$POLY_DIR" + fi + fi + +elif [ "$1" == "stop" ]; then + LOG="$LOG_PATH/planet.log" + echo "Stopping osrm server..." >> "$LOG" + pkill osrm-routed || true + +elif [ "$1" == "online" ]; then + PLANET="${PLANET:-$HOME/planet/planet-latest.o5m}" + OSMCTOOLS="${OSMCTOOLS:-$HOME/osmctools}" + [ ! -d "$OSMCTOOLS" ] && OSMCTOOLS="$INTDIR" + [ ! -x "$OSMCTOOLS/osmconvert" ] && cc -x c -lz -O3 "$OMIM_PATH/tools/osmctools/osmconvert.c" -o "$OSMCTOOLS/osmconvert" + + PBF="$INTDIR/planet.pbf" + "$OSMCTOOLS/osmconvert" "$PLANET" --hash-memory=2000 --out-pbf -o="$PBF" + + OSRM_PATH="${OSRM_PATH:-$OMIM_PATH/3party/osrm/osrm-backend}" + OSRM_BUILD_PATH="${OSRM_BUILD_PATH:-$OMIM_PATH/../osrm-backend-release}" + [ ! -x "$OSRM_BUILD_PATH/osrm-extract" ] && fail "Please compile OSRM binaries to $OSRM_BUILD_PATH" + + OSRM_THREADS=${OSRM_THREADS:-15} + OSRM_MEMORY=${OSRM_MEMORY:-50} + EXTRACT_CFG="$INTDIR/extractor.ini" + PREPARE_CFG="$INTDIR/contractor.ini" + echo "threads = $OSRM_THREADS" > "$EXTRACT_CFG" + echo "memory = $OSRM_MEMORY" > "$PREPARE_CFG" + echo "threads = $OSRM_THREADS" >> "$PREPARE_CFG" + PROFILE="${PROFILE:-$OSRM_PATH/profiles/car.lua}" + [ $# -gt 1 ] && PROFILE="$2" + [ ! -r "$PROFILE" ] && fail "Lua profile $PROFILE is not found" + + export STXXLCFG="$HOME/.stxxl" + OSRM_FILE="$INTDIR/planet.osrm" + RESTRICTIONS_FILE="$OSRM_FILE.restrictions" + LOG="$LOG_PATH/planet.log" + rm -f "$OSRM_FILE" + "$OSRM_BUILD_PATH/osrm-extract" --config "$EXTRACT_CFG" --profile "$PROFILE" "$PBF" >> "$LOG" 2>&1 + "$OSRM_BUILD_PATH/osrm-prepare" --config "$PREPARE_CFG" --profile "$PROFILE" "$OSRM_FILE" -r "$RESTRICTIONS_FILE" >> "$LOG" 2>&1 + if [ -s "$OSRM_FILE" ]; then + [ -z "$KEEP_INTDIR" ] && rm -f "$PBF" + else + echo "Failed to create $OSRM_FILE" >> "$LOG" + fi + +elif [ "$1" == "serve" ]; then + OSRM_PATH="${OSRM_PATH:-$OMIM_PATH/3party/osrm/osrm-backend}" + OSRM_BUILD_PATH="${OSRM_BUILD_PATH:-$OMIM_PATH/../osrm-backend-release}" + [ ! -x "$OSRM_BUILD_PATH/osrm-extract" ] && fail "Please compile OSRM binaries to $OSRM_BUILD_PATH" + + OSRM_THREADS=${OSRM_THREADS:-15} + OSRM_MEMORY=${OSRM_MEMORY:-50} + + export STXXLCFG="$HOME/.stxxl" + OSRM_FILE="$INTDIR/planet.osrm" + PORT="10012" + RESTRICTIONS_FILE="$OSRM_FILE.restrictions" + LOG="$LOG_PATH/planet.log" + if [ -s "$OSRM_FILE" ]; then + echo "Starting: $OSRM_FILE" + "$OSRM_BUILD_PATH/osrm-routed" "$OSRM_FILE" --borders "$OMIM_PATH/data/" --port "$PORT" >> "$LOG" 2>&1 & + + echo "Waiting until OSRM server starts:" + until $(curl --output /dev/null --silent --head --fail http://localhost:$PORT/mapsme); do + printf '.' >> "$LOG" + sleep 5 + done + else + echo "Can't find OSRM file: $OSRM_FILE" + fi + +else + fail "Incorrect parameter: $1" +fi +exit 0