some performance improvements in the graph and tags preparation proce…

…ss, added version info to the built memory mapped files, please re-prepare your network with every new version as the data structure might have changed, also disabled the edge_ids export by default (can be enabled manually, in that case, please prepare the network with simplify=osm-aware in advance)
iisys-hof · Oct 10, 2024 · 3d51aa9 · 3d51aa9
1 parent 3d91d25
commit 3d51aa9
Show file tree

Hide file tree

Showing 24 changed files with 481 additions and 192 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.28)
 
 # project
 project(map_matching_2
-        VERSION 1.0.0
+        VERSION 1.0.1
         DESCRIPTION "Map Matching 2"
         LANGUAGES CXX)
 

diff --git a/README.md b/README.md
diff --git a/data/floating-car-data/README.md b/data/floating-car-data/README.md
@@ -54,3 +54,13 @@ Also given is an example on how to use custom tags for OpenStreetMap, for exampl
 ```
 
 The results will be in the `results` folder that is automatically created.
+
+Results on our test system (accuracy is the weighted mean correct fraction in percent):
+
+| Mode          | Time (s) | Max RAM (MiB) | Accuracy (%) |
+|:--------------|---------:|--------------:|-------------:|
+| Prepare       |    39.41 |         2,508 |          N/A |
+| Match         |     0.67 |           488 |          N/A |
+| Compare       |     0.46 |            32 |        99.58 |
+| Match (RAW)   |     5.70 |         1,692 |          N/A |
+| Prepare (All) |    47.84 |         2,899 |          N/A |
diff --git a/data/gis-cup/README.md b/data/gis-cup/README.md
@@ -49,3 +49,13 @@ for MSVC: `run\msvc\release\bin\map_matching_2.exe`.
 ```
 
 The results will be in the `results` folder that is automatically created.
+
+Results (with ground truth corrections from above applied) on our test system
+(accuracy is the weighted mean correct fraction in percent):
+
+| Mode                 | Time (s) | Max RAM (MiB) | Accuracy (%) |
+|:---------------------|---------:|--------------:|-------------:|
+| Prepare              |    20.63 |         1,327 |          N/A |
+| Convert Ground Truth |     0.48 |           354 |          N/A |
+| Match                |     0.79 |           244 |          N/A |
+| Compare              |     0.28 |            24 |        98.59 |
diff --git a/data/hengfeng-li/README.md b/data/hengfeng-li/README.md
@@ -31,7 +31,7 @@ for MSVC: `run\msvc\release\bin\map_matching_2.exe`.
 # prepare osm network
 ./map_matching_2 --conf data/hengfeng-li/conf/prepare_osm.conf
 
-# prepare ground truth network
+# prepare ground truth network (not simplified!)
 ./map_matching_2 --conf data/hengfeng-li/conf/prepare_ground_truth.conf
 
 # convert ground truth route
@@ -51,3 +51,16 @@ for MSVC: `run\msvc\release\bin\map_matching_2.exe`.
 ```
 
 The results will be in the `results` folder that is automatically created.
+
+Results on our test system (accuracy is the weighted mean correct fraction in percent):
+
+| Mode                   | Time (s) | Max RAM (MiB) | Accuracy (%) |
+|:-----------------------|---------:|--------------:|-------------:|
+| Prepare                |     1.65 |           160 |          N/A |
+| Prepare (OSM)          |     3.07 |           261 |          N/A |
+| Prepare (Ground Truth) |     1.51 |           166 |          N/A |
+| Convert Ground Truth   |     0.22 |            84 |          N/A |
+| Match                  |     0.49 |            72 |          N/A |
+| Match (OSM)            |     0.79 |            84 |          N/A |
+| Compare                |     0.23 |            26 |        99.80 |
+| Compare (OSM)          |     0.23 |            24 |        99.86 |
diff --git a/data/kubicka-et-al/README.md b/data/kubicka-et-al/README.md
@@ -22,7 +22,7 @@ for MSVC: `run\msvc\release\bin\map_matching_2.exe`.
 # prepare networks
 ./map_matching_2 --conf data/kubicka-et-al/conf/prepare.conf
 
-# prepare ground truth networks
+# prepare ground truth networks (not simplified!)
 ./map_matching_2 --conf data/kubicka-et-al/conf/prepare_ground_truth.conf
 
 # convert ground truth routes
@@ -31,7 +31,7 @@ for MSVC: `run\msvc\release\bin\map_matching_2.exe`.
 # match tracks
 ./map_matching_2 --conf data/kubicka-et-al/conf/match.conf
 
-# match tracks with export-edges mode
+# match tracks with export-edges mode (uses the slower ground truth network!)
 ./map_matching_2 --conf data/kubicka-et-al/conf/match_edges.conf
 
 # compare matches with ground truth
@@ -42,3 +42,15 @@ for MSVC: `run\msvc\release\bin\map_matching_2.exe`.
 ```
 
 The results will be in the `results` folder that is automatically created.
+
+Results on our test system (accuracy is the weighted mean correct fraction in percent):
+
+| Mode                   | Time (s) | Max RAM (MiB) | Accuracy (%) |
+|:-----------------------|---------:|--------------:|-------------:|
+| Prepare                |    31.46 |         3,075 |          N/A |
+| Prepare (Ground Truth) |    29.24 |         3,038 |          N/A |
+| Convert Ground Truth   |     8.57 |         1,437 |          N/A |
+| Match                  |     7.83 |         1,278 |          N/A |
+| Match (Export-Edges)   |    25.38 |         3,137 |          N/A |
+| Compare                |     0.94 |            51 |        98.73 |
+| Compare (Export-Edges) |     0.88 |            44 |        98.93 |
diff --git a/data/newson-krumm/README.md b/data/newson-krumm/README.md
@@ -34,3 +34,12 @@ for MSVC: `run\msvc\release\bin\map_matching_2.exe`.
 ```
 
 The results will be in the `results` folder that is automatically created.
+
+Results on our test system (accuracy is the weighted mean correct fraction in percent):
+
+| Mode                 | Time (s) | Max RAM (MiB) | Accuracy (%) |
+|:---------------------|---------:|--------------:|-------------:|
+| Prepare              |     4.02 |           254 |          N/A |
+| Convert Ground Truth |     1.32 |            50 |          N/A |
+| Match                |     2.09 |            98 |          N/A |
+| Compare              |     0.51 |            22 |        99.89 |
diff --git a/deploy/appimage/package.sh b/deploy/appimage/package.sh
@@ -92,7 +92,7 @@ fi
 
 # build appimage
 VERSION="$version" "./$appimagetool" "$appdir" || exit 1
-appimage=$(ls map_matching_2*.AppImage)
+appimage=$(ls -rt map_matching_2*.AppImage | tail -n 1)
 chmod +x "$appimage"
 
 # clean up

diff --git a/docker/build/clang/19-ubuntu-2004/Dockerfile b/docker/build/clang/19-ubuntu-2004/Dockerfile
@@ -0,0 +1,35 @@
+# Build and run from project directory:
+# docker build -t map_matching_2-clang-ubuntu-2004-build -f docker/build/clang/19-ubuntu-2004/Dockerfile .
+# docker run -u $(id -u):$(id -g) -v .:/tmp/map-matching-2 -it --rm map_matching_2-clang-ubuntu-2004-build
+
+FROM ubuntu:20.04 AS build
+
+ARG DEBIAN_FRONTEND="noninteractive"
+# only use major LLVM version
+ARG LLVM_VERSION=19
+ARG CMAKE_VERSION=3.28.6
+
+RUN apt-get update && apt-get dist-upgrade -y && \
+    apt-get install --no-install-recommends -y \
+        tzdata make lsb-release ca-certificates wget software-properties-common gnupg libssl-dev && \
+    # LLVM
+    wget https://apt.llvm.org/llvm.sh && \
+    chmod +x llvm.sh && \
+    ./llvm.sh $LLVM_VERSION all && \
+    rm llvm.sh && \
+    update-alternatives  --install /usr/bin/clang   clang   /usr/bin/clang-$LLVM_VERSION 100 \
+                         --slave   /usr/bin/clang++ clang++ /usr/bin/clang++-$LLVM_VERSION \
+                         --slave   /usr/bin/lldb    lldb    /usr/bin/lldb-$LLVM_VERSION && \
+    # CMake
+    wget "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}.tar.gz" && \
+    tar -xzf "cmake-${CMAKE_VERSION}.tar.gz" && \
+    rm -f "cmake-${CMAKE_VERSION}.tar.gz" && \
+    cd "cmake-${CMAKE_VERSION}" && \
+    ./bootstrap --parallel=$(nproc) && make -j $(nproc) && make install && \
+    cd .. && rm -rf "cmake-${CMAKE_VERSION}" && \
+    update-alternatives --install /usr/bin/cmake cmake /usr/local/bin/cmake 100 && \
+    # debug tools
+    apt-get install --no-install-recommends -y \
+        gdb ninja-build valgrind \
+        locales-all dos2unix rsync tar python3 python3-dev linux-tools-$(uname -r) time git && \
+    rm -rf /var/lib/apt/lists/*
diff --git a/docker/build/clang/19/Dockerfile b/docker/build/clang/19/Dockerfile
@@ -0,0 +1,37 @@
+# Build and run from project directory:
+# docker build -t map_matching_2-clang-build -f docker/build/clang/19/Dockerfile .
+# docker run -u $(id -u):$(id -g) -v .:/tmp/map-matching-2 -it --rm map_matching_2-clang-build
+
+FROM debian:12 AS build
+
+ARG DEBIAN_FRONTEND="noninteractive"
+# only use major LLVM version
+ARG LLVM_VERSION=19
+ARG CMAKE_VERSION=3.28.6
+
+RUN apt-get update && apt-get dist-upgrade -y && \
+    apt-get install --no-install-recommends -y \
+        tzdata make lsb-release ca-certificates wget software-properties-common gnupg libssl-dev && \
+    # https://github.com/hof/bookworm-apt-add-repository-issue
+    [ ! -e /etc/apt/sources.list ] && echo "#" | tee /etc/apt/sources.list && \
+    # LLVM
+    wget https://apt.llvm.org/llvm.sh && \
+    chmod +x llvm.sh && \
+    ./llvm.sh $LLVM_VERSION all && \
+    rm llvm.sh && \
+    update-alternatives  --install /usr/bin/clang   clang   /usr/bin/clang-$LLVM_VERSION 100 \
+                         --slave   /usr/bin/clang++ clang++ /usr/bin/clang++-$LLVM_VERSION \
+                         --slave   /usr/bin/lldb    lldb    /usr/bin/lldb-$LLVM_VERSION && \
+    # CMake
+    wget "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}.tar.gz" && \
+    tar -xzf "cmake-${CMAKE_VERSION}.tar.gz" && \
+    rm -f "cmake-${CMAKE_VERSION}.tar.gz" && \
+    cd "cmake-${CMAKE_VERSION}" && \
+    ./bootstrap --parallel=$(nproc) && make -j $(nproc) && make install && \
+    cd .. && rm -rf "cmake-${CMAKE_VERSION}" && \
+    update-alternatives --install /usr/bin/cmake cmake /usr/local/bin/cmake 100 && \
+    # debug tools
+    apt-get install --no-install-recommends -y \
+        gdb ninja-build valgrind \
+        locales-all dos2unix rsync tar python3 python3-dev linux-perf time git && \
+    rm -rf /var/lib/apt/lists/*
diff --git a/src/app/CMakeLists.txt b/src/app/CMakeLists.txt
@@ -3,11 +3,6 @@ set(THREADS_PREFER_PTHREAD_FLAG ON)
 find_package(Threads REQUIRED)
 find_package(Boost REQUIRED COMPONENTS thread program_options)
 
-configure_file(
-        ${CMAKE_CURRENT_SOURCE_DIR}/include/app/version.hpp.in
-        ${CMAKE_CURRENT_BINARY_DIR}/include/app/version.hpp
-        @ONLY)
-
 add_library(app STATIC)
 
 target_sources(app

diff --git a/src/app/src/app/options.cpp b/src/app/src/app/options.cpp
@@ -15,7 +15,6 @@
 
 #include "app/options.hpp"
 #include "app/general.hpp"
-#include "app/version.hpp"
 
 #include <format>
 #include <thread>
@@ -28,6 +27,7 @@
 #include "io/network/osm_pattern.hpp"
 #include "compare/comparator/compare_output_settings.hpp"
 #include "matching/matcher/matcher_output_settings.hpp"
+#include "util/version.hpp"
 
 namespace map_matching_2::app {
 
@@ -468,7 +468,7 @@ namespace map_matching_2::app {
                         "  \ttrack_length: track length in meter\n"
                         "  \tprepared_length: prepared track length in meter\n"
                         "  \tmatch_length: match length in meter\n"
-                        "  \tedge_ids: list of original edge IDs")
+                        "  \tedge_ids: list of original edge IDs (off by default; if used, combine with simplify osm-aware")
                 ("export-edges", po::value<bool>(&data.export_edges)->default_value(false, "off"),
                         "export matched routes based on the complete network edges instead of the actual parts, "
                         "this is only useful if the routes are later compared to matches that come from a list of edge IDs, "
@@ -793,7 +793,9 @@ namespace map_matching_2::app {
                         "  \tsimplify, but don't combine roads with different osm ids and tags,\n"
                         "  \tthis mode is usually not recommended, except special .csv extracts are of interest\n"
                         "  \tor when the map matching metrics later down the line respect the osm tags,\n"
-                        "  \twhich they do not by design currently\n"
+                        "  \twhich they do not by design currently;\n"
+                        "  \thowever, use this when you export edge_ids with the matches later,\n"
+                        "  \tas only then the ids are correctly preserved during simplification\n"
                         "default is \"all\"")
                 ("remove-unconnected", po::value<bool>(&data.remove_unconnected)->default_value(false, "off"),
                         "remove weakly unconnected components so that only the largest subgraph remains;\n"
@@ -1371,7 +1373,7 @@ namespace map_matching_2::app {
 
     bool show_version(const po::variables_map &vm) {
         if (vm.count("version")) {
-            std::cout << "Version: " << version() << std::endl;
+            std::cout << "Version: " << util::version() << std::endl;
             return true;
         }
         return false;

diff --git a/src/library/CMakeLists.txt b/src/library/CMakeLists.txt
@@ -4,6 +4,11 @@ find_package(Threads REQUIRED)
 find_package(Boost REQUIRED COMPONENTS serialization thread chrono)
 find_package(Osmium REQUIRED COMPONENTS io)
 
+configure_file(
+        ${CMAKE_CURRENT_SOURCE_DIR}/src/util/version.cpp.in
+        ${CMAKE_CURRENT_BINARY_DIR}/src/util/version.cpp
+        @ONLY)
+
 add_library(library STATIC)
 
 target_sources(library
@@ -23,7 +28,8 @@ target_sources(library
         src/io/importer.cpp
         src/matching/matcher/match_task.cpp
         src/matching/defect.cpp
-        src/util/compiler.cpp)
+        src/util/compiler.cpp
+        src/util/version.cpp)
 
 if (EXPLICIT_TEMPLATES)
     target_sources(library
@@ -134,7 +140,8 @@ endif ()
 
 target_include_directories(library
         PUBLIC
-        ${CMAKE_CURRENT_SOURCE_DIR}/include)
+        ${CMAKE_CURRENT_SOURCE_DIR}/include
+        ${CMAKE_CURRENT_BINARY_DIR}/include)
 
 target_include_directories(library SYSTEM
         PUBLIC

diff --git a/src/library/include/graph/adjacency_list.hpp b/src/library/include/graph/adjacency_list.hpp
@@ -724,7 +724,8 @@ namespace map_matching_2::graph {
         }
 
         [[nodiscard]] vertex_descriptor _add_vertex(vertex_type &&vertex) {
-            return try_add(vertices, std::move(vertex));
+            const auto &additional_tidy = []() constexpr -> void {};
+            return try_add(vertices, additional_tidy, std::move(vertex));
         }
 
         edge_descriptor add_edge(const vertex_descriptor &source_desc, const vertex_descriptor &target_desc) requires
@@ -756,7 +757,8 @@ namespace map_matching_2::graph {
             const auto index = vertex_index(source_desc);
             // std::cout << "From " << vertex_index(source_desc) << " to " << vertex_index(target_desc) << std::endl;
             // std::cout << "Size Before: " << get_vertex(source_desc).out_edges.size() << std::endl;
-            edge_descriptor edge_desc = try_add(edges, std::move(edge));
+            const auto &additional_tidy = []() constexpr -> void {};
+            edge_descriptor edge_desc = try_add(edges, additional_tidy, std::move(edge));
 
             edge_type &edge_r = get_edge(edge_desc);
             auto edges_tidy = [&]() {
@@ -767,7 +769,7 @@ namespace map_matching_2::graph {
             };
 
             vertex_type &source_vertex = get_vertex(source_desc);
-            try_add(source_vertex.out_edges, edge_descriptor{edge_desc}, edges_tidy);
+            try_add(source_vertex.out_edges, edges_tidy, edge_descriptor{edge_desc});
 
             if constexpr (is_undirected_v or is_bidirectional_v) {
                 auto source_out_edges_tidy = [&]() {
@@ -782,7 +784,7 @@ namespace map_matching_2::graph {
 
                 if constexpr (is_undirected_v) {
                     if (std::addressof(source_vertex) != std::addressof(target_vertex)) {
-                        try_add(target_vertex.out_edges, edge_descriptor{edge_desc}, source_out_edges_tidy);
+                        try_add(target_vertex.out_edges, source_out_edges_tidy, edge_descriptor{edge_desc});
                     }
                 }
 
@@ -797,7 +799,7 @@ namespace map_matching_2::graph {
                         source_out_edges_tidy();
                     };
 
-                    try_add(target_vertex.in_edges, edge_descriptor{edge_desc}, target_out_edges_tidy);
+                    try_add(target_vertex.in_edges, target_out_edges_tidy, edge_descriptor{edge_desc});
                 }
             }
 
@@ -936,29 +938,23 @@ namespace map_matching_2::graph {
             }
         }
 
-        template<typename Container, typename AdditionalTidy> requires
-            requires(Container container, typename Container::value_type value) {
-                { container.emplace_back(value) };
+        template<typename Container, typename AdditionalTidy, typename... Args> requires
+            requires(Container container, Args &&... args) {
+                { container.emplace_back(std::forward<Args>(args)...) };
                 { container.pop_back() };
             }
         std::conditional_t<util::is_random_access_v<Container>,
             typename Container::size_type, typename Container::iterator>
-        try_add(Container &container, typename Container::value_type &&value, const AdditionalTidy &additional_tidy) {
-            std::exception_ptr eptr = nullptr;
-
+        try_add(Container &container, const AdditionalTidy &additional_tidy, Args &&... args) {
             typename Container::size_type current_size = container.size();
             try {
-                container.emplace_back(std::move(value));
+                container.emplace_back(std::forward<Args>(args)...);
             } catch (...) {
-                eptr = std::current_exception();
                 while (container.size() > current_size) {
                     container.pop_back();
                 }
                 additional_tidy();
-            }
-
-            if (eptr) {
-                std::rethrow_exception(eptr);
+                throw;
             }
 
             if constexpr (util::is_random_access_v<Container>) {
@@ -968,19 +964,6 @@ namespace map_matching_2::graph {
             }
         }
 
-        template<typename Container> requires
-            requires(Container container, typename Container::value_type value) {
-                { container.emplace_back(value) };
-                { container.pop_back() };
-            }
-        std::conditional_t<util::is_random_access_v<Container>,
-            typename Container::size_type, typename Container::iterator>
-        try_add(Container &container, typename Container::value_type &&value) {
-            const auto &additional_tidy = []() constexpr -> void {};
-
-            return try_add(container, std::move(value), additional_tidy);
-        }
-
         void reindex() requires
             (not util::is_random_access_v<vertex_container> and not util::is_random_access_v<edge_container>) {
             if constexpr (not util::is_random_access_v<vertex_container>) {

diff --git a/src/library/include/io/helper/osm_handler_helper.hpp b/src/library/include/io/helper/osm_handler_helper.hpp
@@ -118,10 +118,8 @@ namespace map_matching_2::io::helper {
                 const auto node_location = osm_node.location();
                 const std::size_t index = vertices().size();
                 vertices().emplace_back(osm_vertex_type{
-                        vertex_type{
-                                osm_node.id(), osm_point_type{node_location.lon(), node_location.lat()},
-                                reprojector_variant, tags(osm_node.tags())
-                        }
+                        osm_node.id(), osm_point_type{node_location.lon(), node_location.lat()},
+                        reprojector_variant, tags(osm_node.tags())
                 });
                 return index;
             };