Speed up the "dump" phase (Pass 2) of osm2rdf (#122)

Speed up the "dump" phase (Pass 2) of `osm2rdf` via a number of improvements:

1. Move instead of copy `osmium::Buffer` objects directly into dedicated worker threads and operate on the original `libosmium` objects the entire time. Apart from the copy, this also avoids synchronization overhead (previously, each OSM object was a dedicated task). This was non-trivial to achieve, as several `libosmium` handlers were previously applied iteratively to a finished `osmium::Buffer` object, augmenting the objects in pipeline-like fashion until they were finally passed to the handler that tasked them away to the threads. Several of these handlers assumed that the objects arrive ordered by their OSM id, which is no longer guaranteed when buffers are handled in parallel. We solve this by separating these handlers into a separate internal pass.
2. Replace our internal C-API calls to `zlib` by `zlib-ng`. 
3. Improve the handling and formatting of attributes. In particular, avoid copies by operating directly on the raw C-strings from the `osmium::Buffer`. Also drop the use of `strftime` and generate the time strings directly.
4. Cherry-pick the smaller spatialjoin IDs proposed in #108 to avoid full IRI writes to disk.
5. Update `libspatialjoin`, which now folds the ID directly into the sweeper event if it fits into 64 bit (which it does for every OSM node), eliminating the writing and lookup of the ID for points.

In total, the changes above **sped up the dump phase for `switzerland-latest.osm.pbf` by a factor of almost 5** (4 min -> 50s), when using `--output-compression gz` and otherwise the default options.

On the side, improve the progress bar by "weighting" individual tasks by their estimated duration. In particular, each way is now weighted by the number of its member nodes.

Author: patrickbr

.gitmodules

@@ -14,3 +14,6 @@
 [submodule "vendor/spatialjoin"]
 	path = vendor/spatialjoin
 	url = https://github.com/ad-freiburg/spatialjoin.git
+[submodule "vendor/zlib-ng/zlib-ng"]
+	path = vendor/zlib-ng/zlib-ng
+	url = https://github.com/zlib-ng/zlib-ng.git

CMakeLists.txt

@@ -42,15 +42,18 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
 # set(CMAKE_VERBOSE_MAKEFILE ON)
 
-option(ENABLE_GEOMETRY_STATISTIC "Write geometry statistics if enable" 0)
+option(ENABLE_GEOMETRY_STATISTIC "Write geometry statistics if enabled" 0)
+
+# Force static libraries
+set(BUILD_SHARED_LIBS OFF)
 
 # Enable verbose makefile
 if (ENABLE_GEOMETRY_STATISTIC)
 	# Enable geometry statistics
 	add_definitions(-DENABLE_GEOMETRY_STATISTIC)
 endif()
 
-add_compile_options(-Wall -Wextra -Wno-missing-field-initializers)
+add_compile_options(-Wall -Wextra -Wno-missing-field-initializers -Wno-stringop-overread)
 add_compile_options(-DGTEST_HAS_TR1_TUPLE=0 -DGTEST_USE_OWN_TR1_TUPLE=0)
 
 add_compile_options(-march=native)
@@ -62,9 +65,18 @@ set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fsanitize=address,undefined
 # ----------------------------------------------------------------------------
 find_package(EXPAT REQUIRED)
 find_package(BZip2 REQUIRED)
-find_package(ZLIB REQUIRED)
 find_package(OpenMP)
 
+# zlib-ng
+set(ZLIB_COMPAT ON)
+set(WITH_GTEST OFF)
+set(ZLIB_ENABLE_TESTS OFF)
+add_subdirectory(vendor/zlib-ng/zlib-ng)
+
+set(ZLIB_USE_STATIC_LIBS ON)
+set(ZLIB_INCLUDE_DIR "${CMAKE_BINARY_DIR}/vendor/zlib-ng/zlib-ng")
+set(ZLIB_LIBRARY "${CMAKE_BINARY_DIR}/vendor/zlib-ng/zlib-ng/libz.a")
+
 # Disable installation of google stuff
 set(INSTALL_GMOCK OFF)
 set(INSTALL_GTEST OFF)

apps/CMakeLists.txt

@@ -17,6 +17,7 @@
 # along with osm2rdf.  If not, see <https://www.gnu.org/licenses/>.
 
 add_executable(osm2rdf osm2rdf.cpp)
+
 target_link_libraries(osm2rdf PRIVATE osm2rdf_library spatialjoin-dev pb_util)
 
 add_executable(osm2rdf-stats osm2rdf-stats.cpp)

apps/osm2rdf.cpp

@@ -26,7 +26,6 @@
 #include "osm2rdf/config/ExitCode.h"
 #include "osm2rdf/osm/OsmiumHandler.h"
 #include "osm2rdf/ttl/Writer.h"
-#include "osm2rdf/util/Ram.h"
 #include "osm2rdf/util/Time.h"
 #include "osmium/util/memory.hpp"
 
@@ -89,14 +88,6 @@ int main(int argc, char** argv) {
   config.fromArgs(argc, argv);
   std::cerr << config.getInfo(osm2rdf::util::formattedTimeSpacer) << std::endl;
 
-  std::cerr << osm2rdf::util::currentTimeFormatted() << "Free ram: "
-            << osm2rdf::util::ram::available() /
-                   (osm2rdf::util::ram::GIGA * 1.0)
-            << "G/"
-            << osm2rdf::util::ram::physPages() /
-                   (osm2rdf::util::ram::GIGA * 1.0)
-            << "G" << std::endl;
-
 #if defined(_OPENMP)
   omp_set_num_threads(config.numThreads);
 #endif

include/osm2rdf/config/Config.h

@@ -82,8 +82,6 @@ struct Config {
 
   bool addSpatialRelsForUntaggedNodes = true;
 
-  bool separate = true;
-
   std::string iriPrefixForUntaggedNodes =
       osm2rdf::ttl::constants::IRI_PREFIX__OSM_NODE_UNTAGGED;
 

include/osm2rdf/osm/Area.h

@@ -44,9 +44,8 @@ struct Area {
   [[nodiscard]] const ::util::geo::DMultiPolygon& geom() const noexcept;
   [[nodiscard]] double geomArea() const noexcept;
   [[nodiscard]] const ::util::geo::DBox& envelope() const noexcept;
-  [[nodiscard]] double envelopeArea() const noexcept;
-  [[nodiscard]] const ::util::geo::DPolygon& convexHull() const noexcept;
-  [[nodiscard]] const ::util::geo::DPolygon& orientedBoundingBox() const noexcept;
+  [[nodiscard]] ::util::geo::DPolygon convexHull() const noexcept;
+  [[nodiscard]] ::util::geo::DPolygon orientedBoundingBox() const noexcept;
   [[nodiscard]] const ::util::geo::DPoint centroid() const noexcept;
   [[nodiscard]] bool fromWay() const noexcept;
   [[nodiscard]] bool hasName() const noexcept;
@@ -61,13 +60,9 @@ struct Area {
   id_t _id;
   // The OSM id
   id_t _objId;
-  bool _hasName = false;
   double _geomArea = 0;
-  double _envelopeArea = 0;
   ::util::geo::DMultiPolygon _geom;
   ::util::geo::DBox _envelope;
-  ::util::geo::DPolygon _convexHull;
-  ::util::geo::DPolygon _obb;
 };
 
 }  // namespace osm2rdf::osm

include/osm2rdf/osm/CountHandler.h

@@ -35,6 +35,8 @@ class CountHandler : public osmium::handler::Handler {
   size_t numNodes() const;
   size_t numRelations() const;
   size_t numWays() const;
+  size_t weightedNumWays() const;
+  size_t weightedNumRelations() const;
 
   size_t minNodeId() const { return _minId; };
   size_t maxNodeId() const { return _maxId; };
@@ -43,6 +45,8 @@ class CountHandler : public osmium::handler::Handler {
   size_t _numNodes = 0;
   size_t _numRelations = 0;
   size_t _numWays = 0;
+  size_t _weightedNumWays = 0;
+  size_t _weightedNumRelations = 0;
   bool _firstPassDone = false;
   size_t _minId = std::numeric_limits<size_t>::max();
   size_t _maxId = 0;

include/osm2rdf/osm/FactHandler.h

@@ -47,24 +47,27 @@ class FactHandler {
   void setLocationHandler(osm2rdf::osm::LocationHandler* locationHandler);
   // Add data
   void area(const osm2rdf::osm::Area& area);
-  void node(const osm2rdf::osm::Node& node);
+  void node(const osmium::Node& node);
   void relation(const osm2rdf::osm::Relation& relation);
   void way(const osm2rdf::osm::Way& way);
 
-  template <typename G>
   void writeGeometry(const std::string& s, const std::string& p,
-                          const G& g);
+                     const ::util::geo::DPoint& g);
+  template <typename G>
+  void writeGeometry(const std::string& s, const std::string& p, const G& g);
 
  protected:
   void writeBox(const std::string& s, const std::string& p,
                 const ::util::geo::DBox& box);
   FRIEND_TEST(OSM_FactHandler, writeBoxPrecision1);
   FRIEND_TEST(OSM_FactHandler, writeBoxPrecision2);
 
+  void writeMeta(const std::string& s, const osmium::Node& object);
+
   template <typename T>
   void writeMeta(const std::string& s, const T& object);
 
-  void writeTag(const std::string& s, const osm2rdf::osm::Tag& tag);
+  void writeTag(const std::string& s, const char* key, const char* val);
   FRIEND_TEST(OSM_FactHandler, writeTag_AdminLevel);
   FRIEND_TEST(OSM_FactHandler, writeTag_AdminLevel_nonInteger);
   FRIEND_TEST(OSM_FactHandler, writeTag_AdminLevel_nonInteger2);
@@ -77,7 +80,7 @@ class FactHandler {
   FRIEND_TEST(OSM_FactHandler, writeTag_KeyIRI);
   FRIEND_TEST(OSM_FactHandler, writeTag_KeyNotIRI);
 
-  void writeTagList(const std::string& s, const osm2rdf::osm::TagList& tags);
+  void writeTagList(const std::string& s, const osmium::TagList& tags);
   FRIEND_TEST(OSM_FactHandler, writeTagList);
   FRIEND_TEST(OSM_FactHandler, writeTagListWikidata);
   FRIEND_TEST(OSM_FactHandler, writeTagListRefSingle);
@@ -106,10 +109,15 @@ class FactHandler {
   FRIEND_TEST(OSM_FactHandler, writeTagListStartDateYearMonthDay5);
 
   bool hasSuffix(const std::string& s, const std::string& suffix) const;
+  bool hasSuffix(const char* s, const char* suffix, size_t suffixSize) const;
 
   const osm2rdf::config::Config _config;
   osm2rdf::ttl::Writer<W>* _writer;
   osm2rdf::osm::LocationHandler* _locationHandler;
+  bool _separateUntaggedNodePrefixes = false;
+  std::string _datasetId, _relNamespace, _wayNamespace, _changesetNamespace,
+      _iriXSDDouble, _iriWKTLiteral, _iriXSDInteger, _tagTripleCountIRI,
+      _areaIRI;
 };
 
 }  // namespace osm2rdf::osm

include/osm2rdf/osm/GeometryHandler.h

@@ -50,7 +50,7 @@ class GeometryHandler {
 
   // Add data
   void area(const osm2rdf::osm::Area& area);
-  void node(const osm2rdf::osm::Node& node);
+  void node(const osmium::Node& node);
   void relation(const osm2rdf::osm::Relation& relation);
   void way(const osm2rdf::osm::Way& way);
 
@@ -69,8 +69,6 @@ class GeometryHandler {
   sj::Sweeper _sweeper;
   std::vector<sj::WriteBatch> _parseBatches;
 
-  std::string areaNS(AreaFromType type) const;
-
   static ::util::geo::I32Point transform(const ::util::geo::DPoint& loc);
 
   static ::util::geo::I32Box transform(const ::util::geo::DBox& box);
@@ -79,11 +77,15 @@ class GeometryHandler {
   static ::util::geo::I32MultiPolygon transform(
       const ::util::geo::DMultiPolygon& area);
 
-  void writeRelCb(size_t t, const std::string& a, const std::string& b,
-                  const std::string& pred);
+  std::string getSweeperId(uint64_t oid, char type);
+  std::string getFullID(const char* id, size_t n);
+
+  void writeRelCb(size_t t, const char* a, size_t an, const char* b, size_t bn,
+                  const char* pred, size_t predn);
   void progressCb(size_t progr);
 
   osm2rdf::util::ProgressBar _progressBar;
+  bool _separateUntaggedNodePrefixes = false;
 };
 
 }  // namespace osm2rdf::osm

include/osm2rdf/osm/LocationHandler.h

@@ -41,6 +41,7 @@ class LocationHandler : public osmium::handler::Handler {
   virtual ~LocationHandler() {}
   virtual void node(const osmium::Node& node) = 0;
   virtual void way(osmium::Way& way) = 0;
+  virtual void finalizeNodes() = 0;
   [[nodiscard]] virtual osmium::Location get_node_location(
       const osmium::object_id_type id) const = 0;
   [[nodiscard]] virtual bool get_node_is_tagged(
@@ -57,6 +58,7 @@ class LocationHandlerImpl : public LocationHandler {
                                size_t nodeIdMin, size_t nodeIdMax);
   void node(const osmium::Node& node);
   void way(osmium::Way& way);
+  void finalizeNodes() {_nodesFinalized = true; };
   [[nodiscard]] osmium::Location get_node_location(
       const osmium::object_id_type nodeId) const;
   [[nodiscard]] bool get_node_is_tagged(
@@ -65,6 +67,7 @@ class LocationHandlerImpl : public LocationHandler {
  protected:
   T _index;
   osm2rdf::osm::handler::NodeLocationsForWays<T> _handler;
+  bool _nodesFinalized = false;
 };
 
 template <>
@@ -76,6 +79,7 @@ class LocationHandlerImpl<osmium::index::map::SparseFileArray<
                                size_t nodeIdMin, size_t nodeIdMax);
   void node(const osmium::Node& node);
   void way(osmium::Way& way);
+  void finalizeNodes() {_nodesFinalized = true; };
   [[nodiscard]] osmium::Location get_node_location(
       const osmium::object_id_type nodeId) const;
   [[nodiscard]] bool get_node_is_tagged(
@@ -90,6 +94,7 @@ class LocationHandlerImpl<osmium::index::map::SparseFileArray<
       osmium::index::map::SparseFileArray<osmium::unsigned_object_id_type,
                                           osm2rdf::osm::Location>>
       _handler;
+  bool _nodesFinalized = false;
 };
 
 template <>
@@ -101,6 +106,7 @@ class LocationHandlerImpl<osmium::index::map::DenseFileArray<
                                size_t nodeIdMin, size_t nodeIdMax);
   void node(const osmium::Node& node);
   void way(osmium::Way& way);
+  void finalizeNodes() {_nodesFinalized = true; };
   [[nodiscard]] osmium::Location get_node_location(
       const osmium::object_id_type nodeId) const;
   [[nodiscard]] bool get_node_is_tagged(
@@ -115,6 +121,7 @@ class LocationHandlerImpl<osmium::index::map::DenseFileArray<
       osmium::index::map::DenseFileArray<osmium::unsigned_object_id_type,
                                          osm2rdf::osm::Location>>
       _handler;
+  bool _nodesFinalized = false;
 };
 
 template <>
@@ -126,6 +133,7 @@ class LocationHandlerImpl<osm2rdf::osm::DenseMemIndex<
                                size_t nodeIdMin, size_t nodeIdMax);
   void node(const osmium::Node& node);
   void way(osmium::Way& way);
+  void finalizeNodes() {_nodesFinalized = true; };
   [[nodiscard]] osmium::Location get_node_location(
       const osmium::object_id_type nodeId) const;
   [[nodiscard]] bool get_node_is_tagged(
@@ -138,6 +146,7 @@ class LocationHandlerImpl<osm2rdf::osm::DenseMemIndex<
   osm2rdf::osm::handler::NodeLocationsForWays<osm2rdf::osm::DenseMemIndex<
       osmium::unsigned_object_id_type, osm2rdf::osm::Location>>
       _handler;
+  bool _nodesFinalized = false;
 };
 
 using LocationHandlerRAMDense = LocationHandlerImpl<osm2rdf::osm::DenseMemIndex<