From 6b1a9ffa6d6ed64264e0b7a2f3f9a0859540faf1 Mon Sep 17 00:00:00 2001 From: Leonard Lausen Date: Mon, 8 Oct 2018 04:06:26 +0000 Subject: [PATCH 1/3] Update npy alignment to 64bit following numpy v1.14 https://github.com/numpy/numpy/pull/9025/ --- cnpy.h | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/cnpy.h b/cnpy.h index 0d3bb4c..c7a0c23 100644 --- a/cnpy.h +++ b/cnpy.h @@ -247,10 +247,12 @@ namespace cnpy { } if(shape.size() == 1) dict += ","; dict += "), }"; - //pad with spaces so that preamble+dict is modulo 16 bytes. preamble is 10 bytes. dict needs to end with \n - int remainder = 16 - (10 + dict.size()) % 16; - dict.insert(dict.end(),remainder,' '); - dict.back() = '\n'; + // pad with spaces so that preamble+dict is modulo 64 bytes. preamble is + // 10 bytes. dict needs to end with \n + int remainder = 64 - (10 + dict.size() + 1) % 64; + dict.insert(dict.end(), remainder, ' '); + dict.push_back('\n'); + assert((dict.size() + 10) % 64 == 0); std::vector header; header += (char) 0x93; From 9e7f2c3cd949d504d9aec17ebf20c87e22b21efd Mon Sep 17 00:00:00 2001 From: Leonard Lausen Date: Mon, 8 Oct 2018 07:21:20 +0000 Subject: [PATCH 2/3] Switch to libzip --- .gitmodules | 3 + CMakeLists.txt | 31 +++++--- cmake/FindLibZip.cmake | 18 +++++ cnpy.cpp | 170 +++++++++++------------------------------ cnpy.h | 128 ++++++++++--------------------- extern/libzippp | 1 + 6 files changed, 129 insertions(+), 222 deletions(-) create mode 100644 .gitmodules create mode 100644 cmake/FindLibZip.cmake create mode 160000 extern/libzippp diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..ac5c1d0 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "extern/libzippp"] + path = extern/libzippp + url = https://github.com/leezu/libzippp diff --git a/CMakeLists.txt b/CMakeLists.txt index 9eb550f..5845e87 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,20 +1,33 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 3.0 FATAL_ERROR) -if(COMMAND cmake_policy) - cmake_policy(SET CMP0003 NEW) -endif(COMMAND cmake_policy) +CMAKE_MINIMUM_REQUIRED(VERSION 3.1) project(CNPY) -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") +# Update submodules as needed +find_package(Git QUIET) +if(GIT_FOUND) + option(GIT_SUBMODULE "Check submodules during build" ON) + if(GIT_SUBMODULE) + message(STATUS "Submodule update") + execute_process(COMMAND ${GIT_EXECUTABLE} submodule update --init + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + RESULT_VARIABLE GIT_SUBMOD_RESULT) + if(NOT GIT_SUBMOD_RESULT EQUAL "0") + message(FATAL_ERROR "git submodule update --init failed with ${GIT_SUBMOD_RESULT}, please checkout submodules") + endif() + endif() +endif() option(ENABLE_STATIC "Build static (.a) library" ON) -find_package(ZLIB REQUIRED) - -include_directories(${ZLIB_INCLUDE_DIRS}) +SET(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake" ${CMAKE_MODULE_PATH}) +FIND_PACKAGE(LibZip REQUIRED) add_library(cnpy SHARED "cnpy.cpp") -target_link_libraries(cnpy ${ZLIB_LIBRARIES}) +target_include_directories(cnpy PUBLIC ${LIBZIP_INCLUDE_DIR_ZIP}) +target_include_directories(cnpy PUBLIC "extern/libzippp/") +target_link_libraries(cnpy ${LIBZIP_LIBRARIES}) +target_compile_features(cnpy PRIVATE cxx_std_14) + install(TARGETS "cnpy" LIBRARY DESTINATION lib PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) if(ENABLE_STATIC) diff --git a/cmake/FindLibZip.cmake b/cmake/FindLibZip.cmake new file mode 100644 index 0000000..7993210 --- /dev/null +++ b/cmake/FindLibZip.cmake @@ -0,0 +1,18 @@ +find_package(PkgConfig) +pkg_check_modules(PC_LIBZIP QUIET libzip) +set(LIBZIP_DEFINITIONS ${PC_LIBZIP_CFLAGS_OTHER}) + +find_path(LIBZIP_INCLUDE_DIR zip.h + HINTS ${PC_LIBZIP_INCLUDEDIR} ${PC_LIBZIPP_INCLUDE_DIRS} + PATH_SUFFIXES zip ) + +find_library(LIBZIP_LIBRARY NAMES zip libzip + HINTS ${PC_LIBZIP_LIBDIR} ${PC_LIBZIP_LIBRARY_DIRS} ) + +set(LIBZIP_LIBRARIES ${LIBZIP_LIBRARY} ) +set(LIBZIP_INCLUDE_DIRS ${LIBZIP_INCLUDE_DIR} ) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(libzip DEFAULT_MSG LIBZIP_LIBRARY LIBZIP_INCLUDE_DIR) + +mark_as_advanced(LIBZIP_INCLUDE_DIR LIBZIP_LIBRARY ) diff --git a/cnpy.cpp b/cnpy.cpp index 2d28578..c284ca5 100644 --- a/cnpy.cpp +++ b/cnpy.cpp @@ -59,12 +59,15 @@ template<> std::vector& cnpy::operator+=(std::vector& lhs, const cha return lhs; } -void cnpy::parse_npy_header(unsigned char* buffer,size_t& word_size, std::vector& shape, bool& fortran_order) { - //std::string magic_string(buffer,6); - uint8_t major_version = *reinterpret_cast(buffer+6); - uint8_t minor_version = *reinterpret_cast(buffer+7); - uint16_t header_len = *reinterpret_cast(buffer+8); - std::string header(reinterpret_cast(buffer+9),header_len); +void cnpy::parse_npy_header(std::string &buffer, size_t &word_size, + std::vector &shape, bool &fortran_order) { + assert(buffer.substr(0, 6) == "\x93NUMPY"); + uint8_t major_version = *reinterpret_cast(buffer.data() + 6); + uint8_t minor_version = *reinterpret_cast(buffer.data() + 7); + uint16_t header_len = + (*reinterpret_cast(buffer.data() + 8)) | + ((*reinterpret_cast(buffer.data() + 9)) << 8); + std::string header = buffer.substr(10, header_len); size_t loc1, loc2; @@ -188,140 +191,55 @@ cnpy::NpyArray load_the_npy_file(FILE* fp) { return arr; } -cnpy::NpyArray load_the_npz_array(FILE* fp, uint32_t compr_bytes, uint32_t uncompr_bytes) { +cnpy::NpyArray load_the_npz_array(libzip::file &file, uint64_t size) { + std::string buffer = file.read(size); - std::vector buffer_compr(compr_bytes); - std::vector buffer_uncompr(uncompr_bytes); - size_t nread = fread(&buffer_compr[0],1,compr_bytes,fp); - if(nread != compr_bytes) - throw std::runtime_error("load_the_npy_file: failed fread"); - - int err; - z_stream d_stream; - - d_stream.zalloc = Z_NULL; - d_stream.zfree = Z_NULL; - d_stream.opaque = Z_NULL; - d_stream.avail_in = 0; - d_stream.next_in = Z_NULL; - err = inflateInit2(&d_stream, -MAX_WBITS); - - d_stream.avail_in = compr_bytes; - d_stream.next_in = &buffer_compr[0]; - d_stream.avail_out = uncompr_bytes; - d_stream.next_out = &buffer_uncompr[0]; - - err = inflate(&d_stream, Z_FINISH); - err = inflateEnd(&d_stream); - - std::vector shape; - size_t word_size; - bool fortran_order; - cnpy::parse_npy_header(&buffer_uncompr[0],word_size,shape,fortran_order); + std::vector shape; + size_t word_size; + bool fortran_order; + cnpy::parse_npy_header(buffer, word_size, + shape, fortran_order); - cnpy::NpyArray array(shape, word_size, fortran_order); + cnpy::NpyArray array(shape, word_size, fortran_order); - size_t offset = uncompr_bytes - array.num_bytes(); - memcpy(array.data(),&buffer_uncompr[0]+offset,array.num_bytes()); + uint64_t offset = size - array.num_bytes(); + memcpy(array.data(), buffer.data() + offset, + array.num_bytes()); - return array; + return array; } cnpy::npz_t cnpy::npz_load(std::string fname) { - FILE* fp = fopen(fname.c_str(),"rb"); + libzip::archive zip(fname, ZIP_RDONLY); + cnpy::npz_t arrays; - if(!fp) { - throw std::runtime_error("npz_load: Error! Unable to open file "+fname+"!"); + for (auto stat : zip) { + if (!(stat.valid & (ZIP_STAT_NAME | ZIP_STAT_SIZE))) { + // Skip files without name or size + continue; } + std::string name(stat.name); + auto file = zip.open(name); - cnpy::npz_t arrays; - - while(1) { - std::vector local_header(30); - size_t headerres = fread(&local_header[0],sizeof(char),30,fp); - if(headerres != 30) - throw std::runtime_error("npz_load: failed fread"); - - //if we've reached the global header, stop reading - if(local_header[2] != 0x03 || local_header[3] != 0x04) break; - - //read in the variable name - uint16_t name_len = *(uint16_t*) &local_header[26]; - std::string varname(name_len,' '); - size_t vname_res = fread(&varname[0],sizeof(char),name_len,fp); - if(vname_res != name_len) - throw std::runtime_error("npz_load: failed fread"); - - //erase the lagging .npy - varname.erase(varname.end()-4,varname.end()); - - //read in the extra field - uint16_t extra_field_len = *(uint16_t*) &local_header[28]; - if(extra_field_len > 0) { - std::vector buff(extra_field_len); - size_t efield_res = fread(&buff[0],sizeof(char),extra_field_len,fp); - if(efield_res != extra_field_len) - throw std::runtime_error("npz_load: failed fread"); - } - - uint16_t compr_method = *reinterpret_cast(&local_header[0]+8); - uint32_t compr_bytes = *reinterpret_cast(&local_header[0]+18); - uint32_t uncompr_bytes = *reinterpret_cast(&local_header[0]+22); - - if(compr_method == 0) {arrays[varname] = load_the_npy_file(fp);} - else {arrays[varname] = load_the_npz_array(fp,compr_bytes,uncompr_bytes);} - } + // erase the lagging .npy + std::string varname = name.substr(0, name.size() - 4); + arrays[varname] = load_the_npz_array(file, stat.size); - fclose(fp); - return arrays; + } + + return arrays; } cnpy::NpyArray cnpy::npz_load(std::string fname, std::string varname) { - FILE* fp = fopen(fname.c_str(),"rb"); - - if(!fp) throw std::runtime_error("npz_load: Unable to open file "+fname); - - while(1) { - std::vector local_header(30); - size_t header_res = fread(&local_header[0],sizeof(char),30,fp); - if(header_res != 30) - throw std::runtime_error("npz_load: failed fread"); - - //if we've reached the global header, stop reading - if(local_header[2] != 0x03 || local_header[3] != 0x04) break; - - //read in the variable name - uint16_t name_len = *(uint16_t*) &local_header[26]; - std::string vname(name_len,' '); - size_t vname_res = fread(&vname[0],sizeof(char),name_len,fp); - if(vname_res != name_len) - throw std::runtime_error("npz_load: failed fread"); - vname.erase(vname.end()-4,vname.end()); //erase the lagging .npy - - //read in the extra field - uint16_t extra_field_len = *(uint16_t*) &local_header[28]; - fseek(fp,extra_field_len,SEEK_CUR); //skip past the extra field - - uint16_t compr_method = *reinterpret_cast(&local_header[0]+8); - uint32_t compr_bytes = *reinterpret_cast(&local_header[0]+18); - uint32_t uncompr_bytes = *reinterpret_cast(&local_header[0]+22); - - if(vname == varname) { - NpyArray array = (compr_method == 0) ? load_the_npy_file(fp) : load_the_npz_array(fp,compr_bytes,uncompr_bytes); - fclose(fp); - return array; - } - else { - //skip past the data - uint32_t size = *(uint32_t*) &local_header[22]; - fseek(fp,size,SEEK_CUR); - } - } - - fclose(fp); - - //if we get here, we haven't found the variable in the file - throw std::runtime_error("npz_load: Variable name "+varname+" not found in "+fname); + libzip::archive zip(fname, ZIP_RDONLY); + std::string varfilename = varname + ".npy"; + libzip::stat stat = zip.stat(varfilename); + if (!(stat.valid & ZIP_STAT_SIZE)) { + throw std::runtime_error("npz_load: No size associated with " + varname); + } + + auto file = zip.open(varfilename); + return load_the_npz_array(file, stat.size); } cnpy::NpyArray cnpy::npy_load(std::string fname) { diff --git a/cnpy.h b/cnpy.h index c7a0c23..fc7275c 100644 --- a/cnpy.h +++ b/cnpy.h @@ -6,6 +6,7 @@ #define LIBCNPY_H_ #include +#include #include #include #include @@ -19,6 +20,8 @@ #include #include +#include "./zip.hpp" + namespace cnpy { struct NpyArray { @@ -66,7 +69,7 @@ namespace cnpy { char map_type(const std::type_info& t); template std::vector create_npy_header(const std::vector& shape); void parse_npy_header(FILE* fp,size_t& word_size, std::vector& shape, bool& fortran_order); - void parse_npy_header(unsigned char* buffer,size_t& word_size, std::vector& shape, bool& fortran_order); + void parse_npy_header(std::string &buffer,size_t& word_size, std::vector& shape, bool& fortran_order); void parse_zip_footer(FILE* fp, uint16_t& nrecs, size_t& global_header_size, size_t& global_header_offset); npz_t npz_load(std::string fname); NpyArray npz_load(std::string fname, std::string varname); @@ -130,94 +133,45 @@ namespace cnpy { fclose(fp); } - template void npz_save(std::string zipname, std::string fname, const T* data, const std::vector& shape, std::string mode = "w") - { - //first, append a .npy to the fname - fname += ".npy"; - - //now, on with the show - FILE* fp = NULL; - uint16_t nrecs = 0; - size_t global_header_offset = 0; - std::vector global_header; - - if(mode == "a") fp = fopen(zipname.c_str(),"r+b"); - - if(fp) { - //zip file exists. we need to add a new npy file to it. - //first read the footer. this gives us the offset and size of the global header - //then read and store the global header. - //below, we will write the the new data at the start of the global header then append the global header and footer below it - size_t global_header_size; - parse_zip_footer(fp,nrecs,global_header_size,global_header_offset); - fseek(fp,global_header_offset,SEEK_SET); - global_header.resize(global_header_size); - size_t res = fread(&global_header[0],sizeof(char),global_header_size,fp); - if(res != global_header_size){ - throw std::runtime_error("npz_save: header read error while adding to existing zip"); - } - fseek(fp,global_header_offset,SEEK_SET); + template + void npz_save(std::string zipname, std::string fname, const T *data, + const std::vector &shape, std::string mode = "w") { + libzip::flags_t flag = 0; + if (mode == "a") { + } else if (mode == "w") { + flag = ZIP_TRUNCATE | ZIP_CREATE; + } else { + throw std::runtime_error("npz_save: Invalid mode."); + } + libzip::archive zip(zipname, flag); + + std::vector npy_header = create_npy_header(shape); + + // Prepare buffer + size_t nels = std::accumulate(shape.begin(), shape.end(), 1, + std::multiplies()); + size_t nbytes = nels * sizeof(T) + npy_header.size(); + + + auto ptr = static_cast(std::malloc(nbytes)); + if (ptr == nullptr) + throw std::runtime_error(std::strerror(errno)); + std::memcpy(ptr, npy_header.data(), npy_header.size()); + std::memcpy(ptr + npy_header.size(), data, nels * sizeof(T)); + + // Create source buffer functor - memory will be freed automatically + libzip::source source = + [ ptr, nbytes ](struct zip * archive) -> struct zip_source * { + auto src = zip_source_buffer(archive, ptr, nbytes, 1); + if (src == nullptr) { + std::free(ptr); + throw std::runtime_error(zip_strerror(archive)); } - else { - fp = fopen(zipname.c_str(),"wb"); - } - - std::vector npy_header = create_npy_header(shape); + return src; + }; - size_t nels = std::accumulate(shape.begin(),shape.end(),1,std::multiplies()); - size_t nbytes = nels*sizeof(T) + npy_header.size(); - - //get the CRC of the data to be added - uint32_t crc = crc32(0L,(uint8_t*)&npy_header[0],npy_header.size()); - crc = crc32(crc,(uint8_t*)data,nels*sizeof(T)); - - //build the local header - std::vector local_header; - local_header += "PK"; //first part of sig - local_header += (uint16_t) 0x0403; //second part of sig - local_header += (uint16_t) 20; //min version to extract - local_header += (uint16_t) 0; //general purpose bit flag - local_header += (uint16_t) 0; //compression method - local_header += (uint16_t) 0; //file last mod time - local_header += (uint16_t) 0; //file last mod date - local_header += (uint32_t) crc; //crc - local_header += (uint32_t) nbytes; //compressed size - local_header += (uint32_t) nbytes; //uncompressed size - local_header += (uint16_t) fname.size(); //fname length - local_header += (uint16_t) 0; //extra field length - local_header += fname; - - //build global header - global_header += "PK"; //first part of sig - global_header += (uint16_t) 0x0201; //second part of sig - global_header += (uint16_t) 20; //version made by - global_header.insert(global_header.end(),local_header.begin()+4,local_header.begin()+30); - global_header += (uint16_t) 0; //file comment length - global_header += (uint16_t) 0; //disk number where file starts - global_header += (uint16_t) 0; //internal file attributes - global_header += (uint32_t) 0; //external file attributes - global_header += (uint32_t) global_header_offset; //relative offset of local file header, since it begins where the global header used to begin - global_header += fname; - - //build footer - std::vector footer; - footer += "PK"; //first part of sig - footer += (uint16_t) 0x0605; //second part of sig - footer += (uint16_t) 0; //number of this disk - footer += (uint16_t) 0; //disk where footer starts - footer += (uint16_t) (nrecs+1); //number of records on this disk - footer += (uint16_t) (nrecs+1); //total number of records - footer += (uint32_t) global_header.size(); //nbytes of global headers - footer += (uint32_t) (global_header_offset + nbytes + local_header.size()); //offset of start of global headers, since global header now starts after newly written array - footer += (uint16_t) 0; //zip file comment length - - //write everything - fwrite(&local_header[0],sizeof(char),local_header.size(),fp); - fwrite(&npy_header[0],sizeof(char),npy_header.size(),fp); - fwrite(data,sizeof(T),nels,fp); - fwrite(&global_header[0],sizeof(char),global_header.size(),fp); - fwrite(&footer[0],sizeof(char),footer.size(),fp); - fclose(fp); + // Write everything + zip.add(source, fname + ".npy"); } template void npy_save(std::string fname, const std::vector data, std::string mode = "w") { diff --git a/extern/libzippp b/extern/libzippp new file mode 160000 index 0000000..ba9aec4 --- /dev/null +++ b/extern/libzippp @@ -0,0 +1 @@ +Subproject commit ba9aec43f7760c3384bbe36f9a5454912f61307a From 82ff25140ef6e61bd16ad23b32170b455ee1a52a Mon Sep 17 00:00:00 2001 From: Leonard Lausen Date: Mon, 8 Oct 2018 11:56:02 +0000 Subject: [PATCH 3/3] Default to storing files uncompressed in .npz --- cnpy.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cnpy.h b/cnpy.h index fc7275c..e06771d 100644 --- a/cnpy.h +++ b/cnpy.h @@ -135,7 +135,8 @@ namespace cnpy { template void npz_save(std::string zipname, std::string fname, const T *data, - const std::vector &shape, std::string mode = "w") { + const std::vector &shape, std::string mode = "w", + int32_t comp = ZIP_CM_STORE) { libzip::flags_t flag = 0; if (mode == "a") { } else if (mode == "w") { @@ -171,7 +172,8 @@ namespace cnpy { }; // Write everything - zip.add(source, fname + ".npy"); + uint64_t index = zip.add(source, fname + ".npy"); + zip.set_file_compression(index, comp); } template void npy_save(std::string fname, const std::vector data, std::string mode = "w") {