Overhaul for streaming data directly from zip (optimization)

CMakeLists.txt

@@ -174,6 +174,7 @@ endif()
 # ── Core library ────────────────────────────────────────────────────────────
 add_library(trx
     src/trx.cpp
+    src/legacy_io.cpp
     src/detail/dtype_helpers.cpp
     include/trx/trx.h
     include/trx/trx.tpp
@@ -220,6 +221,16 @@ if(TRX_BUILD_TESTS)
     if(NOT GTest_FOUND)
         find_package(GTest QUIET)
     endif()
+    if(NOT GTest_FOUND)
+        message(STATUS "GTest not found; fetching v1.14.0")
+        FetchContent_Declare(
+            googletest
+            GIT_REPOSITORY https://github.com/google/googletest.git
+            GIT_TAG v1.14.0
+        )
+        FetchContent_MakeAvailable(googletest)
+        set(GTest_FOUND TRUE)
+    endif()
     if(GTest_FOUND)
         enable_testing()
         add_subdirectory(tests)

examples/trxinfo.cpp

@@ -17,6 +17,8 @@
 #include "cli_colors.h"
 
 namespace {
+using json = trx::json;
+
 std::string format_json_array(const json &value) {
   if (!value.is_array()) {
     return "n/a";
@@ -152,7 +154,7 @@ void print_trx_info(const trx::AnyTrxFile &trx, const std::string &path, bool is
       }
     }
   } else {
-    std::cout << "  " << colorize(colors, colors.cyan, "Data per group") << ": none\n";
+    std::cout << "  " << trx_cli::colorize(colors, colors.cyan, "Data per group") << ": none\n";
   }
 }
 

include/trx/legacy_io.h

@@ -0,0 +1,63 @@
+#ifndef TRX_LEGACY_IO_H
+#define TRX_LEGACY_IO_H
+
+#include <string>
+#include <vector>
+#include <cstdint>
+#include <memory>
+#include <trx/trx.h>
+
+namespace trx {
+namespace legacy {
+
+struct Tractogram {
+    std::vector<float> pts;
+    std::vector<uint64_t> offsets;
+    json11::Json header;
+    std::shared_ptr<trx::AnyTrxFile> original_trx;
+};
+
+#pragma pack(push, 1)
+struct TrkHeader {
+    char magic_number[6];
+    int16_t dimensions[3];
+    float voxel_sizes[3];
+    float origin[3];
+    int16_t nb_scalars_per_point;
+    char scalar_name[10][20];
+    int16_t nb_properties_per_streamline;
+    char property_name[10][20];
+    float voxel_to_rasmm[4][4];
+    char reserved[444];
+    char voxel_order[4];
+    char pad2[4];
+    float image_orientation_patient[6];
+    char pad1[2];
+    char invert_x;
+    char invert_y;
+    char invert_z;
+    char swap_xy;
+    char swap_yz;
+    char swap_zx;
+    int32_t nb_streamlines;
+    int32_t version;
+    int32_t hdr_size;
+};
+#pragma pack(pop)
+
+bool load_trx(const std::string &filename, Tractogram &tr);
+bool load_trk(const std::string &filename, Tractogram &tr);
+bool load_tck(const std::string &filename, Tractogram &tr);
+bool load_vtk(const std::string &filename, Tractogram &tr);
+
+bool load_nifti_header(const std::string &ref_path, json11::Json &out_header);
+
+bool save_trx(const Tractogram &tr, const std::string &out_path, const std::string &ref_nifti_path = "");
+bool save_trk(const Tractogram &tr, const std::string &out_path, const std::string &original_filename = "", const std::string &ref_nifti_path = "");
+bool save_tck(const Tractogram &tr, const std::string &out_path);
+bool save_vtk(const Tractogram &tr, const std::string &out_path);
+
+} // namespace legacy
+} // namespace trx
+
+#endif // TRX_LEGACY_IO_H

include/trx/trx.h

@@ -40,9 +40,8 @@
 
 namespace trx {
 namespace fs = std::filesystem;
-}
-
 using json = json11::Json;
+}
 
 namespace trx {
 enum class TrxSaveMode { Auto, Archive, Directory };
@@ -297,7 +296,7 @@ template <typename DT> class TrxFile {
   std::unique_ptr<TrxFile<DT>> deepcopy();
 
   /**
-   * @brief Remove the ununsed portion of preallocated memmaps
+   * @brief Remove the unused portion of preallocated memmaps
    *
    * @param nb_streamlines The number of streamlines to keep
    * @param nb_vertices The number of vertices to keep

include/trx/trx.tpp

@@ -152,14 +152,32 @@ template <class Matrix> void write_binary(const std::string &filename, const Mat
 }
 template <class Matrix> void read_binary(const std::string &filename, Matrix &matrix) {
   std::ifstream in(filename, std::ios::in | std::ios::binary);
-  typename Matrix::Index rows = 0, cols = 0;
-  auto *rows_ptr = reinterpret_cast<char *>(&rows); // check_syntax off
-  auto *cols_ptr = reinterpret_cast<char *>(&cols); // check_syntax off
-  in.read(rows_ptr, sizeof(typename Matrix::Index));
-  in.read(cols_ptr, sizeof(typename Matrix::Index));
-  matrix.resize(rows, cols);
-  auto *matrix_ptr = reinterpret_cast<char *>(matrix.data()); // check_syntax off
-  in.read(matrix_ptr, rows * cols * sizeof(typename Matrix::Scalar));
+  if (!in.is_open()) {
+    throw TrxIOError("Failed to open binary file for reading: " + filename);
+  }
+  typename Matrix::Index rows = matrix.rows(), cols = matrix.cols();
+  if (rows == 0 && cols == 0) {
+    in.seekg(0, std::ios::end);
+    std::streamsize file_size = in.tellg();
+    in.seekg(0, std::ios::beg);
+    if (file_size > 0 && sizeof(typename Matrix::Scalar) > 0) {
+      rows = file_size / sizeof(typename Matrix::Scalar);
+      cols = 1;
+      matrix.resize(rows, cols);
+    }
+  } else if (rows == 0 && cols > 0) {
+    in.seekg(0, std::ios::end);
+    std::streamsize file_size = in.tellg();
+    in.seekg(0, std::ios::beg);
+    if (file_size > 0 && sizeof(typename Matrix::Scalar) > 0) {
+      rows = file_size / (cols * sizeof(typename Matrix::Scalar));
+      matrix.resize(rows, cols);
+    }
+  }
+  if (rows > 0 && cols > 0) {
+    auto *data = reinterpret_cast<char *>(matrix.data()); // check_syntax off
+    in.read(data, rows * cols * sizeof(typename Matrix::Scalar));
+  }
   in.close();
 }
 
@@ -244,7 +262,7 @@ TrxFile<DT>::TrxFile(int nb_vertices, int nb_streamlines, const TrxFile<DT> *ini
       throw TrxArgumentError("Can't use init_as without declaring nb_vertices and nb_streamlines");
     }
 
-    // will remove as completely unecessary. using as placeholders
+    // will remove as completely unnecessary. using as placeholders
     this->header = {};
     this->streamlines.reset();
 
@@ -432,7 +450,7 @@ TrxFile<DT>::_create_trx_from_pointer(json header,
 
     auto [base, dim, ext] = trx::detail::_split_ext_with_dimensionality(elem_filename);
 
-    long long mem_adress = std::get<0>(x->second);
+    long long mem_address = std::get<0>(x->second);
     long long size = std::get<1>(x->second);
 
     if (base == "positions" && (folder.empty() || folder == ".")) {
@@ -446,7 +464,7 @@ TrxFile<DT>::_create_trx_from_pointer(json header,
 
       std::tuple<int, int> shape = std::make_tuple(static_cast<int>(trx->header["NB_VERTICES"].int_value()), 3);
       trx->streamlines->mmap_pos =
-          trx::_create_memmap(filename, shape, "r+", ext, mem_adress);
+          trx::_create_memmap(filename, shape, "r+", ext, mem_address);
 
       trx::detail::remap(trx->streamlines->_data, trx->streamlines->mmap_pos.data(), shape);
     }
@@ -466,7 +484,7 @@ TrxFile<DT>::_create_trx_from_pointer(json header,
       const int offsets_rows = missing_sentinel ? (nb_str + 1) : static_cast<int>(size);
       std::tuple<int, int> shape = std::make_tuple(offsets_rows, 1);
       trx->streamlines->mmap_off = trx::_create_memmap(filename, std::make_tuple(static_cast<int>(size), 1), "r+",
-                                                       ext, mem_adress);
+                                                       ext, mem_address);
 
       if (ext == "uint64") {
         if (missing_sentinel) {
@@ -509,7 +527,7 @@ TrxFile<DT>::_create_trx_from_pointer(json header,
       } else {
         shape = std::make_tuple(static_cast<int>(trx->header["NB_STREAMLINES"].int_value()), nb_scalar);
       }
-      trx->data_per_streamline[base]->mmap = trx::_create_memmap(filename, shape, "r+", ext, mem_adress);
+      trx->data_per_streamline[base]->mmap = trx::_create_memmap(filename, shape, "r+", ext, mem_address);
       const std::string expected_dtype = dtype_from_scalar<DT>();
       if (ext == expected_dtype) {
         trx::detail::remap(trx->data_per_streamline[base]->_matrix, trx->data_per_streamline[base]->mmap.data(), shape);
@@ -534,7 +552,7 @@ TrxFile<DT>::_create_trx_from_pointer(json header,
       } else {
         shape = std::make_tuple(static_cast<int>(trx->header["NB_VERTICES"].int_value()), nb_scalar);
       }
-      trx->data_per_vertex[base]->mmap_pos = trx::_create_memmap(filename, shape, "r+", ext, mem_adress);
+      trx->data_per_vertex[base]->mmap_pos = trx::_create_memmap(filename, shape, "r+", ext, mem_address);
       const std::string expected_dtype = dtype_from_scalar<DT>();
       if (ext == expected_dtype) {
         trx::detail::remap(trx->data_per_vertex[base]->_data, trx->data_per_vertex[base]->mmap_pos.data(), shape);
@@ -564,7 +582,7 @@ TrxFile<DT>::_create_trx_from_pointer(json header,
       std::string sub_folder = path_basename(folder);
 
       trx->data_per_group[sub_folder][data_name] = std::make_unique<MMappedMatrix<DT>>();
-      trx->data_per_group[sub_folder][data_name]->mmap = trx::_create_memmap(filename, shape, "r+", ext, mem_adress);
+      trx->data_per_group[sub_folder][data_name]->mmap = trx::_create_memmap(filename, shape, "r+", ext, mem_address);
       const std::string expected_dtype = dtype_from_scalar<DT>();
       if (ext == expected_dtype) {
         trx::detail::remap(trx->data_per_group[sub_folder][data_name]->_matrix,
@@ -593,7 +611,7 @@ TrxFile<DT>::_create_trx_from_pointer(json header,
       info.rows = std::get<0>(shape);
       info.cols = std::get<1>(shape);
       info.dtype = ext;
-      info.mem_offset = mem_adress;
+      info.mem_offset = mem_address;
       trx->group_backing_info_[base] = std::move(info);
     } else {
       throw TrxFormatError("Entry is not part of a valid TRX structure: " + elem_filename);

main.cpp

@@ -0,0 +1,68 @@
+#include <iostream>
+#include <string>
+#include <trx/legacy_io.h>
+#include <trx/trx.h>
+
+int main(int argc, char** argv) {
+    std::string input_file;
+    std::string output_file;
+    std::string ref_path;
+
+    for (int i = 1; i < argc; ++i) {
+        std::string arg = argv[i];
+        if (arg == "--ref") {
+            if (i + 1 < argc) {
+                ref_path = argv[++i];
+            } else {
+                std::cerr << "Error: --ref requires an argument\n";
+                return 1;
+            }
+        } else if (input_file.empty()) {
+            input_file = arg;
+        } else if (output_file.empty()) {
+            output_file = arg;
+        }
+    }
+
+    if (input_file.empty() || output_file.empty()) {
+        std::cerr << "Usage: convert <input> <output> [--ref <nifti_file>]\n";
+        return 1;
+    }
+
+    trx::legacy::Tractogram tr;
+    bool success = false;
+
+    auto ends_with = [](const std::string& str, const std::string& suffix) {
+        return str.size() >= suffix.size() && str.compare(str.size() - suffix.size(), suffix.size(), suffix) == 0;
+    };
+
+    if (ends_with(input_file, ".trx")) success = trx::legacy::load_trx(input_file, tr);
+    else if (ends_with(input_file, ".trk")) success = trx::legacy::load_trk(input_file, tr);
+    else if (ends_with(input_file, ".tck")) success = trx::legacy::load_tck(input_file, tr);
+    else if (ends_with(input_file, ".vtk")) success = trx::legacy::load_vtk(input_file, tr);
+
+    if (!success) {
+        std::cerr << "Error loading input file\n";
+        return 1;
+    }
+
+    bool is_tck_vtk = ends_with(input_file, ".tck") || ends_with(input_file, ".vtk");
+    bool is_trx_trk = ends_with(output_file, ".trx") || ends_with(output_file, ".trk");
+
+    if (is_tck_vtk && is_trx_trk && ref_path.empty()) {
+        std::cerr << "Error: TCK/VTK -> TRX/TRK conversion requires --ref <nifti_file>\n";
+        return 1;
+    }
+
+    success = false;
+    if (ends_with(output_file, ".trx")) success = trx::legacy::save_trx(tr, output_file, ref_path);
+    else if (ends_with(output_file, ".trk")) success = trx::legacy::save_trk(tr, output_file, input_file, ref_path);
+    else if (ends_with(output_file, ".tck")) success = trx::legacy::save_tck(tr, output_file);
+    else if (ends_with(output_file, ".vtk")) success = trx::legacy::save_vtk(tr, output_file);
+
+    if (!success) {
+        std::cerr << "Error saving output file\n";
+        return 1;
+    }
+    return 0;
+}