rebase & clean up HDF5DataLayer Prefetch

Adapt HDF5DataLayer Prefetch to BVLC#2836

Author: ronghanghu

include/caffe/util/hdf5.hpp

@@ -10,6 +10,31 @@
 
 namespace caffe {
 
+/**
+ * @brief Shapes a Blob to read "num" rows of HDF5 data.  If num == -1, take
+ *        the num of the HDF5 dataset.
+ *
+ * @param file_id      the HDF5 file handle
+ * @param dataset_name the name of the HDF5 dataset to read
+ * @param num          the number of rows to read: either num >= 0,
+ *                     or num == -1 for the number of rows in the HDF5 dataset
+ * @param blob         the Blob to shape
+ *
+ * The HDF5 dataset could be N(>=1) dimensions as long as N doesn't exceed
+ * Blob's maximum dimension.
+ */
+template <typename Dtype>
+void HDF5PrepareBlob(hid_t file_id, const char* dataset_name, int num,
+                     Blob<Dtype>* blob);
+
+/**
+ * @brief Reads rows [offset, offset + data->num() - 1] into Blob* data, which
+ *        must have been pre-shaped using HDF5PrepareBlob (or otherwise).
+ */
+template <typename Dtype>
+int HDF5ReadRowsToBlob(hid_t file_id, const char* dataset_name,
+                       int h5_offset, int blob_offset, Blob<Dtype>* blob);
+
 template <typename Dtype>
 void hdf5_load_nd_dataset_helper(
     hid_t file_id, const char* dataset_name_, int min_dim, int max_dim,

include/caffe/util/io.hpp

@@ -136,34 +136,6 @@ cv::Mat DecodeDatumToCVMat(const Datum& datum, bool is_color);
 
 void CVMatToDatum(const cv::Mat& cv_img, Datum* datum);
 
-/**
- * @brief Shapes a Blob to read "num" rows of HDF5 data.  If num == -1, take
- *        the num of the HDF5 dataset.
- *
- * @param file_id      the HDF5 file handle
- * @param dataset_name the name of the HDF5 dataset to read
- * @param num          the number of rows to read: either num >= 0,
- *                     or num == -1 for the number of rows in the HDF5 dataset
- * @param blob         the Blob to shape
- *
- * The HDF5 dataset could be N(>=1) dimensions as long as N doesn't exceed Blob's maximum dimension.
- */
-template <typename Dtype>
-void HDF5PrepareBlob(hid_t file_id, const char* dataset_name, int num,
-                     Blob<Dtype>* blob);
-
-/**
- * @brief Reads rows [offset, offset + data->num() - 1] into Blob* data, which
- *        must have been pre-shaped using HDF5PrepareBlob (or otherwise).
- */
-template <typename Dtype>
-int HDF5ReadRowsToBlob(hid_t file_id, const char* dataset_name,
-                       int h5_offset, int blob_offset, Blob<Dtype>* blob);
-
-template <typename Dtype>
-void hdf5_save_nd_dataset(
-    const hid_t file_id, const string& dataset_name, const Blob<Dtype>& blob);
-
 }  // namespace caffe
 
 #endif   // CAFFE_UTIL_IO_H_

src/caffe/layers/hdf5_data_layer.cpp

@@ -8,13 +8,12 @@
 #include <string>
 #include <vector>
 
-#include "hdf5.h"
-#include "hdf5_hl.h"
 #include "stdint.h"
 
 #include "caffe/data_layers.hpp"
 #include "caffe/layer.hpp"
 #include "caffe/util/hdf5.hpp"
+#include "caffe/util/io.hpp"
 
 namespace caffe {
 

src/caffe/util/hdf5.cpp

@@ -5,6 +5,98 @@
 
 namespace caffe {
 
+// Verifies format of data stored in HDF5 file and reshapes blob accordingly.
+template <typename Dtype>
+void HDF5PrepareBlob(hid_t file_id, const char* dataset_name, int num,
+                     Blob<Dtype>* blob) {
+  // Verify that the dataset exists.
+  CHECK(H5LTfind_dataset(file_id, dataset_name))
+      << "Failed to find HDF5 dataset " << dataset_name;
+  herr_t status;
+  int ndims;
+  CHECK_LE(0, H5LTget_dataset_ndims(file_id, dataset_name, &ndims))
+      << "Failed to get dataset ndims for " << dataset_name;
+  CHECK_GE(ndims, 1) << "HDF5 dataset must have at least 1 dimension.";
+  CHECK_LE(ndims, kMaxBlobAxes)
+      << "HDF5 dataset must have at most "
+      << kMaxBlobAxes << " dimensions, to fit in a Blob.";
+
+  // Verify that the data format is what we expect: float or double.
+  std::vector<hsize_t> dims(ndims);
+  H5T_class_t h5_class;
+  status = H5LTget_dataset_info(
+      file_id, dataset_name, dims.data(), &h5_class, NULL);
+  CHECK_GE(status, 0) << "Failed to get dataset info for " << dataset_name;
+  CHECK_EQ(h5_class, H5T_FLOAT) << "Expected float or double data";
+  CHECK_GE(num, -1) << "num must be -1 (to indicate the number of rows"
+                       "in the dataset) or non-negative.";
+
+  vector<int> blob_dims(dims.size());
+  blob_dims[0] = (num == -1) ? dims[0] : num;
+  for (int i = 1; i < dims.size(); ++i) {
+    blob_dims[i] = dims[i];
+  }
+  blob->Reshape(blob_dims);
+}
+
+template
+void HDF5PrepareBlob<float>(hid_t file_id, const char* dataset_name, int num,
+                            Blob<float>* blob);
+
+template
+void HDF5PrepareBlob<double>(hid_t file_id, const char* dataset_name, int num,
+                             Blob<double>* blob);
+
+template <typename Dtype>
+int HDF5ReadRowsToBlob(hid_t file_id, const char* dataset_name,
+                       int h5_offset, int blob_offset, Blob<Dtype>* blob) {
+  int ndims;
+  CHECK_LE(0, H5LTget_dataset_ndims(file_id, dataset_name, &ndims))
+      << "Failed to get dataset ndims for " << dataset_name;
+  std::vector<hsize_t> dims(ndims);
+  H5T_class_t h5_class;
+  herr_t status = H5LTget_dataset_info(
+      file_id, dataset_name, dims.data(), &h5_class, NULL);
+  CHECK_GE(status, 0) << "Failed to get dataset info for " << dataset_name;
+  CHECK_EQ(h5_class, H5T_FLOAT) << "Expected float or double data";
+  hid_t dataset = H5Dopen2(file_id, dataset_name, H5P_DEFAULT);
+  hid_t dataspace = H5Dget_space(dataset);
+  vector<hsize_t> slab_start(ndims, 0);
+  slab_start[0] = h5_offset;
+  const int num_rows_available = dims[0] - h5_offset;
+  const int num_rows = std::min(blob->num() - blob_offset, num_rows_available);
+  if (num_rows <= 0) {
+    return 0;
+  }
+  vector<hsize_t> slab_count(ndims, num_rows);
+  for (int i = 1; i < ndims; ++i) {
+    slab_count[i] = dims[i];
+  }
+  status = H5Sselect_hyperslab(dataspace, H5S_SELECT_SET,
+      slab_start.data(), NULL, slab_count.data(), NULL);
+  CHECK_GE(status, 0) << "Failed to select slab.";
+  hid_t memspace = H5Screate_simple(ndims, slab_count.data(), NULL);
+  const int data_size = blob->count() / blob->num();
+  // separate multiplication to avoid a possible overflow
+  const int blob_offset_size = blob_offset * data_size;
+  hid_t type = (sizeof(Dtype) == 4) ? H5T_NATIVE_FLOAT : H5T_NATIVE_DOUBLE;
+  status = H5Dread(dataset, type, memspace, dataspace, H5P_DEFAULT,
+                   blob->mutable_cpu_data() + blob_offset_size);
+  CHECK_GE(status, 0) << "Failed to read dataset " << dataset_name;
+  H5Dclose(dataset);
+  H5Sclose(dataspace);
+  H5Sclose(memspace);
+  return num_rows;
+}
+
+template
+int HDF5ReadRowsToBlob<float>(hid_t file_id, const char* dataset_name,
+    int h5_offset, int blob_offset, Blob<float>* data);
+
+template
+int HDF5ReadRowsToBlob<double>(hid_t file_id, const char* dataset_name,
+    int h5_offset, int blob_offset, Blob<double>* data);
+
 // Verifies format of data stored in HDF5 file and reshapes blob accordingly.
 template <typename Dtype>
 void hdf5_load_nd_dataset_helper(
@@ -59,7 +151,7 @@ void hdf5_save_nd_dataset<float>(
     const hid_t file_id, const string& dataset_name, const Blob<float>& blob,
     bool write_diff) {
   int num_axes = blob.num_axes();
-  hsize_t *dims = new hsize_t[num_axes];
+  std::vector<hsize_t> dims(num_axes);
   for (int i = 0; i < num_axes; ++i) {
     dims[i] = blob.shape(i);
   }
@@ -70,17 +162,16 @@ void hdf5_save_nd_dataset<float>(
     data = blob.cpu_data();
   }
   herr_t status = H5LTmake_dataset_float(
-      file_id, dataset_name.c_str(), num_axes, dims, data);
+      file_id, dataset_name.c_str(), num_axes, dims.data(), data);
   CHECK_GE(status, 0) << "Failed to make float dataset " << dataset_name;
-  delete[] dims;
 }
 
 template <>
 void hdf5_save_nd_dataset<double>(
     hid_t file_id, const string& dataset_name, const Blob<double>& blob,
     bool write_diff) {
   int num_axes = blob.num_axes();
-  hsize_t *dims = new hsize_t[num_axes];
+  std::vector<hsize_t> dims(num_axes);
   for (int i = 0; i < num_axes; ++i) {
     dims[i] = blob.shape(i);
   }
@@ -91,9 +182,8 @@ void hdf5_save_nd_dataset<double>(
     data = blob.cpu_data();
   }
   herr_t status = H5LTmake_dataset_double(
-      file_id, dataset_name.c_str(), num_axes, dims, data);
+      file_id, dataset_name.c_str(), num_axes, dims.data(), data);
   CHECK_GE(status, 0) << "Failed to make double dataset " << dataset_name;
-  delete[] dims;
 }
 
 string hdf5_load_string(hid_t loc_id, const string& dataset_name) {

src/caffe/util/io.cpp

@@ -228,122 +228,4 @@ void CVMatToDatum(const cv::Mat& cv_img, Datum* datum) {
   datum->set_data(buffer);
 }
 
-// Verifies format of data stored in HDF5 file and reshapes blob accordingly.
-template <typename Dtype>
-void HDF5PrepareBlob(hid_t file_id, const char* dataset_name, int num,
-                     Blob<Dtype>* blob) {
-  // Verify that the dataset exists.
-  CHECK(H5LTfind_dataset(file_id, dataset_name))
-      << "Failed to find HDF5 dataset " << dataset_name;
-  herr_t status;
-  int ndims;
-  CHECK_LE(0, H5LTget_dataset_ndims(file_id, dataset_name, &ndims))
-      << "Failed to get dataset ndims for " << dataset_name;
-  CHECK_GE(ndims, 1) << "HDF5 dataset must have at least 1 dimension.";
-  CHECK_LE(ndims, kMaxBlobAxes)
-      << "HDF5 dataset must have at most "
-      << kMaxBlobAxes << " dimensions, to fit in a Blob.";
-
-  // Verify that the data format is what we expect: float or double.
-  std::vector<hsize_t> dims(ndims);
-  H5T_class_t h5_class;
-  status = H5LTget_dataset_info(
-      file_id, dataset_name, dims.data(), &h5_class, NULL);
-  CHECK_GE(status, 0) << "Failed to get dataset info for " << dataset_name;
-  CHECK_EQ(h5_class, H5T_FLOAT) << "Expected float or double data";
-  CHECK_GE(num, -1) << "num must be -1 (to indicate the number of rows"
-                       "in the dataset) or non-negative.";
-
-  vector<int> blob_dims(dims.size());
-  blob_dims[0] = (num == -1) ? dims[0] : num;
-  for (int i = 1; i < dims.size(); ++i) {
-    blob_dims[i] = dims[i];
-  }
-  blob->Reshape(blob_dims);
-}
-
-template
-void HDF5PrepareBlob<float>(hid_t file_id, const char* dataset_name, int num,
-                            Blob<float>* blob);
-
-template
-void HDF5PrepareBlob<double>(hid_t file_id, const char* dataset_name, int num,
-                             Blob<double>* blob);
-
-template <typename Dtype>
-int HDF5ReadRowsToBlob(hid_t file_id, const char* dataset_name,
-                       int h5_offset, int blob_offset, Blob<Dtype>* blob) {
-  int ndims;
-  CHECK_LE(0, H5LTget_dataset_ndims(file_id, dataset_name, &ndims))
-      << "Failed to get dataset ndims for " << dataset_name;
-  std::vector<hsize_t> dims(ndims);
-  H5T_class_t h5_class;
-  herr_t status = H5LTget_dataset_info(
-      file_id, dataset_name, dims.data(), &h5_class, NULL);
-  CHECK_GE(status, 0) << "Failed to get dataset info for " << dataset_name;
-  CHECK_EQ(h5_class, H5T_FLOAT) << "Expected float or double data";
-  hid_t dataset = H5Dopen2(file_id, dataset_name, H5P_DEFAULT);
-  hid_t dataspace = H5Dget_space(dataset);
-  vector<hsize_t> slab_start(ndims, 0);
-  slab_start[0] = h5_offset;
-  const int num_rows_available = dims[0] - h5_offset;
-  const int num_rows = std::min(blob->num() - blob_offset, num_rows_available);
-  if (num_rows <= 0) {
-    return 0;
-  }
-  vector<hsize_t> slab_count(ndims, num_rows);
-  for (int i = 1; i < ndims; ++i) {
-    slab_count[i] = dims[i];
-  }
-  status = H5Sselect_hyperslab(dataspace, H5S_SELECT_SET,
-      slab_start.data(), NULL, slab_count.data(), NULL);
-  CHECK_GE(status, 0) << "Failed to select slab.";
-  hid_t memspace = H5Screate_simple(ndims, slab_count.data(), NULL);
-  const int data_size = blob->count() / blob->num();
-  // separate multiplication to avoid a possible overflow
-  const int blob_offset_size = blob_offset * data_size;
-  hid_t type = (sizeof(Dtype) == 4) ? H5T_NATIVE_FLOAT : H5T_NATIVE_DOUBLE;
-  status = H5Dread(dataset, type, memspace, dataspace, H5P_DEFAULT,
-                   blob->mutable_cpu_data() + blob_offset_size);
-  CHECK_GE(status, 0) << "Failed to read dataset " << dataset_name;
-  H5Dclose(dataset);
-  H5Sclose(dataspace);
-  H5Sclose(memspace);
-  return num_rows;
-}
-
-template
-int HDF5ReadRowsToBlob<float>(hid_t file_id, const char* dataset_name,
-    int h5_offset, int blob_offset, Blob<float>* data);
-
-template
-int HDF5ReadRowsToBlob<double>(hid_t file_id, const char* dataset_name,
-    int h5_offset, int blob_offset, Blob<double>* data);
-
-template <>
-void hdf5_save_nd_dataset<float>(
-    const hid_t file_id, const string& dataset_name, const Blob<float>& blob) {
-  hsize_t dims[HDF5_NUM_DIMS];
-  dims[0] = blob.num();
-  dims[1] = blob.channels();
-  dims[2] = blob.height();
-  dims[3] = blob.width();
-  herr_t status = H5LTmake_dataset_float(
-      file_id, dataset_name.c_str(), HDF5_NUM_DIMS, dims, blob.cpu_data());
-  CHECK_GE(status, 0) << "Failed to make float dataset " << dataset_name;
-}
-
-template <>
-void hdf5_save_nd_dataset<double>(
-    const hid_t file_id, const string& dataset_name, const Blob<double>& blob) {
-  hsize_t dims[HDF5_NUM_DIMS];
-  dims[0] = blob.num();
-  dims[1] = blob.channels();
-  dims[2] = blob.height();
-  dims[3] = blob.width();
-  herr_t status = H5LTmake_dataset_double(
-      file_id, dataset_name.c_str(), HDF5_NUM_DIMS, dims, blob.cpu_data());
-  CHECK_GE(status, 0) << "Failed to make double dataset " << dataset_name;
-}
-
 }  // namespace caffe