[关闭]
@cleardusk 2016-03-06T14:47:24.000000Z 字数 6675 阅读 1307

3.3

GjzCV


loss 层

loss function is also knowns as an error, cost, or objective function

logistic function


wiki链接

Softmax function

Logistic regression

手动编译 mnist 的 convert_minist_data 源码

删除部分 leveldb 后的代码:

  1. #include <iostream>
  2. #include <gflags/gflags.h>
  3. #include <glog/logging.h>
  4. #include <google/protobuf/text_format.h>
  5. #include <lmdb.h>
  6. #include <caffe/proto/caffe.pb.h>
  7. #include <stdint.h>
  8. #include <sys/stat.h>
  9. #include <fstream> // NOLINT(readability/streams)
  10. #include <string>
  11. using namespace caffe; // NOLINT(build/namespaces)
  12. using std::string;
  13. DEFINE_string(backend, "lmdb", "The backend for storing the result");
  14. uint32_t swap_endian(uint32_t val) {
  15. val = ((val << 8) & 0xFF00FF00) | ((val >> 8) & 0xFF00FF);
  16. return (val << 16) | (val >> 16);
  17. }
  18. void convert_dataset(const char* image_filename, const char* label_filename,
  19. const char* db_path, const string& db_backend) {
  20. // Open files
  21. std::ifstream image_file(image_filename, std::ios::in | std::ios::binary);
  22. std::ifstream label_file(label_filename, std::ios::in | std::ios::binary);
  23. CHECK(image_file) << "Unable to open file " << image_filename;
  24. CHECK(label_file) << "Unable to open file " << label_filename;
  25. // Read the magic and the meta data
  26. uint32_t magic;
  27. uint32_t num_items;
  28. uint32_t num_labels;
  29. uint32_t rows;
  30. uint32_t cols;
  31. image_file.read(reinterpret_cast<char*>(&magic), 4);
  32. magic = swap_endian(magic);
  33. CHECK_EQ(magic, 2051) << "Incorrect image file magic.";
  34. label_file.read(reinterpret_cast<char*>(&magic), 4);
  35. magic = swap_endian(magic);
  36. CHECK_EQ(magic, 2049) << "Incorrect label file magic.";
  37. image_file.read(reinterpret_cast<char*>(&num_items), 4);
  38. num_items = swap_endian(num_items);
  39. label_file.read(reinterpret_cast<char*>(&num_labels), 4);
  40. num_labels = swap_endian(num_labels);
  41. CHECK_EQ(num_items, num_labels);
  42. image_file.read(reinterpret_cast<char*>(&rows), 4);
  43. rows = swap_endian(rows);
  44. image_file.read(reinterpret_cast<char*>(&cols), 4);
  45. cols = swap_endian(cols);
  46. // lmdb
  47. MDB_env *mdb_env;
  48. MDB_dbi mdb_dbi;
  49. MDB_val mdb_key, mdb_data;
  50. MDB_txn *mdb_txn;
  51. // Open db
  52. if (db_backend == "lmdb") { // lmdb
  53. LOG(INFO) << "Opening lmdb " << db_path;
  54. CHECK_EQ(mkdir(db_path, 0744), 0)
  55. << "mkdir " << db_path << "failed";
  56. CHECK_EQ(mdb_env_create(&mdb_env), MDB_SUCCESS) << "mdb_env_create failed";
  57. CHECK_EQ(mdb_env_set_mapsize(mdb_env, 1099511627776), MDB_SUCCESS) // 1TB
  58. << "mdb_env_set_mapsize failed";
  59. CHECK_EQ(mdb_env_open(mdb_env, db_path, 0, 0664), MDB_SUCCESS)
  60. << "mdb_env_open failed";
  61. CHECK_EQ(mdb_txn_begin(mdb_env, NULL, 0, &mdb_txn), MDB_SUCCESS)
  62. << "mdb_txn_begin failed";
  63. CHECK_EQ(mdb_open(mdb_txn, NULL, 0, &mdb_dbi), MDB_SUCCESS)
  64. << "mdb_open failed. Does the lmdb already exist? ";
  65. } else {
  66. LOG(FATAL) << "Unknown db backend " << db_backend;
  67. }
  68. // Storing to db
  69. char label;
  70. char* pixels = new char[rows * cols];
  71. int count = 0;
  72. const int kMaxKeyLength = 10;
  73. char key_cstr[kMaxKeyLength];
  74. string value;
  75. Datum datum;
  76. datum.set_channels(1);
  77. datum.set_height(rows);
  78. datum.set_width(cols);
  79. LOG(INFO) << "A total of " << num_items << " items.";
  80. LOG(INFO) << "Rows: " << rows << " Cols: " << cols;
  81. for (int item_id = 0; item_id < num_items; ++item_id) {
  82. image_file.read(pixels, rows * cols);
  83. label_file.read(&label, 1);
  84. datum.set_data(pixels, rows*cols);
  85. datum.set_label(label);
  86. snprintf(key_cstr, kMaxKeyLength, "%08d", item_id);
  87. datum.SerializeToString(&value);
  88. string keystr(key_cstr);
  89. // Put in db
  90. if (db_backend == "lmdb") { // lmdb
  91. mdb_data.mv_size = value.size();
  92. mdb_data.mv_data = reinterpret_cast<void*>(&value[0]);
  93. mdb_key.mv_size = keystr.size();
  94. mdb_key.mv_data = reinterpret_cast<void*>(&keystr[0]);
  95. CHECK_EQ(mdb_put(mdb_txn, mdb_dbi, &mdb_key, &mdb_data, 0), MDB_SUCCESS)
  96. << "mdb_put failed";
  97. } else {
  98. LOG(FATAL) << "Unknown db backend " << db_backend;
  99. }
  100. if (++count % 1000 == 0) {
  101. if (db_backend == "lmdb") { // lmdb
  102. CHECK_EQ(mdb_txn_commit(mdb_txn), MDB_SUCCESS)
  103. << "mdb_txn_commit failed";
  104. CHECK_EQ(mdb_txn_begin(mdb_env, NULL, 0, &mdb_txn), MDB_SUCCESS)
  105. << "mdb_txn_begin failed";
  106. } else {
  107. LOG(FATAL) << "Unknown db backend " << db_backend;
  108. }
  109. }
  110. }
  111. // write the last batch
  112. if (count % 1000 != 0) {
  113. if (db_backend == "lmdb") { // lmdb
  114. CHECK_EQ(mdb_txn_commit(mdb_txn), MDB_SUCCESS) << "mdb_txn_commit failed";
  115. mdb_close(mdb_env, mdb_dbi);
  116. mdb_env_close(mdb_env);
  117. } else {
  118. LOG(FATAL) << "Unknown db backend " << db_backend;
  119. }
  120. LOG(ERROR) << "Processed " << count << " files.";
  121. }
  122. delete[] pixels;
  123. }
  124. int main(int argc, char** argv) {
  125. #ifndef GFLAGS_GFLAGS_H_
  126. namespace gflags = google;
  127. #endif
  128. gflags::SetUsageMessage("This script converts the MNIST dataset to\n"
  129. "the lmdb/leveldb format used by Caffe to load data.\n"
  130. "Usage:\n"
  131. " convert_mnist_data [FLAGS] input_image_file input_label_file "
  132. "output_db_file\n"
  133. "The MNIST dataset could be downloaded at\n"
  134. " http://yann.lecun.com/exdb/mnist/\n"
  135. "You should gunzip them after downloading,"
  136. "or directly use data/mnist/get_mnist.sh\n");
  137. gflags::ParseCommandLineFlags(&argc, &argv, true);
  138. const string& db_backend = FLAGS_backend;
  139. if (argc != 4) {
  140. gflags::ShowUsageWithFlagsRestrict(argv[0],
  141. "examples/mnist/convert_mnist_data");
  142. } else {
  143. google::InitGoogleLogging(argv[0]);
  144. convert_dataset(argv[1], argv[2], argv[3], db_backend);
  145. }
  146. return 0;
  147. }

Makefile 文件:

  1. CAFFE_HEADER = ../caffe/include
  2. CAFFE_LIB = ../caffe/lib
  3. SRC_FILES = test.cpp
  4. OUTPUT = test
  5. CXX = g++
  6. CXX_FLAG = -std=c++11 -I${CAFFE_HEADER} -L${CAFFE_LIB} -lcaffe -lglog -lgflags -lprotobuf -pthread -lpthread -llmdb
  7. # LIB_PATH = /usr/lib/x86_64-linux-gnu
  8. # LIB_FLAG := $(shell pkg-config --cflags --libs libgflags libglog protobuf)
  9. default:
  10. ${CXX} ${SRC_FILES} ${CXX_FLAG} -o ${OUTPUT} #${LIB_FLAG} -L${LIB_PATH} -llmdb
  11. clean:
  12. rm -rf ${OUTPUT}
  13. run:
  14. ./${OUTPUT}

注意,还需要在 .zshrc(.bashrc) 中手动添加

  1. export LD_LIBRARY_PATH=/home/gjz/GjzProjects/DeepLearning/mnist_lg_exp/caffe/lib:$LD_LIBRARY_PATH

测试了,能正常工作。以上的解决方案很粗糙,有需求了再弄得细致一点!!
error while loading shared libraries: libcaffe.so 或者 caffe issue#1988

Multilabel classification

这里跟人脸特征点检测有什么区别?

关于 caffe 源码剖析

值得一看的链接:
https://www.zhihu.com/question/27982282
https://github.com/BVLC/caffe/wiki/Development
https://www.quora.com/How-do-we-read-the-source-code-of-Caffe
https://docs.google.com/presentation/d/1AuiPxUy7-Dgb36Q_SN8ToRrq6Nk0X-sOFmr7UnbAAHI/edit#slide=id.g38b6d86c6_12
file:///home/gjz/Downloads/Brewing%20Deep%20Networks%20With%20Caffe.pdf

caffe 中 layer 的类型

caffe.proto 1040 行之后

  1. enum LayerType {
  2. NONE = 0;
  3. ABSVAL = 35;
  4. ACCURACY = 1;
  5. ARGMAX = 30;
  6. BNLL = 2;
  7. CONCAT = 3;
  8. CONTRASTIVE_LOSS = 37;
  9. CONVOLUTION = 4;
  10. DATA = 5;
  11. DECONVOLUTION = 39;
  12. DROPOUT = 6;
  13. DUMMY_DATA = 32;
  14. EUCLIDEAN_LOSS = 7;
  15. ELTWISE = 25;
  16. EXP = 38;
  17. FLATTEN = 8;
  18. HDF5_DATA = 9;
  19. HDF5_OUTPUT = 10;
  20. HINGE_LOSS = 28;
  21. IM2COL = 11;
  22. IMAGE_DATA = 12;
  23. INFOGAIN_LOSS = 13;
  24. INNER_PRODUCT = 14;
  25. LRN = 15;
  26. MEMORY_DATA = 29;
  27. MULTINOMIAL_LOGISTIC_LOSS = 16;
  28. MVN = 34;
  29. POOLING = 17;
  30. POWER = 26;
  31. RELU = 18;
  32. SIGMOID = 19;
  33. SIGMOID_CROSS_ENTROPY_LOSS = 27;
  34. SILENCE = 36;
  35. SOFTMAX = 20;
  36. SOFTMAX_LOSS = 21;
  37. SPLIT = 22;
  38. SLICE = 33;
  39. TANH = 23;
  40. WINDOW_DATA = 24;
  41. THRESHOLD = 31;
  42. }

caffe 加速

github 上的 issues:
https://github.com/BVLC/caffe/issues/79
http://stackoverflow.com/questions/31395729/how-to-enable-multithreading-with-caffe
https://github.com/BVLC/caffe/issues/1539

明天继续

http://caffe.berkeleyvision.org/
caffefkp

安装 cuda:
http://www.r-tutor.com/gpu-computing/cuda-installation/cuda7.5-ubuntu

找驱动:http://www.nvidia.com/Download/index.aspx?lang=en-us#axzz41qi4wOvv
貌似不行,看 cuda 官网:
https://developer.nvidia.com/cuda-toolkit
如何检测显卡驱动有没有装

添加新批注
在作者公开此批注前,只有你和作者可见。
回复批注