[关闭]
@cleardusk 2016-03-06T14:47:24.000000Z 字数 6675 阅读 1611

3.3

GjzCV

loss 层

loss function is also knowns as an error, cost, or objective function

logistic function


wiki链接

Softmax function

Logistic regression

手动编译 mnist 的 convert_minist_data 源码

删除部分 leveldb 后的代码：

  1. #include <iostream>
  2. #include <gflags/gflags.h>
  3. #include <glog/logging.h>
  4. #include <google/protobuf/text_format.h>
  5. #include <lmdb.h>
  6. #include <caffe/proto/caffe.pb.h>
  8. #include <stdint.h>
  9. #include <sys/stat.h>
  10. #include <fstream>  // NOLINT(readability/streams)
  11. #include <string>
  13. using namespace caffe;  // NOLINT(build/namespaces)
  14. using std::string;
  16. DEFINE_string(backend, "lmdb", "The backend for storing the result");
  18. uint32_t swap_endian(uint32_t val) {
  19.     val = ((val << 8) & 0xFF00FF00) | ((val >> 8) & 0xFF00FF);
  20.     return (val << 16) | (val >> 16);
  21. }
  24. void convert_dataset(const char* image_filename, const char* label_filename,
  25.         const char* db_path, const string& db_backend) {
  26.   // Open files
  27.   std::ifstream image_file(image_filename, std::ios::in | std::ios::binary);
  28.   std::ifstream label_file(label_filename, std::ios::in | std::ios::binary);
  29.   CHECK(image_file) << "Unable to open file " << image_filename;
  30.   CHECK(label_file) << "Unable to open file " << label_filename;
  31.   // Read the magic and the meta data
  32.   uint32_t magic;
  33.   uint32_t num_items;
  34.   uint32_t num_labels;
  35.   uint32_t rows;
  36.   uint32_t cols;
  38.   image_file.read(reinterpret_cast<char*>(&magic), 4);
  39.   magic = swap_endian(magic);
  40.   CHECK_EQ(magic, 2051) << "Incorrect image file magic.";
  41.   label_file.read(reinterpret_cast<char*>(&magic), 4);
  42.   magic = swap_endian(magic);
  43.   CHECK_EQ(magic, 2049) << "Incorrect label file magic.";
  44.   image_file.read(reinterpret_cast<char*>(&num_items), 4);
  45.   num_items = swap_endian(num_items);
  46.   label_file.read(reinterpret_cast<char*>(&num_labels), 4);
  47.   num_labels = swap_endian(num_labels);
  48.   CHECK_EQ(num_items, num_labels);
  49.   image_file.read(reinterpret_cast<char*>(&rows), 4);
  50.   rows = swap_endian(rows);
  51.   image_file.read(reinterpret_cast<char*>(&cols), 4);
  52.   cols = swap_endian(cols);
  54.   // lmdb
  55.   MDB_env *mdb_env;
  56.   MDB_dbi mdb_dbi;
  57.   MDB_val mdb_key, mdb_data;
  58.   MDB_txn *mdb_txn;
  60.   // Open db
  61.   if (db_backend == "lmdb") {  // lmdb
  62.     LOG(INFO) << "Opening lmdb " << db_path;
  63.     CHECK_EQ(mkdir(db_path, 0744), 0)
  64.         << "mkdir " << db_path << "failed";
  65.     CHECK_EQ(mdb_env_create(&mdb_env), MDB_SUCCESS) << "mdb_env_create failed";
  66.     CHECK_EQ(mdb_env_set_mapsize(mdb_env, 1099511627776), MDB_SUCCESS)  // 1TB
  67.         << "mdb_env_set_mapsize failed";
  68.     CHECK_EQ(mdb_env_open(mdb_env, db_path, 0, 0664), MDB_SUCCESS)
  69.         << "mdb_env_open failed";
  70.     CHECK_EQ(mdb_txn_begin(mdb_env, NULL, 0, &mdb_txn), MDB_SUCCESS)
  71.         << "mdb_txn_begin failed";
  72.     CHECK_EQ(mdb_open(mdb_txn, NULL, 0, &mdb_dbi), MDB_SUCCESS)
  73.         << "mdb_open failed. Does the lmdb already exist? ";
  74.   } else {
  75.     LOG(FATAL) << "Unknown db backend " << db_backend;
  76.   }
  78.   // Storing to db
  79.   char label;
  80.   char* pixels = new char[rows * cols];
  81.   int count = 0;
  82.   const int kMaxKeyLength = 10;
  83.   char key_cstr[kMaxKeyLength];
  84.   string value;
  86.   Datum datum;
  87.   datum.set_channels(1);
  88.   datum.set_height(rows);
  89.   datum.set_width(cols);
  90.   LOG(INFO) << "A total of " << num_items << " items.";
  91.   LOG(INFO) << "Rows: " << rows << " Cols: " << cols;
  92.   for (int item_id = 0; item_id < num_items; ++item_id) {
  93.     image_file.read(pixels, rows * cols);
  94.     label_file.read(&label, 1);
  95.     datum.set_data(pixels, rows*cols);
  96.     datum.set_label(label);
  97.     snprintf(key_cstr, kMaxKeyLength, "%08d", item_id);
  98.     datum.SerializeToString(&value);
  99.     string keystr(key_cstr);
  101.     // Put in db
  102.     if (db_backend == "lmdb") {  // lmdb
  103.       mdb_data.mv_size = value.size();
  104.       mdb_data.mv_data = reinterpret_cast<void*>(&value[0]);
  105.       mdb_key.mv_size = keystr.size();
  106.       mdb_key.mv_data = reinterpret_cast<void*>(&keystr[0]);
  107.       CHECK_EQ(mdb_put(mdb_txn, mdb_dbi, &mdb_key, &mdb_data, 0), MDB_SUCCESS)
  108.           << "mdb_put failed";
  109.     } else {
  110.       LOG(FATAL) << "Unknown db backend " << db_backend;
  111.     }
  113.     if (++count % 1000 == 0) {
  114.       if (db_backend == "lmdb") {  // lmdb
  115.         CHECK_EQ(mdb_txn_commit(mdb_txn), MDB_SUCCESS)
  116.             << "mdb_txn_commit failed";
  117.         CHECK_EQ(mdb_txn_begin(mdb_env, NULL, 0, &mdb_txn), MDB_SUCCESS)
  118.             << "mdb_txn_begin failed";
  119.       } else {
  120.         LOG(FATAL) << "Unknown db backend " << db_backend;
  121.       }
  122.     }
  123.   }
  124.   // write the last batch
  125.   if (count % 1000 != 0) {
  126.     if (db_backend == "lmdb") {  // lmdb
  127.       CHECK_EQ(mdb_txn_commit(mdb_txn), MDB_SUCCESS) << "mdb_txn_commit failed";
  128.       mdb_close(mdb_env, mdb_dbi);
  129.       mdb_env_close(mdb_env);
  130.     } else {
  131.       LOG(FATAL) << "Unknown db backend " << db_backend;
  132.     }
  133.     LOG(ERROR) << "Processed " << count << " files.";
  134.   }
  135.   delete[] pixels;
  136. }
  138. int main(int argc, char** argv) {
  139. #ifndef GFLAGS_GFLAGS_H_
  140.   namespace gflags = google;
  141. #endif
  143.   gflags::SetUsageMessage("This script converts the MNIST dataset to\n"
  144.         "the lmdb/leveldb format used by Caffe to load data.\n"
  145.         "Usage:\n"
  146.         "    convert_mnist_data [FLAGS] input_image_file input_label_file "
  147.         "output_db_file\n"
  148.         "The MNIST dataset could be downloaded at\n"
  149.         "    http://yann.lecun.com/exdb/mnist/\n"
  150.         "You should gunzip them after downloading,"
  151.         "or directly use data/mnist/get_mnist.sh\n");
  152.   gflags::ParseCommandLineFlags(&argc, &argv, true);
  154.   const string& db_backend = FLAGS_backend;
  156.   if (argc != 4) {
  157.     gflags::ShowUsageWithFlagsRestrict(argv[0],
  158.         "examples/mnist/convert_mnist_data");
  159.   } else {
  160.     google::InitGoogleLogging(argv[0]);
  161.     convert_dataset(argv[1], argv[2], argv[3], db_backend);
  162.   }
  163.   return 0;
  164. }

Makefile 文件：

  1. CAFFE_HEADER = ../caffe/include
  2. CAFFE_LIB = ../caffe/lib
  4. SRC_FILES = test.cpp
  5. OUTPUT = test
  7. CXX = g++
  8. CXX_FLAG = -std=c++11 -I${CAFFE_HEADER} -L${CAFFE_LIB} -lcaffe -lglog -lgflags -lprotobuf -pthread -lpthread -llmdb
  9. # LIB_PATH = /usr/lib/x86_64-linux-gnu
  10. # LIB_FLAG := $(shell pkg-config --cflags --libs libgflags libglog protobuf)
  12. default:
  13.     ${CXX} ${SRC_FILES} ${CXX_FLAG} -o ${OUTPUT} #${LIB_FLAG} -L${LIB_PATH} -llmdb
  15. clean:
  16.     rm -rf ${OUTPUT}
  18. run:
  19.     ./${OUTPUT}

注意，还需要在 .zshrc(.bashrc) 中手动添加

  1. export LD_LIBRARY_PATH=/home/gjz/GjzProjects/DeepLearning/mnist_lg_exp/caffe/lib:$LD_LIBRARY_PATH

测试了，能正常工作。以上的解决方案很粗糙，有需求了再弄得细致一点！！
error while loading shared libraries: libcaffe.so 或者 caffe issue#1988

Multilabel classification

这里跟人脸特征点检测有什么区别？

关于 caffe 源码剖析

值得一看的链接：
https://www.zhihu.com/question/27982282
https://github.com/BVLC/caffe/wiki/Development
https://www.quora.com/How-do-we-read-the-source-code-of-Caffe
https://docs.google.com/presentation/d/1AuiPxUy7-Dgb36Q_SN8ToRrq6Nk0X-sOFmr7UnbAAHI/edit#slide=id.g38b6d86c6_12
file:///home/gjz/Downloads/Brewing%20Deep%20Networks%20With%20Caffe.pdf

caffe 中 layer 的类型

caffe.proto 1040 行之后

  1.   enum LayerType {
  2.     NONE = 0;
  3.     ABSVAL = 35;
  4.     ACCURACY = 1;
  5.     ARGMAX = 30;
  6.     BNLL = 2;
  7.     CONCAT = 3;
  8.     CONTRASTIVE_LOSS = 37;
  9.     CONVOLUTION = 4;
  10.     DATA = 5;
  11.     DECONVOLUTION = 39;
  12.     DROPOUT = 6;
  13.     DUMMY_DATA = 32;
  14.     EUCLIDEAN_LOSS = 7;
  15.     ELTWISE = 25;
  16.     EXP = 38;
  17.     FLATTEN = 8;
  18.     HDF5_DATA = 9;
  19.     HDF5_OUTPUT = 10;
  20.     HINGE_LOSS = 28;
  21.     IM2COL = 11;
  22.     IMAGE_DATA = 12;
  23.     INFOGAIN_LOSS = 13;
  24.     INNER_PRODUCT = 14;
  25.     LRN = 15;
  26.     MEMORY_DATA = 29;
  27.     MULTINOMIAL_LOGISTIC_LOSS = 16;
  28.     MVN = 34;
  29.     POOLING = 17;
  30.     POWER = 26;
  31.     RELU = 18;
  32.     SIGMOID = 19;
  33.     SIGMOID_CROSS_ENTROPY_LOSS = 27;
  34.     SILENCE = 36;
  35.     SOFTMAX = 20;
  36.     SOFTMAX_LOSS = 21;
  37.     SPLIT = 22;
  38.     SLICE = 33;
  39.     TANH = 23;
  40.     WINDOW_DATA = 24;
  41.     THRESHOLD = 31;
  42.   }

caffe 加速

github 上的 issues:
https://github.com/BVLC/caffe/issues/79
http://stackoverflow.com/questions/31395729/how-to-enable-multithreading-with-caffe
https://github.com/BVLC/caffe/issues/1539

明天继续

http://caffe.berkeleyvision.org/
caffefkp

安装 cuda：
http://www.r-tutor.com/gpu-computing/cuda-installation/cuda7.5-ubuntu

找驱动：http://www.nvidia.com/Download/index.aspx?lang=en-us#axzz41qi4wOvv
貌似不行，看 cuda 官网：
https://developer.nvidia.com/cuda-toolkit
如何检测显卡驱动有没有装

添加新批注
在作者公开此批注前，只有你和作者可见。
回复批注