人脸识别部分内存调用优化

void realloc_if_need(){
size_t reserve_items=std::max(static_cast<int>(max_items/10),512);
size_t allocate_items=0;
bool need_realloc=false;
if(getAllocatedSize()==getSize())
{
allocate_items=std::min(getSize()+reserve_items,reserve_items);
need_realloc=true;
}
if(need_realloc)
{
size_t new_len=allocate_items*(item_size+sizeof(int));
size_t old_len=getAllocatedSize()*(item_size+sizeof(int));
unsigned char* new_buf=new unsigned char[new_len];
memcpy(new_buf,buf,old_len);
buf=new_buf;
}
}

#ifndef _FEATURE_STORAGE_INTERNAL_H_
#define _FEATURE_STORAGE_INTERNAL_H_

#include "feature_storage_comm.h"
#include <vector>
#include <utility>
#include <unordered_map>

class Arena;
class MemoryStorage
{
public:
typedef std::pair<std::string, float> TopNResult;
private:
std::unordered_map<std::string, size_t> data;
std::unordered_map<size_t, std::string> id2map;
size_t max_items;
size_t feat_len;
Arena *arena;
pthread_rwlock_t lock;
public:
MemoryStorage();
~MemoryStorage();

void clear();

void setMaxItems(size_t v, size_t _m_feat_len);

inline size_t getMaxItems() { return max_items; }
inline size_t getSize() { return data.size(); }

bool insert(const std::string &key, const stFeature *f);
bool delete_item(const std::string &key);
const stFeature *get(const std::string &key);
std::vector<TopNResult> searchTopN(size_t topn, const stFeature *f);
private:
void init_arena(size_t v)
{
if(arena)
return;
size_t reserve_items=std::max(static_cast<int>(max_items/10),512);
size_t allocate_items=std::min(v+reserve_items,max_items);
arena=new Arena(sizeof(stFeature)+sizeof(float)* feat_len,max_items,allocate_items);

}
};

#endif

#include "feature_storage.h"
#include <stdio.h>
#include <stdlib.h>
#include <stdarg.h>
#include <cassert>
#include <time.h>
#include <ratio>
#include <chrono>
#include <sstream>
#include <iostream>
#include <thread>
#include <vector>
#include <unistd.h>
#include <stdio.h>
#include <stdlib.h>
#include <time.h>
#include <iomanip>
#include <unistd.h>
#include <assert.h>
using namespace std;

#define FEAT_LEN 128

int main()
{
stFeature *feat = (stFeature*)malloc(sizeof(stFeature)+sizeof(float)*FEAT_LEN);
assert(feat);
feat->len = FEAT_LEN;
feat->magic = ST_FEATURE_MAGIC;
feat->version = 1;
for(int i=0;i<FEAT_LEN;i++)
{
feat->data[i] = (double)i;
}

int capacity = 100*10000;
fs_handle_t fs;
int load_size=100*10000;
int ret = featurestorage_create(capacity,load_size, FEAT_LEN, &fs);
printf("featurestorage_create return:%d\n",  ret);

using namespace std::chrono;
steady_clock::time_point t1 = steady_clock::now();

for(int j=0;j<capacity;j++)
{
stringstream ss;
ss<<"A"<<j;
featurestorage_insert(fs, ss.str().c_str(), feat);
}
steady_clock::time_point t2 = steady_clock::now();
duration<double> time_span = duration_cast<duration<double>>(t2 - t1);
std::cout << "It took me " << time_span.count() << " seconds.";
std::cout << std::endl;
printf("ms size:%d\n", featurestorage_size(fs));

vector<thread> workers;
int cnt = 0;
while(cnt++<2)
{
workers.push_back(thread([&](){
for(int i=0;i<10;i++)
{
stTopNResult rets[10];
steady_clock::time_point t1 = steady_clock::now();
featurestorage_search(fs, feat, 10, rets);
steady_clock::time_point t2 = steady_clock::now();
duration<double> time_span = duration_cast<duration<double>>(t2 - t1);
std::cout <<"["<<std::this_thread::get_id()<<"][" <<fixed<<setprecision(9)<<rets[0].score<<"] It took me "<< time_span.count() << " seconds.";
std::cout << std::endl;
}
}));
}

thread th([&](){
double total = 0.0;
for(int i=0;i<10;i++)
{
stTopNResult rets[10];
steady_clock::time_point t1 = steady_clock::now();
featurestorage_search(fs, feat, 10, rets);
steady_clock::time_point t2 = steady_clock::now();
duration<double> time_span = duration_cast<duration<double>>(t2 - t1);
std::cout <<"["<<std::this_thread::get_id()<<"][" <<fixed<<setprecision(9)<<rets[0].score<<"] It took me "<< time_span.count() << " seconds.";
std::cout << std::endl;
total+=time_span.count();
}
std::cout <<"["<<std::this_thread::get_id()<<"] It took avg "<<fixed<<setprecision(9)<< total/10 << " seconds.";
std::cout << std::endl;
});
th.detach();
getchar();
featurestorage_destroy(fs);
return 0;
}