基于Boost库C++文件夹级别数据批处理

       最近，帮助一个师妹实现了地理坐标的欧式距离计算，其数据存储在文本文件中，然后很多类似的文本文件组成一个文件夹路径，本程序源码主要实现了C++文件便利，存储，以及计算相关功能的实现。下边的贴图是数据的基本格式：文件的前六行是数据说明文件，在处理的时候需要略过，正文数据意义为：纬度，经度，无意义，高程，离1899年的天数，日期和时间。需要做的操作如下：

       1. 从文件中读出数据到一个类中，定义为User，

2. 给定文件夹路径，遍历所有文件

        3. 文件的命名是日期，需要按照日期排序

       4. 相邻的两个文件求得欧式距离，取得最大值

       5. 获取所有相邻文件的最大值

       假设文件有1，2，3，4.则12之间相应求解欧式距离，取得所有欧式距离的最大值，与此类似，取得34最大值，最后取得1234最大值。

关于boost库的安装参照本人前边的博客。

Geolife trajectory

WGS 84

Altitude is in Feet

Reserved 3

0,2,255,My Track,0,0,2,8421376

0

39.984702,116.318417,0,492,39744.1201851852,2008-10-23,02:53:04

39.984683,116.31845,0,492,39744.1202546296,2008-10-23,02:53:10

39.984686,116.318417,0,492,39744.1203125,2008-10-23,02:53:15

39.984688,116.318385,0,492,39744.1203703704,2008-10-23,02:53:20

39.984655,116.318263,0,492,39744.1204282407,2008-10-23,02:53:25

39.984611,116.318026,0,493,39744.1204861111,2008-10-23,02:53:30

39.984608,116.317761,0,493,39744.1205439815,2008-10-23,02:53:35

39.984563,116.317517,0,496,39744.1206018519,2008-10-23,02:53:40

39.984539,116.317294,0,500,39744.1206597222,2008-10-23,02:53:45

39.984606,116.317065,0,505,39744.1207175926,2008-10-23,02:53:50

39.984568,116.316911,0,510,39744.120775463,2008-10-23,02:53:55

User.hpp:实现了操作符重载，

//
// User.hpp
// Test
//
// Created by 秦传庆 on 15/10/23.
// Copyright © 2015年 秦传庆. All rights reserved.
//

#ifndef User_hpp
#define User_hpp
#include <boost/date_time/gregorian/gregorian.hpp>
#include <boost/date_time/posix_time/posix_time.hpp>
#include <sstream>
namespace gregorianDate = boost::gregorian;
using namespace boost::posix_time;
using namespace std;
/*
39.984702,116.318417,0,492,39744.1201851852,2008-10-23,02:53:04
39.984683,116.31845,0,492,39744.1202546296,2008-10-23,02:53:10
39.984686,116.318417,0,492,39744.1203125,2008-10-23,02:53:15
39.984688,116.318385,0,492,39744.1203703704,2008-10-23,02:53:20
39.984655,116.318263,0,492,39744.1204282407,2008-10-23,02:53:25
39.984611,116.318026,0,493,39744.1204861111,2008-10-23,02:53:30
39.984608,116.317761,0,493,39744.1205439815,2008-10-23,02:53:35
*/

class User {
private:
double latitude;
double longitude;
int nonsense;
int altitude;
double daysince18991230;
gregorianDate::date date;
time_duration time;
//ptime date;

public:
friend ostream &operator<<(ostream &os, const User &c);
friend istream &operator>>(istream &is, User &c);
/*Java, setter and geter*/
void setLatitude(double latitude);
double getLatitude();
void setLongitude(double longitude);
double getLongitude();
void setNonsense(int nonsense);
int getNonsense();
void setAltitude(int altitude);
int getAltitude();
void setDaysince18991230(double daysince18991230);
double getDaysince18991230();
void setDate(string date);
void setDate(gregorianDate::date date);
gregorianDate::date getDate();
void setTime(string time);
void setTime(time_duration time);
time_duration getTime();
};
template<class out, class in>
out ConvertDataType(const in& a);
#endif /* User_hpp */


User.cpp

<pre name="code" class="cpp">//
//  User.cpp
//  Test
//
//  Created by 秦传庆 on 15/10/23.
//  Copyright © 2015年 秦传庆. All rights reserved.
//

#include "User.hpp"
void User::setLatitude(double latitude) {
this->latitude = latitude;
}

double User::getLatitude() {
return latitude;
}
void User::setLongitude(double longitude) {
this->longitude = longitude;
}
double User::getLongitude() {
return longitude;
}
void User::setNonsense(int nonsense) {
this->nonsense = nonsense;
}
int User::getNonsense() {
return nonsense;
}
void User::setAltitude(int altitude) {
this->altitude = altitude;
}
int User::getAltitude() {
return altitude;
}
void User::setDaysince18991230(double daysince18991230) {
this->daysince18991230 = daysince18991230;
}
double User::getDaysince18991230() {
return daysince18991230;
}
void User::setDate(string date) {
this->setDate(gregorianDate::from_string(date));
}
void User::setDate(gregorianDate::date date){
this->date = date;
}
gregorianDate::date User::getDate() {
return date;
}
void User::setTime(string time) {
this->setTime(duration_from_string(time));
}
void User::setTime(time_duration time){
this->time = time;
}
time_duration User::getTime() {
return time;
}
ostream &operator<<(ostream &out, const User &user) {
ostream::fmtflags curr_fmt = out.flags();
out.precision(10);
out << user.latitude << ",";
out << user.longitude << ",";
out << user.nonsense << ",";
out << user.altitude << ",";
out << user.daysince18991230 << ",";
out << gregorianDate::to_simple_string(user.date) << ",";
out << to_simple_string(user.time) << endl;
out.flags(curr_fmt);
return out;
}
/*char temp used to go over the ","*/
istream &operator>>(istream &in, User &user) {
char temp;
string temp_string;
istream::fmtflags curr_fmt = in.flags();
in >> user.latitude;
in >> temp;
in >> user.longitude;
in >> temp;
in >> user.nonsense;
in >> temp;
in >> user.altitude;
in >> temp;
in >> user.daysince18991230;
in >> temp;
in >> temp_string;
if ("" == temp_string){
return in;
}
size_t pos = temp_string.find_first_of(",");
user.date = gregorianDate::from_string(temp_string.substr(0, pos));
user.time = duration_from_string(temp_string.substr(pos + 1, temp_string.size()));
in.flags(curr_fmt);
return in;
}

FileIter.hpp: 实现基于boost库的文件遍历

//
//  FileIter.hpp
//  Test
//
//  Created by 秦传庆 on 15/10/23.
//  Copyright © 2015年 秦传庆. All rights reserved.
//

#ifndef FileIter_hpp
#define FileIter_hpp
#include <iostream>
#include <boost/filesystem.hpp>
#include <algorithm>
#include "Utils.hpp"
using namespace std;
using namespace boost::filesystem;
class FileIter {
private:
string path;
vector<string> data;
public:
FileIter(string path);
void walk();
void walk(string path);
vector<string> getVector();
friend bool compare(string file1, string file2);
private:
void walk(boost::filesystem::path p);

};
bool compare(string  file1, string file2);
#endif /* FileIter_hpp */

FileIter.cpp

//
// FileIter.cpp
// Test
//
// Created by 秦传庆 on 15/10/23.
// Copyright © 2015年 秦传庆. All rights reserved.
//

#include "FileIter.hpp"
FileIter::FileIter(string path) {
this->path = path;
}
void FileIter::walk() {
walk(path);
}
void FileIter::walk(string path) {
auto p = boost::filesystem::path(path);
walk(p);
sort(data.begin(), data.end(), compare);
//PrintVector(data);
}
void FileIter::walk(boost::filesystem::path p) {
try {
if (exists(p)) {
if (is_regular_file(p) && !boost::filesystem::is_empty(p)) {
//cout << p << " size is " << file_size(p) << '\n';
data.push_back(p.string());
}else if (is_directory(p)) {
//cout << p << " is a directory containing:\n";
for (directory_entry& x : directory_iterator(p)) {
walk(x.path());
}
}else {
cout << p << " exists, but is not a regular file or directory\n";
}
}else {
cout << p << " does not exist\n";
}
}catch (const filesystem_error& ex) {
cout << ex.what() << '\n';
}

}
vector<string> FileIter::getVector() {
return data;
}
bool compare(string file1, string file2){
file1 = file1.substr(file1.find_last_of("/") + 1, 14);
file2 = file2.substr(file2.find_last_of("/") + 1, 14);
//cout << file1 << "," << file2 << endl;
return file1 < file2;
}

Utils.hpp : 实现了一些功能调试函数，文件是否打开成功，打印容器内容，易于查看。

//
// Utils.hpp
// Test
//
// Created by 秦传庆 on 15/10/25.
// Copyright © 2015年 秦传庆. All rights reserved.
//

#ifndef Utils_hpp
#define Utils_hpp
#define BOOST_LOG_DYN_LINK
#include <fstream>
#include <vector>
#include <iostream>
#include <sstream>
#include <boost/lexical_cast.hpp>
using namespace boost;
using namespace std;

bool FileExist(const fstream& in);

template<class out, class in>
out ConvertDataType(const in& a)
{
/*
std::stringstream temp;
temp<<a;
out b;
temp>>b;
return b;
*/
return lexical_cast<out>(a);
}

template<typename T>
void PrintVector(const vector<T>& data);

template<typename T>
string VectorToString(const vector<T>& data){
string content;
for (auto& temp : data){
content.append(ConvertDataType<std::string>(temp));
}
return content;
}
#include "TestUnit.cpp"

#endif /* Utils_hpp */


Utils.cpp

//
//  Utils.cpp
//  Test
//
//  Created by 秦传庆 on 15/10/25.
//  Copyright © 2015年 秦传庆. All rights reserved.
//

#include "Utils.hpp"
bool FileExist(fstream& in){
if (!in){
std::cerr << "unable open the file, please check it." << std::endl;
return false;
}
return true;
}

TestUnit.cpp : 模版方法打印容器内容

//
//  TestUnit.cpp
//  Test
//
//  Created by 秦传庆 on 15/10/23.
//  Copyright © 2015年 秦传庆. All rights reserved.
//

#include <iostream>
#include <vector>
#include<fstream>
using namespace std;
template<typename T>
void PrintVector(vector<T> data) {
cout << "{" ;
for (auto item : data) {
cout << item << ",";
}
cout << "}" ;
cout << endl;
}

SpaceDistance.hpp : 实现欧式距离计算

//
// SpaceDistance.hpp
// Test
//
// Created by 秦传庆 on 15/10/25.
// Copyright © 2015年 秦传庆. All rights reserved.
//

#ifndef SpaceDistance_hpp
#define SpaceDistance_hpp
#include "User.hpp"
#include "Utils.hpp"
#include "FileIter.hpp"
#include "Logger.hpp"
#include <iostream>
#include <fstream>
#include <vector>
#include <cmath>
#include <algorithm>
#include <boost/timer.hpp>
using namespace std;
class SpaceDistance{
private:
boost::timer time;
FileIter folder;
vector<string> file_names;
fstream file_stream;
vector<User> data_original;
vector<User> data_compare;
//vector<vector<User>> data;
istream_iterator<User> eof;
vector<string>::size_type file_number;
public:
SpaceDistance(string path);
double calculateDistance();
private:
bool checkFileNumber(){
return true;
}
vector<User> getDataFromFile(string file_name);

double calculateDistanceOfTwo();
void validateFiles();

};

#endif /* SpaceDistance_hpp */


SpaceDistance.cpp

//
// SpaceDistance.cpp
// Test
//
// Created by 秦传庆 on 15/10/25.
// Copyright © 2015年 秦传庆. All rights reserved.
//

#include "SpaceDistance.hpp"
using namespace std;

SpaceDistance::SpaceDistance(string path):folder(path){
folder.walk();
file_names = folder.getVector();
//PrintVector(file_names);
validateFiles();
file_number = file_names.size();
}

double SpaceDistance::calculateDistance(){
double distanceOfTwo;
double maxDistance;
string msg;
if (!checkFileNumber()){
return 0.0;
}
vector<double> distance;
/*
for (auto fileName:file_names){
data.push_back(getDataFromFile(fileName));
}
*/
for (auto i = 0; i < file_number - 1; i++) {
time.restart();
data_original = getDataFromFile(file_names[i]);
//data_original = data[i];
for (auto j = i + 1; j < file_number; j++){
data_compare = getDataFromFile(file_names[j]);
//data_compare = data[j];
distanceOfTwo = calculateDistanceOfTwo();
distance.push_back(distanceOfTwo);
//info_log(file_names[i]);
msg.clear();
msg.append(file_names[i]);
msg.append(",");
msg.append(file_names[j]);
msg.append(":");
msg.append(ConvertDataType<std::string>(distanceOfTwo));
BOOST_LOG_FUNCTION();
info_log(msg);
}
msg.clear();
msg.append("Time elapesd: ");
msg.append(ConvertDataType<std::string>(time.elapsed()));
BOOST_LOG_FUNCTION();
info_log(msg);
}
//PrintVector(distance);
//BOOST_LOG_FUNCTION();
maxDistance = *max_element(distance.begin(), distance.end());
//info_log(maxDistance);
return maxDistance;
}
vector<User> SpaceDistance::getDataFromFile(string file_name){
file_stream.open(file_name.c_str(), fstream::in);
if (!FileExist(file_stream)) {
return vector<User>();
}
string temp;
/*
* below for cycle will go over 6 lines in each file.
*/
for(int i = 0; i < 6; i++){
getline(file_stream, temp);
}
istream_iterator<User> item_iter(file_stream);
vector<User> data(item_iter, eof);
file_stream.close();
file_stream.clear();
return data;
}
double SpaceDistance::calculateDistanceOfTwo(){
/*
* here do math of the gero.
*/
vector<double> data;
vector<double> temp;
//auto length = data_original.size() > data_compare.size()? data_compare.size():data_original.size();
double x_original;
double y_original;
double x_compare;
double y_compare;
auto original_length = data_original.size();
auto compare_length = data_compare.size();
for (auto i = 0; i < original_length; i++){
x_original = data_original[i].getLongitude();
y_original = data_original[i].getLatitude();
temp.clear();
for (auto j = 0; j < compare_length; j++){
x_compare = data_compare[j].getLongitude();
y_compare = data_compare[j].getLatitude();
temp.push_back(sqrt(pow((x_original - x_compare), 2) + pow((y_original - y_compare), 2)));
}
//PrintVector(temp);
data.push_back(*max_element(temp.begin(), temp.end()));
}
//PrintVector(data);
//BOOST_LOG_FUNCTION();
//info_log(VectorToString(data));
return *max_element(data.begin(), data.end());
}

void SpaceDistance::validateFiles(){
string extension_string = ".plt";
for (auto iter = file_names.begin(); iter != file_names.end();){
if (extension_string != (*iter).substr((*iter).size() - extension_string.size(), extension_string.size())){
file_names.erase(iter);
} else {
iter++;
}
}
}Logger.hpp  日志记录系统

//
// Logger.hpp
// Test
//
// Created by 秦传庆 on 15/11/2.
// Copyright © 2015年 秦传庆. All rights reserved.
//

#ifndef Logger_hpp
#define Logger_hpp
#define BOOST_LOG_DYN_LINK
#include <iostream>
#include <fstream>
#include <boost/log/expressions.hpp>
#include <boost/smart_ptr/shared_ptr.hpp>
#include <boost/smart_ptr/make_shared_object.hpp>
#include <boost/log/core.hpp>
#include <boost/log/trivial.hpp>
#include <boost/log/sinks/sync_frontend.hpp>
#include <boost/log/sinks/text_file_backend.hpp>
#include <boost/log/sinks/text_ostream_backend.hpp>
#include <boost/log/utility/setup/file.hpp>
#include <boost/log/utility/setup/common_attributes.hpp>
#include <boost/log/sources/severity_logger.hpp>
#include <boost/log/sources/record_ostream.hpp>
#include <boost/log/sources/logger.hpp>
#include <boost/core/null_deleter.hpp>

#include <boost/log/sources/global_logger_storage.hpp>
#include <boost/log/attributes/named_scope.hpp>
#include <boost/log/support/date_time.hpp>
#include <boost/log/detail/format.hpp>
#include <boost/log/detail/thread_id.hpp>

namespace logging = boost::log;
namespace sinks = boost::log::sinks;
namespace src = boost::log::sources;
namespace expr = boost::log::expressions;
namespace attrs = boost::log::attributes;
namespace keywords = boost::log::keywords;
using namespace logging::trivial;

using namespace std;
class Logger {
public:
static void init();
};

template<class T>
void trace_log(const T& content) {
src::severity_logger<severity_level> lg;
BOOST_LOG_SEV(lg, logging::trivial::trace) << content;
}
template<class T>
void debug_log(const T& content) {
src::severity_logger<severity_level> lg;
BOOST_LOG_SEV(lg, logging::trivial::debug) << content;
}
template<class T>
void info_log(const T& content) {
src::severity_logger<severity_level> lg;
BOOST_LOG_SEV(lg, logging::trivial::info) << content;
}
template<class T>
void warning_log(const T& content) {
src::severity_logger<severity_level> lg;
BOOST_LOG_SEV(lg, logging::trivial::warning) << content;
}
template<class T>
void error_log(const T& content) {
src::severity_logger<severity_level> lg;
BOOST_LOG_SEV(lg, logging::trivial::error) << content;
}
template<class T>
void fatal_log(const T& content) {
src::severity_logger<severity_level> lg;
BOOST_LOG_SEV(lg, logging::trivial::fatal) << content;
}
#endif /* Logger_hpp */


Logger.cpp

//
// Logger.cpp
// Test
//
// Created by 秦传庆 on 15/11/2.
// Copyright © 2015年 秦传庆. All rights reserved.
//

#include "Logger.hpp"
void Logger::init() {
typedef sinks::synchronous_sink<sinks::text_ostream_backend> text_sink;
boost::shared_ptr<text_sink> sink = boost::make_shared< text_sink >();
boost::shared_ptr<std::ostream> stream(&std::clog, boost::null_deleter());
sink->locked_backend()->add_stream(stream);

auto pSink = logging::add_file_log
(
keywords::file_name = "/Users/qinchuanqing/Downloads/000/sample_%N.log",
keywords::rotation_size = 10 * 1024 * 1024,
keywords::time_based_rotation = sinks::file::rotation_at_time_point(0, 0, 0),
//keywords::format = "[%LineID%][%TimeStamp%]: %Message%"
keywords::format =
(
expr::stream
<< "[" << expr::format_date_time< boost::posix_time::ptime >("TimeStamp", "%Y-%m-%d %H:%M:%S.%f")
<< " " << expr::format_named_scope("Scope", keywords::format = "%f:%l]")
<< "<" << logging::trivial::severity
<< "> " << expr::smessage
)
);

logging::core::get()->set_filter
(
logging::trivial::severity >= logging::trivial::trace
);
logging::core::get()->add_sink(sink);

pSink->locked_backend()->auto_flush(true);//使日志实时更新
logging::core::get()->add_global_attribute("Scope", attrs::named_scope());
logging::add_common_attributes();
}

main.cpp : 主测试函数：

//
// main.cpp
// Test
//
// Created by 秦传庆 on 15/10/23.
// Copyright © 2015年 秦传庆. All rights reserved.
//

#include "SpaceDistance.hpp"
#include <time.h>
using namespace std;

int main() {

Logger::init();
time_t t_start, t_end;
t_start = time(NULL);
SpaceDistance distance("/Users/qinchuanqing/Downloads/000/Trajectory");
double max_distance = distance.calculateDistance();
BOOST_LOG_FUNCTION();
string msg = "Max number is: ";
msg.append(ConvertDataType<string>(max_distance));
info_log(msg);
msg = "Total time is: ";
t_end = time(NULL);
msg.append(ConvertDataType<string>(difftime(t_end,t_start)));
msg.append(" seconds");
BOOST_LOG_FUNCTION();
trace_log(msg);
return 0;

}