linux读写二进制大文件
2015-01-12 15:58
232 查看
https://raw.githubusercontent.com/ceph/ceph/master/src/common/safe_io.c
一般而言,用pwrite和pread读写磁盘文件不需要用循环,但是在读写超大的文件时就一定需要循环,保证正确读写
下面是自己写的代码,为了验证read/writeBlock的正确性,分两批写入文件,然后又分两批读入文件,实际工程中只要一次读写就行了
一般而言,用pwrite和pread读写磁盘文件不需要用循环,但是在读写超大的文件时就一定需要循环,保证正确读写
// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- // vim: ts=8 sw=2 smarttab /* * Ceph - scalable distributed file system * * Copyright (C) 2011 New Dream Network * * This is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License version 2.1, as published by the Free Software * Foundation. See file COPYING. * */ #define _XOPEN_SOURCE 500 #include <stdio.h> #include <string.h> #include <unistd.h> #include <errno.h> #include <fcntl.h> #include <limits.h> #include "common/safe_io.h" #include "include/compat.h" ssize_t safe_read(int fd, void *buf, size_t count) { size_t cnt = 0; while (cnt < count) { ssize_t r = read(fd, buf, count - cnt); if (r <= 0) { if (r == 0) { // EOF return cnt; } if (errno == EINTR) continue; return -errno; } cnt += r; buf = (char *)buf + r; } return cnt; } ssize_t safe_read_exact(int fd, void *buf, size_t count) { ssize_t ret = safe_read(fd, buf, count); if (ret < 0) return ret; if ((size_t)ret != count) return -EDOM; return 0; } ssize_t safe_write(int fd, const void *buf, size_t count) { while (count > 0) { ssize_t r = write(fd, buf, count); if (r < 0) { if (errno == EINTR) continue; return -errno; } count -= r; buf = (char *)buf + r; } return 0; } ssize_t safe_pread(int fd, void *buf, size_t count, off_t offset) { size_t cnt = 0; char *b = (char*)buf; while (cnt < count) { ssize_t r = pread(fd, b + cnt, count - cnt, offset + cnt); if (r <= 0) { if (r == 0) { // EOF return cnt; } if (errno == EINTR) continue; return -errno; } cnt += r; } return cnt; } ssize_t safe_pread_exact(int fd, void *buf, size_t count, off_t offset) { ssize_t ret = safe_pread(fd, buf, count, offset); if (ret < 0) return ret; if ((size_t)ret != count) return -EDOM; return 0; } ssize_t safe_pwrite(int fd, const void *buf, size_t count, off_t offset) { while (count > 0) { ssize_t r = pwrite(fd, buf, count, offset); if (r < 0) { if (errno == EINTR) continue; return -errno; } count -= r; buf = (char *)buf + r; offset += r; } return 0; } #ifdef CEPH_HAVE_SPLICE ssize_t safe_splice(int fd_in, loff_t *off_in, int fd_out, loff_t *off_out, size_t len, unsigned int flags) { size_t cnt = 0; while (cnt < len) { ssize_t r = splice(fd_in, off_in, fd_out, off_out, len - cnt, flags); if (r <= 0) { if (r == 0) { // EOF return cnt; } if (errno == EINTR) continue; return -errno; } cnt += r; } return cnt; } ssize_t safe_splice_exact(int fd_in, loff_t *off_in, int fd_out, loff_t *off_out, size_t len, unsigned int flags) { ssize_t ret = safe_splice(fd_in, off_in, fd_out, off_out, len, flags); if (ret < 0) return ret; if ((size_t)ret != len) return -EDOM; return 0; } #endif int safe_write_file(const char *base, const char *file, const char *val, size_t vallen) { int ret; char fn[PATH_MAX]; char tmp[PATH_MAX]; int fd; // does the file already have correct content? char oldval[80]; ret = safe_read_file(base, file, oldval, sizeof(oldval)); if (ret == (int)vallen && memcmp(oldval, val, vallen) == 0) return 0; // yes. snprintf(fn, sizeof(fn), "%s/%s", base, file); snprintf(tmp, sizeof(tmp), "%s/%s.tmp", base, file); fd = open(tmp, O_WRONLY|O_CREAT|O_TRUNC, 0644); if (fd < 0) { ret = errno; return -ret; } ret = safe_write(fd, val, vallen); if (ret) { VOID_TEMP_FAILURE_RETRY(close(fd)); return ret; } ret = fsync(fd); if (ret < 0) ret = -errno; VOID_TEMP_FAILURE_RETRY(close(fd)); if (ret < 0) { unlink(tmp); return ret; } ret = rename(tmp, fn); if (ret < 0) { ret = -errno; unlink(tmp); return ret; } fd = open(base, O_RDONLY); if (fd < 0) { ret = -errno; return ret; } ret = fsync(fd); if (ret < 0) ret = -errno; VOID_TEMP_FAILURE_RETRY(close(fd)); return ret; } int safe_read_file(const char *base, const char *file, char *val, size_t vallen) { char fn[PATH_MAX]; int fd, len; snprintf(fn, sizeof(fn), "%s/%s", base, file); fd = open(fn, O_RDONLY); if (fd < 0) { return -errno; } len = safe_read(fd, val, vallen); if (len < 0) { VOID_TEMP_FAILURE_RETRY(close(fd)); return len; } // close sometimes returns errors, but only after write() VOID_TEMP_FAILURE_RETRY(close(fd)); return len; }
下面是自己写的代码,为了验证read/writeBlock的正确性,分两批写入文件,然后又分两批读入文件,实际工程中只要一次读写就行了
#include <unistd.h> #include <iostream> #include <stdlib.h> #include <string.h> #include <stdint.h> #include <string> #include <sys/types.h> #include <sys/stat.h> #include <fcntl.h> #include <fstream> #include <vector> #include <sstream> #include <errno.h> using namespace std; bool writeBlock(int fd, int64_t &fdOft,int8_t *arr, int64_t len) { int64_t num = 0; int64_t oft = 0; int8_t *arr1 = (int8_t*)arr; while (oft < len) { num = pwrite(fd, arr1+oft, len-oft, fdOft+oft); if (num == -1) { return false; } oft += num; } fdOft += oft; return true; } bool readBlock(int fd, int64_t &fdOft, void *arr, int64_t len) { int64_t num = 0; int64_t oft = 0; int8_t *b = (int8_t*)arr; while (oft < len) { num = pread(fd, b + oft, len - oft, fdOft + oft); if (num <= 0) { if (num == 0) { if (oft != len) return false; else return true; } if (errno == EINTR) { continue;//the call was interrupted by a signal befor any data was read } return false; } oft += num; } fdOft += oft; return true; } int main(int argc, char *argv[]) { int64_t len = atol(argv[1]); cout << "len = "<<len <<endl; int fd = open(argv[2], O_RDWR|O_CREAT|O_TRUNC, S_IRWXU); if (fd == -1) { cout << "not open\n"; return -1; } int64_t total = sizeof(double)*len; cout << "total = " << total <<endl; double *arr = (double*)malloc(total); if (arr == NULL) { cout << "malloc err\n"; return -1; } arr[0] = 0.001; for (int64_t i = 1; i < len; ++i) { arr[i] = arr[i-1] + 0.001; } //int64_t num = pwrite(fd,arr,total,0); int64_t fdOft = 0; int64_t first = total/2; int64_t second = total - first; bool flag = writeBlock(fd, fdOft,(int8_t*)arr, first); if (!flag) { cout << "first no\n"; } else { cout << "first yes\n"; } flag = writeBlock(fd, fdOft, (int8_t*)arr + first, second); //cout << num<<endl; if (!flag) { cout << "second no\n"; } else { cout << "second yes\n"; } close(fd); int fdr = open(argv[2], O_RDONLY); double *arr1 = (double*)malloc(total); if (arr1 == NULL) { cout << "malloc error\n"; return -1; } int64_t readOft = 0; if (readBlock(fdr, readOft, arr1, first)) { cout << "first read yes\n"; } if (readBlock(fdr, readOft, (int8_t*)arr1 + first, second)) { cout << "second read yes\n"; } flag = true; for (int64_t i = 0; i < len; ++i) { if (arr1[i] != arr[i]) { flag = false; cout << i << " no equal\n"; } } if (flag) cout << "equal\n"; close(fdr); return 1; }
相关文章推荐
- VB ADO 用二进制方式来读写文件 入数据库 __________请参考(把文件(包括图片)存入数据库)
- 文件二进制的读写
- linux 文件读写
- Linux常用的二进制文件分析方法
- Linux常用的二进制文件分析方法
- VB 写的二进制文件读写数据库操作模块
- C语言二进制文件读写
- Hibernate读写SQL-Server二进制文件
- 文件的二进制读写及文本读写 两个实例
- linux下大文件读写源码示例
- C语言实现二进制文件读写
- 读写二进制文件
- 读写二进制文件
- Linux内存计数详解(读写文件时的缓存机制与windows不同,以致于读写大文件时cached值非常大且居高不下)
- 读写二进制文件,验证正确性
- linux环境下读写超过2G的大文件
- linux环境下如何编辑察看二进制文件?
- linux 下读写文件乱码 解决
- 3-6 读写二进制文件
- linux读写文件速度测试