MemoryMappedFile 内存映射+ Parallel 并行分块 读写大文件
2016-07-13 17:10
941 查看
private static void SpiltFile(string srcFile, int portionSize) { string savedPath = @"\\stcsrv-c81\MMFeedHealthyDatacache\2016_07_10\Feedkeys\No_Process_test.txt"; FileInfo fi = new FileInfo(srcFile); // total size in bytes Int64 size = fi.Length; object locker = new object(); object writeLock = new object(); List<MappedFile> mappedFiles = new List<MappedFile>(); Int64 fileToRead = size;//文件总的大小 portionSize = portionSize * 1024 * 1024; //每块大小 Int64 portion = (Int64)Math.Ceiling(size * 1.0 / portionSize); //分成多少块 Int64 fileOffset = 0; MemoryMappedViewAccessor mmf_reader = null; Stopwatch watch = Stopwatch.StartNew(); watch.Start(); Int64 fileSize = 0; using (MemoryMappedFile mmf = MemoryMappedFile.CreateFromFile(srcFile, FileMode.Open, "xyz", size)) { //using (var writeMap = MemoryMappedFile.CreateFromFile(savedPath, FileMode.Create, "test", size, MemoryMappedFileAccess.ReadWrite)) //{ //bool mutexCreated; //Mutex mutex = new Mutex(true, "testmapmutex", out mutexCreated);//进程间同步 Parallel.For(0, portion, (i, ParallelLoopState) => { //for (int i = 26; i < portion; i++) //{ lock (locker) { fileSize = Math.Min(portionSize, fileToRead - portionSize * i); if (fileSize > 0) { byte[] buffer; using (mmf_reader = mmf.CreateViewAccessor(i * portionSize, fileSize, MemoryMappedFileAccess.Read)) { buffer = new byte[fileSize]; mmf_reader.ReadArray(0, buffer, 0, (int)fileSize); mappedFiles.Add(new MappedFile { Offset = i * portionSize, //fileOffset, Buffer = buffer, FileSize = fileSize }); } //fileToRead -= fileSize; //lock (writeLock) //{ //using (var writeMmf = MemoryMappedFile.OpenExisting("xyz")) //{ // using (var writeAccessor = writeMmf.CreateViewStream(i * portionSize, fileSize)) // { // var w = new BinaryWriter(new FileStream(savedPath, FileMode.Create, FileAccess.Write)); // //writeAccessor.WriteArray(i * portionSize, buffer, 0, buffer.Length); // //writeAccessor.Write(buffer, 0, buffer.Length); // w.Write(buffer); // } //} //using (MemoryMappedViewAccessor writeView = writeMap.CreateViewAccessor()) //{ // writeView.WriteArray(i * portionSize, buffer, 0, (int)fileSize); //} } //} } }); } using (var writeMap = MemoryMappedFile.CreateFromFile(savedPath, FileMode.Create, "test", size, MemoryMappedFileAccess.ReadWrite)) { using (MemoryMappedViewAccessor writeView = writeMap.CreateViewAccessor()) { Parallel.For(0, mappedFiles.Count, i => { try { Monitor.Enter(locker); writeView.WriteArray(mappedFiles[i].Offset, mappedFiles[i].Buffer, 0, (int)mappedFiles[i].FileSize); } catch (Exception) { throw; } finally { Monitor.Exit(locker); } }); } } watch.Stop(); Console.WriteLine(watch.ElapsedMilliseconds); #region MyRegion }
public class MappedFile { public long Offset { get; set; } public byte[] Buffer { get; set; } public long FileSize { get; set; } }
相关文章推荐
- 快速集成Android实现下拉刷新上拉加载更多
- Android 在 SDK 版本5.0,N(API23)阶段startActivity()出错
- [Android]自定义垂直滚动的广告区
- 01-如何集成通用Mapper
- Android 定位系统(GPS)开发详解
- Getting start with OCMock in you unit test
- android之自定义View和ViewGroup(二)(此篇不讲述代码,只讲述原理和结构,带你走进自定义的世界)
- Android异步处理四:AsyncTask的实现原理
- app:transformResourcesWithMergeJavaResForDebug
- Android数据库更新并保留原来数据的实现
- 年度最具特色生活类APP
- android视频播放器Vitamio的使用
- android 流量的统计
- android之surfaceView详解--自定义surfaceView和用于视频surfaceview
- Android M 的动态权限解决方案
- Android权限表
- android之.9.png图片制作
- Android异步处理系列文章索引
- Android 蓝牙开发 博客链接
- android使用ViewPager实现引导页效果