由於項目需要測試windows下的IO性能,因此要寫個小程序,按照要求讀取磁盤上的文件。在讀取文件的時候,測試Windows的IO性能。
主要內容:
- 程序的要求
- 一般的FileStream方式
- 利用kernel32.dll中的CreateFile函數
1. 程序的要求
程序的要求很簡單。
(1)命令行程序
(2)有3個參數,讀取的文件名,一次讀取buffer size,讀取的次數count
(3)如果讀取次數count未到,文件已經讀完,就再次從頭讀取文件。
使用格式如下:
C:\>****.exe “c:\****.bin” 32768 32768
讀取文件“c:\****.bin”,每次讀取4K,讀取32768次,讀取的量大概1G。
2. 一般的FileStream方式
利用FileStream來讀取文件,非常簡單,代碼如下:
using System;using System.Collections.Generic;using System.Text;using System.IO;using System.Reflection;namespace DirectIO{ public class DIOReader { static void Main(string[] args) { long start = DateTime.Now.Ticks; if (args.Length < 3) { Console.WriteLine("parameter error!!"); return; } FileStream input = null; try { int bs = Convert.ToInt32(args[1]); int count = Convert.ToInt32(args[2]); input = new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.None, bs); byte[] b = new byte[bs]; for (int i = 0; i < count; i++) { if (input.Read(b, 0, bs) == 0) input.Seek(0, SeekOrigin.Begin); } Console.WriteLine("Read successed! "); Console.WriteLine(DateTime.Now.Ticks - start); } catch (Exception ex) { Console.WriteLine(ex.Message); } finally { if (input != null) { input.Flush(); input.Close(); // 清除使用的對象 GC.Collect(); GC.Collect(); } } } }}
編譯後的exe文件可以按照既定要求執行,但是對於同一文件,第二次讀取明顯比第一次快很多(大家可以用個1G左右的大文件試試)。第三次讀取,第四次讀取……和第二次差不多,都很快。
基於上述情況,可以判斷是緩存的原因,導致第二次及以後各次都比較快。
但是從代碼中來看,已經執行了input.Flush();input.Close();甚至是GC.Collect();
所以可能是Windows系統或者CLR對文件讀取操作進行了優化,使用了緩存。
3. 利用kernel32.dll中的CreateFile函數
既然上述方法行不通,就得調查新的方法。通過google的查詢,大部分人都是建議用C/C++調用系統API來實現。
不過最後終於找到了用c#實現了無緩存直接讀取磁盤上的文件的方法。其實也是通過DllImport利用了kernel32.dll,不完全是託管代碼。(估計用純託管代碼實現不了)
參考的文章:How do I read a disk directly with .Net?
還有msdn中的CreateFile API
實現代碼就是參考的How do I read a disk directly with .Net?,分爲兩部分
(1)利用CreateFile API構造的可直接讀取磁盤的DeviceStream
using System;using System.Runtime.InteropServices;using System.IO;using Microsoft.Win32.SafeHandles;namespace DirectIO{ public class DeviceStream : Stream, IDisposable { public const short FILE_ATTRIBUTE_NORMAL = 0x80; public const short INVALID_HANDLE_VALUE = -1; public const uint GENERIC_READ = 0x80000000; public const uint NO_BUFFERING = 0x20000000; public const uint GENERIC_WRITE = 0x40000000; public const uint CREATE_NEW = 1; public const uint CREATE_ALWAYS = 2; public const uint OPEN_EXISTING = 3; // Use interop to call the CreateFile function. // For more information about CreateFile, // see the unmanaged MSDN reference library. [DllImport("kernel32.dll", SetLastError = true, CharSet = CharSet.Unicode)] private static extern IntPtr CreateFile(string lpFileName, uint dwDesiredAccess, uint dwShareMode, IntPtr lpSecurityAttributes, uint dwCreationDisposition, uint dwFlagsAndAttributes, IntPtr hTemplateFile); [DllImport("kernel32.dll", SetLastError = true)] private static extern bool ReadFile( IntPtr hFile, // handle to file byte[] lpBuffer, // data buffer int nNumberOfBytesToRead, // number of bytes to read ref int lpNumberOfBytesRead, // number of bytes read IntPtr lpOverlapped // // ref OVERLAPPED lpOverlapped // overlapped buffer ); private SafeFileHandle handleValue = null; private FileStream _fs = null; public DeviceStream(string device) { Load(device); } private void Load(string Path) { if (string.IsNullOrEmpty(Path)) { throw new ArgumentNullException("Path"); } // Try to open the file. IntPtr ptr = CreateFile(Path, GENERIC_READ, 0, IntPtr.Zero, OPEN_EXISTING, NO_BUFFERING, IntPtr.Zero); handleValue = new SafeFileHandle(ptr, true); _fs = new FileStream(handleValue, FileAccess.Read); // If the handle is invalid, // get the last Win32 error // and throw a Win32Exception. if (handleValue.IsInvalid) { Marshal.ThrowExceptionForHR(Marshal.GetHRForLastWin32Error()); } } public override bool CanRead { get { return true; } } public override bool CanSeek { get { return false; } } public override bool CanWrite { get { return false; } } public override void Flush() { return; } public override long Length { get { return -1; } } public override long Position { get { throw new NotImplementedException(); } set { throw new NotImplementedException(); } } /// <summary> /// </summary> /// <param name="buffer">An array of bytes. When this method returns, the buffer contains the specified byte array with the values between offset and /// (offset + count - 1) replaced by the bytes read from the current source. </param> /// <param name="offset">The zero-based byte offset in buffer at which to begin storing the data read from the current stream. </param> /// <param name="count">The maximum number of bytes to be read from the current stream.</param> /// <returns></returns> public override int Read(byte[] buffer, int offset, int count) { int BytesRead = 0; var BufBytes = new byte[count]; if (!ReadFile(handleValue.DangerousGetHandle(), BufBytes, count, ref BytesRead, IntPtr.Zero)) { Marshal.ThrowExceptionForHR(Marshal.GetHRForLastWin32Error()); } for (int i = 0; i < BytesRead; i++) { buffer[offset + i] = BufBytes[i]; } return BytesRead; } public override int ReadByte() { int BytesRead = 0; var lpBuffer = new byte[1]; if (!ReadFile( handleValue.DangerousGetHandle(), // handle to file lpBuffer, // data buffer 1, // number of bytes to read ref BytesRead, // number of bytes read IntPtr.Zero )) { Marshal.ThrowExceptionForHR(Marshal.GetHRForLastWin32Error()); ;} return lpBuffer[0]; } public override long Seek(long offset, SeekOrigin origin) { throw new NotImplementedException(); } public override void SetLength(long value) { throw new NotImplementedException(); } public override void Write(byte[] buffer, int offset, int count) { throw new NotImplementedException(); } public override void Close() { handleValue.Close(); handleValue.Dispose(); handleValue = null; base.Close(); } private bool disposed = false; new void Dispose() { Dispose(true); base.Dispose(); GC.SuppressFinalize(this); } private new void Dispose(bool disposing) { // Check to see if Dispose has already been called. if (!this.disposed) { if (disposing) { if (handleValue != null) { _fs.Dispose(); handleValue.Close(); handleValue.Dispose(); handleValue = null; } } // Note disposing has been done. disposed = true; } } }}
注意和原文相比,改動了一個地方。即加了個NO_BUFFERING的參數,並在調用CreateFile時使用了這個參數。
IntPtr ptr = CreateFile(Path, GENERIC_READ, 0, IntPtr.Zero, OPEN_EXISTING, NO_BUFFERING, IntPtr.Zero);
之前沒有加這個參數的時候,在xp上測試還是第二次比第一次快很多。
(2)完成指定要求的DIOReader
using System;using System.Collections.Generic;using System.Text;using System.IO;using System.Reflection;namespace DirectIO{ public class DIOReader { static void Main(string[] args) { long start = DateTime.Now.Ticks; if (args.Length < 3) { Console.WriteLine("parameter error!!"); return; } BinaryReader input = null; try { int bs = Convert.ToInt32(args[1]); int count = Convert.ToInt32(args[2]); input = new BinaryReader(new DeviceStream(args[0])); byte[] b = new byte[bs]; for (int i = 0; i < count; i++) { if (input.Read(b, 0, bs) == 0) input.BaseStream.Seek(0, SeekOrigin.Begin); } Console.WriteLine("Read successed! "); Console.WriteLine("Total cost " + (new TimeSpan(DateTime.Now.Ticks - start)).TotalSeconds + " seconds"); } catch (Exception ex) { Console.WriteLine(ex.Message); } finally { if (input != null) { input.Close(); } //Console.ReadKey(true); } } }}
這樣,就完成了類似linux上Direct IO模式讀取文件的操作。
通過這個例子可以看出,C#不僅可以開發上層的應用,也可以結合一些非託管的dll完成更加底層的操作。