.NET批量大数据插入性能分析及比较(2.普通插入与拼接sql批量插入)

首先自然是遍历DataTable逐条数据插入

public class DbOperation
    {
        private static string connectionString = ConfigurationManager.ConnectionStrings["ConnectToSql"].ConnectionString;
        private static string asyncconnectionString = ConfigurationManager.ConnectionStrings["ConnectToSqlAsync"].ConnectionString;

        #region 逐条数据插入
        public static bool ExecuteInsert(DataTable dt, int batchSize)
        {
            int count = dt.Rows.Count;
            bool flag = false;
            try
            {
                using (SqlConnection cn = new SqlConnection(connectionString))
                {
                    using (SqlCommand cmd = new SqlCommand("Insert into TestTable(Id, Name) Values(@Id, @Name)", cn))
                    {
                        cn.Open();
                        for (int i = 0; i < count; i++)
                        {
                            cmd.Parameters.AddWithValue("@Id", dt.Rows[i]["Id"]);
                            cmd.Parameters.AddWithValue("@Name", dt.Rows[i]["Name"]);
                            cmd.ExecuteNonQuery();
                            cmd.Parameters.Clear();
                        }
                        flag = true;
                    }
                }
            }
            catch (Exception ex)
            {
                LogHelper.Error(ex.Message);
                return false;
            }
            return flag;
        }
        #endregion
    }

结果如下：Use SqlServer Insert;RecordCount:40000;BatchSize:1;Time:62329;





可以根据批量大小拼接sql减少服务器的往返次数，我们测试下在批处理10,20,50,100,200下的插入时间

#region 拼接sql语句插入
        public static bool ExecuteBatchInsert(DataTable dt, int batchSize)
        {
            int count = dt.Rows.Count;
            StringBuilder sql = new StringBuilder(220);
            bool flag = false;
            SqlConnection cn = null;
            SqlCommand cmd = null;
            try
            {
                cn = new SqlConnection(connectionString);
                cmd = new SqlCommand();
                cmd.Connection = cn;
                cn.Open();
                for (int i = 0; i < count; i += batchSize)
                {
                    for (int j = i; j < i + batchSize && j < count; j++)
                    {
                        sql.AppendFormat("Insert into TestTable(Id, Name) Values({0}, '{1}');", dt.Rows[j]["Id"], dt.Rows[j]["Name"]);
                    }
                    //LogHelper.Info(sql.ToString());
                    cmd.CommandText = sql.ToString();
                    cmd.ExecuteNonQuery();
                    sql.Clear();
                }
                flag = true;
            }
            catch (Exception ex)
            {
                LogHelper.Error(sql.ToString(), ex);
                return false;
            }
            finally
            {
                if (cn != null)
                {
                    if (cn.State == ConnectionState.Open)
                    {
                        cn.Close();
                    }
                    cn.Dispose();
                }
                if (cmd != null) cmd.Dispose();
            }
            return flag;
        }
        #endregion

结果如下：

Use SqlServer Batch Insert;RecordCount:40000;BatchSize:10;Time:30010;

Use SqlServer Batch Insert;RecordCount:40000;BatchSize:20;Time:21858;

Use SqlServer Batch Insert;RecordCount:40000;BatchSize:50;Time:17564;

Use SqlServer Batch Insert;RecordCount:40000;BatchSize:100;Time:19869;

Use SqlServer Batch Insert;RecordCount:40000;BatchSize:200;Time:17936;

插入时间为前者的28% ~ 48%看来减少服务器的往返次数还是有效果的，批量50条基本是平衡点。





全文链接:

.NET批量大数据插入性能分析及比较(1.准备工作)

.NET批量大数据插入性能分析及比较(2.普通插入与拼接sql批量插入)

.NET批量大数据插入性能分析及比较(3.使用事务)

.NET批量大数据插入性能分析及比较(4.使用DataAdapter批量插入)

.NET批量大数据插入性能分析及比较(5.使用SqlBulkCopy)

.NET批量大数据插入性能分析及比较(6.使用表值参数)