test/MiniExcel
1
0
Fork 0
mirror of https://gitee.com/dotnetchina/MiniExcel.git synced 2024-11-29 18:38:08 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

optimize template performance

Browse Source
This commit is contained in:
wei 2021-04-14 15:02:43 +08:00
parent 00d5231ebc
commit d11a3aafeb
8 changed files with 250 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

91
benchmarks/MiniExcel.Benchmarks/DebugConfig.cs Normal file
View File

@ -0,0 +1,91 @@
using BenchmarkDotNet.Analysers;
using BenchmarkDotNet.Columns;
using BenchmarkDotNet.Configs;
using BenchmarkDotNet.Diagnosers;
using BenchmarkDotNet.Exporters;
using BenchmarkDotNet.Filters;
using BenchmarkDotNet.Jobs;
using BenchmarkDotNet.Loggers;
using BenchmarkDotNet.Order;
using BenchmarkDotNet.Reports;
using BenchmarkDotNet.Toolchains.InProcess.Emit;
using BenchmarkDotNet.Validators;
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
namespace MiniExcelLibs.Benchmarks
{
public class DebugConfig : ManualConfig
{
public new IOrderer Orderer => DefaultOrderer.Instance;
public new SummaryStyle SummaryStyle => SummaryStyle.Default;
public new ConfigUnionRule UnionRule => ConfigUnionRule.Union;
public new string ArtifactsPath => Path.Combine(Directory.GetCurrentDirectory(), "BenchmarkDotNet.Artifacts");
public new CultureInfo CultureInfo => null;
public new ConfigOptions Options => ConfigOptions.KeepBenchmarkFiles | ConfigOptions.DisableOptimizationsValidator;
public new IEnumerable<Job> GetJobs()
{
return new Job[1]
{
JobMode<Job>.Default.WithToolchain(new InProcessEmitToolchain(TimeSpan.FromHours(1.0), logOutput: true))
};
}
public new IEnumerable<IValidator> GetValidators()
{
return Array.Empty<IValidator>();
}
public new IEnumerable<IColumnProvider> GetColumnProviders()
{
return DefaultColumnProviders.Instance;
}
public new IEnumerable<IExporter> GetExporters()
{
return Array.Empty<IExporter>();
}
public new IEnumerable<ILogger> GetLoggers()
{
return new ILogger[1]
{
ConsoleLogger.Default
};
}
public new IEnumerable<IDiagnoser> GetDiagnosers()
{
return Array.Empty<IDiagnoser>();
}
public new IEnumerable<IAnalyser> GetAnalysers()
{
return Array.Empty<IAnalyser>();
}
public new IEnumerable<HardwareCounter> GetHardwareCounters()
{
return Array.Empty<HardwareCounter>();
}
public new IEnumerable<IFilter> GetFilters()
{
return Array.Empty<IFilter>();
}
public new IEnumerable<BenchmarkLogicalGroupRule> GetLogicalGroupRules()
{
return Array.Empty<BenchmarkLogicalGroupRule>();
}
}
}

6
benchmarks/MiniExcel.Benchmarks/MiniExcel.Benchmarks.csproj
View File

@ -8,10 +8,14 @@
<ItemGroup>
<PackageReference Include="BenchmarkDotNet" Version="0.12.1" />
<PackageReference Include="ClosedXML" Version="0.95.4" />
<PackageReference Include="ClosedXML.Report" Version="0.2.1" />
<PackageReference Include="DocumentFormat.OpenXml" Version="2.12.3" />
<PackageReference Include="EPPlus" Version="5.6.1" />
<PackageReference Include="ExcelDataReader" Version="3.6.0" />
<PackageReference Include="MiniExcel" Version="0.8.0" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\src\MiniExcel\MiniExcelLibs.csproj" />
</ItemGroup>
</Project>

45
benchmarks/MiniExcel.Benchmarks/Program.cs
View File

@ -25,10 +25,14 @@ namespace MiniExcelLibs.Benchmarks
{
#if !DEBUG
//new BenchmarkSwitcher(typeof(Program).Assembly).Run(args, new Config());
BenchmarkRunner.Run<XlsxBenchmark>();
//BenchmarkRunner.Run<XlsxBenchmark>();
BenchmarkRunner.Run<TemplateXlsxBenchmark>();
#else
//BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args, new DebugInProcessConfig());
new XlsxBenchmark().ClosedXml_Query_Test();
//BenchmarkSwitcher.FromTypes(new[] { typeof(TemplateXlsxBenchmark) }).Run(args, new DebugInProcessConfig() );
//BenchmarkSwitcher.FromAssembly(typeof(TemplateXlsxBenchmark).Assembly).Run(args, new DebugConfig());
new TemplateXlsxBenchmark().MiniExcel_Template_Generate_Test();
//new XlsxBenchmark().MiniExcelCreateTest();
#endif
Console.Read();
}
@ -49,6 +53,41 @@ namespace MiniExcelLibs.Benchmarks
#endif
}
public class TemplateXlsxBenchmark : BenchmarkBase
{
[Benchmark(Description = "MiniExcel Template Generate")]
public void MiniExcel_Template_Generate_Test()
{
{
var path = Path.Combine(Path.GetTempPath(), $"{Guid.NewGuid().ToString()}.xlsx");
var templatePath = @"D:\git\MiniExcel\samples\xlsx\TestTemplateBasicIEmumerableFill.xlsx";
var value = new
{
employees = Enumerable.Range(1, 1000000).Select(s => new { name = "Jack", department = "HR" })
};
MiniExcel.SaveAsByTemplate(path, templatePath, value);
}
}
[Benchmark(Description = "ClosedXml.Report Template Generate")]
public void ClosedXml_Report_Template_Generate_Test()
{
{
var path = Path.Combine(Path.GetTempPath(), $"{Guid.NewGuid().ToString()}.xlsx");
var templatePath = @"D:\git\MiniExcel\samples\xlsx\TestTemplateBasicIEmumerableFill_ClosedXML_Report.xlsx";
var template = new ClosedXML.Report.XLTemplate(templatePath);
var value = new
{
employees = Enumerable.Range(1, 1000000).Select(s => new { name = "Jack", department = "HR" })
};
template.AddVariable(value);
template.Generate();
template.SaveAs(path);
}
}
}
public class XlsxBenchmark: BenchmarkBase
{
[GlobalSetup]

49
drafts/【Try】Magicode.IE Excel Template.linq Normal file
View File

@ -0,0 +1,49 @@
<Query Kind="Program">
<NuGetReference>Dapper</NuGetReference>
<NuGetReference>Magicodes.IE.Excel</NuGetReference>
<NuGetReference>MiniExcel</NuGetReference>
<NuGetReference>Newtonsoft.Json</NuGetReference>
<NuGetReference>System.Data.SqlClient</NuGetReference>
<Namespace>Dapper</Namespace>
<Namespace>MiniExcelLibs</Namespace>
<Namespace>Newtonsoft.Json</Namespace>
<Namespace>Magicodes.ExporterAndImporter.Core</Namespace>
<Namespace>Magicodes.ExporterAndImporter.Excel</Namespace>
<RemoveNamespace>System.Data</RemoveNamespace>
<RemoveNamespace>System.Diagnostics</RemoveNamespace>
<RemoveNamespace>System.Linq.Expressions</RemoveNamespace>
<RemoveNamespace>System.Text</RemoveNamespace>
<RemoveNamespace>System.Text.RegularExpressions</RemoveNamespace>
<RemoveNamespace>System.Threading</RemoveNamespace>
<RemoveNamespace>System.Transactions</RemoveNamespace>
<RemoveNamespace>System.Xml</RemoveNamespace>
<RemoveNamespace>System.Xml.Linq</RemoveNamespace>
<RemoveNamespace>System.Xml.XPath</RemoveNamespace>
</Query>
void Main()
{
{
var path = Path.Combine(Path.GetTempPath(), $"{Guid.NewGuid().ToString()}.xlsx");
var templatePath = @"D:\git\MiniExcel\samples\xlsx\TestTemplateBasicIEmumerableFill_Magicodes_IE.xlsx";
var value = new
{
employees = Enumerable.Range(1, 1000).Select(s => new { name = "Jack", department = "HR" }).ToList()
};
//创建Excel导出对象
IExportFileByTemplate exporter = new ExcelExporter();
exporter.ExportByTemplate(path, value, templatePath).GetAwaiter().GetResult();
}
}
/*
error :
![image](https://user-images.githubusercontent.com/12729184/114646437-d47c8c80-9d0d-11eb-8d5f-a78c61a84536.png)
![image](https://user-images.githubusercontent.com/12729184/114646441-d6465000-9d0d-11eb-887a-940413c2fbc8.png)
![image](https://user-images.githubusercontent.com/12729184/114646466-e0684e80-9d0d-11eb-8d81-f56903cc20fb.png)
*/
// You can define other methods, fields, classes and namespaces here

BIN
samples/xlsx/TestTemplateBasicIEmumerableFill_Magicodes_IE.xlsx Normal file
View File

Binary file not shown.

85
src/MiniExcel/OpenXml/ExcelOpenXmlTemplate.cs
View File

@ -15,13 +15,16 @@ namespace MiniExcelLibs.OpenXml
internal class ExcelOpenXmlTemplate
{
private const string _ns = "http://schemas.openxmlformats.org/spreadsheetml/2006/main";
private static readonly XmlNamespaceManager _ns;
private static readonly Regex _isExpressionRegex;
static ExcelOpenXmlTemplate()
{
_isExpressionRegex = new Regex("(?<={{).*?(?=}})");
_ns = new XmlNamespaceManager(new NameTable());
_ns.AddNamespace("x", "http://schemas.openxmlformats.org/spreadsheetml/2006/main");
}
internal static void SaveAsByTemplateImpl(Stream stream, string templatePath, object value)
{
//only support xlsx
@ -42,7 +45,6 @@ namespace MiniExcelLibs.OpenXml
}
//TODO:DataTable & DapperRow
//TODO: copy new bytes
using (var templateStream = File.Open(templatePath, FileMode.Open, FileAccess.Read, FileShare.Read))
{
@ -62,25 +64,12 @@ namespace MiniExcelLibs.OpenXml
foreach (var sheet in sheets)
{
var sheetStream = sheet.Open();
var doc = new System.Xml.XmlDocument();
doc.Load(sheetStream);
XmlNamespaceManager ns = new XmlNamespaceManager(doc.NameTable);
ns.AddNamespace("x", "http://schemas.openxmlformats.org/spreadsheetml/2006/main");
var worksheet = doc.SelectSingleNode("/x:worksheet", ns);
var rows = doc.SelectNodes($"/x:worksheet/x:sheetData/x:row", ns);
sheetStream.Dispose();
var fullName = sheet.FullName;
sheet.Delete();
ZipArchiveEntry entry = _archive.ZipFile.CreateEntry(fullName);
using (var zipStream = entry.Open())
{
ExcelOpenXmlTemplate.GenerateSheetXml(zipStream, doc.InnerXml, values, sharedStrings);
ExcelOpenXmlTemplate.GenerateSheetXmlImpl(sheet,zipStream, sheetStream, values, sharedStrings);
//doc.Save(zipStream); //don't do it beacause : ![image](https://user-images.githubusercontent.com/12729184/114361127-61a5d100-9ba8-11eb-9bb9-34f076ee28a2.png)
}
}
@ -89,43 +78,26 @@ namespace MiniExcelLibs.OpenXml
_archive.Dispose();
}
}
private static Type GetIEnumerableRuntimeValueType(object v)
{
if (v == null)
throw new InvalidDataException("input parameter value can't be null");
foreach (var tv in v as IEnumerable)
{
if (tv != null)
{
return tv.GetType();
}
}
throw new InvalidDataException("can't get parameter type information");
}
public static void GenerateSheetXml(Stream stream, string sheetXml, Dictionary<string, object> inputMaps,List<string> sharedStrings, XmlWriterSettings xmlWriterSettings = null)
internal static void GenerateSheetXmlImpl(ZipArchiveEntry sheetZipEntry,Stream stream, Stream sheetStream, Dictionary<string, object> inputMaps,List<string> sharedStrings, XmlWriterSettings xmlWriterSettings = null)
{
var doc = new XmlDocument();
doc.LoadXml(sheetXml);
doc.Load(sheetStream);
sheetStream.Dispose();
sheetZipEntry.Delete(); // ZipArchiveEntry can't update directly, so need to delete then create logic
XmlNamespaceManager ns = new XmlNamespaceManager(doc.NameTable);
ns.AddNamespace("x", "http://schemas.openxmlformats.org/spreadsheetml/2006/main");
var worksheet = doc.SelectSingleNode("/x:worksheet", ns);
var sheetData = doc.SelectSingleNode("/x:worksheet/x:sheetData", ns);
var worksheet = doc.SelectSingleNode("/x:worksheet", _ns);
var sheetData = doc.SelectSingleNode("/x:worksheet/x:sheetData", _ns);
// ==== update sharedstring ====
var rows = sheetData.SelectNodes($"x:row", ns);
var rows = sheetData.SelectNodes($"x:row", _ns);
foreach (XmlElement row in rows)
{
var cs = row.SelectNodes($"x:c", ns);
var cs = row.SelectNodes($"x:c", _ns);
foreach (XmlElement c in cs)
{
var t = c.GetAttribute("t");
var v = c.SelectSingleNode("x:v", ns);
var v = c.SelectSingleNode("x:v", _ns);
if (v == null || v.InnerText == null) //![image](https://user-images.githubusercontent.com/12729184/114363496-075a3f80-9bab-11eb-9883-8e3fec10765c.png)
continue;
@ -145,7 +117,7 @@ namespace MiniExcelLibs.OpenXml
// ==== Dimension ====
// note : dimension need to put on the top ![image](https://user-images.githubusercontent.com/12729184/114507911-5dd88400-9c66-11eb-94c6-82ed7bdb5aab.png)
var dimension = doc.SelectSingleNode("/x:worksheet/x:dimension", ns) as XmlElement;
var dimension = doc.SelectSingleNode("/x:worksheet/x:dimension", _ns) as XmlElement;
// update dimension
if (dimension == null)
throw new NotImplementedException("Excel Dimension Xml is null, please issue file for me. https://github.com/shps951023/MiniExcel/issues");
@ -157,9 +129,9 @@ namespace MiniExcelLibs.OpenXml
{
IEnumerable ienumerable = null;
foreach (XmlElement c in row.SelectNodes($"x:c", ns))
foreach (XmlElement c in row.SelectNodes($"x:c", _ns))
{
var v = c.SelectSingleNode("x:v", ns);
var v = c.SelectSingleNode("x:v", _ns);
if (v?.InnerText == null)
continue;
@ -212,7 +184,7 @@ namespace MiniExcelLibs.OpenXml
writer.Write("<sheetData>");
int originRowIndex;
int rowIndexDiff = 0;
foreach (XmlElement row in newSheetData.SelectNodes($"x:row", ns))
foreach (XmlElement row in newSheetData.SelectNodes($"x:row", _ns))
{
var rowCotainIEnumerable = false;
IEnumerable ienumerable = null;
@ -227,14 +199,14 @@ namespace MiniExcelLibs.OpenXml
// check if contains IEnumerble row
{
var cs = row.SelectNodes($"x:c", ns);
var cs = row.SelectNodes($"x:c", _ns);
foreach (XmlElement c in cs)
{
var cr = c.GetAttribute("r");
var letter = new String(cr.Where(Char.IsLetter).ToArray());
c.SetAttribute("r", $"{letter}{{{{{{MiniExcel_RowIndex}}}}}}");
var v = c.SelectSingleNode("x:v", ns);
var v = c.SelectSingleNode("x:v", _ns);
if (v?.InnerText == null)
continue;
@ -336,6 +308,21 @@ namespace MiniExcelLibs.OpenXml
writer.Write(contents[1]);
}
}
private static Type GetIEnumerableRuntimeValueType(object v)
{
if (v == null)
throw new InvalidDataException("input parameter value can't be null");
foreach (var tv in v as IEnumerable)
{
if (tv != null)
{
return tv.GetType();
}
}
throw new InvalidDataException("can't get parameter type information");
}
private static string CleanXml(string xml)
{
//TODO: need to optimize

48
tests/MiniExcelTests/MiniExcelTemplateTests.cs
View File

@ -1,4 +1,5 @@
using MiniExcelLibs;
using MiniExcelLibs.Tests.Utils;
using System;
using System.Collections.Generic;
using System.IO;
@ -32,6 +33,9 @@ namespace MiniExcelTests
Assert.Equal(true, rows[1].C);
Assert.Equal(123, rows[1].D);
Assert.Equal("Jack has 123 points", rows[1].E);
var demension = Helpers.GetFirstSheetDimensionRefValue(path);
Assert.Equal("A1:E2", demension);
}
{
@ -53,38 +57,12 @@ namespace MiniExcelTests
Assert.Equal(true, rows[1].C);
Assert.Equal(123, rows[1].D);
Assert.Equal("Jack has 123 points", rows[1].E);
var demension = Helpers.GetFirstSheetDimensionRefValue(path);
Assert.Equal("A1:E2", demension);
}
}
//[Fact]
//public void PerformanceTest()
//{
// // MiniExcel
// {
// var path = Path.Combine(Path.GetTempPath(), $"{Guid.NewGuid().ToString()}.xlsx");
// var templatePath = @"..\..\..\..\..\samples\xlsx\TestTemplateBasicIEmumerableFill.xlsx";
// var value = new
// {
// employees = Enumerable.Range(1, 1000000).Select(s => new { name = "Jack", department = "HR" })
// };
// MiniExcel.SaveAsByTemplate(path, templatePath, value);
// }
// // ClosexXml.Report
// {
// var path = Path.Combine(Path.GetTempPath(), $"{Guid.NewGuid().ToString()}.xlsx");
// var templatePath = @"..\..\..\..\..\samples\xlsx\TestTemplateBasicIEmumerableFill_ClosedXML_Report.xlsx";
// var template = new ClosedXML.Report.XLTemplate(templatePath);
// var value = new
// {
// employees = Enumerable.Range(1, 1000000).Select(s => new { name = "Jack", department = "HR" })
// };
// template.AddVariable(value);
// template.Generate();
// template.SaveAs(path);
// }
//}
[Fact]
public void TestIEnumerable()
{
@ -105,6 +83,9 @@ namespace MiniExcelTests
}
};
MiniExcel.SaveAsByTemplate(path, templatePath, value);
var demension = Helpers.GetFirstSheetDimensionRefValue(path);
Assert.Equal("A1:B7", demension);
}
{
@ -124,6 +105,9 @@ namespace MiniExcelTests
}
};
MiniExcel.SaveAsByTemplate(path, templatePath, value);
var demension = Helpers.GetFirstSheetDimensionRefValue(path);
Assert.Equal("A1:B7", demension);
}
}
@ -167,6 +151,9 @@ namespace MiniExcelTests
Assert.Equal("IT", rows[7].C);
Assert.Equal("Keaton", rows[8].B);
Assert.Equal("IT", rows[8].C);
var demension = Helpers.GetFirstSheetDimensionRefValue(path);
Assert.Equal("A1:C9", demension);
}
@ -207,6 +194,9 @@ namespace MiniExcelTests
Assert.Equal("IT", rows[7].C);
Assert.Equal("Keaton", rows[8].B);
Assert.Equal("IT", rows[8].C);
var demension = Helpers.GetFirstSheetDimensionRefValue(path);
Assert.Equal("A1:C9", demension);
}
}

11
tests/MiniExcelTests/Utils/Helpers.cs
View File

@ -3,6 +3,7 @@
**/
namespace MiniExcelLibs.Tests.Utils
{
using MiniExcelLibs.OpenXml;
using System;
using System.Collections.Generic;
using System.Dynamic;
@ -13,7 +14,9 @@ namespace MiniExcelLibs.Tests.Utils
using System.Reflection;
using System.Text;
using System.Text.RegularExpressions;
using System.Xml;
using System.Xml.Linq;
using System.Xml.XPath;
internal static class Helpers
{
@ -46,6 +49,8 @@ namespace MiniExcelLibs.Tests.Utils
internal static string GetFirstSheetDimensionRefValue(string path)
{
var ns = new XmlNamespaceManager(new NameTable());
ns.AddNamespace("x", "http://schemas.openxmlformats.org/spreadsheetml/2006/main");
string refV;
using (var stream = File.OpenRead(path))
using (ZipArchive archive = new ZipArchive(stream, ZipArchiveMode.Read, false, Encoding.UTF8))
@ -53,9 +58,9 @@ namespace MiniExcelLibs.Tests.Utils
var sheet = archive.Entries.Single(w => w.FullName.StartsWith("xl/worksheets/sheet1", StringComparison.OrdinalIgnoreCase));
using (var sheetStream = sheet.Open())
{
var dimension = XElement.Load(sheetStream)
.Descendants("dimension");
refV = dimension.Single().Attribute("ref").Value;
var doc = XDocument.Load(sheetStream); ;
var dimension = doc.XPathSelectElement("/x:worksheet/x:dimension", ns);
refV = dimension.Attribute("ref").Value;
}
}