Thanks to visit codestin.com
Credit goes to github.com

Skip to content
This repository was archived by the owner on Aug 2, 2023. It is now read-only.

Api brotli changes #1621

Merged
merged 26 commits into from
Jun 27, 2017
Merged

Api brotli changes #1621

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
f2da3ba
CaterburyPerf
Vedin Jun 16, 2017
da92380
flush/compress
Vedin Jun 19, 2017
790191c
huge changes
Vedin Jun 21, 2017
f3b3e58
space
Vedin Jun 21, 2017
c87498f
small
Vedin Jun 21, 2017
e152fd1
flush/compress
Vedin Jun 19, 2017
d06e4b4
huge changes
Vedin Jun 21, 2017
fb317f5
space
Vedin Jun 21, 2017
8fef201
small
Vedin Jun 21, 2017
d31a1c6
Merge branch 'APIBrotliChanges2' of https://github.com/Vedin/corefxla…
Vedin Jun 21, 2017
78a1a75
resolve merge conflicts
Vedin Jun 21, 2017
d9b7587
unsaved chagnes
Vedin Jun 21, 2017
5a6f037
test fix depends on APIchanges
Vedin Jun 21, 2017
fb51724
resolve issues
Vedin Jun 21, 2017
1972d8b
Less alocation, change State
Vedin Jun 22, 2017
6739e69
issue
Vedin Jun 22, 2017
59f3981
remove unnecessary
Vedin Jun 22, 2017
959a119
resolve
Vedin Jun 22, 2017
7f8b3d4
clean up changes, change state
Vedin Jun 23, 2017
d4ddd1f
merge conflicts
Vedin Jun 23, 2017
38ef5ec
resolve
Vedin Jun 23, 2017
649f42c
guidelines
Vedin Jun 23, 2017
516f4c6
fix bug
Vedin Jun 25, 2017
e1aa814
resolve issues
Vedin Jun 26, 2017
1120e4f
change access
Jun 26, 2017
2f71a9b
small issues
Vedin Jun 26, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
CaterburyPerf
  • Loading branch information
@Vedin
Vedin committed Jun 16, 2017
commit f2da3baee55e108680874391a41b34df213f8502
13 changes: 11 additions & 2 deletions src/System.IO.Compression.Brotli/System/IO/Compression/BrotliStream.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,11 +36,20 @@ public partial class BrotliStream : Stream

public override bool CanTimeout => true;

public override int ReadTimeout { get ; set; }
public override int ReadTimeout { get; set; }

public override int WriteTimeout { get; set; }

public BrotliStream(Stream baseStream, CompressionMode mode, bool leaveOpen, int bufferSize, uint windowSize, CompressionLevel quality) : this(baseStream, mode, leaveOpen, bufferSize)
public BrotliStream(Stream baseStream, CompressionMode mode, bool leaveOpen, int bufferSize, CompressionLevel quality) : this(baseStream, mode, leaveOpen, bufferSize)
{
if (_mode == CompressionMode.Decompress) throw new System.IO.IOException(BrotliEx.QualityAndWinSize);
Copy link
Contributor

@ahsonkhan ahsonkhan Jun 21, 2017 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: inconsistent use of brace brackets.

You don't need the else statement.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

fixed

else
{
_encoder.SetQuality((uint)Brotli.GetQualityFromCompressionLevel(quality));
}
}

public BrotliStream(Stream baseStream, CompressionMode mode, bool leaveOpen, int bufferSize, CompressionLevel quality, uint windowSize) : this(baseStream, mode, leaveOpen, bufferSize)
{
if (_mode == CompressionMode.Decompress) throw new System.IO.IOException(BrotliEx.QualityAndWinSize);
else
Expand Down
141 changes: 24 additions & 117 deletions tests/System.IO.Compression.Tests/BrotliPerfomanceTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,115 +5,22 @@
using System.Collections.Generic;
using System.IO.Compression;
using System.Runtime.CompilerServices;
using System.Security.Cryptography;
using System.Threading.Tasks;
using Xunit;

namespace System.IO.Compression.Tests
{
public class BrotliPerfomanceTests
{
public static string GetTestFilePath(int? index = null, string memberName = null, int lineNumber = 0)
{
return Path.Combine(Path.GetTempPath(), string.Format(
index.HasValue ? "{0}_{1}_{2}_{3}" : "{0}_{1}_{2}",
memberName ?? "TestBase", lineNumber, Path.GetRandomFileName(),
index.GetValueOrDefault()));
}

private static string CreateCompressedFile(CompressionType type)
{
const int fileSize = 1000000;
string filePath = GetTestFilePath() + ".br";
switch (type)
{
case CompressionType.CryptoRandom:
using (RandomNumberGenerator rand = RandomNumberGenerator.Create())
{
byte[] bytes = new byte[fileSize];
rand.GetBytes(bytes);
using (FileStream output = File.Create(filePath))
using (BrotliStream zip = new BrotliStream(output, CompressionMode.Compress))
zip.Write(bytes, 0, bytes.Length);
}
break;
case CompressionType.RepeatedSegments:
{
byte[] bytes = new byte[fileSize / 1000];
new Random(128453).NextBytes(bytes);
using (FileStream output = File.Create(filePath))
using (BrotliStream zip = new BrotliStream(output, CompressionMode.Compress))
for (int i = 0; i < 1000; i++)
zip.Write(bytes, 0, bytes.Length);
}
break;
case CompressionType.NormalData:
{
byte[] bytes = new byte[fileSize];
new Random(128453).NextBytes(bytes);
using (FileStream output = File.Create(filePath))
using (BrotliStream zip = new BrotliStream(output, CompressionMode.Compress))
zip.Write(bytes, 0, bytes.Length);
}
break;
}
return filePath;
}

private static byte[] CreateBytesToCompress(CompressionType type)
{
const int fileSize = 500000;
byte[] bytes = new byte[fileSize];
switch (type)
{
case CompressionType.CryptoRandom:
using (RandomNumberGenerator rand = RandomNumberGenerator.Create())
rand.GetBytes(bytes);
break;
case CompressionType.RepeatedSegments:
{
byte[] small = new byte[1000];
new Random(123453).NextBytes(small);
for (int i = 0; i < fileSize / 1000; i++)
{
small.CopyTo(bytes, 1000 * i);
}
}
break;
case CompressionType.VeryRepetitive:
{
byte[] small = new byte[100];
new Random(123453).NextBytes(small);
for (int i = 0; i < fileSize / 100; i++)
{
small.CopyTo(bytes, 100 * i);
}
break;
}
case CompressionType.NormalData:
new Random(123453).NextBytes(bytes);
break;
}
return bytes;
}

public enum CompressionType
{
CryptoRandom,
RepeatedSegments,
VeryRepetitive,
NormalData
}

private const int Iter = 1;
private const int Iter = 1000;

[Benchmark(InnerIterationCount = Iter)]
[InlineData(CompressionType.CryptoRandom)]
[InlineData(CompressionType.RepeatedSegments)]
[InlineData(CompressionType.NormalData)]
public void DecompressUsingStream(CompressionType type)
[InlineData(Util.CompressionType.CryptoRandom)]
[InlineData(Util.CompressionType.RepeatedSegments)]
[InlineData(Util.CompressionType.NormalData)]
public void DecompressUsingStream(Util.CompressionType type)
{
string testFilePath = CreateCompressedFile(type);
string testFilePath = Util.CreateCompressedFile(type);
int bufferSize = 1024;
var bytes = new byte[bufferSize];
using (MemoryStream brStream = new MemoryStream(File.ReadAllBytes(testFilePath)))
Expand All @@ -135,16 +42,16 @@ public void DecompressUsingStream(CompressionType type)
}

[Benchmark]
[InlineData(CompressionType.CryptoRandom)]
[InlineData(CompressionType.RepeatedSegments)]
[InlineData(CompressionType.VeryRepetitive)]
[InlineData(CompressionType.NormalData)]
public void CompressUsingStream(CompressionType type)
[InlineData(Util.CompressionType.CryptoRandom)]
[InlineData(Util.CompressionType.RepeatedSegments)]
[InlineData(Util.CompressionType.VeryRepetitive)]
[InlineData(Util.CompressionType.NormalData)]
public void CompressUsingStream(Util.CompressionType type)
{
byte[] bytes = CreateBytesToCompress(type);
byte[] bytes = Util.CreateBytesToCompress(type);
foreach (var iteration in Benchmark.Iterations)
{
string filePath = GetTestFilePath();
string filePath = Util.GetTestFilePath();
FileStream output = File.Create(filePath);
using (BrotliStream brotliCompressStream = new BrotliStream(output, CompressionMode.Compress))
{
Expand All @@ -158,12 +65,12 @@ public void CompressUsingStream(CompressionType type)
}

[Benchmark(InnerIterationCount = Iter)]
[InlineData(CompressionType.CryptoRandom)]
[InlineData(CompressionType.RepeatedSegments)]
[InlineData(CompressionType.NormalData)]
public void Decompress(CompressionType type)
[InlineData(Util.CompressionType.CryptoRandom)]
[InlineData(Util.CompressionType.RepeatedSegments)]
[InlineData(Util.CompressionType.NormalData)]
public void Decompress(Util.CompressionType type)
{
string testFilePath = CreateCompressedFile(type);
string testFilePath = Util.CreateCompressedFile(type);
int bufferSize = 1000000;
byte[] data = File.ReadAllBytes(testFilePath);
var bytes = new byte[bufferSize];
Expand All @@ -175,13 +82,13 @@ public void Decompress(CompressionType type)
}

[Benchmark]
[InlineData(CompressionType.CryptoRandom)]
[InlineData(CompressionType.RepeatedSegments)]
[InlineData(CompressionType.VeryRepetitive)]
[InlineData(CompressionType.NormalData)]
public void Compress(CompressionType type)
[InlineData(Util.CompressionType.CryptoRandom)]
[InlineData(Util.CompressionType.RepeatedSegments)]
[InlineData(Util.CompressionType.VeryRepetitive)]
[InlineData(Util.CompressionType.NormalData)]
public void Compress(Util.CompressionType type)
{
byte[] bytes = CreateBytesToCompress(type);
byte[] bytes = Util.CreateBytesToCompress(type);
foreach (var iteration in Benchmark.Iterations)
{
byte[] compressed = new byte[bytes.Length];
Expand Down
98 changes: 98 additions & 0 deletions tests/System.IO.Compression.Tests/BrotliPerformanceCanterburyCorpus.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.
using Microsoft.Xunit.Performance;
using System.Collections.Generic;
using System.IO.Compression;
using System.Runtime.CompilerServices;
using System.Security.Cryptography;
using System.Threading.Tasks;
using Xunit;

namespace System.IO.Compression.Tests
{
public class BrotliPerformanceCanterburyCorpus
{
public static IEnumerable<object[]> CanterburyCorpus()
{
foreach (CompressionLevel compressionLevel in Enum.GetValues(typeof(CompressionLevel)))
{
foreach (int innerIterations in new int[] { 1, 10 })
{
yield return new object[] { innerIterations, "alice29.txt", compressionLevel };
yield return new object[] { innerIterations, "asyoulik.txt", compressionLevel };
yield return new object[] { innerIterations, "cp.html", compressionLevel };
yield return new object[] { innerIterations, "fields.c", compressionLevel };
yield return new object[] { innerIterations, "grammar.lsp", compressionLevel };
yield return new object[] { innerIterations, "kennedy.xls", compressionLevel };
yield return new object[] { innerIterations, "lcet10.txt", compressionLevel };
yield return new object[] { innerIterations, "plrabn12.txt", compressionLevel };
yield return new object[] { innerIterations, "ptt5", compressionLevel };
yield return new object[] { innerIterations, "sum", compressionLevel };
yield return new object[] { innerIterations, "xargs.1", compressionLevel };
}
}
}

/// <summary>
/// Benchmark tests to measure the performance of individually compressing each file in the
/// Canterbury Corpus
/// </summary>
[Benchmark]
[MemberData(nameof(CanterburyCorpus))]
public void Compress_Canterbury(int innerIterations, string fileName, CompressionLevel compressLevel)
{
byte[] bytes = File.ReadAllBytes(Path.Combine("BrotliTestData", "Canterbury", fileName));
int bufferSize = 8096;
FileStream[] fileStreams = new FileStream[innerIterations];
BrotliStream[] brotliStream = new BrotliStream[innerIterations];
string[] paths = new string[innerIterations];
foreach (var iteration in Benchmark.Iterations)
{
for (int i = 0; i < innerIterations; i++)
{
paths[i] = Util.GetTestFilePath();
fileStreams[i] = File.Create(paths[i]);
}
using (iteration.StartMeasurement())
for (int i = 0; i < innerIterations; i++)
{
brotliStream[i] = new BrotliStream(fileStreams[i], CompressionMode.Compress, true, bufferSize, compressLevel);
brotliStream[i].Write(bytes, 0, bytes.Length);
brotliStream[i].Flush();
brotliStream[i].Dispose();
fileStreams[i].Dispose();
}
for (int i = 0; i < innerIterations; i++)
File.Delete(paths[i]);
}
}

/// <summary>
/// Benchmark tests to measure the performance of individually decompressing each file in the
/// Canterbury Corpus
/// </summary>
[Benchmark]
[MemberData(nameof(CanterburyCorpus))]
public void Decompress_Canterbury(int innerIterations, string fileName, CompressionLevel level)
{
string zipFilePath = Path.Combine("BrotliTestData", "Canterbury", "BrotliCompressed", fileName + level.ToString() + ".br");
string sourceFilePath = Path.Combine("BrotliTestData", "Canterbury", fileName);
byte[] outputRead = new byte[new FileInfo(sourceFilePath).Length];
MemoryStream[] memories = new MemoryStream[innerIterations];
foreach (var iteration in Benchmark.Iterations)
{
for (int i = 0; i < innerIterations; i++)
memories[i] = new MemoryStream(File.ReadAllBytes(zipFilePath));

using (iteration.StartMeasurement())
for (int i = 0; i < innerIterations; i++)
using (BrotliStream decompressBrotli = new BrotliStream(memories[i], CompressionMode.Decompress))
decompressBrotli.Read(outputRead, 0, outputRead.Length);

for (int i = 0; i < innerIterations; i++)
memories[i].Dispose();
}
}
}
}
Binary file added ...tem.IO.Compression.Tests/BrotliTestData/Canterbury/BrotliCompressed/alice29.txtFastest.br
View file
Open in desktop
Binary file not shown.
Binary file added ....Compression.Tests/BrotliTestData/Canterbury/BrotliCompressed/alice29.txtNoCompression.br
View file
Open in desktop
Binary file not shown.
Binary file added ...tem.IO.Compression.Tests/BrotliTestData/Canterbury/BrotliCompressed/alice29.txtOptimal.br
View file
Open in desktop
Binary file not shown.
Binary file added ...em.IO.Compression.Tests/BrotliTestData/Canterbury/BrotliCompressed/asyoulik.txtFastest.br
View file
Open in desktop
Binary file not shown.
Binary file added ...Compression.Tests/BrotliTestData/Canterbury/BrotliCompressed/asyoulik.txtNoCompression.br
View file
Open in desktop
Binary file not shown.
Binary file added ...em.IO.Compression.Tests/BrotliTestData/Canterbury/BrotliCompressed/asyoulik.txtOptimal.br
View file
Open in desktop
Binary file not shown.
Binary file added .../System.IO.Compression.Tests/BrotliTestData/Canterbury/BrotliCompressed/cp.htmlFastest.br
View file
Open in desktop
Binary file not shown.
Binary file added ...m.IO.Compression.Tests/BrotliTestData/Canterbury/BrotliCompressed/cp.htmlNoCompression.br
View file
Open in desktop
Binary file not shown.
Binary file added .../System.IO.Compression.Tests/BrotliTestData/Canterbury/BrotliCompressed/cp.htmlOptimal.br
View file
Open in desktop
Binary file not shown.
Binary file added ...System.IO.Compression.Tests/BrotliTestData/Canterbury/BrotliCompressed/fields.cFastest.br
View file
Open in desktop
Binary file not shown.
Binary file added ....IO.Compression.Tests/BrotliTestData/Canterbury/BrotliCompressed/fields.cNoCompression.br
View file
Open in desktop
Binary file not shown.
Binary file added ...System.IO.Compression.Tests/BrotliTestData/Canterbury/BrotliCompressed/fields.cOptimal.br
View file
Open in desktop
Binary file not shown.
Binary file added ...tem.IO.Compression.Tests/BrotliTestData/Canterbury/BrotliCompressed/grammar.lspFastest.br
View file
Open in desktop
Binary file not shown.
Binary file added ....Compression.Tests/BrotliTestData/Canterbury/BrotliCompressed/grammar.lspNoCompression.br
View file
Open in desktop
Binary file not shown.
Binary file added ...tem.IO.Compression.Tests/BrotliTestData/Canterbury/BrotliCompressed/grammar.lspOptimal.br
View file
Open in desktop
Binary file not shown.
Binary file added ...tem.IO.Compression.Tests/BrotliTestData/Canterbury/BrotliCompressed/kennedy.xlsFastest.br
View file
Open in desktop
Binary file not shown.
Binary file added ....Compression.Tests/BrotliTestData/Canterbury/BrotliCompressed/kennedy.xlsNoCompression.br
View file
Open in desktop
Binary file not shown.
Binary file added ...tem.IO.Compression.Tests/BrotliTestData/Canterbury/BrotliCompressed/kennedy.xlsOptimal.br
View file
Open in desktop
Binary file not shown.
Binary file added ...stem.IO.Compression.Tests/BrotliTestData/Canterbury/BrotliCompressed/lcet10.txtFastest.br
View file
Open in desktop
Binary file not shown.
Binary file added ...O.Compression.Tests/BrotliTestData/Canterbury/BrotliCompressed/lcet10.txtNoCompression.br
View file
Open in desktop
Binary file not shown.
Binary file added ...stem.IO.Compression.Tests/BrotliTestData/Canterbury/BrotliCompressed/lcet10.txtOptimal.br
View file
Open in desktop
Binary file not shown.
Binary file added ...em.IO.Compression.Tests/BrotliTestData/Canterbury/BrotliCompressed/plrabn12.txtFastest.br
View file
Open in desktop
Binary file not shown.
Binary file added ...Compression.Tests/BrotliTestData/Canterbury/BrotliCompressed/plrabn12.txtNoCompression.br
View file
Open in desktop
Binary file not shown.
Binary file added ...em.IO.Compression.Tests/BrotliTestData/Canterbury/BrotliCompressed/plrabn12.txtOptimal.br
View file
Open in desktop
Binary file not shown.
Binary file added tests/System.IO.Compression.Tests/BrotliTestData/Canterbury/BrotliCompressed/ptt5Fastest.br
View file
Open in desktop
Binary file not shown.
Binary file added ...stem.IO.Compression.Tests/BrotliTestData/Canterbury/BrotliCompressed/ptt5NoCompression.br
View file
Open in desktop
Binary file not shown.
Binary file added tests/System.IO.Compression.Tests/BrotliTestData/Canterbury/BrotliCompressed/ptt5Optimal.br
View file
Open in desktop
Binary file not shown.
Binary file added tests/System.IO.Compression.Tests/BrotliTestData/Canterbury/BrotliCompressed/sumFastest.br
View file
Open in desktop
Binary file not shown.
Binary file added ...ystem.IO.Compression.Tests/BrotliTestData/Canterbury/BrotliCompressed/sumNoCompression.br
View file
Open in desktop
Binary file not shown.
Binary file added tests/System.IO.Compression.Tests/BrotliTestData/Canterbury/BrotliCompressed/sumOptimal.br
View file
Open in desktop
Binary file not shown.
Binary file added .../System.IO.Compression.Tests/BrotliTestData/Canterbury/BrotliCompressed/xargs.1Fastest.br
View file
Open in desktop
Binary file not shown.
Binary file added ...m.IO.Compression.Tests/BrotliTestData/Canterbury/BrotliCompressed/xargs.1NoCompression.br
View file
Open in desktop
Binary file not shown.
Binary file added .../System.IO.Compression.Tests/BrotliTestData/Canterbury/BrotliCompressed/xargs.1Optimal.br
View file
Open in desktop
Binary file not shown.
Loading