Skip to content

Feat/add multiple threshold #379 #1123

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Mar 30, 2021
Merged

Feat/add multiple threshold #379 #1123

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion Documentation/MSBuildIntegration.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,12 @@ The above command will automatically fail the build if the line, branch or metho
dotnet test /p:CollectCoverage=true /p:Threshold=80 /p:ThresholdType=line
```

You can specify multiple values for `ThresholdType` by separating them with commas. Valid values include `line`, `branch` and `method`.
You can specify multiple values for `ThresholdType` by separating them with commas. Valid values include `line`, `branch` and `method`.
You can do the same for `Threshold` as well.

```bash
dotnet test /p:CollectCoverage=true /p:Threshold="80,100,70", /p:ThresholdType="line,branch,method"
```

By default, Coverlet will validate the threshold value against the coverage result of each module. The `/p:ThresholdStat` option allows you to change this behaviour and can have any of the following values:

Expand Down
55 changes: 43 additions & 12 deletions src/coverlet.console/Program.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
using System.ComponentModel;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Text;

using ConsoleTables;
Expand Down Expand Up @@ -129,13 +130,13 @@ static int Main(string[] args)
process.WaitForExit();

var dOutput = output.HasValue() ? output.Value() : Directory.GetCurrentDirectory() + Path.DirectorySeparatorChar.ToString();
var dThreshold = threshold.HasValue() ? double.Parse(threshold.Value()) : 0;
var dThresholdTypes = thresholdTypes.HasValue() ? thresholdTypes.Values : new List<string>(new string[] { "line", "branch", "method" });
var dThresholdStat = thresholdStat.HasValue() ? Enum.Parse<ThresholdStatistic>(thresholdStat.Value(), true) : Enum.Parse<ThresholdStatistic>("minimum", true);

logger.LogInformation("\nCalculating coverage result...");

var result = coverage.GetCoverageResult();

var directory = Path.GetDirectoryName(dOutput);
if (directory == string.Empty)
{
Expand Down Expand Up @@ -173,27 +174,57 @@ static int Main(string[] args)
}
}

var thresholdTypeFlags = ThresholdTypeFlags.None;

var thresholdTypeFlagQueue = new Queue<ThresholdTypeFlags>();
foreach (var thresholdType in dThresholdTypes)
{
if (thresholdType.Equals("line", StringComparison.OrdinalIgnoreCase))
{
thresholdTypeFlags |= ThresholdTypeFlags.Line;
thresholdTypeFlagQueue.Enqueue(ThresholdTypeFlags.Line);
}
else if (thresholdType.Equals("branch", StringComparison.OrdinalIgnoreCase))
{
thresholdTypeFlags |= ThresholdTypeFlags.Branch;
thresholdTypeFlagQueue.Enqueue(ThresholdTypeFlags.Branch);
}
else if (thresholdType.Equals("method", StringComparison.OrdinalIgnoreCase))
{
thresholdTypeFlags |= ThresholdTypeFlags.Method;
thresholdTypeFlagQueue.Enqueue(ThresholdTypeFlags.Method);
}
}

Dictionary<ThresholdTypeFlags, double> thresholdTypeFlagValues = new Dictionary<ThresholdTypeFlags, double>();
if (threshold.HasValue() && threshold.Value().Contains(','))
{
var thresholdValues = threshold.Value().Split(',', StringSplitOptions.RemoveEmptyEntries).Select(t => t.Trim());
if (thresholdValues.Count() != thresholdTypeFlagQueue.Count())
{
throw new Exception($"Threshold type flag count ({thresholdTypeFlagQueue.Count()}) and values count ({thresholdValues.Count()}) doesn't match");
}

foreach (var thresholdValue in thresholdValues)
{
if (double.TryParse(thresholdValue, out var value))
{
thresholdTypeFlagValues[thresholdTypeFlagQueue.Dequeue()] = value;
}
else
{
throw new Exception($"Invalid threshold value must be numeric");
}
}
}
else
{
double thresholdValue = threshold.HasValue() ? double.Parse(threshold.Value()) : 0;

while (thresholdTypeFlagQueue.Any())
{
thresholdTypeFlagValues[thresholdTypeFlagQueue.Dequeue()] = thresholdValue;
}
}

var coverageTable = new ConsoleTable("Module", "Line", "Branch", "Method");
var summary = new CoverageSummary();
int numModules = result.Modules.Count;

var linePercentCalculation = summary.CalculateLineCoverage(result.Modules);
var branchPercentCalculation = summary.CalculateBranchCoverage(result.Modules);
Expand Down Expand Up @@ -230,26 +261,26 @@ static int Main(string[] args)
{
exitCode += (int)CommandExitCodes.TestFailed;
}
thresholdTypeFlags = result.GetThresholdTypesBelowThreshold(summary, dThreshold, thresholdTypeFlags, dThresholdStat);

var thresholdTypeFlags = result.GetThresholdTypesBelowThreshold(summary, thresholdTypeFlagValues, dThresholdStat);
if (thresholdTypeFlags != ThresholdTypeFlags.None)
{
exitCode += (int)CommandExitCodes.CoverageBelowThreshold;
var exceptionMessageBuilder = new StringBuilder();
if ((thresholdTypeFlags & ThresholdTypeFlags.Line) != ThresholdTypeFlags.None)
{
exceptionMessageBuilder.AppendLine($"The {dThresholdStat.ToString().ToLower()} line coverage is below the specified {dThreshold}");
exceptionMessageBuilder.AppendLine($"The {dThresholdStat.ToString().ToLower()} line coverage is below the specified {thresholdTypeFlagValues[ThresholdTypeFlags.Line]}");
}

if ((thresholdTypeFlags & ThresholdTypeFlags.Branch) != ThresholdTypeFlags.None)
{
exceptionMessageBuilder.AppendLine($"The {dThresholdStat.ToString().ToLower()} branch coverage is below the specified {dThreshold}");
exceptionMessageBuilder.AppendLine($"The {dThresholdStat.ToString().ToLower()} branch coverage is below the specified {thresholdTypeFlagValues[ThresholdTypeFlags.Branch]}");
}

if ((thresholdTypeFlags & ThresholdTypeFlags.Method) != ThresholdTypeFlags.None)
{
exceptionMessageBuilder.AppendLine($"The {dThresholdStat.ToString().ToLower()} method coverage is below the specified {dThreshold}");
exceptionMessageBuilder.AppendLine($"The {dThresholdStat.ToString().ToLower()} method coverage is below the specified {thresholdTypeFlagValues[ThresholdTypeFlags.Method]}");
}

throw new Exception(exceptionMessageBuilder.ToString());
}

Expand Down
51 changes: 21 additions & 30 deletions src/coverlet.core/CoverageResult.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ internal void Merge(Modules modules)
}
}

public ThresholdTypeFlags GetThresholdTypesBelowThreshold(CoverageSummary summary, double threshold, ThresholdTypeFlags thresholdTypes, ThresholdStatistic thresholdStat)
public ThresholdTypeFlags GetThresholdTypesBelowThreshold(CoverageSummary summary, Dictionary<ThresholdTypeFlags, double> thresholdTypeFlagValues, ThresholdStatistic thresholdStat)
{
var thresholdTypeFlags = ThresholdTypeFlags.None;
switch (thresholdStat)
Expand All @@ -123,23 +123,20 @@ public ThresholdTypeFlags GetThresholdTypesBelowThreshold(CoverageSummary summar
double line = summary.CalculateLineCoverage(module.Value).Percent;
double branch = summary.CalculateBranchCoverage(module.Value).Percent;
double method = summary.CalculateMethodCoverage(module.Value).Percent;

if ((thresholdTypes & ThresholdTypeFlags.Line) != ThresholdTypeFlags.None)
if (thresholdTypeFlagValues.TryGetValue(ThresholdTypeFlags.Line, out var lineThresholdValue) && lineThresholdValue > line)
{
if (line < threshold)
thresholdTypeFlags |= ThresholdTypeFlags.Line;
thresholdTypeFlags |= ThresholdTypeFlags.Line;
}

if ((thresholdTypes & ThresholdTypeFlags.Branch) != ThresholdTypeFlags.None)
if (thresholdTypeFlagValues.TryGetValue(ThresholdTypeFlags.Branch, out var branchThresholdValue) && branchThresholdValue > branch)
{
if (branch < threshold)
thresholdTypeFlags |= ThresholdTypeFlags.Branch;
thresholdTypeFlags |= ThresholdTypeFlags.Branch;
}

if ((thresholdTypes & ThresholdTypeFlags.Method) != ThresholdTypeFlags.None)
if (thresholdTypeFlagValues.TryGetValue(ThresholdTypeFlags.Method, out var methodThresholdValue) && methodThresholdValue > method)
{
if (method < threshold)
thresholdTypeFlags |= ThresholdTypeFlags.Method;
thresholdTypeFlags |= ThresholdTypeFlags.Method;
}
}
}
Expand All @@ -149,23 +146,20 @@ public ThresholdTypeFlags GetThresholdTypesBelowThreshold(CoverageSummary summar
double line = summary.CalculateLineCoverage(Modules).AverageModulePercent;
double branch = summary.CalculateBranchCoverage(Modules).AverageModulePercent;
double method = summary.CalculateMethodCoverage(Modules).AverageModulePercent;

if ((thresholdTypes & ThresholdTypeFlags.Line) != ThresholdTypeFlags.None)
if (thresholdTypeFlagValues.TryGetValue(ThresholdTypeFlags.Line, out var lineThresholdValue) && lineThresholdValue > line)
{
if (line < threshold)
thresholdTypeFlags |= ThresholdTypeFlags.Line;
thresholdTypeFlags |= ThresholdTypeFlags.Line;
}

if ((thresholdTypes & ThresholdTypeFlags.Branch) != ThresholdTypeFlags.None)
if (thresholdTypeFlagValues.TryGetValue(ThresholdTypeFlags.Branch, out var branchThresholdValue) && branchThresholdValue > branch)
{
if (branch < threshold)
thresholdTypeFlags |= ThresholdTypeFlags.Branch;
thresholdTypeFlags |= ThresholdTypeFlags.Branch;
}

if ((thresholdTypes & ThresholdTypeFlags.Method) != ThresholdTypeFlags.None)
if (thresholdTypeFlagValues.TryGetValue(ThresholdTypeFlags.Method, out var methodThresholdValue) && methodThresholdValue > method)
{
if (method < threshold)
thresholdTypeFlags |= ThresholdTypeFlags.Method;
thresholdTypeFlags |= ThresholdTypeFlags.Method;
}
}
break;
Expand All @@ -175,22 +169,19 @@ public ThresholdTypeFlags GetThresholdTypesBelowThreshold(CoverageSummary summar
double branch = summary.CalculateBranchCoverage(Modules).Percent;
double method = summary.CalculateMethodCoverage(Modules).Percent;

if ((thresholdTypes & ThresholdTypeFlags.Line) != ThresholdTypeFlags.None)
if (thresholdTypeFlagValues.TryGetValue(ThresholdTypeFlags.Line, out var lineThresholdValue) && lineThresholdValue > line)
{
if (line < threshold)
thresholdTypeFlags |= ThresholdTypeFlags.Line;
thresholdTypeFlags |= ThresholdTypeFlags.Line;
}

if ((thresholdTypes & ThresholdTypeFlags.Branch) != ThresholdTypeFlags.None)
if (thresholdTypeFlagValues.TryGetValue(ThresholdTypeFlags.Branch, out var branchThresholdValue) && branchThresholdValue > branch)
{
if (branch < threshold)
thresholdTypeFlags |= ThresholdTypeFlags.Branch;
thresholdTypeFlags |= ThresholdTypeFlags.Branch;
}

if ((thresholdTypes & ThresholdTypeFlags.Method) != ThresholdTypeFlags.None)
if (thresholdTypeFlagValues.TryGetValue(ThresholdTypeFlags.Method, out var methodThresholdValue) && methodThresholdValue > method)
{
if (method < threshold)
thresholdTypeFlags |= ThresholdTypeFlags.Method;
thresholdTypeFlags |= ThresholdTypeFlags.Method;
}
}
break;
Expand Down
55 changes: 44 additions & 11 deletions src/coverlet.msbuild.tasks/CoverageResultTask.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ public class CoverageResultTask : BaseTask
public string OutputFormat { get; set; }

[Required]
public double Threshold { get; set; }
public string Threshold { get; set; }

[Required]
public string ThresholdType { get; set; }
Expand Down Expand Up @@ -124,25 +124,56 @@ public override bool Execute()

ReportItems = coverageReportPaths.ToArray();

var thresholdTypeFlags = ThresholdTypeFlags.None;
var thresholdStat = ThresholdStatistic.Minimum;
var thresholdTypeFlagQueue = new Queue<ThresholdTypeFlags>();

foreach (var thresholdType in ThresholdType.Split(',').Select(t => t.Trim()))
{
if (thresholdType.Equals("line", StringComparison.OrdinalIgnoreCase))
{
thresholdTypeFlags |= ThresholdTypeFlags.Line;
thresholdTypeFlagQueue.Enqueue(ThresholdTypeFlags.Line);
}
else if (thresholdType.Equals("branch", StringComparison.OrdinalIgnoreCase))
{
thresholdTypeFlags |= ThresholdTypeFlags.Branch;
thresholdTypeFlagQueue.Enqueue(ThresholdTypeFlags.Branch);
}
else if (thresholdType.Equals("method", StringComparison.OrdinalIgnoreCase))
{
thresholdTypeFlags |= ThresholdTypeFlags.Method;
thresholdTypeFlagQueue.Enqueue(ThresholdTypeFlags.Method);
}
}

Dictionary<ThresholdTypeFlags, double> thresholdTypeFlagValues = new Dictionary<ThresholdTypeFlags, double>();
if (Threshold.Contains(','))
{
var thresholdValues = Threshold.Split(new char[] {','}, StringSplitOptions.RemoveEmptyEntries).Select(t => t.Trim());
if(thresholdValues.Count() != thresholdTypeFlagQueue.Count())
{
throw new Exception($"Threshold type flag count ({thresholdTypeFlagQueue.Count()}) and values count ({thresholdValues.Count()}) doesn't match");
}

foreach (var threshold in thresholdValues)
{
if (double.TryParse(threshold, out var value))
{
thresholdTypeFlagValues[thresholdTypeFlagQueue.Dequeue()] = value;
}
else
{
throw new Exception($"Invalid threshold value must be numeric");
}
}
}
else
{
double thresholdValue = double.Parse(Threshold);

while (thresholdTypeFlagQueue.Any())
{
thresholdTypeFlagValues[thresholdTypeFlagQueue.Dequeue()] = thresholdValue;
}
}

var thresholdStat = ThresholdStatistic.Minimum;
if (ThresholdStat.Equals("average", StringComparison.OrdinalIgnoreCase))
{
thresholdStat = ThresholdStatistic.Average;
Expand All @@ -154,7 +185,6 @@ public override bool Execute()

var coverageTable = new ConsoleTable("Module", "Line", "Branch", "Method");
var summary = new CoverageSummary();
int numModules = result.Modules.Count;

var linePercentCalculation = summary.CalculateLineCoverage(result.Modules);
var branchPercentCalculation = summary.CalculateBranchCoverage(result.Modules);
Expand Down Expand Up @@ -189,23 +219,26 @@ public override bool Execute()

Console.WriteLine(coverageTable.ToStringAlternative());

thresholdTypeFlags = result.GetThresholdTypesBelowThreshold(summary, Threshold, thresholdTypeFlags, thresholdStat);
var thresholdTypeFlags = result.GetThresholdTypesBelowThreshold(summary, thresholdTypeFlagValues, thresholdStat);
if (thresholdTypeFlags != ThresholdTypeFlags.None)
{
var exceptionMessageBuilder = new StringBuilder();
if ((thresholdTypeFlags & ThresholdTypeFlags.Line) != ThresholdTypeFlags.None)
{
exceptionMessageBuilder.AppendLine($"The {thresholdStat.ToString().ToLower()} line coverage is below the specified {Threshold}");
exceptionMessageBuilder.AppendLine(
$"The {thresholdStat.ToString().ToLower()} line coverage is below the specified {thresholdTypeFlagValues[ThresholdTypeFlags.Line]}");
}

if ((thresholdTypeFlags & ThresholdTypeFlags.Branch) != ThresholdTypeFlags.None)
{
exceptionMessageBuilder.AppendLine($"The {thresholdStat.ToString().ToLower()} branch coverage is below the specified {Threshold}");
exceptionMessageBuilder.AppendLine(
$"The {thresholdStat.ToString().ToLower()} branch coverage is below the specified {thresholdTypeFlagValues[ThresholdTypeFlags.Branch]}");
}

if ((thresholdTypeFlags & ThresholdTypeFlags.Method) != ThresholdTypeFlags.None)
{
exceptionMessageBuilder.AppendLine($"The {thresholdStat.ToString().ToLower()} method coverage is below the specified {Threshold}");
exceptionMessageBuilder.AppendLine(
$"The {thresholdStat.ToString().ToLower()} method coverage is below the specified {thresholdTypeFlagValues[ThresholdTypeFlags.Method]}");
}

throw new Exception(exceptionMessageBuilder.ToString());
Expand Down
Loading