From aaad18b812206dc3e59ec21c55ac9357b46900b3 Mon Sep 17 00:00:00 2001 From: Jake Radzikowski Date: Wed, 29 Jun 2022 17:07:54 -0700 Subject: [PATCH 01/12] Initial check-in of AutoML Interactive Extension --- Microsoft.ML.sln | 13 ++- eng/Versions.props | 5 +- .../KernelExtension.cs | 75 ++++++++++++++ .../Microsoft.ML.AutoML.Interactive.csproj | 20 ++++ .../NotebookMonitor.cs | 97 +++++++++++++++++++ .../Microsoft.ML.AutoML.csproj | 21 ++++ 6 files changed, 228 insertions(+), 3 deletions(-) create mode 100644 src/Microsoft.ML.AutoML.Interactive/KernelExtension.cs create mode 100644 src/Microsoft.ML.AutoML.Interactive/Microsoft.ML.AutoML.Interactive.csproj create mode 100644 src/Microsoft.ML.AutoML.Interactive/NotebookMonitor.cs diff --git a/Microsoft.ML.sln b/Microsoft.ML.sln index 8b6a05ffda..5b90ca306b 100644 --- a/Microsoft.ML.sln +++ b/Microsoft.ML.sln @@ -121,6 +121,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.AutoML", "src\ EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.AutoML.Samples", "docs\samples\Microsoft.ML.AutoML.Samples\Microsoft.ML.AutoML.Samples.csproj", "{A6924919-9E37-4023-8B7F-E85C8E3CC9B3}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.AutoML.Interactive", "src\Microsoft.ML.AutoML.Interactive\Microsoft.ML.AutoML.Interactive.csproj", "{3B00090A-B5E4-4570-BCD0-B4CD5D499394}" +EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.Samples.GPU", "docs\samples\Microsoft.ML.Samples.GPU\Microsoft.ML.Samples.GPU.csproj", "{3C8F910B-7F23-4D25-B521-6D5AC9570ADD}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.Featurizers", "src\Microsoft.ML.Featurizers\Microsoft.ML.Featurizers.csproj", "{E2DD0721-5B0F-4606-8182-4C7EFB834518}" @@ -155,7 +157,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.SearchSpace.Te EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.AutoML.SourceGenerator", "tools-local\Microsoft.ML.AutoML.SourceGenerator\Microsoft.ML.AutoML.SourceGenerator.csproj", "{C804B990-390E-41D7-8FF1-6774495D70E2}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.ML.TorchSharp", "src\Microsoft.ML.TorchSharp\Microsoft.ML.TorchSharp.csproj", "{FF0BD187-4451-4A3B-934B-2AE3454896E2}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.TorchSharp", "src\Microsoft.ML.TorchSharp\Microsoft.ML.TorchSharp.csproj", "{FF0BD187-4451-4A3B-934B-2AE3454896E2}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -597,6 +599,14 @@ Global {A6924919-9E37-4023-8B7F-E85C8E3CC9B3}.Release|Any CPU.Build.0 = Release|Any CPU {A6924919-9E37-4023-8B7F-E85C8E3CC9B3}.Release|x64.ActiveCfg = Release|Any CPU {A6924919-9E37-4023-8B7F-E85C8E3CC9B3}.Release|x64.Build.0 = Release|Any CPU + {3B00090A-B5E4-4570-BCD0-B4CD5D499394}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {3B00090A-B5E4-4570-BCD0-B4CD5D499394}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3B00090A-B5E4-4570-BCD0-B4CD5D499394}.Debug|x64.ActiveCfg = Debug|Any CPU + {3B00090A-B5E4-4570-BCD0-B4CD5D499394}.Debug|x64.Build.0 = Debug|Any CPU + {3B00090A-B5E4-4570-BCD0-B4CD5D499394}.Release|Any CPU.ActiveCfg = Release|Any CPU + {3B00090A-B5E4-4570-BCD0-B4CD5D499394}.Release|Any CPU.Build.0 = Release|Any CPU + {3B00090A-B5E4-4570-BCD0-B4CD5D499394}.Release|x64.ActiveCfg = Release|Any CPU + {3B00090A-B5E4-4570-BCD0-B4CD5D499394}.Release|x64.Build.0 = Release|Any CPU {3C8F910B-7F23-4D25-B521-6D5AC9570ADD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {3C8F910B-7F23-4D25-B521-6D5AC9570ADD}.Debug|Any CPU.Build.0 = Debug|Any CPU {3C8F910B-7F23-4D25-B521-6D5AC9570ADD}.Debug|x64.ActiveCfg = Debug|Any CPU @@ -796,6 +806,7 @@ Global {C2652287-CD6D-40FB-B042-95FB56D09DB8} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4} {E48285BF-F49A-4EA3-AED0-1BDDBF77EB80} = {09EADF06-BE25-4228-AB53-95AE3E15B530} {A6924919-9E37-4023-8B7F-E85C8E3CC9B3} = {DA452A53-2E94-4433-B08C-041EDEC729E6} + {3B00090A-B5E4-4570-BCD0-B4CD5D499394} = {09EADF06-BE25-4228-AB53-95AE3E15B530} {3C8F910B-7F23-4D25-B521-6D5AC9570ADD} = {DA452A53-2E94-4433-B08C-041EDEC729E6} {E2DD0721-5B0F-4606-8182-4C7EFB834518} = {09EADF06-BE25-4228-AB53-95AE3E15B530} {56CB0850-7341-4D71-9AE4-9EFC472D93DD} = {09EADF06-BE25-4228-AB53-95AE3E15B530} diff --git a/eng/Versions.props b/eng/Versions.props index da4fc00545..5178e7ece8 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -35,8 +35,8 @@ 2.3.1 3.3.0 3.9.0 - 1.0.0-beta.22103.1 - 1.0.0-beta.22103.1 + 1.0.0-beta.22314.1 + 1.0.0-beta.22314.1 0.4.1 1.10.0 0.0.0.12 @@ -44,6 +44,7 @@ 2.1.0 10.0.3 2.1.3 + 0.0.1 1.3.3 0.20.1 2 diff --git a/src/Microsoft.ML.AutoML.Interactive/KernelExtension.cs b/src/Microsoft.ML.AutoML.Interactive/KernelExtension.cs new file mode 100644 index 0000000000..fb7e76c6ae --- /dev/null +++ b/src/Microsoft.ML.AutoML.Interactive/KernelExtension.cs @@ -0,0 +1,75 @@ +using Microsoft.AspNetCore.Html; +using Microsoft.Data.Analysis; +using Microsoft.DotNet.Interactive; +using Microsoft.DotNet.Interactive.Formatting; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.Json; +using System.Threading.Tasks; +using Plotly.NET.CSharp; +using static Microsoft.DotNet.Interactive.Formatting.PocketViewTags; + + +namespace Microsoft.ML.AutoML +{ + public class KernelExtension : IKernelExtension + { +#pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously + public Task OnLoadAsync(Kernel kernel) +#pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously + { + Formatter.Register((monitor, writer) => + { + WriteSummary(monitor, writer); + WriteChart(monitor, writer); + WriteTable(monitor, writer); + }, "text/html"); + + return Task.CompletedTask; + } + + private static void WriteSummary(NotebookMonitor monitor, TextWriter writer) + { + + var summary = new List(); + + if (monitor.BestTrial != null) + { + summary.Add(h3("Best Run")); + summary.Add(p($"Trial: {monitor.BestTrial.TrialSettings.TrialId}")); + summary.Add(p($"Trainer: {monitor.BestTrial.TrialSettings.Pipeline}".Replace("Unknown=>", ""))); + } + if (monitor.ActiveTrial != null) + { + + var activeRunParam = JsonSerializer.Serialize(monitor.ActiveTrial.Parameter, new JsonSerializerOptions() { WriteIndented = true, }); + + summary.Add(h3("Active Run")); + summary.Add(p($"Trial: {monitor.ActiveTrial.TrialId}")); + summary.Add(p($"Trainer: {monitor.ActiveTrial.Pipeline}".Replace("Unknown=>", ""))); + summary.Add(p($"Parameters: {activeRunParam}")); + } + + writer.Write(div(summary)); + } + + private static void WriteChart(NotebookMonitor monitor, TextWriter writer) + { + var x = monitor.CompletedTrials.Select(x => x.TrialSettings.TrialId); + var y = monitor.CompletedTrials.Select(x => x.Metric); + + var chart = Chart.Point(x, y, "Hello") + .WithTraceInfo(ShowLegend: false) + .WithXAxisStyle(TitleText: "Trial", ShowGrid: false) + .WithYAxisStyle(TitleText: "Metric", ShowGrid: false); + + Formatter.GetPreferredFormatterFor(typeof(Plotly.NET.GenericChart.GenericChart), "text/html").Format(chart, writer); + } + + private static void WriteTable(NotebookMonitor notebookMonitor, TextWriter writer) + { + Formatter.GetPreferredFormatterFor(typeof(DataFrame), "text/html").Format(notebookMonitor.DataFrame, writer); + } + } +} \ No newline at end of file diff --git a/src/Microsoft.ML.AutoML.Interactive/Microsoft.ML.AutoML.Interactive.csproj b/src/Microsoft.ML.AutoML.Interactive/Microsoft.ML.AutoML.Interactive.csproj new file mode 100644 index 0000000000..15040401ee --- /dev/null +++ b/src/Microsoft.ML.AutoML.Interactive/Microsoft.ML.AutoML.Interactive.csproj @@ -0,0 +1,20 @@ + + + + net6.0 + false + + + + + + + + + + + + + + + diff --git a/src/Microsoft.ML.AutoML.Interactive/NotebookMonitor.cs b/src/Microsoft.ML.AutoML.Interactive/NotebookMonitor.cs new file mode 100644 index 0000000000..aa683df453 --- /dev/null +++ b/src/Microsoft.ML.AutoML.Interactive/NotebookMonitor.cs @@ -0,0 +1,97 @@ +using Microsoft.DotNet.Interactive; +using System.Collections.Generic; +using Microsoft.Data.Analysis; +using System; +using System.Threading.Tasks; +using System.Text.Json; + + +namespace Microsoft.ML.AutoML +{ + public class NotebookMonitor : IMonitor + { + private DisplayedValue? ValueToUpdate; + private DateTime _lastUpdate = DateTime.MinValue; + + public TrialResult? BestTrial { get; set; } + public TrialResult? MostRecentTrial { get; set; } + public TrialSettings? ActiveTrial { get; set; } + public List CompletedTrials { get; set; } + public DataFrame DataFrame { get; set; } + + public NotebookMonitor() + { + this.CompletedTrials = new List(); + this.DataFrame = new DataFrame(new PrimitiveDataFrameColumn("Trial"), new PrimitiveDataFrameColumn("Metric"), new StringDataFrameColumn("Trainer"), new StringDataFrameColumn("Parameters")); + } + + public void ReportBestTrial(TrialResult result) + { + this.BestTrial = result; + Update(); + } + + public void ReportCompletedTrial(TrialResult result) + { + this.MostRecentTrial = result; + this.CompletedTrials.Add(result); + + var activeRunParam = JsonSerializer.Serialize(result.TrialSettings.Parameter, new JsonSerializerOptions() { WriteIndented = false, }); + + this.DataFrame.Append(new List>() + { + new KeyValuePair("Trial",result.TrialSettings.TrialId), + new KeyValuePair("Metric", result.Metric), + new KeyValuePair("Trainer",result.TrialSettings.Pipeline.ToString().Replace("Unknown=>","")), + new KeyValuePair("Parameters",activeRunParam), + }, true); + Update(); + } + + public void ReportFailTrial(TrialResult result) + { + // TODO figure out what to do with failed trials. + Update(); + } + + public void ReportRunningTrial(TrialSettings setting) + { + this.ActiveTrial = setting; + Update(); + } + + private bool updatePending = false; + public void Update() + { + Task.Run(async () => + { + if (updatePending == true) + { + // Keep waiting + } + else + { + int timeRemaining = 5000 - (int)(DateTime.Now.Millisecond - this._lastUpdate.Millisecond); + updatePending = true; + if (timeRemaining > 0) + { + await Task.Delay(timeRemaining); + } + if (this.ValueToUpdate != null) + { + updatePending = false; + this._lastUpdate = DateTime.Now; + this.ValueToUpdate.Update(this); + } + } + }); + + } + + public void SetUpdate(DisplayedValue valueToUpdate) + { + this.ValueToUpdate = valueToUpdate; + Update(); + } + } +} diff --git a/src/Microsoft.ML.AutoML/Microsoft.ML.AutoML.csproj b/src/Microsoft.ML.AutoML/Microsoft.ML.AutoML.csproj index 36a14ec5ce..ac73e557dc 100644 --- a/src/Microsoft.ML.AutoML/Microsoft.ML.AutoML.csproj +++ b/src/Microsoft.ML.AutoML/Microsoft.ML.AutoML.csproj @@ -6,8 +6,29 @@ Microsoft.ML.AutoML ML.NET AutoML: Optimizes an ML pipeline for your dataset, by automatically locating the best feature engineering, model, and hyperparameters $(TargetsForTfmSpecificBuildOutput);CopyProjectReferencesToPackage + + + $(NoWarn);1591;NU5100;MSML_GeneralName;MSML_ParameterLocalVarName;MSML_PrivateFieldName;MSML_TypeParamName;SA1028;SA1507;SX1101;MSML_NoInstanceInitializers + $(TargetsForTfmSpecificContentInPackage);AddAutoMLInteractiveToInteractiveExtensionsFolder + + + + + + + + + + <_ItemsToIncludeForInteractive Update="@(_ItemsToIncludeForInteractive)" PackagePath="interactive-extensions/dotnet" /> + + + + From ce5f67d5f2fc27466f2203422ebe066db1bec4d3 Mon Sep 17 00:00:00 2001 From: Jake Radzikowski Date: Wed, 29 Jun 2022 17:10:59 -0700 Subject: [PATCH 02/12] Remove waring disable code --- src/Microsoft.ML.AutoML.Interactive/KernelExtension.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Microsoft.ML.AutoML.Interactive/KernelExtension.cs b/src/Microsoft.ML.AutoML.Interactive/KernelExtension.cs index fb7e76c6ae..3812f7d64b 100644 --- a/src/Microsoft.ML.AutoML.Interactive/KernelExtension.cs +++ b/src/Microsoft.ML.AutoML.Interactive/KernelExtension.cs @@ -15,9 +15,7 @@ namespace Microsoft.ML.AutoML { public class KernelExtension : IKernelExtension { -#pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously public Task OnLoadAsync(Kernel kernel) -#pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously { Formatter.Register((monitor, writer) => { From 77cc34d47304d5363f32e12930e07db51078fe45 Mon Sep 17 00:00:00 2001 From: Jake Radzikowski Date: Wed, 29 Jun 2022 23:42:00 -0700 Subject: [PATCH 03/12] clean up warnings --- ...ion.cs => AutoMLMonitorKernelExtension.cs} | 24 ++++++++++++---- .../NotebookMonitor.cs | 28 +++++++++++-------- 2 files changed, 35 insertions(+), 17 deletions(-) rename src/Microsoft.ML.AutoML.Interactive/{KernelExtension.cs => AutoMLMonitorKernelExtension.cs} (75%) diff --git a/src/Microsoft.ML.AutoML.Interactive/KernelExtension.cs b/src/Microsoft.ML.AutoML.Interactive/AutoMLMonitorKernelExtension.cs similarity index 75% rename from src/Microsoft.ML.AutoML.Interactive/KernelExtension.cs rename to src/Microsoft.ML.AutoML.Interactive/AutoMLMonitorKernelExtension.cs index 3812f7d64b..a7897ac7f6 100644 --- a/src/Microsoft.ML.AutoML.Interactive/KernelExtension.cs +++ b/src/Microsoft.ML.AutoML.Interactive/AutoMLMonitorKernelExtension.cs @@ -1,6 +1,11 @@ -using Microsoft.AspNetCore.Html; +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.AspNetCore.Html; using Microsoft.Data.Analysis; using Microsoft.DotNet.Interactive; +using Microsoft.DotNet.Interactive.Commands; using Microsoft.DotNet.Interactive.Formatting; using System.Collections.Generic; using System.IO; @@ -13,9 +18,9 @@ namespace Microsoft.ML.AutoML { - public class KernelExtension : IKernelExtension + public class AutoMLMonitorKernelExtension : IKernelExtension { - public Task OnLoadAsync(Kernel kernel) + public async Task OnLoadAsync(Kernel kernel) { Formatter.Register((monitor, writer) => { @@ -24,7 +29,16 @@ public Task OnLoadAsync(Kernel kernel) WriteTable(monitor, writer); }, "text/html"); - return Task.CompletedTask; + if (Kernel.Root?.FindKernel("csharp") is { } csKernel) + { + await LoadExtensionApiAsync(csKernel); + } + } + + private static async Task LoadExtensionApiAsync(Kernel cSharpKernel) + { + await cSharpKernel.SendAsync(new SubmitCode($@"#r ""{typeof(AutoMLMonitorKernelExtension).Assembly.Location}"" +using {typeof(NotebookMonitor).Namespace};")); } private static void WriteSummary(NotebookMonitor monitor, TextWriter writer) @@ -70,4 +84,4 @@ private static void WriteTable(NotebookMonitor notebookMonitor, TextWriter write Formatter.GetPreferredFormatterFor(typeof(DataFrame), "text/html").Format(notebookMonitor.DataFrame, writer); } } -} \ No newline at end of file +} diff --git a/src/Microsoft.ML.AutoML.Interactive/NotebookMonitor.cs b/src/Microsoft.ML.AutoML.Interactive/NotebookMonitor.cs index aa683df453..d2468a2ebb 100644 --- a/src/Microsoft.ML.AutoML.Interactive/NotebookMonitor.cs +++ b/src/Microsoft.ML.AutoML.Interactive/NotebookMonitor.cs @@ -1,4 +1,8 @@ -using Microsoft.DotNet.Interactive; +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.DotNet.Interactive; using System.Collections.Generic; using Microsoft.Data.Analysis; using System; @@ -10,12 +14,12 @@ namespace Microsoft.ML.AutoML { public class NotebookMonitor : IMonitor { - private DisplayedValue? ValueToUpdate; + private DisplayedValue _valueToUpdate; private DateTime _lastUpdate = DateTime.MinValue; - public TrialResult? BestTrial { get; set; } - public TrialResult? MostRecentTrial { get; set; } - public TrialSettings? ActiveTrial { get; set; } + public TrialResult BestTrial { get; set; } + public TrialResult MostRecentTrial { get; set; } + public TrialSettings ActiveTrial { get; set; } public List CompletedTrials { get; set; } public DataFrame DataFrame { get; set; } @@ -60,28 +64,28 @@ public void ReportRunningTrial(TrialSettings setting) Update(); } - private bool updatePending = false; + private bool _updatePending = false; public void Update() { Task.Run(async () => { - if (updatePending == true) + if (_updatePending == true) { // Keep waiting } else { int timeRemaining = 5000 - (int)(DateTime.Now.Millisecond - this._lastUpdate.Millisecond); - updatePending = true; + _updatePending = true; if (timeRemaining > 0) { await Task.Delay(timeRemaining); } - if (this.ValueToUpdate != null) + if (this._valueToUpdate != null) { - updatePending = false; + _updatePending = false; this._lastUpdate = DateTime.Now; - this.ValueToUpdate.Update(this); + this._valueToUpdate.Update(this); } } }); @@ -90,7 +94,7 @@ public void Update() public void SetUpdate(DisplayedValue valueToUpdate) { - this.ValueToUpdate = valueToUpdate; + this._valueToUpdate = valueToUpdate; Update(); } } From e2278c4eae33ff68f2542a91e20f70e77463757b Mon Sep 17 00:00:00 2001 From: Jake Radzikowski Date: Thu, 30 Jun 2022 16:56:23 -0700 Subject: [PATCH 04/12] Add chart header and work around Plotly.NET issue. --- .../AutoMLMonitorKernelExtension.cs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/Microsoft.ML.AutoML.Interactive/AutoMLMonitorKernelExtension.cs b/src/Microsoft.ML.AutoML.Interactive/AutoMLMonitorKernelExtension.cs index a7897ac7f6..0123ee23e4 100644 --- a/src/Microsoft.ML.AutoML.Interactive/AutoMLMonitorKernelExtension.cs +++ b/src/Microsoft.ML.AutoML.Interactive/AutoMLMonitorKernelExtension.cs @@ -76,7 +76,18 @@ private static void WriteChart(NotebookMonitor monitor, TextWriter writer) .WithXAxisStyle(TitleText: "Trial", ShowGrid: false) .WithYAxisStyle(TitleText: "Metric", ShowGrid: false); + var chartHeader = new List(); + chartHeader.Add(h1("Plot metrics over trials.")); + writer.Write(div(chartHeader)); + + Formatter.GetPreferredFormatterFor(typeof(Plotly.NET.GenericChart.GenericChart), "text/html").Format(chart, writer); + + // Works around issue with earlier versions of Plotly.NET - https://github.com/plotly/Plotly.NET/pull/305 + if (writer.ToString().EndsWith(""); + } } private static void WriteTable(NotebookMonitor notebookMonitor, TextWriter writer) From b3884cc05c8b1e4a5db4335d04521b4a31d610ac Mon Sep 17 00:00:00 2001 From: Jake Radzikowski Date: Thu, 30 Jun 2022 16:58:26 -0700 Subject: [PATCH 05/12] Fix chart title. --- .../AutoMLMonitorKernelExtension.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.ML.AutoML.Interactive/AutoMLMonitorKernelExtension.cs b/src/Microsoft.ML.AutoML.Interactive/AutoMLMonitorKernelExtension.cs index 0123ee23e4..bcf745256f 100644 --- a/src/Microsoft.ML.AutoML.Interactive/AutoMLMonitorKernelExtension.cs +++ b/src/Microsoft.ML.AutoML.Interactive/AutoMLMonitorKernelExtension.cs @@ -71,13 +71,13 @@ private static void WriteChart(NotebookMonitor monitor, TextWriter writer) var x = monitor.CompletedTrials.Select(x => x.TrialSettings.TrialId); var y = monitor.CompletedTrials.Select(x => x.Metric); - var chart = Chart.Point(x, y, "Hello") + var chart = Chart.Point(x, y, "Plot Metrics over Trials.") .WithTraceInfo(ShowLegend: false) .WithXAxisStyle(TitleText: "Trial", ShowGrid: false) .WithYAxisStyle(TitleText: "Metric", ShowGrid: false); var chartHeader = new List(); - chartHeader.Add(h1("Plot metrics over trials.")); + chartHeader.Add(h1("Plot Metrics over Trials.")); writer.Write(div(chartHeader)); From 4b65d8b62530188515f955f12f81796875e06509 Mon Sep 17 00:00:00 2001 From: Jake Radzikowski Date: Thu, 30 Jun 2022 17:11:48 -0700 Subject: [PATCH 06/12] Improve styling and add header to Table. --- .../AutoMLMonitorKernelExtension.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.ML.AutoML.Interactive/AutoMLMonitorKernelExtension.cs b/src/Microsoft.ML.AutoML.Interactive/AutoMLMonitorKernelExtension.cs index bcf745256f..91a49e3b92 100644 --- a/src/Microsoft.ML.AutoML.Interactive/AutoMLMonitorKernelExtension.cs +++ b/src/Microsoft.ML.AutoML.Interactive/AutoMLMonitorKernelExtension.cs @@ -77,7 +77,7 @@ private static void WriteChart(NotebookMonitor monitor, TextWriter writer) .WithYAxisStyle(TitleText: "Metric", ShowGrid: false); var chartHeader = new List(); - chartHeader.Add(h1("Plot Metrics over Trials.")); + chartHeader.Add(h3("Plot Metrics over Trials")); writer.Write(div(chartHeader)); @@ -92,6 +92,9 @@ private static void WriteChart(NotebookMonitor monitor, TextWriter writer) private static void WriteTable(NotebookMonitor notebookMonitor, TextWriter writer) { + var tableHeader = new List(); + tableHeader.Add(h3("All Trials Table")); + writer.Write(div(tableHeader)); Formatter.GetPreferredFormatterFor(typeof(DataFrame), "text/html").Format(notebookMonitor.DataFrame, writer); } } From ee268eee15d68a5afc176c7cb29b4af3c0f2cce7 Mon Sep 17 00:00:00 2001 From: Jake Radzikowski Date: Tue, 5 Jul 2022 13:47:20 -0700 Subject: [PATCH 07/12] Update comments and remove some items from NoWarn. --- src/Microsoft.ML.AutoML/Microsoft.ML.AutoML.csproj | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Microsoft.ML.AutoML/Microsoft.ML.AutoML.csproj b/src/Microsoft.ML.AutoML/Microsoft.ML.AutoML.csproj index ac73e557dc..78e4b7e857 100644 --- a/src/Microsoft.ML.AutoML/Microsoft.ML.AutoML.csproj +++ b/src/Microsoft.ML.AutoML/Microsoft.ML.AutoML.csproj @@ -9,17 +9,17 @@ - $(NoWarn);1591;NU5100;MSML_GeneralName;MSML_ParameterLocalVarName;MSML_PrivateFieldName;MSML_TypeParamName;SA1028;SA1507;SX1101;MSML_NoInstanceInitializers + $(NoWarn);1591;NU5100; $(TargetsForTfmSpecificContentInPackage);AddAutoMLInteractiveToInteractiveExtensionsFolder - + - + From 5f97544321ae7dba4dbd80092a38f569137d25e1 Mon Sep 17 00:00:00 2001 From: Jake Radzikowski Date: Tue, 5 Jul 2022 13:57:02 -0700 Subject: [PATCH 08/12] Remove trailing semicolon --- src/Microsoft.ML.AutoML/Microsoft.ML.AutoML.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.ML.AutoML/Microsoft.ML.AutoML.csproj b/src/Microsoft.ML.AutoML/Microsoft.ML.AutoML.csproj index 78e4b7e857..2a223df06f 100644 --- a/src/Microsoft.ML.AutoML/Microsoft.ML.AutoML.csproj +++ b/src/Microsoft.ML.AutoML/Microsoft.ML.AutoML.csproj @@ -11,7 +11,7 @@ 1591: Documentation warnings NU5100: Warning that gets triggered because a .dll is not placed under lib folder on package. This is by design as we want AutoML Interactive to be under interactive-extensions folder. --> - $(NoWarn);1591;NU5100; + $(NoWarn);1591;NU5100 $(TargetsForTfmSpecificContentInPackage);AddAutoMLInteractiveToInteractiveExtensionsFolder From 11228f37157cda51407d2f8d3999f7bcbe670983 Mon Sep 17 00:00:00 2001 From: Jake Radzikowski Date: Tue, 5 Jul 2022 14:03:22 -0700 Subject: [PATCH 09/12] Resolve PR feedback --- .../AutoMLMonitorKernelExtension.cs | 2 +- .../NotebookMonitor.cs | 26 +++++++++---------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/Microsoft.ML.AutoML.Interactive/AutoMLMonitorKernelExtension.cs b/src/Microsoft.ML.AutoML.Interactive/AutoMLMonitorKernelExtension.cs index 91a49e3b92..69b685e22d 100644 --- a/src/Microsoft.ML.AutoML.Interactive/AutoMLMonitorKernelExtension.cs +++ b/src/Microsoft.ML.AutoML.Interactive/AutoMLMonitorKernelExtension.cs @@ -95,7 +95,7 @@ private static void WriteTable(NotebookMonitor notebookMonitor, TextWriter write var tableHeader = new List(); tableHeader.Add(h3("All Trials Table")); writer.Write(div(tableHeader)); - Formatter.GetPreferredFormatterFor(typeof(DataFrame), "text/html").Format(notebookMonitor.DataFrame, writer); + Formatter.GetPreferredFormatterFor(typeof(DataFrame), "text/html").Format(notebookMonitor.TrialData, writer); } } } diff --git a/src/Microsoft.ML.AutoML.Interactive/NotebookMonitor.cs b/src/Microsoft.ML.AutoML.Interactive/NotebookMonitor.cs index d2468a2ebb..e6c9b228e9 100644 --- a/src/Microsoft.ML.AutoML.Interactive/NotebookMonitor.cs +++ b/src/Microsoft.ML.AutoML.Interactive/NotebookMonitor.cs @@ -21,28 +21,28 @@ public class NotebookMonitor : IMonitor public TrialResult MostRecentTrial { get; set; } public TrialSettings ActiveTrial { get; set; } public List CompletedTrials { get; set; } - public DataFrame DataFrame { get; set; } + public DataFrame TrialData { get; set; } public NotebookMonitor() { - this.CompletedTrials = new List(); - this.DataFrame = new DataFrame(new PrimitiveDataFrameColumn("Trial"), new PrimitiveDataFrameColumn("Metric"), new StringDataFrameColumn("Trainer"), new StringDataFrameColumn("Parameters")); + CompletedTrials = new List(); + TrialData = new DataFrame(new PrimitiveDataFrameColumn("Trial"), new PrimitiveDataFrameColumn("Metric"), new StringDataFrameColumn("Trainer"), new StringDataFrameColumn("Parameters")); } public void ReportBestTrial(TrialResult result) { - this.BestTrial = result; + BestTrial = result; Update(); } public void ReportCompletedTrial(TrialResult result) { - this.MostRecentTrial = result; - this.CompletedTrials.Add(result); + MostRecentTrial = result; + CompletedTrials.Add(result); var activeRunParam = JsonSerializer.Serialize(result.TrialSettings.Parameter, new JsonSerializerOptions() { WriteIndented = false, }); - this.DataFrame.Append(new List>() + TrialData.Append(new List>() { new KeyValuePair("Trial",result.TrialSettings.TrialId), new KeyValuePair("Metric", result.Metric), @@ -60,7 +60,7 @@ public void ReportFailTrial(TrialResult result) public void ReportRunningTrial(TrialSettings setting) { - this.ActiveTrial = setting; + ActiveTrial = setting; Update(); } @@ -75,17 +75,17 @@ public void Update() } else { - int timeRemaining = 5000 - (int)(DateTime.Now.Millisecond - this._lastUpdate.Millisecond); + int timeRemaining = 5000 - (int)(DateTime.UtcNow.Millisecond - _lastUpdate.Millisecond); _updatePending = true; if (timeRemaining > 0) { await Task.Delay(timeRemaining); } - if (this._valueToUpdate != null) + if (_valueToUpdate != null) { _updatePending = false; - this._lastUpdate = DateTime.Now; - this._valueToUpdate.Update(this); + _lastUpdate = DateTime.UtcNow; + _valueToUpdate.Update(this); } } }); @@ -94,7 +94,7 @@ public void Update() public void SetUpdate(DisplayedValue valueToUpdate) { - this._valueToUpdate = valueToUpdate; + _valueToUpdate = valueToUpdate; Update(); } } From 4348d97d640be648137472360732e995b19f5cf7 Mon Sep 17 00:00:00 2001 From: Jake Radzikowski Date: Tue, 5 Jul 2022 14:41:52 -0700 Subject: [PATCH 10/12] Move _updatePending = false outside of if check. --- src/Microsoft.ML.AutoML.Interactive/NotebookMonitor.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.ML.AutoML.Interactive/NotebookMonitor.cs b/src/Microsoft.ML.AutoML.Interactive/NotebookMonitor.cs index e6c9b228e9..9442767aec 100644 --- a/src/Microsoft.ML.AutoML.Interactive/NotebookMonitor.cs +++ b/src/Microsoft.ML.AutoML.Interactive/NotebookMonitor.cs @@ -77,13 +77,15 @@ public void Update() { int timeRemaining = 5000 - (int)(DateTime.UtcNow.Millisecond - _lastUpdate.Millisecond); _updatePending = true; + if (timeRemaining > 0) { await Task.Delay(timeRemaining); } + + _updatePending = false; if (_valueToUpdate != null) { - _updatePending = false; _lastUpdate = DateTime.UtcNow; _valueToUpdate.Update(this); } From 5cfbec4a3e419c98be851317b85467d2c77f724a Mon Sep 17 00:00:00 2001 From: Jake Radzikowski Date: Tue, 5 Jul 2022 23:47:19 -0700 Subject: [PATCH 11/12] Resolve PR Feedback --- .../AutoMLMonitorKernelExtension.cs | 14 +++++----- .../NotebookMonitor.cs | 27 +++++++------------ 2 files changed, 17 insertions(+), 24 deletions(-) diff --git a/src/Microsoft.ML.AutoML.Interactive/AutoMLMonitorKernelExtension.cs b/src/Microsoft.ML.AutoML.Interactive/AutoMLMonitorKernelExtension.cs index 69b685e22d..7868a24b4c 100644 --- a/src/Microsoft.ML.AutoML.Interactive/AutoMLMonitorKernelExtension.cs +++ b/src/Microsoft.ML.AutoML.Interactive/AutoMLMonitorKernelExtension.cs @@ -48,19 +48,21 @@ private static void WriteSummary(NotebookMonitor monitor, TextWriter writer) if (monitor.BestTrial != null) { - summary.Add(h3("Best Run")); - summary.Add(p($"Trial: {monitor.BestTrial.TrialSettings.TrialId}")); + var bestTrialParam = JsonSerializer.Serialize(monitor.BestTrial.TrialSettings.Parameter, new JsonSerializerOptions() { WriteIndented = true, }); + summary.Add(h3("Best Trial")); + summary.Add(p($"Id: {monitor.BestTrial.TrialSettings.TrialId}")); summary.Add(p($"Trainer: {monitor.BestTrial.TrialSettings.Pipeline}".Replace("Unknown=>", ""))); + summary.Add(p($"Parameters: {bestTrialParam}")); } if (monitor.ActiveTrial != null) { - var activeRunParam = JsonSerializer.Serialize(monitor.ActiveTrial.Parameter, new JsonSerializerOptions() { WriteIndented = true, }); + var activeTrialParam = JsonSerializer.Serialize(monitor.ActiveTrial.Parameter, new JsonSerializerOptions() { WriteIndented = true, }); - summary.Add(h3("Active Run")); - summary.Add(p($"Trial: {monitor.ActiveTrial.TrialId}")); + summary.Add(h3("Active Trial")); + summary.Add(p($"Id: {monitor.ActiveTrial.TrialId}")); summary.Add(p($"Trainer: {monitor.ActiveTrial.Pipeline}".Replace("Unknown=>", ""))); - summary.Add(p($"Parameters: {activeRunParam}")); + summary.Add(p($"Parameters: {activeTrialParam}")); } writer.Write(div(summary)); diff --git a/src/Microsoft.ML.AutoML.Interactive/NotebookMonitor.cs b/src/Microsoft.ML.AutoML.Interactive/NotebookMonitor.cs index 9442767aec..7031610371 100644 --- a/src/Microsoft.ML.AutoML.Interactive/NotebookMonitor.cs +++ b/src/Microsoft.ML.AutoML.Interactive/NotebookMonitor.cs @@ -8,7 +8,7 @@ using System; using System.Threading.Tasks; using System.Text.Json; - +using System.Threading; namespace Microsoft.ML.AutoML { @@ -64,31 +64,22 @@ public void ReportRunningTrial(TrialSettings setting) Update(); } - private bool _updatePending = false; + private int _updatePending = 0; public void Update() { Task.Run(async () => { - if (_updatePending == true) - { - // Keep waiting - } - else + if (Interlocked.CompareExchange(ref _updatePending, 1, 0) == 0) // _updatePending is int initialized with 0 { - int timeRemaining = 5000 - (int)(DateTime.UtcNow.Millisecond - _lastUpdate.Millisecond); - _updatePending = true; - - if (timeRemaining > 0) + DateTime n = DateTime.UtcNow; + if (n - _lastUpdate < TimeSpan.FromSeconds(5)) { - await Task.Delay(timeRemaining); + await Task.Delay(n - _lastUpdate); } - _updatePending = false; - if (_valueToUpdate != null) - { - _lastUpdate = DateTime.UtcNow; - _valueToUpdate.Update(this); - } + _valueToUpdate.Update(this); + _lastUpdate = n; + _updatePending = 0; } }); From 02ffadd75bb25fa7be3ea97d538d7b7c87d2eca1 Mon Sep 17 00:00:00 2001 From: Jake Radzikowski Date: Wed, 6 Jul 2022 09:55:33 -0700 Subject: [PATCH 12/12] Pull out ActionThrottler into class and fix delay bug. --- .../ActionThrottler.cs | 50 +++++++++++++++++++ .../NotebookMonitor.cs | 38 ++++++-------- 2 files changed, 65 insertions(+), 23 deletions(-) create mode 100644 src/Microsoft.ML.AutoML.Interactive/ActionThrottler.cs diff --git a/src/Microsoft.ML.AutoML.Interactive/ActionThrottler.cs b/src/Microsoft.ML.AutoML.Interactive/ActionThrottler.cs new file mode 100644 index 0000000000..16c15b947a --- /dev/null +++ b/src/Microsoft.ML.AutoML.Interactive/ActionThrottler.cs @@ -0,0 +1,50 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.ML.AutoML +{ + internal class ActionThrottler + { + private readonly Action _action; + private readonly TimeSpan _minDelay; + + private DateTime _nextUpdateTime = DateTime.MinValue; + private int _updatePending = 0; + + /// + /// This constructor initializes an ActionThrottler that ensures runs no more than once per . + /// + /// The action to thorttle. + /// Timespan to indicate the minimum delay between each time action is executed. + public ActionThrottler(Action action, TimeSpan minDelay) + { + _minDelay = minDelay; + _action = action; + } + + + public async Task ExecuteAsync() + { + if (Interlocked.CompareExchange(ref _updatePending, 1, 0) == 0) // _updatePending is int initialized with 0 + { + DateTime currentTime = DateTime.UtcNow; + + if (_nextUpdateTime > currentTime) + { + await Task.Delay(_nextUpdateTime - currentTime); + } + _action(); + _nextUpdateTime = DateTime.UtcNow + _minDelay; + _updatePending = 0; + } + } + } +} diff --git a/src/Microsoft.ML.AutoML.Interactive/NotebookMonitor.cs b/src/Microsoft.ML.AutoML.Interactive/NotebookMonitor.cs index 7031610371..6e401d87d7 100644 --- a/src/Microsoft.ML.AutoML.Interactive/NotebookMonitor.cs +++ b/src/Microsoft.ML.AutoML.Interactive/NotebookMonitor.cs @@ -14,8 +14,8 @@ namespace Microsoft.ML.AutoML { public class NotebookMonitor : IMonitor { + private readonly ActionThrottler _updateThrottler; private DisplayedValue _valueToUpdate; - private DateTime _lastUpdate = DateTime.MinValue; public TrialResult BestTrial { get; set; } public TrialResult MostRecentTrial { get; set; } @@ -27,12 +27,14 @@ public NotebookMonitor() { CompletedTrials = new List(); TrialData = new DataFrame(new PrimitiveDataFrameColumn("Trial"), new PrimitiveDataFrameColumn("Metric"), new StringDataFrameColumn("Trainer"), new StringDataFrameColumn("Parameters")); + _updateThrottler = new ActionThrottler(Update, TimeSpan.FromSeconds(5)); } public void ReportBestTrial(TrialResult result) { BestTrial = result; - Update(); + + ThrottledUpdate(); } public void ReportCompletedTrial(TrialResult result) @@ -49,46 +51,36 @@ public void ReportCompletedTrial(TrialResult result) new KeyValuePair("Trainer",result.TrialSettings.Pipeline.ToString().Replace("Unknown=>","")), new KeyValuePair("Parameters",activeRunParam), }, true); - Update(); + + ThrottledUpdate(); } public void ReportFailTrial(TrialResult result) { // TODO figure out what to do with failed trials. - Update(); + ThrottledUpdate(); } public void ReportRunningTrial(TrialSettings setting) { ActiveTrial = setting; - Update(); + ThrottledUpdate(); } - private int _updatePending = 0; - public void Update() + private void ThrottledUpdate() { - Task.Run(async () => - { - if (Interlocked.CompareExchange(ref _updatePending, 1, 0) == 0) // _updatePending is int initialized with 0 - { - DateTime n = DateTime.UtcNow; - if (n - _lastUpdate < TimeSpan.FromSeconds(5)) - { - await Task.Delay(n - _lastUpdate); - } - - _valueToUpdate.Update(this); - _lastUpdate = n; - _updatePending = 0; - } - }); + Task.Run(async () => await _updateThrottler.ExecuteAsync()); + } + public void Update() + { + _valueToUpdate.Update(this); } public void SetUpdate(DisplayedValue valueToUpdate) { _valueToUpdate = valueToUpdate; - Update(); + ThrottledUpdate(); } } }