Watch file-based certificates for changes (#49979)

* Watch file-based certificates for changes

When `IConfiguration` specifies certificates by path and `reloadOnChange` is true, monitor those paths for changes and trigger reloads.  The reload behavior is the same as if the path in the configuration file were changed, rather than the contents of the referenced file.

- Does not apply to code-based configuration, which never triggers reloads
- Does not trigger on deletion - that would just tear down the server
- Does not look through symlinks to determine whether the target has changed
- On by default (when `reloadOnChange` is true) but can be disabled with app context switch `Microsoft.AspNetCore.Server.Kestrel.DisableCertificateFileWatching`

Fixes #32351

* Annotate CertificateConfig with MemberNotNullWhenAttribute

* Index log messages from 1

* Make log message placeholders PascalCase

* Append 'Unsynchronized' to test hook names

* Add explanatory comment

* Fix tests now that structured logging property names are PascalCase

* Add a test at the KestrelConfigurationLoader level

* Try a different way of updating a file's last write time to address CI flakiness

* Speculatively use deterministically distinct names in flaky tests

* Make more aggressive changes to files

* Format log messages consistently

* Remove redundant @s

* Add an abstraction layer between CertificatePathWatcher and the file system to make testing more reliable

* Clarify use of delays in KestrelConfigurationLoaderTests.CertificateChangedOnDisk

* Address PR feedback

* Preemptively add CertificateChangedOnDisk to the helix retry list

Author: amcasey

eng/test-configuration.json

@@ -21,6 +21,7 @@
     {"testName": {"contains": "InjectedStartup_DefaultApplicationNameIsEntryAssembly"}},
     {"testName": {"contains": "HEADERS_Received_SecondRequest_ConnectProtocolReset"}},
     {"testName": {"contains": "ClientUsingOldCallWithNewProtocol"}},
+    {"testName": {"contains": "CertificateChangedOnDisk"}},
     {"testAssembly": {"contains": "IIS"}},
     {"testAssembly": {"contains": "Template"}},
     {"failureMessage": {"contains":"(Site is started but no worker process found)"}},

src/Servers/Kestrel/Core/src/Internal/CertificatePathWatcher.cs

@@ -0,0 +1,328 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Diagnostics;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.FileProviders;
+using Microsoft.Extensions.FileProviders.Physical;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Primitives;
+
+namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal;
+
+internal sealed partial class CertificatePathWatcher : IDisposable
+{
+    private readonly Func<string, IFileProvider?> _fileProviderFactory;
+    private readonly string _contentRootDir;
+    private readonly ILogger<CertificatePathWatcher> _logger;
+
+    private readonly object _metadataLock = new();
+
+    /// <remarks>Acquire <see cref="_metadataLock"/> before accessing.</remarks>
+    private readonly Dictionary<string, DirectoryWatchMetadata> _metadataForDirectory = new();
+    /// <remarks>Acquire <see cref="_metadataLock"/> before accessing.</remarks>
+    private readonly Dictionary<string, FileWatchMetadata> _metadataForFile = new();
+
+    private ConfigurationReloadToken _reloadToken = new();
+    private bool _disposed;
+
+    public CertificatePathWatcher(IHostEnvironment hostEnvironment, ILogger<CertificatePathWatcher> logger)
+        : this(
+              hostEnvironment.ContentRootPath,
+              logger,
+              dir => Directory.Exists(dir) ? new PhysicalFileProvider(dir, ExclusionFilters.None) : null)
+    {
+    }
+
+    /// <remarks>
+    /// For testing.
+    /// </remarks>
+    internal CertificatePathWatcher(string contentRootPath, ILogger<CertificatePathWatcher> logger, Func<string, IFileProvider?> fileProviderFactory)
+    {
+        _contentRootDir = contentRootPath;
+        _logger = logger;
+        _fileProviderFactory = fileProviderFactory;
+    }
+
+    /// <summary>
+    /// Returns a token that will fire when any watched <see cref="CertificateConfig"/> is changed on disk.
+    /// The affected <see cref="CertificateConfig"/> will have <see cref="CertificateConfig.FileHasChanged"/>
+    /// set to <code>true</code>.
+    /// </summary>
+    public IChangeToken GetChangeToken()
+    {
+        return _reloadToken;
+    }
+
+    /// <summary>
+    /// Update the set of <see cref="CertificateConfig"/>s being watched for file changes.
+    /// If a given <see cref="CertificateConfig"/> appears in both lists, it is first removed and then added.
+    /// </summary>
+    /// <remarks>
+    /// Does not consider targets when watching files that are symlinks.
+    /// </remarks>
+    public void UpdateWatches(List<CertificateConfig> certificateConfigsToRemove, List<CertificateConfig> certificateConfigsToAdd)
+    {
+        var addSet = new HashSet<CertificateConfig>(certificateConfigsToAdd, ReferenceEqualityComparer.Instance);
+        var removeSet = new HashSet<CertificateConfig>(certificateConfigsToRemove, ReferenceEqualityComparer.Instance);
+
+        // Don't remove anything we're going to re-add anyway.
+        // Don't remove such items from addSet to guard against the (hypothetical) possibility
+        // that a caller might remove a config that isn't already present.
+        removeSet.ExceptWith(certificateConfigsToAdd);
+
+        if (addSet.Count == 0 && removeSet.Count == 0)
+        {
+            return;
+        }
+
+        lock (_metadataLock)
+        {
+            // Adds before removes to increase the chances of watcher reuse.
+            // Since removeSet doesn't contain any of these configs, this won't change the semantics.
+            foreach (var certificateConfig in addSet)
+            {
+                AddWatchUnsynchronized(certificateConfig);
+            }
+
+            foreach (var certificateConfig in removeSet)
+            {
+                RemoveWatchUnsynchronized(certificateConfig);
+            }
+        }
+    }
+
+    /// <summary>
+    /// Start watching a certificate's file path for changes.
+    /// <paramref name="certificateConfig"/> must have <see cref="CertificateConfig.IsFileCert"/> set to <code>true</code>.
+    /// </summary>
+    /// <remarks>
+    /// Internal for testing.
+    /// </remarks>
+    internal void AddWatchUnsynchronized(CertificateConfig certificateConfig)
+    {
+        Debug.Assert(certificateConfig.IsFileCert, "AddWatch called on non-file cert");
+
+        var path = Path.Combine(_contentRootDir, certificateConfig.Path);
+        var dir = Path.GetDirectoryName(path)!;
+
+        if (!_metadataForDirectory.TryGetValue(dir, out var dirMetadata))
+        {
+            // If we wanted to detected deletions of this whole directory (which we don't since we ignore deletions),
+            // we'd probably need to watch the whole directory hierarchy
+
+            var fileProvider = _fileProviderFactory(dir);
+            if (fileProvider is null)
+            {
+                _logger.DirectoryDoesNotExist(dir, path);
+                return;
+            }
+
+            dirMetadata = new DirectoryWatchMetadata(fileProvider);
+            _metadataForDirectory.Add(dir, dirMetadata);
+
+            _logger.CreatedDirectoryWatcher(dir);
+        }
+
+        if (!_metadataForFile.TryGetValue(path, out var fileMetadata))
+        {
+            // PhysicalFileProvider appears to be able to tolerate non-existent files, as long as the directory exists
+
+            var disposable = ChangeToken.OnChange(
+                () => dirMetadata.FileProvider.Watch(Path.GetFileName(path)),
+                static tuple => tuple.Item1.OnChange(tuple.Item2),
+                ValueTuple.Create(this, path));
+
+            fileMetadata = new FileWatchMetadata(disposable);
+            _metadataForFile.Add(path, fileMetadata);
+            dirMetadata.FileWatchCount++;
+
+            // We actually don't care if the file doesn't exist - we'll watch in case it is created
+            fileMetadata.LastModifiedTime = GetLastModifiedTimeOrMinimum(path, dirMetadata.FileProvider);
+
+            _logger.CreatedFileWatcher(path);
+        }
+
+        if (!fileMetadata.Configs.Add(certificateConfig))
+        {
+            _logger.ReusedObserver(path);
+            return;
+        }
+
+        _logger.AddedObserver(path);
+
+        _logger.ObserverCount(path, fileMetadata.Configs.Count);
+        _logger.FileCount(dir, dirMetadata.FileWatchCount);
+    }
+
+    private DateTimeOffset GetLastModifiedTimeOrMinimum(string path, IFileProvider fileProvider)
+    {
+        try
+        {
+            return fileProvider.GetFileInfo(Path.GetFileName(path)).LastModified;
+        }
+        catch (Exception e)
+        {
+            _logger.LastModifiedTimeError(path, e);
+        }
+
+        return DateTimeOffset.MinValue;
+    }
+
+    private void OnChange(string path)
+    {
+        // Block until any in-progress updates are complete
+        lock (_metadataLock)
+        {
+            if (!_metadataForFile.TryGetValue(path, out var fileMetadata))
+            {
+                _logger.UntrackedFileEvent(path);
+                return;
+            }
+
+            // Existence implied by the fact that we're tracking the file
+            var dirMetadata = _metadataForDirectory[Path.GetDirectoryName(path)!];
+
+            // We ignore file changes that don't advance the last modified time.
+            // For example, if we lose access to the network share the file is
+            // stored on, we don't notify our listeners because no one wants
+            // their endpoint/server to shutdown when that happens.
+            // We also anticipate that a cert file might be renamed to cert.bak
+            // before a new cert is introduced with the old name.
+            // This also helps us in scenarios where the underlying file system
+            // reports more than one change for a single logical operation.
+            var lastModifiedTime = GetLastModifiedTimeOrMinimum(path, dirMetadata.FileProvider);
+            if (lastModifiedTime > fileMetadata.LastModifiedTime)
+            {
+                fileMetadata.LastModifiedTime = lastModifiedTime;
+            }
+            else
+            {
+                if (lastModifiedTime == DateTimeOffset.MinValue)
+                {
+                    _logger.EventWithoutLastModifiedTime(path);
+                }
+                else if (lastModifiedTime == fileMetadata.LastModifiedTime)
+                {
+                    _logger.RedundantEvent(path);
+                }
+                else
+                {
+                    _logger.OutOfOrderEvent(path);
+                }
+                return;
+            }
+
+            var configs = fileMetadata.Configs;
+            foreach (var config in configs)
+            {
+                config.FileHasChanged = true;
+            }
+        }
+
+        // AddWatch and RemoveWatch don't affect the token, so this doesn't need to be under the semaphore.
+        // It does however need to be synchronized, since there could be multiple concurrent events.
+        var previousToken = Interlocked.Exchange(ref _reloadToken, new());
+        previousToken.OnReload();
+    }
+
+    /// <summary>
+    /// Stop watching a certificate's file path for changes (previously started by <see cref="AddWatchUnsynchronized"/>.
+    /// <paramref name="certificateConfig"/> must have <see cref="CertificateConfig.IsFileCert"/> set to <code>true</code>.
+    /// </summary>
+    /// <remarks>
+    /// Internal for testing.
+    /// </remarks>
+    internal void RemoveWatchUnsynchronized(CertificateConfig certificateConfig)
+    {
+        Debug.Assert(certificateConfig.IsFileCert, "RemoveWatch called on non-file cert");
+
+        var path = Path.Combine(_contentRootDir, certificateConfig.Path);
+        var dir = Path.GetDirectoryName(path)!;
+
+        if (!_metadataForFile.TryGetValue(path, out var fileMetadata))
+        {
+            _logger.UnknownFile(path);
+            return;
+        }
+
+        var configs = fileMetadata.Configs;
+
+        if (!configs.Remove(certificateConfig))
+        {
+            _logger.UnknownObserver(path);
+            return;
+        }
+
+        _logger.RemovedObserver(path);
+
+        // If we found fileMetadata, there must be a containing/corresponding dirMetadata
+        var dirMetadata = _metadataForDirectory[dir];
+
+        if (configs.Count == 0)
+        {
+            fileMetadata.Dispose();
+            _metadataForFile.Remove(path);
+            dirMetadata.FileWatchCount--;
+
+            _logger.RemovedFileWatcher(path);
+
+            if (dirMetadata.FileWatchCount == 0)
+            {
+                dirMetadata.Dispose();
+                _metadataForDirectory.Remove(dir);
+
+                _logger.RemovedDirectoryWatcher(dir);
+            }
+        }
+
+        _logger.ObserverCount(path, configs.Count);
+        _logger.FileCount(dir, dirMetadata.FileWatchCount);
+    }
+
+    /// <remarks>Test hook</remarks>
+    internal int TestGetDirectoryWatchCountUnsynchronized() => _metadataForDirectory.Count;
+
+    /// <remarks>Test hook</remarks>
+    internal int TestGetFileWatchCountUnsynchronized(string dir) => _metadataForDirectory.TryGetValue(dir, out var metadata) ? metadata.FileWatchCount : 0;
+
+    /// <remarks>Test hook</remarks>
+    internal int TestGetObserverCountUnsynchronized(string path) => _metadataForFile.TryGetValue(path, out var metadata) ? metadata.Configs.Count : 0;
+
+    void IDisposable.Dispose()
+    {
+        if (_disposed)
+        {
+            return;
+        }
+        _disposed = true;
+
+        foreach (var dirMetadata in _metadataForDirectory.Values)
+        {
+            dirMetadata.Dispose();
+        }
+
+        foreach (var fileMetadata in _metadataForFile.Values)
+        {
+            fileMetadata.Dispose();
+        }
+    }
+
+    private sealed class DirectoryWatchMetadata(IFileProvider fileProvider) : IDisposable
+    {
+        public readonly IFileProvider FileProvider = fileProvider;
+        public int FileWatchCount;
+
+        public void Dispose() => (FileProvider as IDisposable)?.Dispose();
+    }
+
+    private sealed class FileWatchMetadata(IDisposable disposable) : IDisposable
+    {
+        public readonly IDisposable Disposable = disposable;
+        public readonly HashSet<CertificateConfig> Configs = new(ReferenceEqualityComparer.Instance);
+        public DateTimeOffset LastModifiedTime = DateTimeOffset.MinValue;
+
+        public void Dispose() => Disposable.Dispose();
+    }
+}