From 4516114ac5a0e3dc4b92b2b75a266e55c7fba2ba Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Thu, 27 Mar 2025 08:35:40 -0700 Subject: [PATCH 1/2] =?UTF-8?q?Log=20contextual=20requests=20that=20affect?= =?UTF-8?q?=20sourcekitd=E2=80=99s=20global=20state?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This way we can log them when a sourcekitd request crashes and we can thus replay these contextual requests when diagnosing the crash. --- Sources/SKTestSupport/SkipUnless.swift | 5 +- Sources/SourceKitD/SourceKitD.swift | 55 +++++++++++++++++- Sources/SourceKitLSP/Rename.swift | 9 +-- .../Swift/CodeCompletionSession.swift | 41 +++++++------ Sources/SourceKitLSP/Swift/CursorInfo.swift | 3 +- .../Swift/DiagnosticReportManager.swift | 3 +- .../Swift/GeneratedInterfaceManager.swift | 22 +++---- .../SourceKitLSP/Swift/MacroExpansion.swift | 6 +- .../Swift/RefactoringResponse.swift | 3 +- .../Swift/RelatedIdentifiers.swift | 3 +- .../SourceKitLSP/Swift/SemanticTokens.swift | 3 +- .../Swift/SwiftLanguageService.swift | 32 +++++------ .../SourceKitLSP/Swift/VariableTypeInfo.swift | 3 +- Tests/SourceKitDTests/SourceKitDTests.swift | 20 +++++-- .../SwiftSourceKitPluginTests.swift | 57 +++++++------------ 15 files changed, 154 insertions(+), 111 deletions(-) diff --git a/Sources/SKTestSupport/SkipUnless.swift b/Sources/SKTestSupport/SkipUnless.swift index d35d8762d..81784a5b4 100644 --- a/Sources/SKTestSupport/SkipUnless.swift +++ b/Sources/SKTestSupport/SkipUnless.swift @@ -259,13 +259,14 @@ package actor SkipUnless { ) do { let response = try await sourcekitd.send( + \.codeCompleteSetPopularAPI, sourcekitd.dictionary([ - sourcekitd.keys.request: sourcekitd.requests.codeCompleteSetPopularAPI, sourcekitd.keys.codeCompleteOptions: [ sourcekitd.keys.useNewAPI: 1 - ], + ] ]), timeout: defaultTimeoutDuration, + documentUrl: nil, fileContents: nil ) return response[sourcekitd.keys.useNewAPI] == 1 diff --git a/Sources/SourceKitD/SourceKitD.swift b/Sources/SourceKitD/SourceKitD.swift index 8ad0ed857..c073627dd 100644 --- a/Sources/SourceKitD/SourceKitD.swift +++ b/Sources/SourceKitD/SourceKitD.swift @@ -284,6 +284,43 @@ package actor SourceKitD { } } + private struct ContextualRequest { + enum Kind { + case editorOpen + case codeCompleteOpen + } + let kind: Kind + let request: SKDRequestDictionary + } + + private var contextualRequests: [URL: [ContextualRequest]] = [:] + + private func recordContextualRequest( + requestUid: sourcekitd_api_uid_t, + request: SKDRequestDictionary, + documentUrl: URL? + ) { + guard let documentUrl else { + return + } + switch requestUid { + case requests.editorOpen: + contextualRequests[documentUrl] = [ContextualRequest(kind: .editorOpen, request: request)] + case requests.editorClose: + contextualRequests[documentUrl] = nil + case requests.codeCompleteOpen: + contextualRequests[documentUrl, default: []].removeAll(where: { $0.kind == .codeCompleteOpen }) + contextualRequests[documentUrl, default: []].append(ContextualRequest(kind: .codeCompleteOpen, request: request)) + case requests.codeCompleteClose: + contextualRequests[documentUrl, default: []].removeAll(where: { $0.kind == .codeCompleteOpen }) + if contextualRequests[documentUrl]?.isEmpty ?? false { + contextualRequests[documentUrl] = nil + } + default: + break + } + } + /// - Parameters: /// - request: The request to send to sourcekitd. /// - timeout: The maximum duration how long to wait for a response. If no response is returned within this time, @@ -291,10 +328,15 @@ package actor SourceKitD { /// - fileContents: The contents of the file that the request operates on. If sourcekitd crashes, the file contents /// will be logged. package func send( + _ requestUid: KeyPath, _ request: SKDRequestDictionary, timeout: Duration, + documentUrl: URL?, fileContents: String? ) async throws -> SKDResponseDictionary { + request.set(keys.request, to: requests[keyPath: requestUid]) + recordContextualRequest(requestUid: requests[keyPath: requestUid], request: request, documentUrl: documentUrl) + let sourcekitdResponse = try await withTimeout(timeout) { return try await withCancellableCheckedThrowingContinuation { (continuation) -> SourceKitDRequestHandle? in logger.info( @@ -337,13 +379,24 @@ package actor SourceKitD { guard let dict = sourcekitdResponse.value else { if sourcekitdResponse.error == .connectionInterrupted { - let log = """ + var log = """ Request: \(request.description) File contents: \(fileContents ?? "") """ + + if let documentUrl { + let contextualRequests = (contextualRequests[documentUrl] ?? []).filter { $0.request !== request } + for (index, contextualRequest) in contextualRequests.enumerated() { + log += """ + + Contextual request \(index + 1) / \(contextualRequests.count): + \(contextualRequest.request.description) + """ + } + } let chunks = splitLongMultilineMessage(message: log) for (index, chunk) in chunks.enumerated() { logger.fault( diff --git a/Sources/SourceKitLSP/Rename.swift b/Sources/SourceKitLSP/Rename.swift index b2384294a..66be313bf 100644 --- a/Sources/SourceKitLSP/Rename.swift +++ b/Sources/SourceKitLSP/Rename.swift @@ -353,7 +353,6 @@ extension SwiftLanguageService { } let req = sourcekitd.dictionary([ - keys.request: sourcekitd.requests.nameTranslation, keys.sourceFile: snapshot.uri.pseudoPath, keys.compilerArgs: await self.buildSettings(for: snapshot.uri, fallbackAfterTimeout: false)?.compilerArgs as [SKDRequestValue]?, @@ -363,7 +362,7 @@ extension SwiftLanguageService { keys.argNames: sourcekitd.array(name.parameters.map { $0.stringOrWildcard }), ]) - let response = try await sendSourcekitdRequest(req, fileContents: snapshot.text) + let response = try await send(sourcekitdRequest: \.nameTranslation, req, snapshot: snapshot) guard let isZeroArgSelector: Int = response[keys.isZeroArgSelector], let selectorPieces: SKDResponseArray = response[keys.selectorPieces] @@ -405,7 +404,6 @@ extension SwiftLanguageService { name: String ) async throws -> String { let req = sourcekitd.dictionary([ - keys.request: sourcekitd.requests.nameTranslation, keys.sourceFile: snapshot.uri.pseudoPath, keys.compilerArgs: await self.buildSettings(for: snapshot.uri, fallbackAfterTimeout: false)?.compilerArgs as [SKDRequestValue]?, @@ -421,7 +419,7 @@ extension SwiftLanguageService { req.set(keys.baseName, to: name) } - let response = try await sendSourcekitdRequest(req, fileContents: snapshot.text) + let response = try await send(sourcekitdRequest: \.nameTranslation, req, snapshot: snapshot) guard let baseName: String = response[keys.baseName] else { throw NameTranslationError.malformedClangToSwiftTranslateNameResponse(response) @@ -886,7 +884,6 @@ extension SwiftLanguageService { ) let skreq = sourcekitd.dictionary([ - keys.request: requests.findRenameRanges, keys.sourceFile: snapshot.uri.pseudoPath, // find-syntactic-rename-ranges is a syntactic sourcekitd request that doesn't use the in-memory file snapshot. // We need to send the source text again. @@ -894,7 +891,7 @@ extension SwiftLanguageService { keys.renameLocations: locations, ]) - let syntacticRenameRangesResponse = try await sendSourcekitdRequest(skreq, fileContents: snapshot.text) + let syntacticRenameRangesResponse = try await send(sourcekitdRequest: \.findRenameRanges, skreq, snapshot: snapshot) guard let categorizedRanges: SKDResponseArray = syntacticRenameRangesResponse[keys.categorizedRanges] else { throw ResponseError.internalError("sourcekitd did not return categorized ranges") } diff --git a/Sources/SourceKitLSP/Swift/CodeCompletionSession.swift b/Sources/SourceKitLSP/Swift/CodeCompletionSession.swift index da8d93ddc..730c6d16c 100644 --- a/Sources/SourceKitLSP/Swift/CodeCompletionSession.swift +++ b/Sources/SourceKitLSP/Swift/CodeCompletionSession.swift @@ -263,7 +263,20 @@ class CodeCompletionSession { self.clientSupportsDocumentationResolve = clientCapabilities.textDocument?.completion?.completionItem?.resolveSupport?.properties.contains("documentation") ?? false + } + private func send( + sourceKitDRequest requestUid: KeyPath & Sendable, + _ request: SKDRequestDictionary, + snapshot: DocumentSnapshot? + ) async throws -> SKDResponseDictionary { + try await sourcekitd.send( + requestUid, + request, + timeout: options.sourcekitdRequestTimeoutOrDefault, + documentUrl: snapshot?.uri.arbitrarySchemeURL, + fileContents: snapshot?.text + ) } private func open( @@ -278,7 +291,6 @@ class CodeCompletionSession { let sourcekitdPosition = snapshot.sourcekitdPosition(of: self.position) let req = sourcekitd.dictionary([ - keys.request: sourcekitd.requests.codeCompleteOpen, keys.line: sourcekitdPosition.line, keys.column: sourcekitdPosition.utf8Column, keys.name: uri.pseudoPath, @@ -287,11 +299,7 @@ class CodeCompletionSession { keys.codeCompleteOptions: optionsDictionary(filterText: filterText), ]) - let dict = try await sourcekitd.send( - req, - timeout: options.sourcekitdRequestTimeoutOrDefault, - fileContents: snapshot.text - ) + let dict = try await send(sourceKitDRequest: \.codeCompleteOpen, req, snapshot: snapshot) self.state = .open guard let completions: SKDResponseArray = dict[keys.results] else { @@ -317,7 +325,6 @@ class CodeCompletionSession { logger.info("Updating code completion session: \(self.description) filter=\(filterText)") let sourcekitdPosition = snapshot.sourcekitdPosition(of: self.position) let req = sourcekitd.dictionary([ - keys.request: sourcekitd.requests.codeCompleteUpdate, keys.line: sourcekitdPosition.line, keys.column: sourcekitdPosition.utf8Column, keys.name: uri.pseudoPath, @@ -325,11 +332,7 @@ class CodeCompletionSession { keys.codeCompleteOptions: optionsDictionary(filterText: filterText), ]) - let dict = try await sourcekitd.send( - req, - timeout: options.sourcekitdRequestTimeoutOrDefault, - fileContents: snapshot.text - ) + let dict = try await send(sourceKitDRequest: \.codeCompleteUpdate, req, snapshot: snapshot) guard let completions: SKDResponseArray = dict[keys.results] else { return CompletionList(isIncomplete: false, items: []) } @@ -370,7 +373,6 @@ class CodeCompletionSession { case .open: let sourcekitdPosition = snapshot.sourcekitdPosition(of: self.position) let req = sourcekitd.dictionary([ - keys.request: sourcekitd.requests.codeCompleteClose, keys.line: sourcekitdPosition.line, keys.column: sourcekitdPosition.utf8Column, keys.sourceFile: snapshot.uri.pseudoPath, @@ -378,7 +380,7 @@ class CodeCompletionSession { keys.codeCompleteOptions: [keys.useNewAPI: 1], ]) logger.info("Closing code completion session: \(self.description)") - _ = try? await sourcekitd.send(req, timeout: options.sourcekitdRequestTimeoutOrDefault, fileContents: nil) + _ = try? await send(sourceKitDRequest: \.codeCompleteClose, req, snapshot: nil) self.state = .closed } } @@ -548,11 +550,16 @@ class CodeCompletionSession { var item = item if let itemId = CompletionItemData(fromLSPAny: item.data)?.itemId { let req = sourcekitd.dictionary([ - sourcekitd.keys.request: sourcekitd.requests.codeCompleteDocumentation, - sourcekitd.keys.identifier: itemId, + sourcekitd.keys.identifier: itemId ]) let documentationResponse = await orLog("Retrieving documentation for completion item") { - try await sourcekitd.send(req, timeout: timeout, fileContents: nil) + try await sourcekitd.send( + \.codeCompleteDocumentation, + req, + timeout: timeout, + documentUrl: nil, + fileContents: nil + ) } if let docString: String = documentationResponse?[sourcekitd.keys.docBrief] { item.documentation = .markupContent(MarkupContent(kind: .markdown, value: docString)) diff --git a/Sources/SourceKitLSP/Swift/CursorInfo.swift b/Sources/SourceKitLSP/Swift/CursorInfo.swift index 835ebfae8..0cb02ba49 100644 --- a/Sources/SourceKitLSP/Swift/CursorInfo.swift +++ b/Sources/SourceKitLSP/Swift/CursorInfo.swift @@ -147,7 +147,6 @@ extension SwiftLanguageService { let keys = self.keys let skreq = sourcekitd.dictionary([ - keys.request: requests.cursorInfo, keys.cancelOnSubsequentRequest: 0, keys.offset: offsetRange.lowerBound, keys.length: offsetRange.upperBound != offsetRange.lowerBound ? offsetRange.count : nil, @@ -160,7 +159,7 @@ extension SwiftLanguageService { appendAdditionalParameters?(skreq) - let dict = try await sendSourcekitdRequest(skreq, fileContents: snapshot.text) + let dict = try await send(sourcekitdRequest: \.cursorInfo, skreq, snapshot: snapshot) var cursorInfoResults: [CursorInfo] = [] if let cursorInfo = CursorInfo(dict, documentManager: documentManager, sourcekitd: sourcekitd) { diff --git a/Sources/SourceKitLSP/Swift/DiagnosticReportManager.swift b/Sources/SourceKitLSP/Swift/DiagnosticReportManager.swift index 6156bcc41..63196eab5 100644 --- a/Sources/SourceKitLSP/Swift/DiagnosticReportManager.swift +++ b/Sources/SourceKitLSP/Swift/DiagnosticReportManager.swift @@ -113,7 +113,6 @@ actor DiagnosticReportManager { let keys = self.keys let skreq = sourcekitd.dictionary([ - keys.request: requests.diagnostics, keys.sourceFile: snapshot.uri.sourcekitdSourceFile, keys.primaryFile: snapshot.uri.primaryFile?.pseudoPath, keys.compilerArgs: compilerArgs as [SKDRequestValue], @@ -122,8 +121,10 @@ actor DiagnosticReportManager { let dict: SKDResponseDictionary do { dict = try await self.sourcekitd.send( + \.diagnostics, skreq, timeout: options.sourcekitdRequestTimeoutOrDefault, + documentUrl: snapshot.uri.arbitrarySchemeURL, fileContents: snapshot.text ) } catch SKDError.requestFailed(let sourcekitdError) { diff --git a/Sources/SourceKitLSP/Swift/GeneratedInterfaceManager.swift b/Sources/SourceKitLSP/Swift/GeneratedInterfaceManager.swift index 42a51a937..4793fd34a 100644 --- a/Sources/SourceKitLSP/Swift/GeneratedInterfaceManager.swift +++ b/Sources/SourceKitLSP/Swift/GeneratedInterfaceManager.swift @@ -67,13 +67,13 @@ actor GeneratedInterfaceManager { let sourcekitd = swiftLanguageService.sourcekitd for documentToClose in documentsToClose { await orLog("Closing generated interface") { - _ = try await swiftLanguageService.sendSourcekitdRequest( + _ = try await swiftLanguageService.send( + sourcekitdRequest: \.editorClose, sourcekitd.dictionary([ - sourcekitd.keys.request: sourcekitd.requests.editorClose, sourcekitd.keys.name: documentToClose, sourcekitd.keys.cancelBuilds: 0, ]), - fileContents: nil + snapshot: nil ) } } @@ -114,7 +114,6 @@ actor GeneratedInterfaceManager { let keys = sourcekitd.keys let skreq = sourcekitd.dictionary([ - keys.request: sourcekitd.requests.editorOpenInterface, keys.moduleName: document.moduleName, keys.groupName: document.groupName, keys.name: document.sourcekitdDocumentName, @@ -123,7 +122,7 @@ actor GeneratedInterfaceManager { .compilerArgs as [SKDRequestValue]?, ]) - let dict = try await swiftLanguageService.sendSourcekitdRequest(skreq, fileContents: nil) + let dict = try await swiftLanguageService.send(sourcekitdRequest: \.editorOpenInterface, skreq, snapshot: nil) guard let contents: String = dict[keys.sourceText] else { throw ResponseError.unknown("sourcekitd response is missing sourceText") @@ -133,13 +132,13 @@ actor GeneratedInterfaceManager { // Another request raced us to create the generated interface. Discard what we computed here and return the cached // value. await orLog("Closing generated interface created during race") { - _ = try await swiftLanguageService.sendSourcekitdRequest( + _ = try await swiftLanguageService.send( + sourcekitdRequest: \.editorClose, sourcekitd.dictionary([ - keys.request: sourcekitd.requests.editorClose, keys.name: document.sourcekitdDocumentName, keys.cancelBuilds: 0, ]), - fileContents: nil + snapshot: nil ) } return cached @@ -191,12 +190,15 @@ actor GeneratedInterfaceManager { let sourcekitd = swiftLanguageService.sourcekitd let keys = sourcekitd.keys let skreq = sourcekitd.dictionary([ - keys.request: sourcekitd.requests.editorFindUSR, keys.sourceFile: document.sourcekitdDocumentName, keys.usr: usr, ]) - let dict = try await swiftLanguageService.sendSourcekitdRequest(skreq, fileContents: details.snapshot.text) + let dict = try await swiftLanguageService.send( + sourcekitdRequest: \.editorFindUSR, + skreq, + snapshot: details.snapshot + ) guard let offset: Int = dict[keys.offset] else { throw ResponseError.unknown("Missing key 'offset'") } diff --git a/Sources/SourceKitLSP/Swift/MacroExpansion.swift b/Sources/SourceKitLSP/Swift/MacroExpansion.swift index d7044287f..34d6d8a2d 100644 --- a/Sources/SourceKitLSP/Swift/MacroExpansion.swift +++ b/Sources/SourceKitLSP/Swift/MacroExpansion.swift @@ -125,7 +125,6 @@ actor MacroExpansionManager { let length = snapshot.utf8OffsetRange(of: range).count let skreq = swiftLanguageService.sourcekitd.dictionary([ - keys.request: swiftLanguageService.requests.semanticRefactoring, // Preferred name for e.g. an extracted variable. // Empty string means sourcekitd chooses a name automatically. keys.name: "", @@ -139,10 +138,7 @@ actor MacroExpansionManager { keys.compilerArgs: buildSettings?.compilerArgs as [SKDRequestValue]?, ]) - let dict = try await swiftLanguageService.sendSourcekitdRequest( - skreq, - fileContents: snapshot.text - ) + let dict = try await swiftLanguageService.send(sourcekitdRequest: \.semanticRefactoring, skreq, snapshot: snapshot) guard let expansions = [RefactoringEdit](dict, snapshot, keys) else { throw SemanticRefactoringError.noEditsNeeded(snapshot.uri) } diff --git a/Sources/SourceKitLSP/Swift/RefactoringResponse.swift b/Sources/SourceKitLSP/Swift/RefactoringResponse.swift index 070a3dded..9d695694f 100644 --- a/Sources/SourceKitLSP/Swift/RefactoringResponse.swift +++ b/Sources/SourceKitLSP/Swift/RefactoringResponse.swift @@ -117,7 +117,6 @@ extension SwiftLanguageService { let utf8Column = snapshot.lineTable.utf8ColumnAt(line: line, utf16Column: utf16Column) let skreq = sourcekitd.dictionary([ - keys.request: self.requests.semanticRefactoring, // Preferred name for e.g. an extracted variable. // Empty string means sourcekitd chooses a name automatically. keys.name: "", @@ -131,7 +130,7 @@ extension SwiftLanguageService { as [SKDRequestValue]?, ]) - let dict = try await sendSourcekitdRequest(skreq, fileContents: snapshot.text) + let dict = try await send(sourcekitdRequest: \.semanticRefactoring, skreq, snapshot: snapshot) guard let refactor = SemanticRefactoring(refactorCommand.title, dict, snapshot, self.keys) else { throw SemanticRefactoringError.noEditsNeeded(uri) } diff --git a/Sources/SourceKitLSP/Swift/RelatedIdentifiers.swift b/Sources/SourceKitLSP/Swift/RelatedIdentifiers.swift index 8e3ec78b0..d72db1b60 100644 --- a/Sources/SourceKitLSP/Swift/RelatedIdentifiers.swift +++ b/Sources/SourceKitLSP/Swift/RelatedIdentifiers.swift @@ -67,7 +67,6 @@ extension SwiftLanguageService { includeNonEditableBaseNames: Bool ) async throws -> RelatedIdentifiersResponse { let skreq = sourcekitd.dictionary([ - keys.request: requests.relatedIdents, keys.cancelOnSubsequentRequest: 0, keys.offset: snapshot.utf8Offset(of: position), keys.sourceFile: snapshot.uri.sourcekitdSourceFile, @@ -77,7 +76,7 @@ extension SwiftLanguageService { as [SKDRequestValue]?, ]) - let dict = try await sendSourcekitdRequest(skreq, fileContents: snapshot.text) + let dict = try await send(sourcekitdRequest: \.relatedIdents, skreq, snapshot: snapshot) guard let results: SKDResponseArray = dict[self.keys.results] else { throw ResponseError.internalError("sourcekitd response did not contain results") diff --git a/Sources/SourceKitLSP/Swift/SemanticTokens.swift b/Sources/SourceKitLSP/Swift/SemanticTokens.swift index f754d1727..c2f41b169 100644 --- a/Sources/SourceKitLSP/Swift/SemanticTokens.swift +++ b/Sources/SourceKitLSP/Swift/SemanticTokens.swift @@ -27,13 +27,12 @@ extension SwiftLanguageService { } let skreq = sourcekitd.dictionary([ - keys.request: requests.semanticTokens, keys.sourceFile: snapshot.uri.sourcekitdSourceFile, keys.primaryFile: snapshot.uri.primaryFile?.pseudoPath, keys.compilerArgs: buildSettings.compilerArgs as [SKDRequestValue], ]) - let dict = try await sendSourcekitdRequest(skreq, fileContents: snapshot.text) + let dict = try await send(sourcekitdRequest: \.semanticTokens, skreq, snapshot: snapshot) guard let skTokens: SKDResponseArray = dict[keys.semanticTokens] else { return nil diff --git a/Sources/SourceKitLSP/Swift/SwiftLanguageService.swift b/Sources/SourceKitLSP/Swift/SwiftLanguageService.swift index 9fcb132f2..14487d274 100644 --- a/Sources/SourceKitLSP/Swift/SwiftLanguageService.swift +++ b/Sources/SourceKitLSP/Swift/SwiftLanguageService.swift @@ -300,14 +300,17 @@ package actor SwiftLanguageService: LanguageService, Sendable { } } - func sendSourcekitdRequest( + func send( + sourcekitdRequest requestUid: KeyPath & Sendable, _ request: SKDRequestDictionary, - fileContents: String? + snapshot: DocumentSnapshot? ) async throws -> SKDResponseDictionary { try await sourcekitd.send( + requestUid, request, timeout: options.sourcekitdRequestTimeoutOrDefault, - fileContents: fileContents + documentUrl: snapshot?.uri.arbitrarySchemeURL, + fileContents: snapshot?.text ) } @@ -424,10 +427,7 @@ extension SwiftLanguageService { /// Tell sourcekitd to crash itself. For testing purposes only. package func crash() async { - let req = sourcekitd.dictionary([ - keys.request: sourcekitd.requests.crashWithExit - ]) - _ = try? await sendSourcekitdRequest(req, fileContents: nil) + _ = try? await send(sourcekitdRequest: \.crashWithExit, sourcekitd.dictionary([:]), snapshot: nil) } // MARK: - Build System Integration @@ -450,7 +450,7 @@ extension SwiftLanguageService { let closeReq = closeDocumentSourcekitdRequest(uri: snapshot.uri) _ = await orLog("Closing document to re-open it") { - try await self.sendSourcekitdRequest(closeReq, fileContents: nil) + try await self.send(sourcekitdRequest: \.editorClose, closeReq, snapshot: nil) } let buildSettings = await buildSettings(for: snapshot.uri, fallbackAfterTimeout: true) @@ -460,7 +460,7 @@ extension SwiftLanguageService { ) self.buildSettingsForOpenFiles[snapshot.uri] = buildSettings _ = await orLog("Re-opening document") { - try await self.sendSourcekitdRequest(openReq, fileContents: snapshot.text) + try await self.send(sourcekitdRequest: \.editorOpen, openReq, snapshot: snapshot) } if await capabilityRegistry.clientSupportsPullDiagnostics(for: .swift) { @@ -492,10 +492,7 @@ extension SwiftLanguageService { } await orLog("Sending dependencyUpdated request to sourcekitd") { - let req = sourcekitd.dictionary([ - keys.request: requests.dependencyUpdated - ]) - _ = try await self.sendSourcekitdRequest(req, fileContents: nil) + _ = try await self.send(sourcekitdRequest: \.dependencyUpdated, sourcekitd.dictionary([:]), snapshot: nil) } // Even after sending the `dependencyUpdated` request to sourcekitd, the code completion session has state from // before the AST update. Close it and open a new code completion session on the next completion request. @@ -514,7 +511,6 @@ extension SwiftLanguageService { compileCommand: SwiftCompileCommand? ) -> SKDRequestDictionary { return sourcekitd.dictionary([ - keys.request: self.requests.editorOpen, keys.name: snapshot.uri.pseudoPath, keys.sourceText: snapshot.text, keys.enableSyntaxMap: 0, @@ -527,7 +523,6 @@ extension SwiftLanguageService { func closeDocumentSourcekitdRequest(uri: DocumentURI) -> SKDRequestDictionary { return sourcekitd.dictionary([ - keys.request: requests.editorClose, keys.name: uri.pseudoPath, keys.cancelBuilds: 0, ]) @@ -549,7 +544,7 @@ extension SwiftLanguageService { buildSettingsForOpenFiles[snapshot.uri] = buildSettings let req = openDocumentSourcekitdRequest(snapshot: snapshot, compileCommand: buildSettings) - _ = try? await self.sendSourcekitdRequest(req, fileContents: snapshot.text) + _ = try? await self.send(sourcekitdRequest: \.editorOpen, req, snapshot: snapshot) await publishDiagnosticsIfNeeded(for: notification.textDocument.uri) } } @@ -566,7 +561,7 @@ extension SwiftLanguageService { await generatedInterfaceManager.close(document: data) case nil: let req = closeDocumentSourcekitdRequest(uri: notification.textDocument.uri) - _ = try? await self.sendSourcekitdRequest(req, fileContents: nil) + _ = try? await self.send(sourcekitdRequest: \.editorClose, req, snapshot: nil) } } @@ -663,7 +658,6 @@ extension SwiftLanguageService { for edit in edits { let req = sourcekitd.dictionary([ - keys.request: self.requests.editorReplaceText, keys.name: notification.textDocument.uri.pseudoPath, keys.enableSyntaxMap: 0, keys.enableStructure: 0, @@ -674,7 +668,7 @@ extension SwiftLanguageService { keys.sourceText: edit.replacement, ]) do { - _ = try await self.sendSourcekitdRequest(req, fileContents: nil) + _ = try await self.send(sourcekitdRequest: \.editorReplaceText, req, snapshot: nil) } catch { logger.fault( """ diff --git a/Sources/SourceKitLSP/Swift/VariableTypeInfo.swift b/Sources/SourceKitLSP/Swift/VariableTypeInfo.swift index 1b48f2d77..edde381cf 100644 --- a/Sources/SourceKitLSP/Swift/VariableTypeInfo.swift +++ b/Sources/SourceKitLSP/Swift/VariableTypeInfo.swift @@ -86,7 +86,6 @@ extension SwiftLanguageService { let snapshot = try await self.latestSnapshot(for: uri) let skreq = sourcekitd.dictionary([ - keys.request: requests.collectVariableType, keys.sourceFile: snapshot.uri.sourcekitdSourceFile, keys.primaryFile: snapshot.uri.primaryFile?.pseudoPath, keys.compilerArgs: await self.buildSettings(for: uri, fallbackAfterTimeout: false)?.compilerArgs @@ -100,7 +99,7 @@ extension SwiftLanguageService { skreq.set(keys.length, to: end - start) } - let dict = try await sendSourcekitdRequest(skreq, fileContents: snapshot.text) + let dict = try await send(sourcekitdRequest: \.collectVariableType, skreq, snapshot: snapshot) guard let skVariableTypeInfos: SKDResponseArray = dict[keys.variableTypeList] else { return [] } diff --git a/Tests/SourceKitDTests/SourceKitDTests.swift b/Tests/SourceKitDTests/SourceKitDTests.swift index 12688c151..0ee8aaa20 100644 --- a/Tests/SourceKitDTests/SourceKitDTests.swift +++ b/Tests/SourceKitDTests/SourceKitDTests.swift @@ -76,7 +76,6 @@ final class SourceKitDTests: XCTestCase { args.append(path) let req = sourcekitd.dictionary([ - keys.request: sourcekitd.requests.editorOpen, keys.name: path, keys.sourceText: """ func foo() {} @@ -84,15 +83,26 @@ final class SourceKitDTests: XCTestCase { keys.compilerArgs: args, ]) - _ = try await sourcekitd.send(req, timeout: defaultTimeoutDuration, fileContents: nil) + _ = try await sourcekitd.send( + \.editorOpen, + req, + timeout: defaultTimeoutDuration, + documentUrl: nil, + fileContents: nil + ) try await fulfillmentOfOrThrow(expectation1, expectation2) let close = sourcekitd.dictionary([ - keys.request: sourcekitd.requests.editorClose, - keys.name: path, + keys.name: path ]) - _ = try await sourcekitd.send(close, timeout: defaultTimeoutDuration, fileContents: nil) + _ = try await sourcekitd.send( + \.editorClose, + close, + timeout: defaultTimeoutDuration, + documentUrl: nil, + fileContents: nil + ) } } diff --git a/Tests/SwiftSourceKitPluginTests/SwiftSourceKitPluginTests.swift b/Tests/SwiftSourceKitPluginTests/SwiftSourceKitPluginTests.swift index 17cd1c8f9..a8ab48c04 100644 --- a/Tests/SwiftSourceKitPluginTests/SwiftSourceKitPluginTests.swift +++ b/Tests/SwiftSourceKitPluginTests/SwiftSourceKitPluginTests.swift @@ -1812,6 +1812,13 @@ struct CompletionRequestFlags: OptionSet { } fileprivate extension SourceKitD { + private func send( + _ requestUid: KeyPath & Sendable, + _ request: SKDRequestDictionary + ) async throws -> SKDResponseDictionary { + try await self.send(requestUid, request, timeout: defaultTimeoutDuration, documentUrl: nil, fileContents: nil) + } + @discardableResult nonisolated func openDocument( _ name: String, @@ -1824,19 +1831,17 @@ fileprivate extension SourceKitD { compilerArguments += ["-sdk", defaultSDKPath] } let req = dictionary([ - keys.request: requests.editorOpen, keys.name: name, keys.sourceText: textWithoutMarkers, keys.syntacticOnly: 1, keys.compilerArgs: compilerArguments as [SKDRequestValue], ]) - _ = try await send(req, timeout: defaultTimeoutDuration, fileContents: nil) + _ = try await send(\.editorOpen, req) return DocumentPositions(markers: markers, textWithoutMarkers: textWithoutMarkers) } nonisolated func editDocument(_ name: String, fromOffset offset: Int, length: Int, newContents: String) async throws { let req = dictionary([ - keys.request: requests.editorReplaceText, keys.name: name, keys.offset: offset, keys.length: length, @@ -1844,20 +1849,19 @@ fileprivate extension SourceKitD { keys.syntacticOnly: 1, ]) - _ = try await send(req, timeout: defaultTimeoutDuration, fileContents: nil) + _ = try await send(\.editorReplaceText, req) } nonisolated func closeDocument(_ name: String) async throws { let req = dictionary([ - keys.request: requests.editorClose, - keys.name: name, + keys.name: name ]) - _ = try await send(req, timeout: defaultTimeoutDuration, fileContents: nil) + _ = try await send(\.editorClose, req) } nonisolated func completeImpl( - requestUID: sourcekitd_api_uid_t, + requestUID: KeyPath & Sendable, path: String, position: Position, filter: String, @@ -1879,7 +1883,6 @@ fileprivate extension SourceKitD { ]) let req = dictionary([ - keys.request: requestUID, keys.line: position.line + 1, // Technically sourcekitd needs a UTF-8 index but we can assume there are no Unicode characters in the tests keys.column: position.utf16index + 1, @@ -1888,7 +1891,7 @@ fileprivate extension SourceKitD { keys.compilerArgs: compilerArguments as [SKDRequestValue]?, ]) - let res = try await send(req, timeout: defaultTimeoutDuration, fileContents: nil) + let res = try await send(requestUID, req) return try CompletionResultSet(res) } @@ -1903,7 +1906,7 @@ fileprivate extension SourceKitD { compilerArguments: [String]? = nil ) async throws -> CompletionResultSet { return try await completeImpl( - requestUID: requests.codeCompleteOpen, + requestUID: \.codeCompleteOpen, path: path, position: position, filter: filter, @@ -1924,7 +1927,7 @@ fileprivate extension SourceKitD { maxResults: Int? = nil ) async throws -> CompletionResultSet { return try await completeImpl( - requestUID: requests.codeCompleteUpdate, + requestUID: \.codeCompleteUpdate, path: path, position: position, filter: filter, @@ -1938,7 +1941,6 @@ fileprivate extension SourceKitD { nonisolated func completeClose(path: String, position: Position) async throws { let req = dictionary([ - keys.request: requests.codeCompleteClose, keys.line: position.line + 1, // Technically sourcekitd needs a UTF-8 index but we can assume there are no Unicode characters in the tests keys.column: position.utf16index + 1, @@ -1946,45 +1948,32 @@ fileprivate extension SourceKitD { keys.codeCompleteOptions: dictionary([keys.useNewAPI: 1]), ]) - _ = try await send(req, timeout: defaultTimeoutDuration, fileContents: nil) + _ = try await send(\.codeCompleteClose, req) } nonisolated func completeDocumentation(id: Int) async throws -> CompletionDocumentation { - let req = dictionary([ - keys.request: requests.codeCompleteDocumentation, - keys.identifier: id, - ]) - - let resp = try await send(req, timeout: defaultTimeoutDuration, fileContents: nil) + let resp = try await send(\.codeCompleteDocumentation, dictionary([keys.identifier: id])) return CompletionDocumentation(resp) } nonisolated func completeDiagnostic(id: Int) async throws -> CompletionDiagnostic? { - let req = dictionary([ - keys.request: requests.codeCompleteDiagnostic, - keys.identifier: id, - ]) - let resp = try await send(req, timeout: defaultTimeoutDuration, fileContents: nil) + let resp = try await send(\.codeCompleteDiagnostic, dictionary([keys.identifier: id])) return CompletionDiagnostic(resp) } nonisolated func dependencyUpdated() async throws { - let req = dictionary([ - keys.request: requests.dependencyUpdated - ]) - _ = try await send(req, timeout: defaultTimeoutDuration, fileContents: nil) + _ = try await send(\.dependencyUpdated, dictionary([:])) } nonisolated func setPopularAPI(popular: [String], unpopular: [String]) async throws { let req = dictionary([ - keys.request: requests.codeCompleteSetPopularAPI, keys.codeCompleteOptions: dictionary([keys.useNewAPI: 1]), keys.popular: popular as [SKDRequestValue], keys.unpopular: unpopular as [SKDRequestValue], ]) - let resp = try await send(req, timeout: defaultTimeoutDuration, fileContents: nil) + let resp = try await send(\.codeCompleteSetPopularAPI, req) XCTAssertEqual(resp[keys.useNewAPI], 1) } @@ -1994,14 +1983,13 @@ fileprivate extension SourceKitD { notoriousModules: [String] ) async throws { let req = dictionary([ - keys.request: requests.codeCompleteSetPopularAPI, keys.codeCompleteOptions: dictionary([keys.useNewAPI: 1]), keys.scopedPopularityTablePath: scopedPopularityDataPath, keys.popularModules: popularModules as [SKDRequestValue], keys.notoriousModules: notoriousModules as [SKDRequestValue], ]) - let resp = try await send(req, timeout: defaultTimeoutDuration, fileContents: nil) + let resp = try await send(\.codeCompleteSetPopularAPI, req) XCTAssertEqual(resp[keys.useNewAPI], 1) } @@ -2019,13 +2007,12 @@ fileprivate extension SourceKitD { ]) } let req = dictionary([ - keys.request: requests.codeCompleteSetPopularAPI, keys.codeCompleteOptions: dictionary([keys.useNewAPI: 1]), keys.symbolPopularity: symbolPopularity as [SKDRequestValue], keys.modulePopularity: modulePopularity as [SKDRequestValue], ]) - let resp = try await send(req, timeout: defaultTimeoutDuration, fileContents: nil) + let resp = try await send(\.codeCompleteSetPopularAPI, req) XCTAssertEqual(resp[keys.useNewAPI], 1) } From c0cd2ac3d673fd3528199011f3f32ecbb53ac9b3 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Thu, 27 Mar 2025 11:00:08 -0700 Subject: [PATCH 2/2] Add contextual request support to `sourcekit-lsp diagnose` Teach `sourcekit-lsp diagnose` how to extract contextual requests from the system log and use them to reduce sourcekitd crashes. --- Sources/Diagnose/CMakeLists.txt | 1 + Sources/Diagnose/DiagnoseCommand.swift | 4 + Sources/Diagnose/MergeSwiftFiles.swift | 1 + Sources/Diagnose/OSLogScraper.swift | 70 ++++++++- Sources/Diagnose/ReduceCommand.swift | 12 +- Sources/Diagnose/ReduceFrontendCommand.swift | 1 + Sources/Diagnose/ReproducerBundle.swift | 14 +- Sources/Diagnose/RequestInfo.swift | 135 ++++++++++++------ .../RunSourcekitdRequestCommand.swift | 67 +++++++-- .../Diagnose/SourceKitDRequestExecutor.swift | 63 +++++--- Sources/Diagnose/SourceReducer.swift | 2 + Sources/Diagnose/Toolchain+PluginPaths.swift | 23 +++ Sources/SKUtilities/CMakeLists.txt | 2 +- Tests/DiagnoseTests/DiagnoseTests.swift | 13 +- 14 files changed, 313 insertions(+), 95 deletions(-) create mode 100644 Sources/Diagnose/Toolchain+PluginPaths.swift diff --git a/Sources/Diagnose/CMakeLists.txt b/Sources/Diagnose/CMakeLists.txt index 6e5b7dabd..3ea6e1159 100644 --- a/Sources/Diagnose/CMakeLists.txt +++ b/Sources/Diagnose/CMakeLists.txt @@ -19,6 +19,7 @@ add_library(Diagnose STATIC StderrStreamConcurrencySafe.swift SwiftFrontendCrashScraper.swift Toolchain+SwiftFrontend.swift + Toolchain+PluginPaths.swift TraceFromSignpostsCommand.swift) set_target_properties(Diagnose PROPERTIES diff --git a/Sources/Diagnose/DiagnoseCommand.swift b/Sources/Diagnose/DiagnoseCommand.swift index 05a5dba8f..232c516c6 100644 --- a/Sources/Diagnose/DiagnoseCommand.swift +++ b/Sources/Diagnose/DiagnoseCommand.swift @@ -13,6 +13,7 @@ package import ArgumentParser import Foundation import LanguageServerProtocolExtensions +import SKLogging import SwiftExtensions import TSCExtensions import ToolchainRegistry @@ -136,6 +137,7 @@ package struct DiagnoseCommand: AsyncParsableCommand { break } catch { // Reducing this request failed. Continue reducing the next one, maybe that one succeeds. + logger.info("Reducing sourcekitd crash failed: \(error.forLogging)") } } } @@ -173,6 +175,7 @@ package struct DiagnoseCommand: AsyncParsableCommand { let executor = OutOfProcessSourceKitRequestExecutor( sourcekitd: sourcekitd, + pluginPaths: toolchain.pluginPaths, swiftFrontend: crashInfo.swiftFrontend, reproducerPredicate: nil ) @@ -457,6 +460,7 @@ package struct DiagnoseCommand: AsyncParsableCommand { let requestInfo = requestInfo let executor = OutOfProcessSourceKitRequestExecutor( sourcekitd: sourcekitd, + pluginPaths: toolchain.pluginPaths, swiftFrontend: swiftFrontend, reproducerPredicate: nil ) diff --git a/Sources/Diagnose/MergeSwiftFiles.swift b/Sources/Diagnose/MergeSwiftFiles.swift index e204008b5..1cfa401cb 100644 --- a/Sources/Diagnose/MergeSwiftFiles.swift +++ b/Sources/Diagnose/MergeSwiftFiles.swift @@ -31,6 +31,7 @@ extension RequestInfo { let compilerArgs = compilerArgs.filter { $0 != "-primary-file" && !$0.hasSuffix(".swift") } + ["$FILE"] let mergedRequestInfo = RequestInfo( requestTemplate: requestTemplate, + contextualRequestTemplates: contextualRequestTemplates, offset: offset, compilerArgs: compilerArgs, fileContents: mergedFile diff --git a/Sources/Diagnose/OSLogScraper.swift b/Sources/Diagnose/OSLogScraper.swift index 776d27e8a..287c79e5b 100644 --- a/Sources/Diagnose/OSLogScraper.swift +++ b/Sources/Diagnose/OSLogScraper.swift @@ -12,6 +12,8 @@ #if canImport(OSLog) import OSLog +import SKLogging +import RegexBuilder /// Reads oslog messages to find recent sourcekitd crashes. struct OSLogScraper { @@ -45,34 +47,90 @@ struct OSLogScraper { #"subsystem CONTAINS "sourcekit-lsp" AND composedMessage CONTAINS "sourcekitd crashed" AND category = %@"#, logCategory ) - var isInFileContentSection = false + enum LogSection { + case request + case fileContents + case contextualRequest + } + var section = LogSection.request var request = "" var fileContents = "" + var contextualRequests: [String] = [] + let sourcekitdCrashedRegex = Regex { + "sourcekitd crashed (" + OneOrMore(.digit) + "/" + OneOrMore(.digit) + ")" + } + let contextualRequestRegex = Regex { + "Contextual request " + OneOrMore(.digit) + " / " + OneOrMore(.digit) + ":" + } + for entry in try getLogEntries(matching: predicate) { for line in entry.composedMessage.components(separatedBy: "\n") { - if line.starts(with: "sourcekitd crashed (") { + if try sourcekitdCrashedRegex.wholeMatch(in: line) != nil { continue } if line == "Request:" { continue } if line == "File contents:" { - isInFileContentSection = true + section = .fileContents + continue + } + if line == "File contents:" { + section = .fileContents + continue + } + if try contextualRequestRegex.wholeMatch(in: line) != nil { + section = .contextualRequest + contextualRequests.append("") continue } if line == "--- End Chunk" { continue } - if isInFileContentSection { - fileContents += line + "\n" - } else { + switch section { + case .request: request += line + "\n" + case .fileContents: + fileContents += line + "\n" + case .contextualRequest: + if !contextualRequests.isEmpty { + contextualRequests[contextualRequests.count - 1] += line + "\n" + } else { + // Should never happen because we have appended at least one element to `contextualRequests` when switching + // to the `contextualRequest` section. + logger.fault("Dropping contextual request line: \(line)") + } } } } var requestInfo = try RequestInfo(request: request) + + let contextualRequestInfos = contextualRequests.compactMap { contextualRequest in + orLog("Processsing contextual request") { + try RequestInfo(request: contextualRequest) + } + }.filter { contextualRequest in + if contextualRequest.fileContents != requestInfo.fileContents { + logger.error("Contextual request concerns a different file than the crashed request. Ignoring it") + return false + } + return true + } + requestInfo.contextualRequestTemplates = contextualRequestInfos.map(\.requestTemplate) + if requestInfo.compilerArgs.isEmpty { + requestInfo.compilerArgs = contextualRequestInfos.last(where: { !$0.compilerArgs.isEmpty })?.compilerArgs ?? [] + } requestInfo.fileContents = fileContents + return requestInfo } diff --git a/Sources/Diagnose/ReduceCommand.swift b/Sources/Diagnose/ReduceCommand.swift index fb3788725..80ce597cc 100644 --- a/Sources/Diagnose/ReduceCommand.swift +++ b/Sources/Diagnose/ReduceCommand.swift @@ -12,6 +12,7 @@ package import ArgumentParser import Foundation +import SourceKitD import ToolchainRegistry import struct TSCBasic.AbsolutePath @@ -68,12 +69,16 @@ package struct ReduceCommand: AsyncParsableCommand { @MainActor package func run() async throws { - guard let sourcekitd = try await toolchain?.sourcekitd else { + guard let toolchain = try await toolchain else { + throw GenericError("Unable to find toolchain") + } + guard let sourcekitd = toolchain.sourcekitd else { throw GenericError("Unable to find sourcekitd.framework") } - guard let swiftFrontend = try await toolchain?.swiftFrontend else { + guard let swiftFrontend = toolchain.swiftFrontend else { throw GenericError("Unable to find sourcekitd.framework") } + let pluginPaths = toolchain.pluginPaths let progressBar = PercentProgressAnimation(stream: stderrStreamConcurrencySafe, header: "Reducing sourcekitd issue") @@ -82,6 +87,7 @@ package struct ReduceCommand: AsyncParsableCommand { let executor = OutOfProcessSourceKitRequestExecutor( sourcekitd: sourcekitd, + pluginPaths: pluginPaths, swiftFrontend: swiftFrontend, reproducerPredicate: nsPredicate ) @@ -96,6 +102,6 @@ package struct ReduceCommand: AsyncParsableCommand { try reduceRequestInfo.fileContents.write(to: reducedSourceFile, atomically: true, encoding: .utf8) print("Reduced Request:") - print(try reduceRequestInfo.request(for: reducedSourceFile)) + print(try reduceRequestInfo.requests(for: reducedSourceFile).joined(separator: "\n\n\n\n")) } } diff --git a/Sources/Diagnose/ReduceFrontendCommand.swift b/Sources/Diagnose/ReduceFrontendCommand.swift index 2d7c5316f..a779237f7 100644 --- a/Sources/Diagnose/ReduceFrontendCommand.swift +++ b/Sources/Diagnose/ReduceFrontendCommand.swift @@ -90,6 +90,7 @@ package struct ReduceFrontendCommand: AsyncParsableCommand { let executor = OutOfProcessSourceKitRequestExecutor( sourcekitd: sourcekitd, + pluginPaths: nil, swiftFrontend: swiftFrontend, reproducerPredicate: nsPredicate ) diff --git a/Sources/Diagnose/ReproducerBundle.swift b/Sources/Diagnose/ReproducerBundle.swift index 3ff842c8a..8fdccb917 100644 --- a/Sources/Diagnose/ReproducerBundle.swift +++ b/Sources/Diagnose/ReproducerBundle.swift @@ -40,12 +40,14 @@ func makeReproducerBundle(for requestInfo: RequestInfo, toolchain: Toolchain, bu + requestInfo.compilerArgs.replacing(["$FILE"], with: ["./input.swift"]).joined(separator: " \\\n") try command.write(to: bundlePath.appendingPathComponent("command.sh"), atomically: true, encoding: .utf8) } else { - let request = try requestInfo.request(for: URL(fileURLWithPath: "/input.swift")) - try request.write( - to: bundlePath.appendingPathComponent("request.yml"), - atomically: true, - encoding: .utf8 - ) + let requests = try requestInfo.requests(for: bundlePath.appendingPathComponent("input.swift")) + for (index, request) in requests.enumerated() { + try request.write( + to: bundlePath.appendingPathComponent("request-\(index).yml"), + atomically: true, + encoding: .utf8 + ) + } } for compilerArg in requestInfo.compilerArgs { // Find the first slash so we are also able to copy files from eg. diff --git a/Sources/Diagnose/RequestInfo.swift b/Sources/Diagnose/RequestInfo.swift index 696a07f13..8bba0b193 100644 --- a/Sources/Diagnose/RequestInfo.swift +++ b/Sources/Diagnose/RequestInfo.swift @@ -17,11 +17,19 @@ import SwiftExtensions /// All the information necessary to replay a sourcektid request. package struct RequestInfo: Sendable { /// The JSON request object. Contains the following dynamic placeholders: - /// - `$OFFSET`: To be replaced by `offset` before running the request - /// - `$FILE`: Will be replaced with a path to the file that contains the reduced source code. /// - `$COMPILER_ARGS`: Will be replaced by the compiler arguments of the request + /// - `$FILE`: Will be replaced with a path to the file that contains the reduced source code. + /// - `$FILE_CONTENTS`: Will be replaced by the contents of the reduced source file inside quotes + /// - `$OFFSET`: To be replaced by `offset` before running the request var requestTemplate: String + /// Requests that should be executed before `requestTemplate` to set up state in sourcekitd so that `requestTemplate` + /// can reproduce an issue, eg. sending an `editor.open` before a `codecomplete.open` so that we have registered the + /// compiler arguments in the SourceKit plugin. + /// + /// These request templates receive the same substitutions as `requestTemplate`. + var contextualRequestTemplates: [String] + /// The offset at which the sourcekitd request should be run. Replaces the /// `$OFFSET` placeholder in the request template. var offset: Int @@ -32,7 +40,7 @@ package struct RequestInfo: Sendable { /// The contents of the file that the sourcekitd request operates on. package var fileContents: String - package func request(for file: URL) throws -> String { + package func requests(for file: URL) throws -> [String] { let encoder = JSONEncoder() encoder.outputFormatting = [.prettyPrinted, .sortedKeys, .withoutEscapingSlashes] guard var compilerArgs = String(data: try encoder.encode(compilerArgs), encoding: .utf8) else { @@ -40,11 +48,15 @@ package struct RequestInfo: Sendable { } // Drop the opening `[` and `]`. The request template already contains them compilerArgs = String(compilerArgs.dropFirst().dropLast()) - return + let quotedFileContents = + try String(data: JSONEncoder().encode(try String(contentsOf: file, encoding: .utf8)), encoding: .utf8) ?? "" + return try (contextualRequestTemplates + [requestTemplate]).map { requestTemplate in requestTemplate - .replacingOccurrences(of: "$OFFSET", with: String(offset)) - .replacingOccurrences(of: "$COMPILER_ARGS", with: compilerArgs) - .replacingOccurrences(of: "$FILE", with: try file.filePath.replacing(#"\"#, with: #"\\"#)) + .replacingOccurrences(of: "$OFFSET", with: String(offset)) + .replacingOccurrences(of: "$COMPILER_ARGS", with: compilerArgs) + .replacingOccurrences(of: "$FILE_CONTENTS", with: quotedFileContents) + .replacingOccurrences(of: "$FILE", with: try file.filePath.replacing(#"\"#, with: #"\\"#)) + } } /// A fake value that is used to indicate that we are reducing a `swift-frontend` issue instead of a sourcekitd issue. @@ -57,8 +69,15 @@ package struct RequestInfo: Sendable { } """ - package init(requestTemplate: String, offset: Int, compilerArgs: [String], fileContents: String) { + package init( + requestTemplate: String, + contextualRequestTemplates: [String], + offset: Int, + compilerArgs: [String], + fileContents: String + ) { self.requestTemplate = requestTemplate + self.contextualRequestTemplates = contextualRequestTemplates self.offset = offset self.compilerArgs = compilerArgs self.fileContents = fileContents @@ -70,42 +89,19 @@ package struct RequestInfo: Sendable { package init(request: String) throws { var requestTemplate = request - // Extract offset - let offsetRegex = Regex { - "key.offset: " - Capture(ZeroOrMore(.digit)) - } - if let offsetMatch = requestTemplate.matches(of: offsetRegex).only { - offset = Int(offsetMatch.1)! - requestTemplate.replace(offsetRegex, with: "key.offset: $OFFSET") - } else { - offset = 0 - } - // If the request contained source text, remove it. We want to pick it up from the file on disk and most (possibly // all) sourcekitd requests use key.sourcefile if key.sourcetext is missing. - requestTemplate.replace(#/ *key.sourcetext: .*\n/#, with: "") + requestTemplate.replace(#/ *key.sourcetext: .*\n/#, with: #"key.sourcetext: $FILE_CONTENTS\#n"#) - // Extract source file - let sourceFileRegex = Regex { - #"key.sourcefile: ""# - Capture(ZeroOrMore(#/[^"]/#)) - "\"" - } - guard let sourceFileMatch = requestTemplate.matches(of: sourceFileRegex).only else { - throw GenericError("Failed to find key.sourcefile in the request") - } - let sourceFilePath = String(sourceFileMatch.1) - requestTemplate.replace(sourceFileMatch.1, with: "$FILE") - - // Extract compiler arguments - let compilerArgsExtraction = try extractCompilerArguments(from: requestTemplate) - requestTemplate = compilerArgsExtraction.template - compilerArgs = compilerArgsExtraction.compilerArgs + let sourceFilePath: URL + (requestTemplate, offset) = try extractOffset(from: requestTemplate) + (requestTemplate, sourceFilePath) = try extractSourceFile(from: requestTemplate) + (requestTemplate, compilerArgs) = try extractCompilerArguments(from: requestTemplate) self.requestTemplate = requestTemplate + self.contextualRequestTemplates = [] - fileContents = try String(contentsOf: URL(fileURLWithPath: sourceFilePath), encoding: .utf8) + fileContents = try String(contentsOf: sourceFilePath, encoding: .utf8) } /// Create a `RequestInfo` that is used to reduce a `swift-frontend issue` @@ -139,6 +135,7 @@ package struct RequestInfo: Sendable { // `mergeSwiftFiles`. self.init( requestTemplate: Self.fakeRequestTemplateForFrontendIssues, + contextualRequestTemplates: [], offset: 0, compilerArgs: frontendArgsWithFilelistInlined, fileContents: "" @@ -146,6 +143,63 @@ package struct RequestInfo: Sendable { } } +private func extractOffset(from requestTemplate: String) throws -> (template: String, offset: Int) { + let offsetRegex = Regex { + "key.offset: " + Capture(ZeroOrMore(.digit)) + } + guard let offsetMatch = requestTemplate.matches(of: offsetRegex).only else { + return (requestTemplate, 0) + } + let requestTemplate = requestTemplate.replacing(offsetRegex, with: "key.offset: $OFFSET") + return (requestTemplate, Int(offsetMatch.1)!) +} + +private func extractSourceFile(from requestTemplate: String) throws -> (template: String, sourceFile: URL) { + var requestTemplate = requestTemplate + let sourceFileRegex = Regex { + #"key.sourcefile: ""# + Capture(ZeroOrMore(#/[^"]/#)) + "\"" + } + let nameRegex = Regex { + #"key.name: ""# + Capture(ZeroOrMore(#/[^"]/#)) + "\"" + } + let sourceFileMatch = requestTemplate.matches(of: sourceFileRegex).only + let nameMatch = requestTemplate.matches(of: nameRegex).only + + let sourceFilePath: String? + if let sourceFileMatch { + sourceFilePath = String(sourceFileMatch.1) + requestTemplate.replace(sourceFileMatch.1, with: "$FILE") + } else { + sourceFilePath = nil + } + + let namePath: String? + if let nameMatch { + namePath = String(nameMatch.1) + requestTemplate.replace(nameMatch.1, with: "$FILE") + } else { + namePath = nil + } + switch (sourceFilePath, namePath) { + case (let sourceFilePath?, let namePath?): + if sourceFilePath != namePath { + throw GenericError("Mismatching find key.sourcefile and key.name in the request") + } + return (requestTemplate, URL(fileURLWithPath: sourceFilePath)) + case (let sourceFilePath?, nil): + return (requestTemplate, URL(fileURLWithPath: sourceFilePath)) + case (nil, let namePath?): + return (requestTemplate, URL(fileURLWithPath: namePath)) + case (nil, nil): + throw GenericError("Failed to find key.sourcefile or key.name in the request") + } +} + private func extractCompilerArguments( from requestTemplate: String ) throws -> (template: String, compilerArgs: [String]) { @@ -160,9 +214,6 @@ private func extractCompilerArguments( } let template = lines[...compilerArgsStartIndex] + ["$COMPILER_ARGS"] + lines[compilerArgsEndIndex...] let compilerArgsJson = "[" + lines[(compilerArgsStartIndex + 1).. SourceKitDRequestResult { + var arguments = [ + ProcessInfo.processInfo.arguments[0], + "debug", + "run-sourcekitd-request", + "--sourcekitd", + try sourcekitd.filePath, + ] + if let pluginPaths { + arguments += [ + "--sourcekit-plugin-path", + try pluginPaths.servicePlugin.filePath, + "--sourcekit-client-plugin-path", + try pluginPaths.clientPlugin.filePath, + ] + } + try request.fileContents.write(to: temporarySourceFile, atomically: true, encoding: .utf8) - let requestString = try request.request(for: temporarySourceFile) - try requestString.write(to: temporaryRequestFile, atomically: true, encoding: .utf8) - - let process = Process( - arguments: [ - ProcessInfo.processInfo.arguments[0], - "debug", - "run-sourcekitd-request", - "--sourcekitd", - try sourcekitd.filePath, + let requestStrings = try request.requests(for: temporarySourceFile) + for (index, requestString) in requestStrings.enumerated() { + let temporaryRequestFile = temporaryDirectory.appendingPathComponent("request-\(index).yml") + try requestString.write( + to: temporaryRequestFile, + atomically: true, + encoding: .utf8 + ) + arguments += [ "--request-file", try temporaryRequestFile.filePath, ] - ) - try process.launch() - let result = try await process.waitUntilExit() + } + let result = try await Process.run(arguments: arguments, workingDirectory: nil) return requestResult(for: result) } } diff --git a/Sources/Diagnose/SourceReducer.swift b/Sources/Diagnose/SourceReducer.swift index aed94b59e..017784dbc 100644 --- a/Sources/Diagnose/SourceReducer.swift +++ b/Sources/Diagnose/SourceReducer.swift @@ -269,6 +269,7 @@ fileprivate class SourceReducer { let reducedRequestInfo = RequestInfo( requestTemplate: requestInfo.requestTemplate, + contextualRequestTemplates: requestInfo.contextualRequestTemplates, offset: adjustedOffset, compilerArgs: requestInfo.compilerArgs, fileContents: reducedSource @@ -632,6 +633,7 @@ fileprivate func getSwiftInterface( """ let requestInfo = RequestInfo( requestTemplate: requestTemplate, + contextualRequestTemplates: [], offset: 0, compilerArgs: compilerArgs, fileContents: "" diff --git a/Sources/Diagnose/Toolchain+PluginPaths.swift b/Sources/Diagnose/Toolchain+PluginPaths.swift new file mode 100644 index 000000000..bcf673f17 --- /dev/null +++ b/Sources/Diagnose/Toolchain+PluginPaths.swift @@ -0,0 +1,23 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import SourceKitD +import ToolchainRegistry + +extension Toolchain { + var pluginPaths: PluginPaths? { + guard let sourceKitClientPlugin, let sourceKitServicePlugin else { + return nil + } + return PluginPaths(clientPlugin: sourceKitClientPlugin, servicePlugin: sourceKitServicePlugin) + } +} diff --git a/Sources/SKUtilities/CMakeLists.txt b/Sources/SKUtilities/CMakeLists.txt index 9d29ca202..ec86a7709 100644 --- a/Sources/SKUtilities/CMakeLists.txt +++ b/Sources/SKUtilities/CMakeLists.txt @@ -23,4 +23,4 @@ target_compile_options(SKUtilitiesForPlugin PRIVATE target_link_libraries(SKUtilitiesForPlugin PRIVATE SKLoggingForPlugin SwiftExtensionsForPlugin - $<$>:Foundation>) \ No newline at end of file + $<$>:Foundation>) diff --git a/Tests/DiagnoseTests/DiagnoseTests.swift b/Tests/DiagnoseTests/DiagnoseTests.swift index 9e7522250..ead060814 100644 --- a/Tests/DiagnoseTests/DiagnoseTests.swift +++ b/Tests/DiagnoseTests/DiagnoseTests.swift @@ -314,14 +314,21 @@ private class InProcessSourceKitRequestExecutor: SourceKitRequestExecutor { func runSourceKitD(request: RequestInfo) async throws -> SourceKitDRequestResult { try request.fileContents.write(to: temporarySourceFile, atomically: true, encoding: .utf8) - let requestString = try request.request(for: temporarySourceFile) - logger.info("Sending request: \(requestString)") + let requestStrings = try request.requests(for: temporarySourceFile) + logger.info("Sending request: \(requestStrings.joined(separator: "\n\n\n\n"))") let sourcekitd = try await SourceKitD.getOrCreate( dylibPath: sourcekitd, pluginPaths: sourceKitPluginPaths ) - let response = try await sourcekitd.run(requestYaml: requestString) + var response: SKDResponse? = nil + for requestString in requestStrings { + response = try await sourcekitd.run(requestYaml: requestString) + } + guard let response else { + logger.error("No request executed") + return .error + } logger.info("Received response: \(response.description)")