From 42e0766567547b01084b5f383b86d076b7a4c246 Mon Sep 17 00:00:00 2001 From: mrange Date: Tue, 20 Jan 2015 19:11:13 +0100 Subject: [PATCH] Implements extended if grammar Origin: https://fslang.uservoice.com/forums/245727-f-language/suggestions/6079342-allow-extended-if-grammar This commit adds the possibility to write condtional compilation expressions like this: #if SILVERLIGHT || NETFX #endif or a bit more advanced: #if (SILVERLIGHT || NETFX) && COMPILED || !DEBUG #endif This commit doesn't add #elif which is an orthogonal problem. --- .gitignore | 9 + .../assemblyinfo.FSharp.Compiler.dll.fs | 3 + src/fsharp-library-unittests-build.proj | 1 + src/fsharp.sln | 6 + src/fsharp/FSComp.txt | 6 +- .../FSharp.Compiler-proto.fsproj | 12 + .../FSharp.Compiler.Unittests.fsproj | 82 ++++++ .../HashIfExpression.fs | 268 ++++++++++++++++++ .../FSharp.Compiler/FSharp.Compiler.fsproj | 16 ++ .../FSharp.Compiler/InternalsVisibleTo.fs | 1 + .../FSharp.LanguageService.Compiler.fsproj | 14 +- src/fsharp/ast.fs | 14 +- src/fsharp/lex.fsl | 32 ++- src/fsharp/pplex.fsl | 70 +++++ src/fsharp/pppars.fsy | 61 ++++ tests/RunTests.cmd | 20 ++ .../ConditionalCompilation/E_MustBeIdent01.fs | 5 +- .../ExtendedIfGrammar.fs | 182 ++++++++++++ .../ConditionalCompilation/env.lst | 2 +- .../Tests.LanguageService.Colorizer.fs | 32 ++- .../Tests.LanguageService.ErrorList.fs | 8 - .../Tests.LanguageService.Squiggles.fs | 6 +- 22 files changed, 815 insertions(+), 35 deletions(-) create mode 100644 src/fsharp/FSharp.Compiler.Unittests/FSharp.Compiler.Unittests.fsproj create mode 100644 src/fsharp/FSharp.Compiler.Unittests/HashIfExpression.fs create mode 100644 src/fsharp/pplex.fsl create mode 100644 src/fsharp/pppars.fsy create mode 100644 tests/fsharpqa/Source/Conformance/LexicalAnalysis/ConditionalCompilation/ExtendedIfGrammar.fs diff --git a/.gitignore b/.gitignore index e60f09fbfd7..4c2cb5c3dac 100644 --- a/.gitignore +++ b/.gitignore @@ -45,12 +45,18 @@ src/fsharp/FSharp.Compiler/ilpars.fsi src/fsharp/FSharp.Compiler/lex.fs src/fsharp/FSharp.Compiler/pars.fs src/fsharp/FSharp.Compiler/pars.fsi +src/fsharp/FSharp.Compiler/pplex.fs +src/fsharp/FSharp.Compiler/pppars.fs +src/fsharp/FSharp.Compiler/pppars.fsi src/fsharp/FSharp.Compiler-proto/illex.fs src/fsharp/FSharp.Compiler-proto/ilpars.fs src/fsharp/FSharp.Compiler-proto/ilpars.fsi src/fsharp/FSharp.Compiler-proto/lex.fs src/fsharp/FSharp.Compiler-proto/pars.fs src/fsharp/FSharp.Compiler-proto/pars.fsi +src/fsharp/FSharp.Compiler-proto/pplex.fs +src/fsharp/FSharp.Compiler-proto/pppars.fs +src/fsharp/FSharp.Compiler-proto/pppars.fsi *~ tests/projects/Sample_VS2012_FSharp_ConsoleApp_net45_with_resource/Sample_VS2012_FSharp_ConsoleApp_net45/Sample_VS2012_FSharp_ConsoleApp_net45.sln tests/projects/Sample_VS2012_FSharp_ConsoleApp_net45_with_resource/Sample_VS2012_FSharp_ConsoleApp_net45/Sample_VS2012_FSharp_ConsoleApp_net45.userprefs @@ -68,6 +74,9 @@ src/fsharp/FSharp.LanguageService.Compiler/illex.* src/fsharp/FSharp.LanguageService.Compiler/ilpars.* src/fsharp/FSharp.LanguageService.Compiler/lex.* src/fsharp/FSharp.LanguageService.Compiler/pars.* +src/fsharp/FSharp.LanguageService.Compiler/pplex.fs +src/fsharp/FSharp.LanguageService.Compiler/pppars.fs +src/fsharp/FSharp.LanguageService.Compiler/pppars.fsi tests/fsharp/typecheck/sigs/*.dll tests/fsharp/typecheck/sigs/*.exe vsintegration/src/unittests/Unittests.fsi diff --git a/src/assemblyinfo/assemblyinfo.FSharp.Compiler.dll.fs b/src/assemblyinfo/assemblyinfo.FSharp.Compiler.dll.fs index 957bb6ca5fa..990a59b48d7 100644 --- a/src/assemblyinfo/assemblyinfo.FSharp.Compiler.dll.fs +++ b/src/assemblyinfo/assemblyinfo.FSharp.Compiler.dll.fs @@ -22,6 +22,7 @@ open System.Reflection // Note: internals visible to unit test DLLs in Retail (and all) builds. [] +[] [] [] [] @@ -45,6 +46,7 @@ open System.Reflection [] [] [] +[] #endif #if STRONG_NAME_FSHARP_COMPILER_WITH_TEST_KEY @@ -62,6 +64,7 @@ open System.Reflection [] [] [] +[] #endif diff --git a/src/fsharp-library-unittests-build.proj b/src/fsharp-library-unittests-build.proj index 202385b1fe3..8f3d0e6433a 100644 --- a/src/fsharp-library-unittests-build.proj +++ b/src/fsharp-library-unittests-build.proj @@ -9,6 +9,7 @@ + diff --git a/src/fsharp.sln b/src/fsharp.sln index 5c92f68583f..0e9eb059b59 100644 --- a/src/fsharp.sln +++ b/src/fsharp.sln @@ -31,6 +31,8 @@ Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "FSharp.Data.TypeProviders", EndProject Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "FsiAnyCPU", "fsharp\fsiAnyCpu\FsiAnyCPU.fsproj", "{8B3E283D-B5FE-4055-9D80-7E3A32F3967B}" EndProject +Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "FSharp.Compiler.Unittests", "fsharp\FSharp.Compiler.Unittests\FSharp.Compiler.Unittests.fsproj", "{A8D9641A-9170-4CF4-8FE0-6DB8C134E1B5}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -75,6 +77,10 @@ Global {CB7D20C4-6506-406D-9144-5342C3595F03}.Release|Any CPU.Build.0 = Release|Any CPU {8B3E283D-B5FE-4055-9D80-7E3A32F3967B}.Debug|Any CPU.ActiveCfg = Debug|x86 {8B3E283D-B5FE-4055-9D80-7E3A32F3967B}.Release|Any CPU.ActiveCfg = Release|x86 + {A8D9641A-9170-4CF4-8FE0-6DB8C134E1B5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A8D9641A-9170-4CF4-8FE0-6DB8C134E1B5}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A8D9641A-9170-4CF4-8FE0-6DB8C134E1B5}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A8D9641A-9170-4CF4-8FE0-6DB8C134E1B5}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/src/fsharp/FSComp.txt b/src/fsharp/FSComp.txt index 1c400dd7767..1c4aff6391c 100644 --- a/src/fsharp/FSComp.txt +++ b/src/fsharp/FSComp.txt @@ -1064,7 +1064,7 @@ lexHashEndingNoMatchingIf,"#endif has no matching #if" 1169,lexHashIfMustHaveIdent,"#if directive should be immediately followed by an identifier" 1170,lexWrongNestedHashEndif,"Syntax error. Wrong nested #endif, unexpected tokens before it." lexHashBangMustBeFirstInFile,"#! may only appear as the first line at the start of a file." -1171,lexExpectedSingleLineComment,"Expected single line comment or end of line" +1171,pplexExpectedSingleLineComment,"Expected single line comment or end of line" 1172,memberOperatorDefinitionWithNoArguments,"Infix operator member '%s' has no arguments. Expected a tuple of 2 arguments, e.g. static member (+) (x,y) = ..." 1173,memberOperatorDefinitionWithNonPairArgument,"Infix operator member '%s' has %d initial argument(s). Expected a tuple of 2 arguments, e.g. static member (+) (x,y) = ..." 1174,memberOperatorDefinitionWithCurriedArguments,"Infix operator member '%s' has extra curried arguments. Expected a tuple of 2 arguments, e.g. static member (+) (x,y) = ..." @@ -1336,3 +1336,7 @@ descriptionUnavailable,"(description unavailable...)" 3180,abImplicitHeapAllocation,"The mutable local '%s' is implicitly allocated as a reference cell because it has been captured by a closure. This warning is for informational purposes only to indicate where implicit allocations are performed." estApplyStaticArgumentsForMethodNotImplemented,"A type provider implemented GetStaticParametersForMethod, but ApplyStaticArgumentsForMethod was not implemented or invalid" 3181,etErrorApplyingStaticArgumentsToMethod,"An error occured applying the static arguments to a provided method" +3182,pplexUnexpectedChar,"Unexpected character '%s' in preprocessor expression" +3183,ppparsUnexpectedToken,"Unexpected token '%s' in preprocessor expression" +3184,ppparsIncompleteExpression,"Incomplete preprocessor expression" +3185,ppparsMissingToken,"Missing token '%s' in preprocessor expression" diff --git a/src/fsharp/FSharp.Compiler-proto/FSharp.Compiler-proto.fsproj b/src/fsharp/FSharp.Compiler-proto/FSharp.Compiler-proto.fsproj index 4f2d5186657..747e4aaa64b 100644 --- a/src/fsharp/FSharp.Compiler-proto/FSharp.Compiler-proto.fsproj +++ b/src/fsharp/FSharp.Compiler-proto/FSharp.Compiler-proto.fsproj @@ -22,6 +22,16 @@ FSComp.txt + + --lexlib Internal.Utilities.Text.Lexing + pplex.fsl + + + Microsoft.FSharp.Compiler.PPParser + Microsoft.FSharp.Compiler + --internal --lexlib Internal.Utilities.Text.Lexing --parslib Internal.Utilities.Text.Parsing + pppars.fsy + --lexlib Internal.Utilities.Text.Lexing lex.fsl @@ -250,6 +260,7 @@ ast.fs + lexhelp.fsi @@ -257,6 +268,7 @@ lexhelp.fs + sreflect.fsi diff --git a/src/fsharp/FSharp.Compiler.Unittests/FSharp.Compiler.Unittests.fsproj b/src/fsharp/FSharp.Compiler.Unittests/FSharp.Compiler.Unittests.fsproj new file mode 100644 index 00000000000..f48ce0fcbf3 --- /dev/null +++ b/src/fsharp/FSharp.Compiler.Unittests/FSharp.Compiler.Unittests.fsproj @@ -0,0 +1,82 @@ + + + + + ..\.. + {a8d9641a-9170-4cf4-8fe0-6db8c134e1b5} + + + + Debug + AnyCPU + 2.0 + true + true + Library + FSharp.Compiler.Unittests + v3.5 + SystematicUnitTests + + false + false + netcore + + + {CandidateAssemblyFiles}; + {TargetFrameworkDirectory}; + {Registry:Software\Microsoft\.NETFramework,v4.5,AssemblyFoldersEx}; + + + + $(DefineConstants);SILVERLIGHT + $(DefineConstants);EXTENSIONTYPING + + + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 3 + + + pdbonly + true + bin\Release\ + TRACE + prompt + 3 + + + + + true + + + + + + + + + + + + + + + + + + FSharp.Compiler + {2e4d67b4-522d-4cf7-97e4-ba940f0b18f3} + + + FSharp.Core + {ded3bbd7-53f4-428a-8c9f-27968e768605} + True + + + + \ No newline at end of file diff --git a/src/fsharp/FSharp.Compiler.Unittests/HashIfExpression.fs b/src/fsharp/FSharp.Compiler.Unittests/HashIfExpression.fs new file mode 100644 index 00000000000..cf70addb330 --- /dev/null +++ b/src/fsharp/FSharp.Compiler.Unittests/HashIfExpression.fs @@ -0,0 +1,268 @@ +// Copyright (c) Microsoft Open Technologies, Inc. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace FSharp.Compiler.Unittests + +open System +open System.Text + +open NUnit.Framework + +open Internal.Utilities.Text.Lexing +open Microsoft.FSharp.Compiler +open Microsoft.FSharp.Compiler.Lexer +open Microsoft.FSharp.Compiler.Lexhelp +open Microsoft.FSharp.Compiler.ErrorLogger +open Microsoft.FSharp.Compiler.ErrorLogger +open Microsoft.FSharp.Compiler.Ast + +[] +type HashIfExpression() = + + let preludes = [|"#if "; "#elif "|] + let epilogues = [|""; " // Testing"|] + + let ONE = IfdefId "ONE" + let TWO = IfdefId "TWO" + let THREE = IfdefId "THREE" + + let isSet l r = (l &&& r) <> 0 + + let (!!) e = IfdefNot(e) + let (&&&) l r = IfdefAnd(l,r) + let (|||) l r = IfdefOr(l,r) + + let mutable tearDown = fun () -> () + + let exprAsString (e : LexerIfdefExpression) : string = + let sb = StringBuilder() + let append (s : string) = ignore <| sb.Append s + let rec build (e : LexerIfdefExpression) : unit = + match e with + | IfdefAnd (l,r)-> append "("; build l; append " && "; build r; append ")" + | IfdefOr (l,r) -> append "("; build l; append " || "; build r; append ")" + | IfdefNot ee -> append "!"; build ee + | IfdefId nm -> append nm + + build e + + sb.ToString () + + let createParser () = + let errors = ResizeArray() + let warnings = ResizeArray() + + let errorLogger = + { + new ErrorLogger("TestErrorLogger") with + member x.WarnSinkImpl(e) = warnings.Add e + member x.ErrorSinkImpl(e) = errors.Add e + member x.ErrorCount = errors.Count + } + + let stack : LexerIfdefStack = ref [] + let lightSyntax = LightSyntaxStatus(true, false) + let resourceManager = LexResourceManager () + let defines = [] + let startPos = Position.Empty + let args = mkLexargs ("dummy", defines, lightSyntax, resourceManager, stack, errorLogger) + + CompileThreadStatic.ErrorLogger <- errorLogger + + let parser (s : string) = + let lexbuf = LexBuffer.FromChars (s.ToCharArray ()) + lexbuf.StartPos <- startPos + lexbuf.EndPos <- startPos + let tokenStream = PPLexer.tokenstream args + + PPParser.start tokenStream lexbuf + + errors, warnings, parser + + [] + member this.Setup() = + let el = CompileThreadStatic.ErrorLogger + tearDown <- + fun () -> + CompileThreadStatic.BuildPhase <- BuildPhase.DefaultPhase + CompileThreadStatic.ErrorLogger <- el + + CompileThreadStatic.BuildPhase <- BuildPhase.Compile + + [] + member this.TearDown() = + tearDown () + + [] + member this.PositiveParserTestCases()= + + let errors, warnings, parser = createParser () + + let positiveTestCases = + [| + "ONE" , ONE + "ONE//" , ONE + "ONE // Comment" , ONE + "!ONE" , !!ONE + "!!ONE" , !! (!!ONE) + "DEBUG" , (IfdefId "DEBUG") + "!DEBUG" , !! (IfdefId "DEBUG") + "O_s1" , IfdefId "O_s1" + "(ONE)" , (ONE) + "ONE&&TWO" , ONE &&& TWO + "ONE||TWO" , ONE ||| TWO + "( ONE && TWO )" , ONE &&& TWO + "ONE && TWO && THREE" , (ONE &&& TWO) &&& THREE + "ONE || TWO || THREE" , (ONE ||| TWO) ||| THREE + "ONE || TWO && THREE" , ONE ||| (TWO &&& THREE) + "ONE && TWO || THREE" , (ONE &&& TWO) ||| THREE + "ONE || (TWO && THREE)" , ONE ||| (TWO &&& THREE) + "ONE && (TWO || THREE)" , ONE &&& (TWO ||| THREE) + "!ONE || TWO && THREE" , (!!ONE) ||| (TWO &&& THREE) + "ONE && !TWO || THREE" , (ONE &&& (!!TWO)) ||| THREE + "ONE || !(TWO && THREE)" , ONE ||| (!!(TWO &&& THREE)) + "true" , IfdefId "true" + "false" , IfdefId "false" + |] + + let failures = ResizeArray () + let fail = failures.Add + + for test,expected in positiveTestCases do + for prelude in preludes do + let test = prelude + test + for epilogue in epilogues do + let test = test + epilogue + try + let expr = parser test + + if expected <> expr then + fail <| sprintf "'%s', expected %A, actual %A" test (exprAsString expected) (exprAsString expr) + with + | e -> fail <| sprintf "'%s', expected %A, actual %s,%A" test (exprAsString expected) (e.GetType().Name) e.Message + + + let fs = + failures + |> Seq.append (errors |> Seq.map (fun pe -> pe.DebugDisplay ())) + |> Seq.append (warnings |> Seq.map (fun pe -> pe.DebugDisplay ())) + |> Seq.toArray + + let failure = String.Join ("\n", fs) + + Assert.AreEqual("", failure) + + () + + [] + member this.NegativeParserTestCases()= + + let errors, warnings, parser = createParser () + + let negativeTests = + [| + "" + "!" + "&&" + "||" + "@" + "ONE ONE" + "ONE@" + "@ONE" + "$" + "ONE$" + "$ONE" + "ONE!" + "(ONE" + "ONE)" + // TODO: Investigate why this raises a parse failure + // "(ONE ||)" + "ONE&&" + "ONE ||" + "&& ONE" + "||ONE" + "ONE TWO" + "ONE(* Comment" + "ONE(* Comment *)" + "ONE(**)" + "ONE (* Comment" + "ONE (* Comment *)" + "ONE (**)" + "ONE )(@$&%*@^#%#!$)" + |] + + let failures = ResizeArray () + let fail = failures.Add + + for test in negativeTests do + for prelude in preludes do + let test = prelude + test + for epilogue in epilogues do + let test = test + epilogue + try + let bec = errors.Count + let expr = parser test + let aec = errors.Count + + if bec = aec then // No new errors discovered + fail <| sprintf "'%s', expected 'parse error', actual %A" test (exprAsString expr) + with + | e -> fail <| sprintf "'%s', expected 'parse error', actual %s,%A" test (e.GetType().Name) e.Message + + let fs = failures |> Seq.toArray + + let fails = String.Join ("\n", fs) + + Assert.AreEqual("", fails) + + [] + member this.LexerIfdefEvalTestCases()= + + let failures = ResizeArray () + let fail = failures.Add + + for i in 0..7 do + let one = isSet i 1 + let two = isSet i 2 + let three = isSet i 4 + + let lookup s = + match s with + | "ONE" -> one + | "TWO" -> two + | "THREE" -> three + | _ -> false + + let testCases = + [| + ONE , one + !!ONE , not one + !! (!!ONE) , not (not one) + TWO , two + !!TWO , not two + !! (!!TWO) , not (not two) + ONE &&& TWO , one && two + ONE ||| TWO , one || two + (ONE &&& TWO) &&& THREE , (one && two) && three + (ONE ||| TWO) ||| THREE , (one || two) || three + ONE ||| (TWO &&& THREE) , one || (two && three) + (ONE &&& TWO) ||| THREE , (one && two) || three + ONE ||| (TWO &&& THREE) , one || (two && three) + ONE &&& (TWO ||| THREE) , one && (two || three) + (!!ONE) ||| (TWO &&& THREE) , (not one) || (two && three) + (ONE &&& (!!TWO)) ||| THREE , (one && (not two)) || three + ONE ||| (!!(TWO &&& THREE)) , one || (not (two && three)) + |] + + let eval = LexerIfdefEval lookup + for expr, expected in testCases do + let actual = eval expr + + if actual <> expected then + fail <| sprintf "For ONE=%A, TWO=%A, THREE=%A the expression %A is expected to be %A but was %A" one two three (exprAsString expr) expected actual + + + let fs = failures |> Seq.toArray + + let fails = String.Join ("\n", fs) + + Assert.AreEqual("", fails) diff --git a/src/fsharp/FSharp.Compiler/FSharp.Compiler.fsproj b/src/fsharp/FSharp.Compiler/FSharp.Compiler.fsproj index 41f5c106d69..b2e381c0022 100644 --- a/src/fsharp/FSharp.Compiler/FSharp.Compiler.fsproj +++ b/src/fsharp/FSharp.Compiler/FSharp.Compiler.fsproj @@ -247,6 +247,16 @@ ILXErase\cu_erase.fs + + --lexlib Internal.Utilities.Text.Lexing + ParserAndUntypedAST\pplex.fsl + + + Microsoft.FSharp.Compiler.PPParser + Microsoft.FSharp.Compiler + --internal --lexlib Internal.Utilities.Text.Lexing --parslib Internal.Utilities.Text.Parsing + ParserAndUntypedAST\pppars.fsy + --lexlib Internal.Utilities.Text.Lexing ParserAndUntypedAST\lex.fsl @@ -272,6 +282,9 @@ ParserAndUntypedAST\ast.fs + + ParserAndUntypedAST\pppars.fs + ParserAndUntypedAST\pars.fs @@ -281,6 +294,9 @@ ParserAndUntypedAST\lexhelp.fs + + ParserAndUntypedAST\pplex.fs + ParserAndUntypedAST\lex.fs diff --git a/src/fsharp/FSharp.Compiler/InternalsVisibleTo.fs b/src/fsharp/FSharp.Compiler/InternalsVisibleTo.fs index b77e12a0a12..f0d5e127e65 100644 --- a/src/fsharp/FSharp.Compiler/InternalsVisibleTo.fs +++ b/src/fsharp/FSharp.Compiler/InternalsVisibleTo.fs @@ -18,6 +18,7 @@ open System.Reflection [] [] [] +[] do() diff --git a/src/fsharp/FSharp.LanguageService.Compiler/FSharp.LanguageService.Compiler.fsproj b/src/fsharp/FSharp.LanguageService.Compiler/FSharp.LanguageService.Compiler.fsproj index 3309752329d..8bc00ff3d9f 100644 --- a/src/fsharp/FSharp.LanguageService.Compiler/FSharp.LanguageService.Compiler.fsproj +++ b/src/fsharp/FSharp.LanguageService.Compiler/FSharp.LanguageService.Compiler.fsproj @@ -36,6 +36,16 @@ assemblyinfo.FSharp.Compiler.dll.fs + + --lexlib Internal.Utilities.Text.Lexing + pplex.fsl + + + Microsoft.FSharp.Compiler.PPParser + Microsoft.FSharp.Compiler + --internal --lexlib Internal.Utilities.Text.Lexing --parslib Internal.Utilities.Text.Parsing + pppars.fsy + --lexlib Internal.Utilities.Text.Lexing lex.fsl @@ -267,6 +277,8 @@ lexhelp.fs + + sreflect.fsi @@ -459,7 +471,7 @@ - + diff --git a/src/fsharp/ast.fs b/src/fsharp/ast.fs index 4222e3b6698..3af95cf757c 100644 --- a/src/fsharp/ast.fs +++ b/src/fsharp/ast.fs @@ -2016,7 +2016,19 @@ type LexerEndlineContinuation = match x with | LexerEndlineContinuation.Token(ifd) | LexerEndlineContinuation.Skip(ifd, _, _) -> ifd - + +type LexerIfdefExpression = + | IfdefAnd of LexerIfdefExpression*LexerIfdefExpression + | IfdefOr of LexerIfdefExpression*LexerIfdefExpression + | IfdefNot of LexerIfdefExpression + | IfdefId of string + +let rec LexerIfdefEval (lookup : string -> bool) = function + | IfdefAnd (l,r) -> (LexerIfdefEval lookup l) && (LexerIfdefEval lookup r) + | IfdefOr (l,r) -> (LexerIfdefEval lookup l) || (LexerIfdefEval lookup r) + | IfdefNot e -> not (LexerIfdefEval lookup e) + | IfdefId id -> lookup id + /// The parser defines a number of tokens for whitespace and /// comments eliminated by the lexer. These carry a specification of /// a continuation for the lexer for continued processing after we've dealt with diff --git a/src/fsharp/lex.fsl b/src/fsharp/lex.fsl index 36234663702..2c37ee00d93 100644 --- a/src/fsharp/lex.fsl +++ b/src/fsharp/lex.fsl @@ -149,15 +149,17 @@ let shouldStartFile args lexbuf (m:range) err tok = if (m.StartColumn <> 0 || m.StartLine <> 1) then fail args lexbuf err tok else tok -let extractIdentFromHashIf (lexed:string) = - // Skip the '#if' token, then trim whitespace, then find the end of the identifier - let lexed = lexed.Trim() - let trimIf = lexed.Substring(3).Trim() - let identEnd = trimIf.IndexOfAny([| ' '; '\t'; '/' |]) - let identEnd = (if identEnd = -1 then trimIf.Length else identEnd) - trimIf.Substring(0, identEnd) -} +let evalIfDefExpression startPos args (lookup:string->bool) (lexed:string) = + let lexbuf = LexBuffer.FromChars (lexed.ToCharArray ()) + lexbuf.StartPos <- startPos + lexbuf.EndPos <- startPos + let tokenStream = Microsoft.FSharp.Compiler.PPLexer.tokenstream args + let expr = Microsoft.FSharp.Compiler.PPParser.start tokenStream lexbuf + + LexerIfdefEval lookup expr + +} let letter = '\Lu' | '\Ll' | '\Lt' | '\Lm' | '\Lo' | '\Nl' let surrogateChar = '\Cs' let digit = '\Nd' @@ -165,6 +167,8 @@ let hex = ['0'-'9'] | ['A'-'F'] | ['a'-'f'] let truewhite = [' '] let offwhite = ['\t'] let anywhite = truewhite | offwhite +let anychar = [^'\n''\r'] +let anystring = anychar* let op_char = '!'|'$'|'%'|'&'|'*'|'+'|'-'|'.'|'/'|'<'|'='|'>'|'?'|'@'|'^'|'|'|'~'|':' let ignored_op_char = '.' | '$' | '?' let xinteger = @@ -597,15 +601,16 @@ rule token args skip = parse mlCompatWarning (FSComp.SR.lexIndentOffForML()) lexbuf.LexemeRange; if not skip then (HASH_LIGHT (LexCont.Token !args.ifdefStack)) else token args skip lexbuf } - | anywhite* "#if" anywhite+ ident anywhite* ("//" [^'\n''\r']*)? + | anywhite* "#if" anywhite+ anystring { let m = lexbuf.LexemeRange + let lookup id = List.mem id args.defines let lexed = lexeme lexbuf - let id = extractIdentFromHashIf lexed + let isTrue = evalIfDefExpression lexbuf.StartPos args lookup lexed args.ifdefStack := (IfDefIf,m) :: !(args.ifdefStack); // Get the token; make sure it starts at zero position & return let cont, f = - ( if List.mem id args.defines then (LexCont.EndLine(LexerEndlineContinuation.Token(!args.ifdefStack)), endline (LexerEndlineContinuation.Token !args.ifdefStack) args skip) + ( if isTrue then (LexCont.EndLine(LexerEndlineContinuation.Token(!args.ifdefStack)), endline (LexerEndlineContinuation.Token !args.ifdefStack) args skip) else (LexCont.EndLine(LexerEndlineContinuation.Skip(!args.ifdefStack,0,m)), endline (LexerEndlineContinuation.Skip(!args.ifdefStack,0,m)) args skip) ) let tok = shouldStartLine args lexbuf m (FSComp.SR.lexHashIfMustBeFirst()) (HASH_IF(m,lexed,cont)) if not skip then tok else f lexbuf } @@ -646,9 +651,8 @@ rule token args skip = parse // Skips INACTIVE code until if finds #else / #endif matching with the #if or #else and ifdefSkip n m args skip = parse - | anywhite* "#if" anywhite+ ident anywhite* ("//" [^'\n''\r']*)? + | anywhite* "#if" anywhite+ anystring { let m = lexbuf.LexemeRange - let _id = extractIdentFromHashIf (lexeme lexbuf) // If #if is the first thing on the line then increase depth, otherwise skip, because it is invalid (e.g. "(**) #if ...") if (m.StartColumn <> 0) then @@ -721,7 +725,7 @@ and endline cont args skip = parse } | [^'\r' '\n']+ | _ - { let tok = fail args lexbuf (FSComp.SR.lexExpectedSingleLineComment()) (WHITESPACE (LexCont.Token !args.ifdefStack)) + { let tok = fail args lexbuf (FSComp.SR.pplexExpectedSingleLineComment()) (WHITESPACE (LexCont.Token !args.ifdefStack)) if not skip then tok else token args skip lexbuf } and string sargs skip = parse diff --git a/src/fsharp/pplex.fsl b/src/fsharp/pplex.fsl new file mode 100644 index 00000000000..8fe85ee7dea --- /dev/null +++ b/src/fsharp/pplex.fsl @@ -0,0 +1,70 @@ +// Copyright (c) Microsoft Open Technologies, Inc. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +{ + +module internal Microsoft.FSharp.Compiler.PPLexer + +open System + +open Microsoft.FSharp.Compiler.Ast +open Microsoft.FSharp.Compiler.ErrorLogger +open Microsoft.FSharp.Compiler.Lexhelp + +open Internal.Utilities.Text.Lexing + +let lexeme (lexbuf : UnicodeLexing.Lexbuf) = UnicodeLexing.Lexbuf.LexemeString lexbuf + +let fail (args : lexargs) (lexbuf:UnicodeLexing.Lexbuf) e = + let m = lexbuf.LexemeRange + args.errorLogger.ErrorR(Error(e,m)) + PPParser.EOF +} + +let letter = '\Lu' | '\Ll' | '\Lt' | '\Lm' | '\Lo' | '\Nl' +let digit = '\Nd' +let connecting_char = '\Pc' +let combining_char = '\Mn' | '\Mc' +let formatting_char = '\Cf' + +let ident_start_char = + letter | '_' + +let ident_char = + letter + | connecting_char + | combining_char + | formatting_char + | digit + | ['\''] + +let ident = ident_start_char ident_char* +let comment = "//" _* +let mcomment = "(*" _* +let whitespace = [' ' '\t'] + +rule tokenstream args = parse +// -------------------------- +| "#if" { PPParser.PRELUDE } +| "#elif" { PPParser.PRELUDE } +| ident { PPParser.ID(lexeme lexbuf) } +// -------------------------- +| "!" { PPParser.OP_NOT } +| "&&" { PPParser.OP_AND } +| "||" { PPParser.OP_OR } +| "(" { PPParser.LPAREN } +| ")" { PPParser.RPAREN } +// -------------------------- +| whitespace { tokenstream args lexbuf } +// -------------------------- +| comment { PPParser.EOF } +| mcomment { fail args lexbuf (FSComp.SR.pplexExpectedSingleLineComment()) } +| _ { + let lex = lexeme lexbuf + let _ = rest lexbuf + fail args lexbuf (FSComp.SR.pplexUnexpectedChar(lex)) + } +| eof { PPParser.EOF } +// -------------------------- +and rest = parse +| _ { rest lexbuf } +| eof { () } diff --git a/src/fsharp/pppars.fsy b/src/fsharp/pppars.fsy new file mode 100644 index 00000000000..0f76836c366 --- /dev/null +++ b/src/fsharp/pppars.fsy @@ -0,0 +1,61 @@ +// Copyright (c) Microsoft Open Technologies, Inc. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +%{ +open Microsoft.FSharp.Compiler.Ast +open Microsoft.FSharp.Compiler.ErrorLogger + +let dummy = IfdefId("DUMMY") + +let doNothing _ dflt= + dflt + +let fail (ps : Internal.Utilities.Text.Parsing.IParseState) i e = + let f,t = ps.InputRange i + let m = mkSynRange f t + errorR(Error(e,m)) + dummy +%} + + +%start start + +%token ID +%token OP_NOT OP_AND OP_OR LPAREN RPAREN PRELUDE EOF + +%nonassoc RPAREN +%nonassoc PRELUDE +%left OP_OR +%left OP_AND +%left OP_NOT +%nonassoc LPAREN +%nonassoc ID + +%type < LexerIfdefExpression > start + +%% + +start: Full { $1 } + +Recover: + | error { doNothing parseState () } + +Full: + | PRELUDE Expr EOF { $2 } + | Recover { fail parseState 1 (FSComp.SR.ppparsMissingToken("#if/#elif")) } + +Expr: + | LPAREN Expr RPAREN { $2 } + | ID { IfdefId($1) } + | OP_NOT Expr { IfdefNot($2) } + | Expr OP_AND Expr { IfdefAnd($1,$3) } + | Expr OP_OR Expr { IfdefOr($1,$3) } + + | OP_AND Recover { fail parseState 1 (FSComp.SR.ppparsUnexpectedToken("&&")) } + | OP_OR Recover { fail parseState 1 (FSComp.SR.ppparsUnexpectedToken("||")) } + | OP_NOT Recover { fail parseState 1 (FSComp.SR.ppparsUnexpectedToken("!")) } + | LPAREN error RPAREN { doNothing parseState dummy } + | LPAREN Expr Recover { fail parseState 3 (FSComp.SR.ppparsMissingToken(")")) } + | LPAREN Recover { fail parseState 2 (FSComp.SR.ppparsIncompleteExpression()) } + | RPAREN Recover { fail parseState 1 (FSComp.SR.ppparsUnexpectedToken(")")) } + | Expr Recover { fail parseState 2 (FSComp.SR.ppparsIncompleteExpression()) } + | EOF { fail parseState 1 (FSComp.SR.ppparsIncompleteExpression()) } diff --git a/tests/RunTests.cmd b/tests/RunTests.cmd index 908b2fd8ba8..364c317e30e 100644 --- a/tests/RunTests.cmd +++ b/tests/RunTests.cmd @@ -39,6 +39,10 @@ if not exist "%RESULTSDIR%" (mkdir "%RESULTSDIR%") if /I "%2" == "fsharp" (goto :FSHARP) if /I "%2" == "fsharpqa" (goto :FSHARPQA) +if /I "%2" == "compilerunit" ( + set compilerunitsuffix=net40 + goto :COMPILERUNIT +) if /I "%2" == "coreunit" ( set coreunitsuffix=net40 goto :COREUNIT @@ -178,6 +182,22 @@ echo nunit-console.exe /nologo /result=%XMLFILE% /output=%OUTPUTFILE% /err=%ERRO goto :EOF +:COMPILERUNIT + +set XMLFILE=ComplierUnit_%compilerunitsuffix%_Xml.xml +set OUTPUTFILE=ComplierUnit_%compilerunitsuffix%_Output.log +set ERRORFILE=ComplierUnit_%compilerunitsuffix%_Error.log + +where.exe nunit-console.exe > NUL 2> NUL +if errorlevel 1 ( + echo Error: nunit-console.exe is not in the PATH + exit /b 1 +) +echo nunit-console.exe /nologo /result=%XMLFILE% /output=%OUTPUTFILE% /err=%ERRORFILE% /work=%RESULTSDIR% %FSCBINPATH%\..\..\%compilerunitsuffix%\bin\FSharp.Compiler.Unittests.dll + nunit-console.exe /nologo /result=%XMLFILE% /output=%OUTPUTFILE% /err=%ERRORFILE% /work=%RESULTSDIR% %FSCBINPATH%\..\..\%compilerunitsuffix%\bin\FSharp.Compiler.Unittests.dll + +goto :EOF + :IDEUNIT set XMLFILE=IDEUnit_Xml.xml diff --git a/tests/fsharpqa/Source/Conformance/LexicalAnalysis/ConditionalCompilation/E_MustBeIdent01.fs b/tests/fsharpqa/Source/Conformance/LexicalAnalysis/ConditionalCompilation/E_MustBeIdent01.fs index 1a37d88be02..7c9d079bbfa 100644 --- a/tests/fsharpqa/Source/Conformance/LexicalAnalysis/ConditionalCompilation/E_MustBeIdent01.fs +++ b/tests/fsharpqa/Source/Conformance/LexicalAnalysis/ConditionalCompilation/E_MustBeIdent01.fs @@ -1,6 +1,7 @@ -// #Regression #Conformance #LexicalAnalysis +// #Regression #Conformance #LexicalAnalysis // Verify error if preprocessor directive isn't a valid identifier -//#if directive should be immediately followed by an identifier +//Unexpected character '\*' in preprocessor expression +//Incomplete preprocessor expression #if *COMPILED* exit 0 diff --git a/tests/fsharpqa/Source/Conformance/LexicalAnalysis/ConditionalCompilation/ExtendedIfGrammar.fs b/tests/fsharpqa/Source/Conformance/LexicalAnalysis/ConditionalCompilation/ExtendedIfGrammar.fs new file mode 100644 index 00000000000..b6f8b84c206 --- /dev/null +++ b/tests/fsharpqa/Source/Conformance/LexicalAnalysis/ConditionalCompilation/ExtendedIfGrammar.fs @@ -0,0 +1,182 @@ +// #Conformance #LexicalAnalysis + +let success = 1 +let failure = -1 + +#if DEFINED +let e0 = success +#else +let e0 = failure +#endif + +#if UNDEFINED +let e1 = failure +#else +let e1 = success +#endif + +#if DEFINED && UNDEFINED +let e2 = failure +#else +let e2 = success +#endif + +#if UNDEFINED && DEFINED +let e3 = failure +#else +let e3 = success +#endif + +#if DEFINED || UNDEFINED +let e4 = success +#else +let e4 = failure +#endif + +#if UNDEFINED || DEFINED +let e5 = success +#else +let e5 = failure +#endif + +#if !UNDEFINED +let e6 = success +#else +let e6 = failure +#endif + +#if !DEFINED +let e7 = failure +#else +let e7 = success +#endif + +#if !UNDEFINED || DEFINED +let e8 = success +#else +let e8 = failure +#endif + +#if !DEFINED && DEFINED +let e9 = failure +#else +let e9 = success +#endif + +#if DEFINED && DEFINED && UNDEFINED +let e10 = failure +#else +let e10 = success +#endif + +#if UNDEFINED || UNDEFINED || DEFINED +let e11 = success +#else +let e11 = failure +#endif + +#if DEFINED || DEFINED && UNDEFINED +let e12 = success +#else +let e12 = failure +#endif + +#if UNDEFINED && DEFINED || DEFINED +let e13 = success +#else +let e13 = failure +#endif + +#if (DEFINED) +let e14 = success +#else +let e14 = failure +#endif + +#if (DEFINED || DEFINED) && UNDEFINED +let e15 = failure +#else +let e15 = success +#endif + +#if UNDEFINED && (DEFINED || DEFINED) +let e16 = failure +#else +let e16 = success +#endif + +#if DEFINED // A test comment +let e17 = success +#else +let e17 = failure +#endif + +// When it comes to #if true/false are seen as identifiers not values +#if true +let e18 = failure +#else +let e18 = success +#endif + +#if false +let e19 = failure +#else +let e19 = success +#endif + +#if !!DEFINED +let e20 = success +#else +let e20 = failure +#endif + +#if !!!DEFINED +let e21 = failure +#else +let e21 = success +#endif + +#if !!UNDEFINED +let e22 = failure +#else +let e22 = success +#endif + +#if !!!UNDEFINED +let e23 = success +#else +let e23 = failure +#endif + +let verify r e = if r = success then 0 else e + +let result = + 0 + + verify e0 0x000001 + + verify e1 0x000002 + + verify e2 0x000004 + + verify e3 0x000008 + + verify e4 0x000010 + + verify e5 0x000020 + + verify e6 0x000040 + + verify e7 0x000080 + + verify e8 0x000100 + + verify e9 0x000200 + + verify e10 0x000400 + + verify e11 0x000800 + + verify e12 0x001000 + + verify e13 0x002000 + + verify e14 0x004000 + + verify e15 0x008000 + + verify e16 0x010000 + + verify e17 0x020000 + + verify e18 0x040000 + + verify e19 0x080000 + + verify e20 0x100000 + + verify e21 0x200000 + + verify e22 0x400000 + + verify e23 0x800000 + +if result <> 0 then printfn "ExtendedIfGrammar failed: 0x%X" result + +exit result diff --git a/tests/fsharpqa/Source/Conformance/LexicalAnalysis/ConditionalCompilation/env.lst b/tests/fsharpqa/Source/Conformance/LexicalAnalysis/ConditionalCompilation/env.lst index db60a5ae6ce..8ab90626e01 100644 --- a/tests/fsharpqa/Source/Conformance/LexicalAnalysis/ConditionalCompilation/env.lst +++ b/tests/fsharpqa/Source/Conformance/LexicalAnalysis/ConditionalCompilation/env.lst @@ -19,4 +19,4 @@ SOURCE=FSharp01.fs # FSharp01.fs SOURCE=FSharp02.fs # FSharp02.fs SOURCE=OCaml01.fs # OCaml01.fs - + SOURCE=ExtendedIfGrammar.fs SCFLAGS="--define:DEFINED" # ExtendedIfGrammar.fs diff --git a/vsintegration/src/unittests/Tests.LanguageService.Colorizer.fs b/vsintegration/src/unittests/Tests.LanguageService.Colorizer.fs index e0238c328f9..83e7c65b18e 100644 --- a/vsintegration/src/unittests/Tests.LanguageService.Colorizer.fs +++ b/vsintegration/src/unittests/Tests.LanguageService.Colorizer.fs @@ -985,7 +985,35 @@ let z = __LINE__(*Test3*) check "(*If*)" TokenType.Comment check "(*Else*)" TokenType.Comment check "(*Endif*)" TokenType.Comment - + + /// FEATURE: Preprocessor extended grammar basic check. + /// FEATURE: More extensive grammar test is done in compiler unit tests + [] + member public this.``Preprocessor.ExtendedIfGrammar.Basic01``() = + this.VerifyColorizerAtStartOfMarker( + fileContents = """ + #if UNDEFINED || !UNDEFINED // Extended #if + let x = "activeCode" + #else + let x = "inactiveCode" + #endif + """, + marker = "activeCode", + tokenType = TokenType.String) + + [] + member public this.``Preprocessor.ExtendedIfGrammar.Basic02``() = + this.VerifyColorizerAtStartOfMarker( + fileContents = """ + #if UNDEFINED || !UNDEFINED // Extended #if + let x = "activeCode" + #else + let x = "inactiveCode" + #endif + """, + marker = "inactiveCode", + tokenType = TokenType.InactiveCode) + /// #else / #endif in multiline strings is ignored [] member public this.``Preprocessor.DirectivesInString``() = @@ -1040,7 +1068,7 @@ let z = __LINE__(*Test3*) "#endif"] let (_, _, file) = this.CreateSingleFileProject(code) MoveCursorToStartOfMarker(file, "!!COMPILED") - AssertEqual(TokenType.Operator, GetTokenTypeAtCursor(file)) + AssertEqual(TokenType.Identifier, GetTokenTypeAtCursor(file)) // This was an off-by-one bug in the replacement Colorizer diff --git a/vsintegration/src/unittests/Tests.LanguageService.ErrorList.fs b/vsintegration/src/unittests/Tests.LanguageService.ErrorList.fs index e300da13935..f67faabda08 100644 --- a/vsintegration/src/unittests/Tests.LanguageService.ErrorList.fs +++ b/vsintegration/src/unittests/Tests.LanguageService.ErrorList.fs @@ -579,14 +579,6 @@ but here has type TakeCoffeeBreak(this.VS) // Wait for the background compiler to catch up. VerifyErrorListContainedExpectedStr("nonexistent",project) - [] - member public this.``ErrorReporting.WrongPreprocessorIf``() = - let fileContent = """ - #light - #if !!!!COMPILED - #endif""" - this.VerifyErrorListContainedExpectedString(fileContent,"#if") - [] member public this.``BackgroundComplier``() = this.VerifyErrorListCountAtOpenProject( diff --git a/vsintegration/src/unittests/Tests.LanguageService.Squiggles.fs b/vsintegration/src/unittests/Tests.LanguageService.Squiggles.fs index a70d4d8b033..2c5db4199be 100644 --- a/vsintegration/src/unittests/Tests.LanguageService.Squiggles.fs +++ b/vsintegration/src/unittests/Tests.LanguageService.Squiggles.fs @@ -854,17 +854,13 @@ type X() = member public this.``Squiggles.HashIfWithoutIdent``() = this.TestSquiggle true [ "#if"; "#endif" ] "if" "#if directive should be immediately followed by an identifier" - [] - member public this.``Squiggles.HashIfWrongExpr``() = - this.TestSquiggle true [ "#if !IDENT"; "#endif" ] "if" "#if directive should be immediately followed by an identifier" - [] member public this.``Squiggles.HashIfWithMultilineComment``() = this.TestSquiggle true [ "#if IDENT (* aaa *)"; "#endif" ] "(* aaa" "Expected single line comment or end of line" [] member public this.``Squiggles.HashIfWithUnexpected``() = - this.TestSquiggle true [ "#if IDENT whatever"; "#endif" ] "whatever" "Expected single line comment or end of line" + this.TestSquiggle true [ "#if IDENT whatever"; "#endif" ] "whatever" "Incomplete preprocessor expression" // FEATURE: Touching a depended-upon file will cause a intellisense to update in the currently edited file. []