From e50704ec811c360b3168328088e71e025e497ffc Mon Sep 17 00:00:00 2001
From: Glenn Sarti <glennsarti@users.noreply.github.com>
Date: Wed, 13 Jun 2018 21:47:33 +0800
Subject: [PATCH 1/8] (GH-1336) Add syntax aware folding provider

Previously the Powershell extension used the default VSCode indentation based
folding regions.  This commit adds the skeleton for a syntax aware, client-side
folding provider as per the API introduced in VSCode 1.23.0

* The client side detection uses the PowerShell Textmate grammar file and the
  vscode-text node module to parse the text file into tokens which will be
  matched in later commits.

* However due to the way vscode imports the vscode-textmate module we can't
  simply just import it, instead we need to use file based require statements
  https://github.com/Microsoft/vscode/issues/46281

* This also means it's difficult to use any typings exposed in that module.  As
  we only need one interface, this is replicated verbatim in the file, but not
  exported

* Logging is added to help diagnose potential issues
---
 package.json            |   2 +-
 src/features/Folding.ts | 337 ++++++++++++++++++++++++++++++++++++++++
 src/main.ts             |   2 +
 3 files changed, 340 insertions(+), 1 deletion(-)
 create mode 100644 src/features/Folding.ts

diff --git a/package.json b/package.json
index 6616a98e8a..8df8a6197b 100644
--- a/package.json
+++ b/package.json
@@ -5,7 +5,7 @@
   "publisher": "ms-vscode",
   "description": "Develop PowerShell scripts in Visual Studio Code!",
   "engines": {
-    "vscode": "^1.22.0"
+    "vscode": "^1.23.0"
   },
   "license": "SEE LICENSE IN LICENSE.txt",
   "homepage": "https://github.com/PowerShell/vscode-powershell/blob/master/README.md",
diff --git a/src/features/Folding.ts b/src/features/Folding.ts
new file mode 100644
index 0000000000..eed63fe1aa
--- /dev/null
+++ b/src/features/Folding.ts
@@ -0,0 +1,337 @@
+import * as path from "path";
+import * as vscode from "vscode";
+import {
+    DocumentSelector,
+    LanguageClient,
+} from "vscode-languageclient";
+import { IFeature } from "../feature";
+import { Logger } from "../logging";
+
+/**
+ * Defines a grammar file that is in a VS Code Extension
+ */
+interface IExtensionGrammar {
+    /**
+     * The name of the language, e.g. powershell
+     */
+    language?: string;
+    /**
+     * The absolute path to the grammar file
+     */
+    path?: string;
+    /**
+     * The path to the extension
+     */
+    extensionPath?: string;
+}
+
+/**
+ * Defines a VS Code extension with minimal properities for grammar contribution
+ */
+interface IExtensionPackage {
+    /**
+     * Hashtable of items this extension contributes
+     */
+    contributes?: {
+        /**
+         * Array of grammars this extension supports
+         */
+        grammars?: IExtensionGrammar[],
+    };
+}
+
+/**
+ * Defines a grammar token in a text document
+ * Need to reproduce the IToken interface from vscode-textmate due to the odd way it has to be required
+ * https://github.com/Microsoft/vscode-textmate/blob/46af9487e1c8fa78aa1f2e2/release/main.d.ts#L161-L165
+ */
+interface IToken {
+    /**
+     * Zero based offset where the token starts
+     */
+    startIndex: number;
+    /**
+     * Zero based offset where the token ends
+     */
+    readonly endIndex: number;
+    /**
+     * Array of scope names that the token is a member of
+     */
+    readonly scopes: string[];
+}
+
+/**
+ * Defines a list of grammar tokens, typically for an entire text document
+ */
+interface ITokenList extends Array<IToken> { }
+
+/**
+ * Due to how the vscode-textmate library is required, we need to minimally define a Grammar object, which
+ * can be used to tokenize a text document.
+ * https://github.com/Microsoft/vscode-textmate/blob/46af9487e1c8fa78aa1f2e2/release/main.d.ts#L92-L108
+ */
+interface IGrammar {
+    /**
+     * Tokenize `lineText` using previous line state `prevState`.
+     */
+    tokenizeLine(lineText: any, prevState: any): any;
+}
+
+/**
+ * Defines a pair line numbers which describes a potential folding range in a text document
+ */
+class LineNumberRange {
+    /**
+     * The zero-based line number of the start of the range
+     */
+    public startline: number;
+    /**
+     * The zero-based line number of the end of the range
+     */
+    public endline: number;
+    /**
+     * The type of range this represents
+     */
+    public rangeKind: vscode.FoldingRangeKind;
+
+    constructor(
+        rangeKind: vscode.FoldingRangeKind,
+    ) {
+        this.rangeKind = rangeKind;
+    }
+
+    /**
+     * Build the range based on a pair of grammar tokens
+     * @param start     The token where the range starts
+     * @param end       The token where the range ends
+     * @param document  The text document
+     * @returns         Built LineNumberRange object
+     */
+    public fromTokenPair(
+        start: IToken,
+        end: IToken,
+        document: vscode.TextDocument,
+    ): LineNumberRange {
+        this.startline = document.positionAt(start.startIndex).line;
+        this.endline = document.positionAt(end.startIndex).line;
+        return this;
+    }
+
+    /**
+     * Build the range based on a pair of line numbers
+     * @param startLine The line where the range starts
+     * @param endLine   The line where the range ends
+     * @returns         Built LineNumberRange object
+     */
+    public fromLinePair(
+        startLine: number,
+        endLine: number,
+    ): LineNumberRange {
+        this.startline = startLine;
+        this.endline = endLine;
+        return this;
+    }
+
+    /**
+     * Whether this line number range, is a valid folding range in the document
+     * @returns Whether the range passes all validation checks
+     */
+    public isValidRange(): boolean {
+        // Start and end lines must be defined and positive integers
+        if (this.startline == null || this.endline == null) { return false; }
+        if (this.startline < 0 || this.endline < 0) { return false; }
+        // End line number cannot be before the start
+        if (this.startline > this.endline) { return false; }
+        // Folding ranges must span at least 2 lines
+        return (this.endline - this.startline >= 1);
+    }
+
+    /**
+     * Creates a vscode.FoldingRange object based on this object
+     * @returns A Folding Range object for use with the Folding Provider
+     */
+    public toFoldingRange(): vscode.FoldingRange {
+        return new vscode.FoldingRange(this.startline, this.endline, this.rangeKind);
+    }
+}
+
+/**
+ * An array of line number ranges
+ */
+interface ILineNumberRangeList extends Array<LineNumberRange> { }
+
+/**
+ * A PowerShell syntax aware Folding Provider
+ */
+export class FoldingProvider implements vscode.FoldingRangeProvider {
+    private powershellGrammar: IGrammar;
+
+    constructor(
+        powershellGrammar: IGrammar,
+    ) {
+        this.powershellGrammar = powershellGrammar;
+    }
+
+    /**
+     * Given a text document, parse the document and return a list of code folding ranges.
+     * @param document  Text document to parse
+     * @param context   Not used
+     * @param token     Not used
+     */
+    public async provideFoldingRanges(
+        document: vscode.TextDocument,
+        context: vscode.FoldingContext,
+        token: vscode.CancellationToken,
+    ): Promise<vscode.FoldingRange[]> {
+
+        // If the grammar hasn't been setup correctly, return empty result
+        if (this.powershellGrammar == null) { return []; }
+
+        // Convert the document text into a series of grammar tokens
+        const tokens: ITokenList = this.powershellGrammar.tokenizeLine(document.getText(), null).tokens;
+
+        // Parse the token list looking for matching tokens and return
+        // a list of LineNumberRange objects.  Then filter the list and only return matches
+        // that are a valid folding range e.g. It meets a minimum line span limit
+        const foldableRegions = this.extractFoldableRegions(tokens, document)
+            .filter((item) => item.isValidRange());
+
+        // Sort the list of matched tokens, starting at the top of the document,
+        // and ensure that, in the case of multiple ranges starting the same line,
+        // that the largest range (i.e. most number of lines spanned) is sorted
+        // first.  This is needed as vscode will just ignore any duplicate folding
+        // ranges.
+        foldableRegions.sort((a: LineNumberRange, b: LineNumberRange) => {
+            // Initially look at the start line
+            if (a.startline > b.startline) { return 1; }
+            if (a.startline < b.startline) { return -1; }
+            // They have the same start line so now consider the end line.
+            // The biggest line range is sorted first
+            if (a.endline > b.endline) { return -1; }
+            if (a.endline < b.endline) { return 1; }
+            // They're the same
+            return 0;
+        });
+
+        // Convert the matched token list into a FoldingRange[]
+        const foldingRanges = [];
+        foldableRegions.forEach((item) => { foldingRanges.push(item.toFoldingRange()); });
+
+        return foldingRanges;
+    }
+
+    /**
+     * Given a list of tokens, return a list of line number ranges which could be folding regions in the document
+     * @param tokens   List of grammar tokens to parse
+     * @param document The source text document
+     * @returns        A list of LineNumberRange objects of the possible document folding regions
+     */
+    private extractFoldableRegions(
+        tokens: ITokenList,
+        document: vscode.TextDocument,
+    ): ILineNumberRangeList {
+        const matchedTokens: ILineNumberRangeList = [];
+
+        return matchedTokens;
+    }
+}
+
+export class FoldingFeature implements IFeature {
+    private foldingProvider: FoldingProvider;
+
+    /**
+     * Constructs a handler for the FoldingProvider.  It returns success if the required grammar file can not be located
+     * but does not regist a provider.  This causes VS Code to instead still use the indentation based provider
+     * @param logger           The logging object to send messages to
+     * @param documentSelector documentSelector object for this Folding Provider
+     */
+    constructor(private logger: Logger, documentSelector: DocumentSelector) {
+        const grammar: IGrammar = this.grammar(logger);
+
+        // If the PowerShell grammar is not available for some reason, don't register a folding provider,
+        // which reverts VSCode to the default indentation style folding
+        if (grammar == null) {
+            logger.writeWarning("Unable to load the PowerShell grammar file");
+            return;
+        }
+
+        this.foldingProvider = new FoldingProvider(grammar);
+        vscode.languages.registerFoldingRangeProvider(documentSelector, this.foldingProvider);
+
+        logger.write("Syntax Folding Provider registered");
+    }
+
+    /* dispose() is required by the IFeature interface, but is not required by this feature */
+    public dispose(): any { return undefined; }
+
+    /* setLanguageClient() is required by the IFeature interface, but is not required by this feature */
+    public setLanguageClient(languageclient: LanguageClient): void { return undefined; }
+
+    /**
+     * Returns the PowerShell grammar parser, from the vscode-textmate node module
+     * @param logger The logging object to send messages to
+     * @returns      A grammar parser for the PowerShell language is succesful or undefined if an error occured
+     */
+    public grammar(logger: Logger): IGrammar {
+        const tm = this.getCoreNodeModule("vscode-textmate", logger);
+        if (tm == null) { return undefined; }
+        logger.writeDiagnostic(`Loaded the vscode-textmate module`);
+        const registry = new tm.Registry();
+        if (registry == null) { return undefined; }
+        logger.writeDiagnostic(`Created the textmate Registry`);
+        const grammarPath = this.powerShellGrammarPath();
+        if (grammarPath == null) { return undefined; }
+        logger.writeDiagnostic(`PowerShell grammar file specified as ${grammarPath}`);
+        try {
+            return registry.loadGrammarFromPathSync(grammarPath);
+        } catch (err) {
+            logger.writeError(`Error while loading the PowerShell grammar file at ${grammarPath}`, err);
+        }
+    }
+
+    /**
+     * Returns a node module installed within VSCode, or null if it fails.
+     * Some node modules (e.g. vscode-textmate) cannot be required directly, instead the known module locations
+     * must be tried. Documented in https://github.com/Microsoft/vscode/issues/46281
+     * @param moduleName Name of the module to load e.g. vscode-textmate
+     * @param logger     The logging object to send messages to
+     * @returns          The required module, or null if the module cannot be required
+     */
+    private getCoreNodeModule(moduleName: string, logger: Logger) {
+        // Attempt to load the module from known locations
+        const loadLocations: string[] = [
+            `${vscode.env.appRoot}/node_modules.asar/${moduleName}`,
+            `${vscode.env.appRoot}/node_modules/${moduleName}`,
+        ];
+
+        for (const filename of loadLocations) {
+            try {
+                const mod = require(filename);
+                logger.writeDiagnostic(`Succesfully required ${filename}`);
+                return mod;
+            } catch (err) {
+                logger.writeError(`Error while attempting to require ${filename}`, err);
+            }
+        }
+        return null;
+    }
+
+     /**
+      * Search all of the loaded extenions for the PowerShell grammar file
+      * @returns The absolute path to the PowerShell grammar file.  Returns undefined if the path cannot be located.
+      */
+     private powerShellGrammarPath(): string {
+        // Go through all the extension packages and search for PowerShell grammars,
+        // returning the path to the first we find
+        for (const ext of vscode.extensions.all) {
+            if (!(ext.packageJSON && ext.packageJSON.contributes && ext.packageJSON.contributes.grammars)) {
+                continue;
+            }
+            for (const grammar of ext.packageJSON.contributes.grammars) {
+                if (grammar.language !== "powershell") { continue; }
+                return path.join(ext.extensionPath, grammar.path);
+            }
+        }
+        return undefined;
+    }
+}
diff --git a/src/main.ts b/src/main.ts
index b5e7a5db2b..c4095ca190 100644
--- a/src/main.ts
+++ b/src/main.ts
@@ -18,6 +18,7 @@ import { DocumentFormatterFeature } from "./features/DocumentFormatter";
 import { ExamplesFeature } from "./features/Examples";
 import { ExpandAliasFeature } from "./features/ExpandAlias";
 import { ExtensionCommandsFeature } from "./features/ExtensionCommands";
+import { FoldingFeature } from "./features/Folding";
 import { GenerateBugReportFeature } from "./features/GenerateBugReport";
 import { HelpCompletionFeature } from "./features/HelpCompletion";
 import { NewFileOrProjectFeature } from "./features/NewFileOrProject";
@@ -132,6 +133,7 @@ export function activate(context: vscode.ExtensionContext): void {
         new SpecifyScriptArgsFeature(context),
         new HelpCompletionFeature(logger),
         new CustomViewsFeature(),
+        new FoldingFeature(logger, documentSelector),
     ];
 
     sessionManager.setExtensionFeatures(extensionFeatures);

From 09c8adebe135b8f2b39b0035972fd6c96225a315 Mon Sep 17 00:00:00 2001
From: Glenn Sarti <glennsarti@users.noreply.github.com>
Date: Wed, 13 Jun 2018 21:52:53 +0800
Subject: [PATCH 2/8] (GH-1336) Add syntax folding for braces and parentheses

This commit adds detection of text regions bounded by braces { } and parentheses
( ).  This provides syntax aware folding for functions, arrays and hash tables.
---
 src/features/Folding.ts | 48 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 48 insertions(+)

diff --git a/src/features/Folding.ts b/src/features/Folding.ts
index eed63fe1aa..77bb779fef 100644
--- a/src/features/Folding.ts
+++ b/src/features/Folding.ts
@@ -220,6 +220,38 @@ export class FoldingProvider implements vscode.FoldingRangeProvider {
         return foldingRanges;
     }
 
+    /**
+     * Given a start and end textmate scope name, find matching grammar tokens
+     * and pair them together.  Uses a simple stack to take into account nested regions.
+     * @param tokens         List of grammar tokens to parse
+     * @param startScopeName The name of the starting scope to match
+     * @param endScopeName   The name of the ending scope to match
+     * @param matchType      The type of range this matched token pair represents e.g. A comment
+     * @param document       The source text document
+     * @returns              A list of LineNumberRange objects of the matched token scopes
+     */
+    private matchScopeElements(
+        tokens: ITokenList,
+        startScopeName: string,
+        endScopeName: string,
+        matchType: vscode.FoldingRangeKind,
+        document: vscode.TextDocument,
+    ): ILineNumberRangeList {
+        const result = [];
+        const tokenStack = [];
+
+        tokens.forEach((token) => {
+            if (token.scopes.indexOf(startScopeName) !== -1) {
+                tokenStack.push(token);
+            }
+            if (token.scopes.indexOf(endScopeName) !== -1) {
+                result.unshift((new LineNumberRange(matchType)).fromTokenPair(tokenStack.pop(), token, document));
+            }
+        });
+
+        return result;
+    }
+
     /**
      * Given a list of tokens, return a list of line number ranges which could be folding regions in the document
      * @param tokens   List of grammar tokens to parse
@@ -232,6 +264,22 @@ export class FoldingProvider implements vscode.FoldingRangeProvider {
     ): ILineNumberRangeList {
         const matchedTokens: ILineNumberRangeList = [];
 
+        // Find matching braces   { -> }
+        this.matchScopeElements(
+            tokens,
+            "punctuation.section.braces.begin.powershell",
+            "punctuation.section.braces.end.powershell",
+            vscode.FoldingRangeKind.Region, document)
+            .forEach((match) => { matchedTokens.push(match); });
+
+        // Find matching parentheses   ( -> )
+        this.matchScopeElements(
+            tokens,
+            "punctuation.section.group.begin.powershell",
+            "punctuation.section.group.end.powershell",
+            vscode.FoldingRangeKind.Region, document)
+            .forEach((match) => { matchedTokens.push(match); });
+
         return matchedTokens;
     }
 }

From f5ee6d2ef1668ee4a22b2e54220122467063f81a Mon Sep 17 00:00:00 2001
From: Glenn Sarti <glennsarti@users.noreply.github.com>
Date: Wed, 13 Jun 2018 21:55:35 +0800
Subject: [PATCH 3/8] (GH-1336) Add syntax folding for here strings

This commit adds detection of text regions composed of single and double
quoted here strings;  @' '@ and @" "@.
---
 src/features/Folding.ts | 48 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 48 insertions(+)

diff --git a/src/features/Folding.ts b/src/features/Folding.ts
index 77bb779fef..5a93d4b515 100644
--- a/src/features/Folding.ts
+++ b/src/features/Folding.ts
@@ -252,6 +252,40 @@ export class FoldingProvider implements vscode.FoldingRangeProvider {
         return result;
     }
 
+    /**
+     * Given a textmate scope name, find a series of contiguous tokens which contain
+     * that scope name and pair them together.
+     * @param tokens    List of grammar tokens to parse
+     * @param scopeName The name of the scope to match
+     * @param matchType The type of range this region represents e.g. A comment
+     * @param document  The source text document
+     * @returns         A list of LineNumberRange objects of the contiguous token scopes
+     */
+    private matchContiguousScopeElements(
+        tokens: ITokenList,
+        scopeName: string,
+        matchType: vscode.FoldingRangeKind,
+        document: vscode.TextDocument,
+    ): ILineNumberRangeList {
+        const result = [];
+        let startToken;
+
+        tokens.forEach((token, index) => {
+            if (token.scopes.indexOf(scopeName) !== -1) {
+                if (startToken === undefined) { startToken = token; }
+
+                // If we're at the end of the token list, or the next token does not include the scopeName
+                // we've reached the end of the contiguous block.
+                if (((index + 1) >= tokens.length) || (tokens[index + 1].scopes.indexOf(scopeName) === -1)) {
+                    result.push((new LineNumberRange(matchType)).fromTokenPair(startToken, token, document));
+                    startToken = undefined;
+                }
+            }
+        });
+
+        return result;
+    }
+
     /**
      * Given a list of tokens, return a list of line number ranges which could be folding regions in the document
      * @param tokens   List of grammar tokens to parse
@@ -280,6 +314,20 @@ export class FoldingProvider implements vscode.FoldingRangeProvider {
             vscode.FoldingRangeKind.Region, document)
             .forEach((match) => { matchedTokens.push(match); });
 
+        // Find contiguous here strings   @' -> '@
+        this.matchContiguousScopeElements(
+            tokens,
+            "string.quoted.single.heredoc.powershell",
+            vscode.FoldingRangeKind.Region, document)
+            .forEach((match) => { matchedTokens.push(match); });
+
+        // Find contiguous here strings   @" -> "@
+        this.matchContiguousScopeElements(
+            tokens,
+            "string.quoted.double.heredoc.powershell",
+            vscode.FoldingRangeKind.Region, document)
+            .forEach((match) => { matchedTokens.push(match); });
+
         return matchedTokens;
     }
 }

From 1e1518068adf0b4b060994fce83349d181d0766c Mon Sep 17 00:00:00 2001
From: Glenn Sarti <glennsarti@users.noreply.github.com>
Date: Wed, 13 Jun 2018 21:57:24 +0800
Subject: [PATCH 4/8] (GH-1336) Add syntax folding for comments

This commit adds syntax aware folding for comment regions

* Contiguous blocks of line comments `# ....`

* Block comments `<# ... #>`

* Region bound comments `# region ...  # endregion`
---
 src/features/Folding.ts | 151 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 151 insertions(+)

diff --git a/src/features/Folding.ts b/src/features/Folding.ts
index 5a93d4b515..29a7d7bce3 100644
--- a/src/features/Folding.ts
+++ b/src/features/Folding.ts
@@ -286,6 +286,138 @@ export class FoldingProvider implements vscode.FoldingRangeProvider {
         return result;
     }
 
+    /**
+     * Given a zero based offset, find the line text preceeding it in the document
+     * @param offset   Zero based offset in the document
+     * @param document The source text document
+     * @returns        The line text preceeding the offset, not including the preceeding Line Feed
+     */
+    private preceedingText(
+        offset: number,
+        document: vscode.TextDocument,
+    ): string {
+        const endPos = document.positionAt(offset);
+        const startPos = endPos.translate(0, -endPos.character);
+
+        return document.getText(new vscode.Range(startPos, endPos));
+    }
+
+    /**
+     * Given a zero based offset, find the line text after it in the document
+     * @param offset   Zero based offset in the document
+     * @param document The source text document
+     * @returns        The line text after the offset, not including the subsequent Line Feed
+     */
+    private subsequentText(
+        offset: number,
+        document: vscode.TextDocument,
+    ): string {
+        const startPos: vscode.Position = document.positionAt(offset);
+        const endPos: vscode.Position = document.lineAt(document.positionAt(offset)).range.end;
+        return document.getText(new vscode.Range(startPos, endPos));
+    }
+
+    /**
+     * Finding blocks of comment tokens is more complicated as the newline characters are not
+     * classed as comments.  To workaround this we search for the comment character `#` scope name
+     * "punctuation.definition.comment.powershell" and then determine contiguous line numbers from there
+     * @param tokens   List of grammar tokens to parse
+     * @param document The source text document
+     * @returns        A list of LineNumberRange objects for blocks of comment lines
+     */
+    private matchBlockCommentScopeElements(
+        tokens: ITokenList,
+        document: vscode.TextDocument,
+    ): ILineNumberRangeList {
+        const result = [];
+
+        const emptyLine = /^[\s]+$/;
+
+        let startLine: number = -1;
+        let nextLine: number = -1;
+
+        tokens.forEach((token) => {
+            if (token.scopes.indexOf("punctuation.definition.comment.powershell") !== -1) {
+                // The punctuation.definition.comment.powershell token matches new-line comments
+                // and inline comments e.g. `$x = 'foo' # inline comment`.  We are only interested
+                // in comments which begin the line i.e. no preceeding text
+                if (emptyLine.test(this.preceedingText(token.startIndex, document))) {
+                    const lineNum = document.positionAt(token.startIndex).line;
+                    // A simple pattern for keeping track of contiguous numbers in a known sorted array
+                    if (startLine === -1) {
+                        startLine = lineNum;
+                    } else if (lineNum !== nextLine) {
+                        result.push(
+                            (
+                                new LineNumberRange(vscode.FoldingRangeKind.Comment)
+                            ).fromLinePair(startLine, nextLine - 1),
+                        );
+                        startLine = lineNum;
+                    }
+                    nextLine = lineNum + 1;
+                }
+            }
+        });
+
+        // If we exit the token array and we're still processing comment lines, then the
+        // comment block simply ends at the end of document
+        if (startLine !== -1) {
+            result.push((new LineNumberRange(vscode.FoldingRangeKind.Comment)).fromLinePair(startLine, nextLine - 1));
+        }
+
+        return result;
+    }
+
+    /**
+     * Create a new token object with an appended scopeName
+     * @param token     The token to append the scope to
+     * @param scopeName The scope name to append
+     * @returns         A copy of the original token, but with the scope appended
+     */
+    private addTokenScope(
+        token: IToken,
+        scopeName: string,
+    ): IToken {
+        // Only a shallow clone is required
+        const tokenClone = Object.assign({}, token);
+        tokenClone.scopes.push(scopeName);
+        return tokenClone;
+    }
+
+    /**
+     * Given a list of grammar tokens, find the tokens that are comments and
+     * the comment text is either `# region` or `# endregion`.  Return a new list of tokens
+     * with custom scope names added, "custom.start.region" and "custom.end.region" respectively
+     * @param tokens   List of grammar tokens to parse
+     * @param document The source text document
+     * @returns        A list of LineNumberRange objects of the line comment region blocks
+     */
+    private extractRegionScopeElements(
+        tokens: ITokenList,
+        document: vscode.TextDocument,
+    ): ITokenList {
+        const result = [];
+
+        const emptyLine = /^[\s]+$/;
+        const startRegionText = /^#\s*region\b/;
+        const endRegionText = /^#\s*endregion\b/;
+
+        tokens.forEach((token) => {
+            if (token.scopes.indexOf("punctuation.definition.comment.powershell") !== -1) {
+                if (emptyLine.test(this.preceedingText(token.startIndex, document))) {
+                    const commentText = this.subsequentText(token.startIndex, document);
+                    if (startRegionText.test(commentText)) {
+                        result.push(this.addTokenScope(token, "custom.start.region"));
+                    }
+                    if (endRegionText.test(commentText)) {
+                        result.push(this.addTokenScope(token, "custom.end.region"));
+                    }
+                }
+            }
+        });
+        return result;
+    }
+
     /**
      * Given a list of tokens, return a list of line number ranges which could be folding regions in the document
      * @param tokens   List of grammar tokens to parse
@@ -328,6 +460,25 @@ export class FoldingProvider implements vscode.FoldingRangeProvider {
             vscode.FoldingRangeKind.Region, document)
             .forEach((match) => { matchedTokens.push(match); });
 
+        // Find matching comment regions   #region -> #endregion
+        this.matchScopeElements(
+            this.extractRegionScopeElements(tokens, document),
+            "custom.start.region",
+            "custom.end.region",
+            vscode.FoldingRangeKind.Region, document)
+            .forEach((match) => { matchedTokens.push(match); });
+
+        // Find blocks of line comments   # comment1\n# comment2\n...
+        this.matchBlockCommentScopeElements(tokens, document).forEach((match) => { matchedTokens.push(match); });
+
+        // Find matching block comments   <# -> #>
+        this.matchScopeElements(
+            tokens,
+            "punctuation.definition.comment.block.begin.powershell",
+            "punctuation.definition.comment.block.end.powershell",
+            vscode.FoldingRangeKind.Comment, document)
+            .forEach((match) => { matchedTokens.push(match); });
+
         return matchedTokens;
     }
 }

From 6a99426f0ff649e8bf73508067074ae6c4f48e6e Mon Sep 17 00:00:00 2001
From: Glenn Sarti <glennsarti@users.noreply.github.com>
Date: Thu, 28 Jun 2018 13:02:17 +0800
Subject: [PATCH 5/8] (GH-1336) Add Code Folding settings and enable by default

Previously the syntax folding was available for all users.  However it was able
to be configured or turned off.  This commit adds a new `codeFolding` settings
section, with the syntax folding feature enabled by default.
---
 package.json            |  5 +++++
 src/features/Folding.ts |  4 ++++
 src/settings.ts         | 11 +++++++++++
 3 files changed, 20 insertions(+)

diff --git a/package.json b/package.json
index 8df8a6197b..497181f2a2 100644
--- a/package.json
+++ b/package.json
@@ -467,6 +467,11 @@
           "default": "",
           "description": "Specifies the path to a PowerShell Script Analyzer settings file. To override the default settings for all projects, enter an absolute path, or enter a path relative to your workspace."
         },
+        "powershell.codeFolding.enable": {
+          "type": "boolean",
+          "default": true,
+          "description": "Enables syntax based code folding. When disabled, the default indentation based code folding is used."
+        },
         "powershell.codeFormatting.preset": {
           "type": "string",
           "enum": [
diff --git a/src/features/Folding.ts b/src/features/Folding.ts
index 29a7d7bce3..ebbf8d65cf 100644
--- a/src/features/Folding.ts
+++ b/src/features/Folding.ts
@@ -6,6 +6,7 @@ import {
 } from "vscode-languageclient";
 import { IFeature } from "../feature";
 import { Logger } from "../logging";
+import * as Settings from "../settings";
 
 /**
  * Defines a grammar file that is in a VS Code Extension
@@ -495,6 +496,9 @@ export class FoldingFeature implements IFeature {
     constructor(private logger: Logger, documentSelector: DocumentSelector) {
         const grammar: IGrammar = this.grammar(logger);
 
+        const settings = Settings.load();
+        if (!(settings.codeFolding && settings.codeFolding.enable)) { return; }
+
         // If the PowerShell grammar is not available for some reason, don't register a folding provider,
         // which reverts VSCode to the default indentation style folding
         if (grammar == null) {
diff --git a/src/settings.ts b/src/settings.ts
index 35ef50cffa..42fe34f803 100644
--- a/src/settings.ts
+++ b/src/settings.ts
@@ -29,6 +29,10 @@ export interface IBugReportingSettings {
     project: string;
 }
 
+export interface ICodeFoldingSettings {
+    enable?: boolean;
+}
+
 export interface ICodeFormattingSettings {
     preset: CodeFormattingPreset;
     openBraceOnSameLine: boolean;
@@ -72,6 +76,7 @@ export interface ISettings {
     scriptAnalysis?: IScriptAnalysisSettings;
     debugging?: IDebuggingSettings;
     developer?: IDeveloperSettings;
+    codeFolding?: ICodeFoldingSettings;
     codeFormatting?: ICodeFormattingSettings;
     integratedConsole?: IIntegratedConsoleSettings;
     bugReporting?: IBugReportingSettings;
@@ -109,6 +114,10 @@ export function load(): ISettings {
         powerShellExeIsWindowsDevBuild: false,
     };
 
+    const defaultCodeFoldingSettings: ICodeFoldingSettings = {
+        enable: true,
+    };
+
     const defaultCodeFormattingSettings: ICodeFormattingSettings = {
         preset: CodeFormattingPreset.Custom,
         openBraceOnSameLine: true,
@@ -150,6 +159,8 @@ export function load(): ISettings {
             configuration.get<IDebuggingSettings>("debugging", defaultDebuggingSettings),
         developer:
             getWorkspaceSettingsWithDefaults<IDeveloperSettings>(configuration, "developer", defaultDeveloperSettings),
+        codeFolding:
+            configuration.get<ICodeFoldingSettings>("codeFolding", defaultCodeFoldingSettings),
         codeFormatting:
             configuration.get<ICodeFormattingSettings>("codeFormatting", defaultCodeFormattingSettings),
         integratedConsole:

From 0434ff862533b756fcf8c25984a3d0d455a49971 Mon Sep 17 00:00:00 2001
From: Glenn Sarti <glennsarti@users.noreply.github.com>
Date: Mon, 18 Jun 2018 20:33:41 +0800
Subject: [PATCH 6/8] (GH-1336) Add integration tests for the Folding Provider

Previously there were no tests to verify the folding provider.  Due to the
provider depending on 3rd party libraries (vscode-textmate and PowerShell
grammar file) these tests will provide a degree of detection if breaking changes
occur.
---
 test/features/folding.test.ts | 57 +++++++++++++++++++++++++++++++++++
 test/fixtures/folding.ps1     | 47 +++++++++++++++++++++++++++++
 test/test_utils.ts            | 29 ++++++++++++++++++
 3 files changed, 133 insertions(+)
 create mode 100644 test/features/folding.test.ts
 create mode 100644 test/fixtures/folding.ps1
 create mode 100644 test/test_utils.ts

diff --git a/test/features/folding.test.ts b/test/features/folding.test.ts
new file mode 100644
index 0000000000..4806f574ab
--- /dev/null
+++ b/test/features/folding.test.ts
@@ -0,0 +1,57 @@
+import * as assert from "assert";
+import * as path from "path";
+import * as vscode from "vscode";
+import { DocumentSelector } from "vscode-languageclient";
+import * as folding from "../../src/features/Folding";
+import { MockLogger } from "../test_utils";
+
+const fixturePath = path.join(__dirname, "..", "..", "..", "test", "fixtures");
+
+function assertFoldingRegions(result, expected): void {
+    assert.equal(result.length, expected.length);
+
+    for (let i = 0; i < expected.length; i++) {
+        const failMessage = `expected ${JSON.stringify(expected[i])}, actual ${JSON.stringify(result[i])}`;
+        assert.equal(result[i].start, expected[i].start, failMessage);
+        assert.equal(result[i].end, expected[i].end, failMessage);
+        assert.equal(result[i].kind, expected[i].kind, failMessage);
+    }
+}
+
+suite("Features", () => {
+
+    suite("Folding Provider", () => {
+        const logger: MockLogger = new MockLogger();
+        const mockSelector: DocumentSelector = [
+            { language: "powershell", scheme: "file" },
+        ];
+        const psGrammar = (new folding.FoldingFeature(logger, mockSelector)).grammar(logger);
+        const provider = (new folding.FoldingProvider(psGrammar));
+
+        test("Can detect the PowerShell Grammar", () => {
+            assert.notEqual(psGrammar, null);
+        });
+
+        test("Can detect all of the foldable regions in a document", async () => {
+            // Integration test against the test fixture 'folding.ps1' that contains
+            // all of the different types of folding available
+            const uri = vscode.Uri.file(path.join(fixturePath, "folding.ps1"));
+            const document = await vscode.workspace.openTextDocument(uri);
+            const result = await provider.provideFoldingRanges(document, null, null);
+
+            const expected = [
+                { start: 1,  end: 6,  kind: 1 },
+                { start: 7,  end: 46, kind: 3 },
+                { start: 8,  end: 13, kind: 1 },
+                { start: 14, end: 17, kind: 3 },
+                { start: 21, end: 23, kind: 1 },
+                { start: 25, end: 35, kind: 3 },
+                { start: 27, end: 31, kind: 3 },
+                { start: 37, end: 39, kind: 3 },
+                { start: 42, end: 45, kind: 3 },
+            ];
+
+            assertFoldingRegions(result, expected);
+        });
+    });
+});
diff --git a/test/fixtures/folding.ps1 b/test/fixtures/folding.ps1
new file mode 100644
index 0000000000..c023b75264
--- /dev/null
+++ b/test/fixtures/folding.ps1
@@ -0,0 +1,47 @@
+function short-func {};
+<#
+.SYNOPSIS
+  Displays a list of WMI Classes based upon a search criteria
+.EXAMPLE
+ Get-WmiClasses -class disk -ns rootcimv2"
+#>
+function New-VSCodeCannotFold {
+<#
+.SYNOPSIS
+  Displays a list of WMI Classes based upon a search criteria
+.EXAMPLE
+ Get-WmiClasses -class disk -ns rootcimv2"
+#>
+  $I = @'
+cannot fold
+
+'@
+
+  # this won't be folded
+
+  # This should be foldable
+  # This should be foldable
+  # This should be foldable
+
+  #region This fools the indentation folding.
+  Write-Host "Hello"
+    # region
+    Write-Host "Hello"
+    # comment1
+    Write-Host "Hello"
+    #endregion
+    Write-Host "Hello"
+    # comment2
+    Write-Host "Hello"
+    # endregion
+
+  $c = {
+    Write-Host "Hello"
+  }
+
+  # Array fools indentation folding
+  $d = @(
+  'element1',
+  'elemet2'
+  )
+}
diff --git a/test/test_utils.ts b/test/test_utils.ts
new file mode 100644
index 0000000000..5a0d04edba
--- /dev/null
+++ b/test/test_utils.ts
@@ -0,0 +1,29 @@
+import { Logger, LogLevel } from "../src/logging";
+
+export class MockLogger extends Logger {
+    // Note - This is not a true mock as the constructor is inherited and causes errors due to trying load
+    // the "PowerShell Extension Logs" multiple times.  Ideally logging should be via an interface and then
+    // we can mock correctly.
+
+    public dispose() { return undefined; }
+
+    public getLogFilePath(baseName: string): string { return "mock"; }
+
+    public writeAtLevel(logLevel: LogLevel, message: string, ...additionalMessages: string[]) { return undefined; }
+
+    public write(message: string, ...additionalMessages: string[]) { return undefined; }
+
+    public writeDiagnostic(message: string, ...additionalMessages: string[]) { return undefined; }
+
+    public writeVerbose(message: string, ...additionalMessages: string[]) { return undefined; }
+
+    public writeWarning(message: string, ...additionalMessages: string[]) { return undefined; }
+
+    public writeAndShowWarning(message: string, ...additionalMessages: string[]) { return undefined; }
+
+    public writeError(message: string, ...additionalMessages: string[]) { return undefined; }
+
+    public writeAndShowError(message: string, ...additionalMessages: string[]) { return undefined; }
+
+    public startNewLog(minimumLogLevel: string = "Normal") { return undefined; }
+}

From 5156173f3718c4ab7db91dc393d6a5cedf036cfa Mon Sep 17 00:00:00 2001
From: Glenn Sarti <glennsarti@users.noreply.github.com>
Date: Wed, 20 Jun 2018 19:59:33 +0800
Subject: [PATCH 7/8] (maint) Modify tslint configuration for test files

Previously tslint was raising errors in Travis CI saying that the excluded test
fixtures directory was not included in the project.  This was by design however
it appears to be a known bug https://github.com/palantir/tslint/issues/3793.

This commit removes the exclude for test files from linting and adds a tslint
directive in the default index.ts file.  A tslint directive is used instead of
solving the issue because this is the default testing file for VS Code extesions
and shouldn't really be modified unless absolutely necessary.  In this instance
it was safer for a tslint directive.
---
 package.json  | 2 +-
 test/index.ts | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/package.json b/package.json
index 497181f2a2..755861ba5e 100644
--- a/package.json
+++ b/package.json
@@ -52,7 +52,7 @@
     "vscode.powershell"
   ],
   "scripts": {
-    "compile": "tsc -v && tsc -p ./ && tslint -p ./ -e test/*",
+    "compile": "tsc -v && tsc -p ./ && tslint -p ./",
     "compile-watch": "tsc -watch -p ./",
     "postinstall": "node ./node_modules/vscode/bin/install",
     "test": "node ./node_modules/vscode/bin/test"
diff --git a/test/index.ts b/test/index.ts
index 8ff1de2e82..0490604c84 100644
--- a/test/index.ts
+++ b/test/index.ts
@@ -1,3 +1,4 @@
+// tslint:disable no-var-requires
 let testRunner = require("vscode/lib/testrunner");
 
 // See https://github.com/mochajs/mocha/wiki/Using-mocha-programmatically#set-options for options

From d9a9e6d34ebbe297fc7486247be0ee8c3ced3456 Mon Sep 17 00:00:00 2001
From: Tyler Leonhardt <tylerl0706@gmail.com>
Date: Fri, 29 Jun 2018 12:01:59 -0700
Subject: [PATCH 8/8] add copyright headers

---
 src/features/Folding.ts       | 4 ++++
 test/features/folding.test.ts | 4 ++++
 test/index.ts                 | 4 ++++
 test/test_utils.ts            | 4 ++++
 4 files changed, 16 insertions(+)

diff --git a/src/features/Folding.ts b/src/features/Folding.ts
index ebbf8d65cf..59f316b927 100644
--- a/src/features/Folding.ts
+++ b/src/features/Folding.ts
@@ -1,3 +1,7 @@
+/*---------------------------------------------------------
+ * Copyright (C) Microsoft Corporation. All rights reserved.
+ *--------------------------------------------------------*/
+
 import * as path from "path";
 import * as vscode from "vscode";
 import {
diff --git a/test/features/folding.test.ts b/test/features/folding.test.ts
index 4806f574ab..fd54f654bd 100644
--- a/test/features/folding.test.ts
+++ b/test/features/folding.test.ts
@@ -1,3 +1,7 @@
+/*---------------------------------------------------------
+ * Copyright (C) Microsoft Corporation. All rights reserved.
+ *--------------------------------------------------------*/
+
 import * as assert from "assert";
 import * as path from "path";
 import * as vscode from "vscode";
diff --git a/test/index.ts b/test/index.ts
index 0490604c84..aa8ec699be 100644
--- a/test/index.ts
+++ b/test/index.ts
@@ -1,4 +1,8 @@
 // tslint:disable no-var-requires
+/*---------------------------------------------------------
+ * Copyright (C) Microsoft Corporation. All rights reserved.
+ *--------------------------------------------------------*/
+
 let testRunner = require("vscode/lib/testrunner");
 
 // See https://github.com/mochajs/mocha/wiki/Using-mocha-programmatically#set-options for options
diff --git a/test/test_utils.ts b/test/test_utils.ts
index 5a0d04edba..fce7d8d56b 100644
--- a/test/test_utils.ts
+++ b/test/test_utils.ts
@@ -1,3 +1,7 @@
+/*---------------------------------------------------------
+ * Copyright (C) Microsoft Corporation. All rights reserved.
+ *--------------------------------------------------------*/
+
 import { Logger, LogLevel } from "../src/logging";
 
 export class MockLogger extends Logger {