refactor(Cypress): add nodemodules

node_modules/gherkin/lib/gherkin/ast_builder.js

@@ -0,0 +1,264 @@
+var AstNode = require('./ast_node');
+var Errors = require('./errors');
+
+module.exports = function AstBuilder () {
+
+  var stack = [new AstNode('None')];
+  var comments = [];
+
+  this.reset = function () {
+    stack = [new AstNode('None')];
+    comments = [];
+  };
+
+  this.startRule = function (ruleType) {
+    stack.push(new AstNode(ruleType));
+  };
+
+  this.endRule = function (ruleType) {
+    var node = stack.pop();
+    var transformedNode = transformNode(node);
+    currentNode().add(node.ruleType, transformedNode);
+  };
+
+  this.build = function (token) {
+    if(token.matchedType === 'Comment') {
+      comments.push({
+        type: 'Comment',
+        location: getLocation(token),
+        text: token.matchedText
+      });
+    } else {
+      currentNode().add(token.matchedType, token);
+    }
+  };
+
+  this.getResult = function () {
+    return currentNode().getSingle('GherkinDocument');
+  };
+
+  function currentNode () {
+    return stack[stack.length - 1];
+  }
+
+  function getLocation (token, column) {
+    return !column ? token.location : {line: token.location.line, column: column};
+  }
+
+  function getTags (node) {
+    var tags = [];
+    var tagsNode = node.getSingle('Tags');
+    if (!tagsNode) return tags;
+    tagsNode.getTokens('TagLine').forEach(function (token) {
+      token.matchedItems.forEach(function (tagItem) {
+        tags.push({
+          type: 'Tag',
+          location: getLocation(token, tagItem.column),
+          name: tagItem.text
+        });
+      });
+
+    });
+    return tags;
+  }
+
+  function getCells(tableRowToken) {
+    return tableRowToken.matchedItems.map(function (cellItem) {
+      return {
+        type: 'TableCell',
+        location: getLocation(tableRowToken, cellItem.column),
+        value: cellItem.text
+      }
+    });
+  }
+
+  function getDescription (node) {
+    return node.getSingle('Description');
+  }
+
+  function getSteps (node) {
+    return node.getItems('Step');
+  }
+
+  function getTableRows(node) {
+    var rows = node.getTokens('TableRow').map(function (token) {
+      return {
+        type: 'TableRow',
+        location: getLocation(token),
+        cells: getCells(token)
+      };
+    });
+    ensureCellCount(rows);
+    return rows;
+  }
+
+  function ensureCellCount(rows) {
+    if(rows.length == 0) return;
+    var cellCount = rows[0].cells.length;
+
+    rows.forEach(function (row) {
+      if (row.cells.length != cellCount) {
+        throw Errors.AstBuilderException.create("inconsistent cell count within the table", row.location);
+      }
+    });
+  }
+
+  function transformNode(node) {
+    switch(node.ruleType) {
+      case 'Step':
+        var stepLine = node.getToken('StepLine');
+        var stepArgument = node.getSingle('DataTable') || node.getSingle('DocString') || undefined;
+
+        return {
+          type: node.ruleType,
+          location: getLocation(stepLine),
+          keyword: stepLine.matchedKeyword,
+          text: stepLine.matchedText,
+          argument: stepArgument
+        }
+      case 'DocString':
+        var separatorToken = node.getTokens('DocStringSeparator')[0];
+        var contentType = separatorToken.matchedText.length > 0 ? separatorToken.matchedText : undefined;
+        var lineTokens = node.getTokens('Other');
+        var content = lineTokens.map(function (t) {return t.matchedText}).join("\n");
+
+        var result = {
+          type: node.ruleType,
+          location: getLocation(separatorToken),
+          content: content
+        };
+        // conditionally add this like this (needed to make tests pass on node 0.10 as well as 4.0)
+        if(contentType) {
+          result.contentType = contentType;
+        }
+        return result;
+      case 'DataTable':
+        var rows = getTableRows(node);
+        return {
+          type: node.ruleType,
+          location: rows[0].location,
+          rows: rows,
+        }
+      case 'Background':
+        var backgroundLine = node.getToken('BackgroundLine');
+        var description = getDescription(node);
+        var steps = getSteps(node);
+
+        return {
+          type: node.ruleType,
+          location: getLocation(backgroundLine),
+          keyword: backgroundLine.matchedKeyword,
+          name: backgroundLine.matchedText,
+          description: description,
+          steps: steps
+        };
+      case 'Scenario_Definition':
+        var tags = getTags(node);
+        var scenarioNode = node.getSingle('Scenario');
+        if(scenarioNode) {
+          var scenarioLine = scenarioNode.getToken('ScenarioLine');
+          var description = getDescription(scenarioNode);
+          var steps = getSteps(scenarioNode);
+
+          return {
+            type: scenarioNode.ruleType,
+            tags: tags,
+            location: getLocation(scenarioLine),
+            keyword: scenarioLine.matchedKeyword,
+            name: scenarioLine.matchedText,
+            description: description,
+            steps: steps
+          };
+        } else {
+          var scenarioOutlineNode = node.getSingle('ScenarioOutline');
+          if(!scenarioOutlineNode) throw new Error('Internal grammar error');
+
+          var scenarioOutlineLine = scenarioOutlineNode.getToken('ScenarioOutlineLine');
+          var description = getDescription(scenarioOutlineNode);
+          var steps = getSteps(scenarioOutlineNode);
+          var examples = scenarioOutlineNode.getItems('Examples_Definition');
+
+          return {
+            type: scenarioOutlineNode.ruleType,
+            tags: tags,
+            location: getLocation(scenarioOutlineLine),
+            keyword: scenarioOutlineLine.matchedKeyword,
+            name: scenarioOutlineLine.matchedText,
+            description: description,
+            steps: steps,
+            examples: examples
+          };
+        }
+      case 'Examples_Definition':
+        var tags = getTags(node);
+        var examplesNode = node.getSingle('Examples');
+        var examplesLine = examplesNode.getToken('ExamplesLine');
+        var description = getDescription(examplesNode);
+        var exampleTable = examplesNode.getSingle('Examples_Table')
+
+        return {
+          type: examplesNode.ruleType,
+          tags: tags,
+          location: getLocation(examplesLine),
+          keyword: examplesLine.matchedKeyword,
+          name: examplesLine.matchedText,
+          description: description,
+          tableHeader: exampleTable != undefined ? exampleTable.tableHeader : undefined,
+          tableBody: exampleTable != undefined ? exampleTable.tableBody : undefined
+        };
+      case 'Examples_Table':
+        var rows = getTableRows(node)
+
+        return {
+          tableHeader: rows != undefined ? rows[0] : undefined,
+          tableBody: rows != undefined ? rows.slice(1) : undefined
+        };
+      case 'Description':
+        var lineTokens = node.getTokens('Other');
+        // Trim trailing empty lines
+        var end = lineTokens.length;
+        while (end > 0 && lineTokens[end-1].line.trimmedLineText === '') {
+            end--;
+        }
+        lineTokens = lineTokens.slice(0, end);
+
+        var description = lineTokens.map(function (token) { return token.matchedText}).join("\n");
+        return description;
+
+      case 'Feature':
+        var header = node.getSingle('Feature_Header');
+        if(!header) return null;
+        var tags = getTags(header);
+        var featureLine = header.getToken('FeatureLine');
+        if(!featureLine) return null;
+        var children = []
+        var background = node.getSingle('Background');
+        if(background) children.push(background);
+        children = children.concat(node.getItems('Scenario_Definition'));
+        var description = getDescription(header);
+        var language = featureLine.matchedGherkinDialect;
+
+        return {
+          type: node.ruleType,
+          tags: tags,
+          location: getLocation(featureLine),
+          language: language,
+          keyword: featureLine.matchedKeyword,
+          name: featureLine.matchedText,
+          description: description,
+          children: children,
+        };
+      case 'GherkinDocument':
+        var feature = node.getSingle('Feature');
+
+        return {
+          type: node.ruleType,
+          feature: feature,
+          comments: comments
+        };
+      default:
+        return node;
+    }
+  }
+
+};

node_modules/gherkin/lib/gherkin/ast_node.js

@@ -0,0 +1,28 @@
+function AstNode (ruleType) {
+  this.ruleType = ruleType;
+  this._subItems = {};
+}
+
+AstNode.prototype.add = function (ruleType, obj) {
+  var items = this._subItems[ruleType];
+  if(items === undefined) this._subItems[ruleType] = items = [];
+  items.push(obj);
+}
+
+AstNode.prototype.getSingle = function (ruleType) {
+  return (this._subItems[ruleType] || [])[0];
+}
+
+AstNode.prototype.getItems = function (ruleType) {
+  return this._subItems[ruleType] || [];
+}
+
+AstNode.prototype.getToken = function (tokenType) {
+  return this.getSingle(tokenType);
+}
+
+AstNode.prototype.getTokens = function (tokenType) {
+  return this._subItems[tokenType] || [];
+}
+
+module.exports = AstNode;

node_modules/gherkin/lib/gherkin/count_symbols.js

@@ -0,0 +1,6 @@
+// https://mathiasbynens.be/notes/javascript-unicode
+var regexAstralSymbols = /[\uD800-\uDBFF][\uDC00-\uDFFF]/g;
+
+module.exports = function countSymbols(string) {
+  return string.replace(regexAstralSymbols, '_').length;
+}

node_modules/gherkin/lib/gherkin/dialects.js

@@ -0,0 +1 @@
+module.exports = require('./gherkin-languages.json');

node_modules/gherkin/lib/gherkin/errors.js

@@ -0,0 +1,61 @@
+var Errors = {};
+
+[
+  'ParserException',
+  'CompositeParserException',
+  'UnexpectedTokenException',
+  'UnexpectedEOFException',
+  'AstBuilderException',
+  'NoSuchLanguageException'
+].forEach(function (name) {
+
+  function ErrorProto (message) {
+    this.message = message || ('Unspecified ' + name);
+    if (Error.captureStackTrace) {
+      Error.captureStackTrace(this, arguments.callee);
+    }
+  }
+
+  ErrorProto.prototype = Object.create(Error.prototype);
+  ErrorProto.prototype.name = name;
+  ErrorProto.prototype.constructor = ErrorProto;
+  Errors[name] = ErrorProto;
+});
+
+Errors.CompositeParserException.create = function(errors) {
+  var message = "Parser errors:\n" + errors.map(function (e) { return e.message; }).join("\n");
+  var err = new Errors.CompositeParserException(message);
+  err.errors = errors;
+  return err;
+};
+
+Errors.UnexpectedTokenException.create = function(token, expectedTokenTypes, stateComment) {
+  var message = "expected: " + expectedTokenTypes.join(', ') + ", got '" + token.getTokenValue().trim() + "'";
+  var location = !token.location.column
+    ? {line: token.location.line, column: token.line.indent + 1 }
+    : token.location;
+  return createError(Errors.UnexpectedEOFException, message, location);
+};
+
+Errors.UnexpectedEOFException.create = function(token, expectedTokenTypes, stateComment) {
+  var message = "unexpected end of file, expected: " + expectedTokenTypes.join(', ');
+  return createError(Errors.UnexpectedTokenException, message, token.location);
+};
+
+Errors.AstBuilderException.create = function(message, location) {
+  return createError(Errors.AstBuilderException, message, location);
+};
+
+Errors.NoSuchLanguageException.create = function(language, location) {
+  var message = "Language not supported: " + language;
+  return createError(Errors.NoSuchLanguageException, message, location);
+};
+
+function createError(Ctor, message, location) {
+  var fullMessage = "(" + location.line + ":" + location.column + "): " + message;
+  var error = new Ctor(fullMessage);
+  error.location = location;
+  return error;
+}
+
+module.exports = Errors;

node_modules/gherkin/lib/gherkin/generate_events.js

@@ -0,0 +1,76 @@
+var Parser = require('./parser')
+var Compiler = require('./pickles/compiler')
+
+var compiler = new Compiler()
+var parser = new Parser()
+parser.stopAtFirstError = false
+
+function generateEvents(data, uri, types, language) {
+  types = Object.assign({
+    'source': true,
+    'gherkin-document': true,
+    'pickle': true
+  }, types || {})
+
+  result = []
+
+  try {
+    if (types['source']) {
+      result.push({
+        type: 'source',
+        uri: uri,
+        data: data,
+        media: {
+          encoding: 'utf-8',
+          type: 'text/x.cucumber.gherkin+plain'
+        }
+      })
+    }
+
+    if (!types['gherkin-document'] && !types['pickle'])
+      return result
+
+    var gherkinDocument = parser.parse(data, language)
+
+    if (types['gherkin-document']) {
+      result.push({
+        type: 'gherkin-document',
+        uri: uri,
+        document: gherkinDocument
+      })
+    }
+
+    if (types['pickle']) {
+      var pickles = compiler.compile(gherkinDocument)
+      for (var p in pickles) {
+        result.push({
+          type: 'pickle',
+          uri: uri,
+          pickle: pickles[p]
+        })
+      }
+    }
+  } catch (err) {
+    var errors = err.errors || [err]
+    for (var e in errors) {
+      result.push({
+        type: "attachment",
+        source: {
+          uri: uri,
+          start: {
+            line: errors[e].location.line,
+            column: errors[e].location.column
+          }
+        },
+        data: errors[e].message,
+        media: {
+          encoding: "utf-8",
+          type: "text/x.cucumber.stacktrace+plain"
+        }
+      })
+    }
+  }
+  return result
+}
+
+module.exports = generateEvents

node_modules/gherkin/lib/gherkin/gherkin_line.js

@@ -0,0 +1,83 @@
+var countSymbols = require('./count_symbols')
+
+function GherkinLine(lineText, lineNumber) {
+  this.lineText = lineText;
+  this.lineNumber = lineNumber;
+  this.trimmedLineText = lineText.replace(/^\s+/g, ''); // ltrim
+  this.isEmpty = this.trimmedLineText.length == 0;
+  this.indent = countSymbols(lineText) - countSymbols(this.trimmedLineText);
+};
+
+GherkinLine.prototype.startsWith = function startsWith(prefix) {
+  return this.trimmedLineText.indexOf(prefix) == 0;
+};
+
+GherkinLine.prototype.startsWithTitleKeyword = function startsWithTitleKeyword(keyword) {
+  return this.startsWith(keyword+':'); // The C# impl is more complicated. Find out why.
+};
+
+GherkinLine.prototype.getLineText = function getLineText(indentToRemove) {
+  if (indentToRemove < 0 || indentToRemove > this.indent) {
+    return this.trimmedLineText;
+  } else {
+    return this.lineText.substring(indentToRemove);
+  }
+};
+
+GherkinLine.prototype.getRestTrimmed = function getRestTrimmed(length) {
+  return this.trimmedLineText.substring(length).trim();
+};
+
+GherkinLine.prototype.getTableCells = function getTableCells() {
+  var cells = [];
+  var col = 0;
+  var startCol = col + 1;
+  var cell = '';
+  var firstCell = true;
+  while (col < this.trimmedLineText.length) {
+    var chr = this.trimmedLineText[col];
+    col++;
+
+    if (chr == '|') {
+      if (firstCell) {
+        // First cell (content before the first |) is skipped
+        firstCell = false;
+      } else {
+        var cellIndent = cell.length - cell.replace(/^\s+/g, '').length;
+        var span = {column: this.indent + startCol + cellIndent, text: cell.trim()};
+        cells.push(span);
+      }
+      cell = '';
+      startCol = col + 1;
+    } else if (chr == '\\') {
+      chr = this.trimmedLineText[col];
+      col += 1;
+      if (chr == 'n') {
+        cell += '\n';
+      } else {
+        if (chr != '|' && chr != '\\') {
+          cell += '\\';
+        }
+        cell += chr;
+      }
+    } else {
+      cell += chr;
+    }
+  }
+
+  return cells;
+};
+
+GherkinLine.prototype.getTags = function getTags() {
+  var column = this.indent + 1;
+  var items = this.trimmedLineText.trim().split('@');
+  items.shift();
+  return items.map(function (item) {
+    var length = item.length;
+    var span = {column: column, text: '@' + item.trim()};
+    column += length + 1;
+    return span;
+  });
+};
+
+module.exports = GherkinLine;

node_modules/gherkin/lib/gherkin/ndjson.js

@@ -0,0 +1,3 @@
+/**
+ * Created by aslakhellesoy on 23/10/2016.
+ */

node_modules/gherkin/lib/gherkin/pickles/compiler.js

@@ -0,0 +1,169 @@
+var countSymbols = require('../count_symbols');
+
+function Compiler() {
+  this.compile = function (gherkin_document) {
+    var pickles = [];
+
+    if (gherkin_document.feature == null) return pickles;
+
+    var feature = gherkin_document.feature;
+    var language = feature.language;
+    var featureTags = feature.tags;
+    var backgroundSteps = [];
+
+    feature.children.forEach(function (scenarioDefinition) {
+      if(scenarioDefinition.type === 'Background') {
+        backgroundSteps = pickleSteps(scenarioDefinition);
+      } else if(scenarioDefinition.type === 'Scenario') {
+        compileScenario(featureTags, backgroundSteps, scenarioDefinition, language, pickles);
+      } else {
+        compileScenarioOutline(featureTags, backgroundSteps, scenarioDefinition, language, pickles);
+      }
+    });
+    return pickles;
+  };
+
+  function compileScenario(featureTags, backgroundSteps, scenario, language, pickles) {
+    var steps = scenario.steps.length == 0 ? [] : [].concat(backgroundSteps);
+
+    var tags = [].concat(featureTags).concat(scenario.tags);
+
+    scenario.steps.forEach(function (step) {
+      steps.push(pickleStep(step));
+    });
+
+    var pickle = {
+      tags: pickleTags(tags),
+      name: scenario.name,
+      language: language,
+      locations: [pickleLocation(scenario.location)],
+      steps: steps
+    };
+    pickles.push(pickle);
+  }
+
+  function compileScenarioOutline(featureTags, backgroundSteps, scenarioOutline, language, pickles) {
+    scenarioOutline.examples.filter(function(e) { return e.tableHeader != undefined; }).forEach(function (examples) {
+      var variableCells = examples.tableHeader.cells;
+      examples.tableBody.forEach(function (values) {
+        var valueCells = values.cells;
+        var steps = scenarioOutline.steps.length == 0 ? [] : [].concat(backgroundSteps);
+        var tags = [].concat(featureTags).concat(scenarioOutline.tags).concat(examples.tags);
+
+        scenarioOutline.steps.forEach(function (scenarioOutlineStep) {
+          var stepText = interpolate(scenarioOutlineStep.text, variableCells, valueCells);
+          var args = createPickleArguments(scenarioOutlineStep.argument, variableCells, valueCells);
+          var pickleStep = {
+            text: stepText,
+            arguments: args,
+            locations: [
+              pickleLocation(values.location),
+              pickleStepLocation(scenarioOutlineStep)
+            ]
+          };
+          steps.push(pickleStep);
+        });
+
+        var pickle = {
+          name: interpolate(scenarioOutline.name, variableCells, valueCells),
+          language: language,
+          steps: steps,
+          tags: pickleTags(tags),
+          locations: [
+            pickleLocation(values.location),
+            pickleLocation(scenarioOutline.location)
+          ]
+        };
+        pickles.push(pickle);
+
+      });
+    });
+  }
+
+  function createPickleArguments(argument, variableCells, valueCells) {
+    var result = [];
+    if (!argument) return result;
+    if (argument.type === 'DataTable') {
+      var table = {
+        rows: argument.rows.map(function (row) {
+          return {
+            cells: row.cells.map(function (cell) {
+              return {
+                location: pickleLocation(cell.location),
+                value: interpolate(cell.value, variableCells, valueCells)
+              };
+            })
+          };
+        })
+      };
+      result.push(table);
+    } else if (argument.type === 'DocString') {
+      var docString = {
+        location: pickleLocation(argument.location),
+        content: interpolate(argument.content, variableCells, valueCells),
+      };
+      if(argument.contentType) {
+        docString.contentType = interpolate(argument.contentType, variableCells, valueCells);
+      }
+      result.push(docString);
+    } else {
+      throw Error('Internal error');
+    }
+    return result;
+  }
+
+  function interpolate(name, variableCells, valueCells) {
+    variableCells.forEach(function (variableCell, n) {
+      var valueCell = valueCells[n];
+      var search = new RegExp('<' + variableCell.value + '>', 'g');
+      // JS Specific - dollar sign needs to be escaped with another dollar sign
+      // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/replace#Specifying_a_string_as_a_parameter
+      var replacement = valueCell.value.replace(new RegExp('\\$', 'g'), '$$$$')
+      name = name.replace(search, replacement);
+    });
+    return name;
+  }
+
+  function pickleSteps(scenarioDefinition) {
+    return scenarioDefinition.steps.map(function (step) {
+      return pickleStep(step);
+    });
+  }
+
+  function pickleStep(step) {
+    return {
+      text: step.text,
+      arguments: createPickleArguments(step.argument, [], []),
+      locations: [pickleStepLocation(step)]
+    }
+  }
+
+  function pickleStepLocation(step) {
+    return {
+      line: step.location.line,
+      column: step.location.column + (step.keyword ? countSymbols(step.keyword) : 0)
+    };
+  }
+
+  function pickleLocation(location) {
+    return {
+      line: location.line,
+      column: location.column
+    }
+  }
+
+  function pickleTags(tags) {
+    return tags.map(function (tag) {
+      return pickleTag(tag);
+    });
+  }
+
+  function pickleTag(tag) {
+    return {
+      name: tag.name,
+      location: pickleLocation(tag.location)
+    };
+  }
+}
+
+module.exports = Compiler;

node_modules/gherkin/lib/gherkin/stream/event_stream.js

@@ -0,0 +1,38 @@
+'use strict'
+
+const Stream = require('stream')
+const generateEvents = require('../generate_events')
+
+/**
+ * Stream that reads a Gherkin document as plain text and writes
+ * events.
+ */
+class EventStream extends Stream.Transform {
+  /**
+   * @param uri the uri of the Gherkin document written to this stream
+   * @param types {object} with keys source,gherkin-document and pickle,
+   *   indicating what kinds of events to emit
+   */
+  constructor(uri, types, language) {
+    super({ objectMode: true })
+    this._uri = uri
+    this._types = types
+    this._language = language
+    this._gherkin = ""
+  }
+
+  _transform(chunk, _, callback) {
+    this._gherkin += chunk
+    callback()
+  }
+
+  _flush(callback) {
+    const events = generateEvents(this._gherkin, this._uri, this._types, this._language)
+    for (const event of events) {
+      this.push(event)
+    }
+    callback()
+  }
+}
+
+module.exports = EventStream

node_modules/gherkin/lib/gherkin/token.js

@@ -0,0 +1,15 @@
+function Token(line, location) {
+  this.line = line;
+  this.location = location;
+  this.isEof = line == null;
+};
+
+Token.prototype.getTokenValue = function () {
+  return this.isEof ? "EOF" : this.line.getLineText(-1);
+};
+
+Token.prototype.detach = function () {
+  // TODO: Detach line, but is this really needed?
+};
+
+module.exports = Token;

node_modules/gherkin/lib/gherkin/token_formatter_builder.js

@@ -0,0 +1,36 @@
+module.exports = function TokenFormatterBuilder() {
+  var tokensText = '';
+
+  this.reset = function () {
+    tokensText = '';
+  };
+
+  this.startRule = function(ruleType) {};
+
+  this.endRule = function(ruleType) {};
+
+  this.build = function(token) {
+    tokensText += formatToken(token) + '\n';
+  };
+
+  this.getResult = function() {
+    return tokensText;
+  }
+
+  function formatToken(token) {
+    if(token.isEof) return 'EOF';
+
+    return "(" +
+    token.location.line +
+    ":" +
+    token.location.column +
+    ")" +
+    token.matchedType +
+    ":" +
+    (typeof token.matchedKeyword === 'string' ? token.matchedKeyword : '') +
+    "/" +
+    (typeof token.matchedText === 'string' ? token.matchedText : '') +
+    "/" +
+    token.matchedItems.map(function (i) { return i.column + ':' + i.text; }).join(',');
+  }
+};

node_modules/gherkin/lib/gherkin/token_matcher.js

@@ -0,0 +1,189 @@
+var DIALECTS = require('./dialects');
+var Errors = require('./errors');
+var LANGUAGE_PATTERN = /^\s*#\s*language\s*:\s*([a-zA-Z\-_]+)\s*$/;
+
+module.exports = function TokenMatcher(defaultDialectName) {
+  defaultDialectName = defaultDialectName || 'en';
+
+  var dialect;
+  var dialectName;
+  var activeDocStringSeparator;
+  var indentToRemove;
+
+  function changeDialect(newDialectName, location) {
+    var newDialect = DIALECTS[newDialectName];
+    if(!newDialect) {
+      throw Errors.NoSuchLanguageException.create(newDialectName, location);
+    }
+
+    dialectName = newDialectName;
+    dialect = newDialect;
+  }
+
+  this.reset = function () {
+    if(dialectName != defaultDialectName) changeDialect(defaultDialectName);
+    activeDocStringSeparator = null;
+    indentToRemove = 0;
+  };
+
+  this.reset();
+
+  this.match_TagLine = function match_TagLine(token) {
+    if(token.line.startsWith('@')) {
+      setTokenMatched(token, 'TagLine', null, null, null, token.line.getTags());
+      return true;
+    }
+    return false;
+  };
+
+  this.match_FeatureLine = function match_FeatureLine(token) {
+    return matchTitleLine(token, 'FeatureLine', dialect.feature);
+  };
+
+  this.match_ScenarioLine = function match_ScenarioLine(token) {
+    return matchTitleLine(token, 'ScenarioLine', dialect.scenario);
+  };
+
+  this.match_ScenarioOutlineLine = function match_ScenarioOutlineLine(token) {
+    return matchTitleLine(token, 'ScenarioOutlineLine', dialect.scenarioOutline);
+  };
+
+  this.match_BackgroundLine = function match_BackgroundLine(token) {
+    return matchTitleLine(token, 'BackgroundLine', dialect.background);
+  };
+
+  this.match_ExamplesLine = function match_ExamplesLine(token) {
+    return matchTitleLine(token, 'ExamplesLine', dialect.examples);
+  };
+
+  this.match_TableRow = function match_TableRow(token) {
+    if (token.line.startsWith('|')) {
+      // TODO: indent
+      setTokenMatched(token, 'TableRow', null, null, null, token.line.getTableCells());
+      return true;
+    }
+    return false;
+  };
+
+  this.match_Empty = function match_Empty(token) {
+    if (token.line.isEmpty) {
+      setTokenMatched(token, 'Empty', null, null, 0);
+      return true;
+    }
+    return false;
+  };
+
+  this.match_Comment = function match_Comment(token) {
+    if(token.line.startsWith('#')) {
+      var text = token.line.getLineText(0); //take the entire line, including leading space
+      setTokenMatched(token, 'Comment', text, null, 0);
+      return true;
+    }
+    return false;
+  };
+
+  this.match_Language = function match_Language(token) {
+    var match;
+    if(match = token.line.trimmedLineText.match(LANGUAGE_PATTERN)) {
+      var newDialectName = match[1];
+      setTokenMatched(token, 'Language', newDialectName);
+
+      changeDialect(newDialectName, token.location);
+      return true;
+    }
+    return false;
+  };
+
+  this.match_DocStringSeparator = function match_DocStringSeparator(token) {
+    return activeDocStringSeparator == null
+      ?
+      // open
+      _match_DocStringSeparator(token, '"""', true) ||
+      _match_DocStringSeparator(token, '```', true)
+      :
+      // close
+      _match_DocStringSeparator(token, activeDocStringSeparator, false);
+  };
+
+  function _match_DocStringSeparator(token, separator, isOpen) {
+    if (token.line.startsWith(separator)) {
+      var contentType = null;
+      if (isOpen) {
+        contentType = token.line.getRestTrimmed(separator.length);
+        activeDocStringSeparator = separator;
+        indentToRemove = token.line.indent;
+      } else {
+        activeDocStringSeparator = null;
+        indentToRemove = 0;
+      }
+
+      // TODO: Use the separator as keyword. That's needed for pretty printing.
+      setTokenMatched(token, 'DocStringSeparator', contentType);
+      return true;
+    }
+    return false;
+  }
+
+  this.match_EOF = function match_EOF(token) {
+    if(token.isEof) {
+      setTokenMatched(token, 'EOF');
+      return true;
+    }
+    return false;
+  };
+
+  this.match_StepLine = function match_StepLine(token) {
+    var keywords = []
+      .concat(dialect.given)
+      .concat(dialect.when)
+      .concat(dialect.then)
+      .concat(dialect.and)
+      .concat(dialect.but);
+    var length = keywords.length;
+    for(var i = 0, keyword; i < length; i++) {
+      var keyword = keywords[i];
+
+      if (token.line.startsWith(keyword)) {
+        var title = token.line.getRestTrimmed(keyword.length);
+        setTokenMatched(token, 'StepLine', title, keyword);
+        return true;
+      }
+    }
+    return false;
+  };
+
+  this.match_Other = function match_Other(token) {
+    var text = token.line.getLineText(indentToRemove); //take the entire line, except removing DocString indents
+    setTokenMatched(token, 'Other', unescapeDocString(text), null, 0);
+    return true;
+  };
+
+  function matchTitleLine(token, tokenType, keywords) {
+    var length = keywords.length;
+    for(var i = 0, keyword; i < length; i++) {
+      var keyword = keywords[i];
+
+      if (token.line.startsWithTitleKeyword(keyword)) {
+        var title = token.line.getRestTrimmed(keyword.length + ':'.length);
+        setTokenMatched(token, tokenType, title, keyword);
+        return true;
+      }
+    }
+    return false;
+  }
+
+  function setTokenMatched(token, matchedType, text, keyword, indent, items) {
+    token.matchedType = matchedType;
+    token.matchedText = text;
+    token.matchedKeyword = keyword;
+    token.matchedIndent = (typeof indent === 'number') ? indent : (token.line == null ? 0 : token.line.indent);
+    token.matchedItems = items || [];
+
+    token.location.column = token.matchedIndent + 1;
+    token.matchedGherkinDialect = dialectName;
+  }
+
+  function unescapeDocString(text) {
+    return activeDocStringSeparator != null ? text.replace("\\\"\\\"\\\"", "\"\"\"") : text;
+  }
+};

node_modules/gherkin/lib/gherkin/token_scanner.js

@@ -0,0 +1,23 @@
+var Token = require('./token');
+var GherkinLine = require('./gherkin_line');
+
+/**
+ * The scanner reads a gherkin doc (typically read from a .feature file) and creates a token for each line. 
+ * The tokens are passed to the parser, which outputs an AST (Abstract Syntax Tree).
+ * 
+ * If the scanner sees a `#` language header, it will reconfigure itself dynamically to look for 
+ * Gherkin keywords for the associated language. The keywords are defined in gherkin-languages.json.
+ */
+module.exports = function TokenScanner(source) {
+  var lines = source.split(/\r?\n/);
+  if(lines.length > 0 && lines[lines.length-1].trim() == '') {
+    lines.pop();
+  }
+  var lineNumber = 0;
+
+  this.read = function () {
+    var line = lines[lineNumber++];
+    var location = {line: lineNumber, column: 0};
+    return line == null ? new Token(null, location) : new Token(new GherkinLine(line, lineNumber), location);
+  }
+};