From b29efe81189e6c33a863a955629c6c96ba2c59a5 Mon Sep 17 00:00:00 2001
From: adon <adon@yahoo-inc.com>
Date: Fri, 10 Jul 2015 15:34:27 +0800
Subject: [PATCH] release as v2.0.1

---
 bower.json             |   2 +-
 dist/context-parser.js | 820 +++++++++++++++++++++++++++++++++++------
 package.json           |   2 +-
 3 files changed, 714 insertions(+), 110 deletions(-)

diff --git a/bower.json b/bower.json
index 6fa6c82..550bf89 100644
--- a/bower.json
+++ b/bower.json
@@ -1,6 +1,6 @@
 {
   "name": "context-parser",
-  "version": "2.0.0",
+  "version": "2.0.1",
   "homepage": "https://github.com/yahoo/context-parser",
   "description": "HTML5 Context Parser",
   "main": "src/context-parser.js",
diff --git a/dist/context-parser.js b/dist/context-parser.js
index c679681..36338f3 100644
--- a/dist/context-parser.js
+++ b/dist/context-parser.js
@@ -8,44 +8,149 @@ Authors: Nera Liu <neraliu@yahoo-inc.com>
          Albert Yu <albertyu@yahoo-inc.com>
          Adonis Fung <adon@yahoo-inc.com>
 */
+/*jshint -W030 */
 (function() {
 "use strict";
 
-var stateMachine = require('./html5-state-machine.js');
+var stateMachine = require('./html5-state-machine.js'),
+    htmlState = stateMachine.State,
+    reInputPreProcessing = /(?:\r\n?|[\x01-\x08\x0B\x0E-\x1F\x7F-\x9F\uFDD0-\uFDEF\uFFFE\uFFFF]|[\uD83F\uD87F\uD8BF\uD8FF\uD93F\uD97F\uD9BF\uD9FF\uDA3F\uDA3F\uDA7F\uDABF\uDAFF\uDB3F\uDB7F\uDBBF\uDBFF][\uDFFE\uDFFF])/g;
 
 /**
  * @class FastParser
  * @constructor FastParser
  */
-function FastParser() {
-    this.state = stateMachine.State.STATE_DATA;  /* Save the current status */
-    this.tagNames = ['', '']; /* Save the current tag name */
-    this.tagNameIdx = '';
-    this.attributeName = ''; /* Save the current attribute name */
-    this.attributeValue = ''; /* Save the current attribute value */
+function FastParser(config) {
+    var self = this, k;
+
+    // deep copy config to this.config
+    self.config = {};
+    if (config) {
+        for (k in config) {
+            self.config[k] = config[k];
+        }
+    }
+    config = self.config;   
+
+    // config enabled by default - no conversion needed
+    // config.enableInputPreProcessing = (config.enableInputPreProcessing !== false);
+
+    self.listeners = {};
+    self.reset();
 }
 
 /**
- * @function FastParser#contextualize
+ * @function FastParser#reset
+ *
+ * @description
+ * Reset all internal states, as if being created with the new operator
+ */
+ FastParser.prototype.reset = function () {
+    var self = this;
+
+    self.state = stateMachine.State.STATE_DATA;  /* Save the current status */
+    self.tags = ['', '']; /* Save the current tag name */
+    self.tagIdx = 0;
+    self.attrName = ''; /* Save the current attribute name */
+    self.attributeValue = null; /* Save the current attribute value */
+    self.input = '';
+    self.inputLen = 0;
+
+    return self;
+ };
+
+/**
+ * @function FastParser#on
  *
- * @param {string} input - The byte stream of the HTML5 web page.
- * @returns {integer} The return code of success or failure of parsing.
+ * @param {string} eventType - the event type 
+ * @param {function} listener - the event listener
+ * @returns this
  *
  * @description
- * <p>The context analyzing function, it analyzes the output context of each character based on
- * the HTML5 WHATWG - https://html.spec.whatwg.org/multipage/</p>
+ * <p>register the given event listener to the given eventType</p>
  *
  */
-FastParser.prototype.contextualize = function(input) {
-    var len = input.length;
-
-    for(var i = 0; i < len; ++i) {
-        i = this.beforeWalk(i, input);
-        if ( i >= len ) { break; }
-        i = this.walk(i, input);
-        if ( i >= len ) { break; }
-        this.afterWalk(input[i], i);
+FastParser.prototype.on = function (eventType, listener) {
+    var l = this.listeners[eventType];
+    if (listener) {
+        if (l) {
+            l.push(listener);
+        } else {
+            this.listeners[eventType] = [listener];
+        }
     }
+    return this;
+};
+
+/**
+ * @function FastParser#once
+ *
+ * @param {string} eventType - the event type (e.g., preWalk, reWalk, postWalk, ...)
+ * @param {function} listener - the event listener
+ * @returns this
+ *
+ * @description
+ * <p>register the given event listener to the given eventType, for which it will be fired only once</p>
+ *
+ */
+FastParser.prototype.once = function(eventType, listener) {
+    var self = this, onceListener;
+    if (listener) {
+        onceListener = function () {
+            self.off(eventType, onceListener);
+            listener.apply(self, arguments);
+        };
+        return this.on(eventType, onceListener);
+    }
+    return this;
+};
+
+/**
+ * @function FastParser#off
+ *
+ * @param {string} eventType - the event type (e.g., preWalk, reWalk, postWalk, ...)
+ * @param {function} listener - the event listener
+ * @returns this
+ *
+ * @description
+ * <p>remove the listener from being fired when the eventType happen</p>
+ *
+ */
+FastParser.prototype.off = function (eventType, listener) {
+    if (listener) {
+        var i, len, listeners = this.listeners[eventType];
+        if (listeners) {
+            for (i = 0; listeners[i]; i++) {
+                if (listeners[i] === listener) {
+                    listeners.splice(i, 1);
+                    break;
+                }
+            }
+        }
+    }
+    return this;
+};
+
+/**
+ * @function FastParser#emit
+ *
+ * @param {string} eventType - the event type (e.g., preWalk, reWalk, postWalk, ...)
+ * @returns this
+ *
+ * @description
+ * <p>fire those listeners correspoding to the given eventType</p>
+ *
+ */
+FastParser.prototype.emit = function (listeners, args) {
+    if (listeners) {
+        var i = -1, len;
+        if ((len = listeners.length)) {
+            while (++i < len) {
+                listeners[i].apply(this, args || []);
+            }
+        }
+    }
+    return this;
 };
 
 /*
@@ -56,7 +161,7 @@ FastParser.prototype.contextualize = function(input) {
  * @returns {integer} the new location of the current character.
  *
  */
-FastParser.prototype.walk = function(i, input) {
+FastParser.prototype.walk = function(i, input, endsWithEOF) {
 
     var ch = input[i],
         symbol = this.lookupChar(ch),
@@ -73,7 +178,7 @@ FastParser.prototype.walk = function(i, input) {
         case 3:  this.appendTagName(ch);  break;
         case 4:  this.resetEndTag(ch);    break;
         case 6:                       /* match end tag token with start tag token's tag name */
-            if(this.tagNames[0] === this.tagNames[1]) {
+            if(this.tags[0].toLowerCase() === this.tags[1].toLowerCase()) {
                 reconsume = 0;  /* see 12.2.4.13 - switch state for the following case, otherwise, reconsume. */
                 this.matchEndTagWithStartTag(symbol);
             }
@@ -86,10 +191,7 @@ FastParser.prototype.walk = function(i, input) {
     }
 
     if (reconsume) {                  /* reconsume the character */
-        if( this.states) {
-            // This is error prone. May need to change the way we walk the stream to avoid this.
-            this.states[i] = this.state; 
-        }
+        this.listeners.reWalk && this.emit(this.listeners.reWalk, [this.state, i, endsWithEOF]);
         return this.walk(i, input);
     }
 
@@ -97,22 +199,22 @@ FastParser.prototype.walk = function(i, input) {
 };
 
 FastParser.prototype.createStartTag = function (ch) {
-    this.tagNameIdx = 0;
-    this.tagNames[0] = ch.toLowerCase();
+    this.tagIdx = 0;
+    this.tags[0] = ch;
 };
 
 FastParser.prototype.createEndTag = function (ch) {
-    this.tagNameIdx = 1;
-    this.tagNames[1] = ch.toLowerCase();
+    this.tagIdx = 1;
+    this.tags[1] = ch;
 };
 
 FastParser.prototype.appendTagName = function (ch) {
-    this.tagNames[this.tagNameIdx] += ch.toLowerCase();
+    this.tags[this.tagIdx] += ch;
 };
 
 FastParser.prototype.resetEndTag = function (ch) {
-    this.tagNameIdx = 1;
-    this.tagNames[1] = '';
+    this.tagIdx = 1;
+    this.tags[1] = '';
 };
 
 FastParser.prototype.matchEndTagWithStartTag = function (symbol) {
@@ -124,8 +226,9 @@ FastParser.prototype.matchEndTagWithStartTag = function (symbol) {
         GREATER-THAN SIGN (>): If the current end tag token is an appropriate end tag token, then switch to the data state and emit the current tag token.
                 Otherwise, treat it as per the 'anything else' entry below.
         */
-        this.tagNames[0] = '';
-        this.tagNames[1] = '';
+        this.tags[0] = '';
+        this.tags[1] = '';
+
         switch (symbol) {
             case stateMachine.Symbol.SPACE: /** Whitespaces */
                 this.state = stateMachine.State.STATE_BEFORE_ATTRIBUTE_NAME;
@@ -141,14 +244,14 @@ FastParser.prototype.matchEndTagWithStartTag = function (symbol) {
 
 FastParser.prototype.matchEscapedScriptTag = function (ch) {
     /* switch to the script data double escaped state if we see <script> inside <script><!-- */    
-    if ( this.tagNames[1] === 'script') {
+    if ( this.tags[1].toLowerCase() === 'script') {
         this.state = stateMachine.State.STATE_SCRIPT_DATA_DOUBLE_ESCAPED;
     }
 };
 
 FastParser.prototype.processTagName = function (ch) {
     /* context transition when seeing <sometag> and switch to Script / Rawtext / RCdata / ... */
-    switch (this.tagNames[0]) {
+    switch (this.tags[0].toLowerCase()) {
         // TODO - give exceptions when non-HTML namespace is used.
         // case 'math':
         // case 'svg':
@@ -176,17 +279,17 @@ FastParser.prototype.processTagName = function (ch) {
 
 FastParser.prototype.createAttributeNameAndValueTag = function (ch) {
     /* new attribute name and value token */
-    this.attributeValue = '';
-    this.attributeName = ch.toLowerCase();
+    this.attributeValue = null;
+    this.attrName = ch;
 };
 
 FastParser.prototype.appendAttributeNameTag = function (ch) {
     /* append to attribute name token */
-    this.attributeName += ch.toLowerCase();
+    this.attrName += ch;
 };
 
 FastParser.prototype.appendAttributeValueTag = function(ch) {
-    this.attributeValue += ch;   
+    this.attributeValue = this.attributeValue === null ? ch : this.attributeValue + ch;
 };
 
 /**
@@ -200,69 +303,283 @@ FastParser.prototype.appendAttributeValueTag = function(ch) {
  * e.g. [A-z] = type 17 (Letter [A-z])</p>
  *
  */
-
-
-
 FastParser.prototype.lookupChar = function(ch) {
     var o = ch.charCodeAt(0);
     if ( o > 122 ) { return 12; }
     return stateMachine.lookupSymbolFromChar[o];
 };
 
+/**
+ * @function FastParser#contextualize
+ *
+ * @param {string} input - the input stream
+ */
+FastParser.prototype.contextualize = function(input, endsWithEOF) {
+    var self = this, listeners = self.listeners, i = -1, lastState;
+
+    // Perform input stream preprocessing
+    // Reference: https://html.spec.whatwg.org/multipage/syntax.html#preprocessing-the-input-stream
+    self.input = self.config.enableInputPreProcessing ? input.replace(reInputPreProcessing, function(m){return m[0] === '\r' ? '\n' : '\uFFFD';}) : input;
+    self.inputLen = self.input.length;
+
+    while (++i < self.inputLen) {
+        lastState = self.state;
+
+        // TODO: endsWithEOF handling
+        listeners.preWalk && this.emit(listeners.preWalk, [lastState, i, endsWithEOF]);
+
+        // these functions are not supposed to alter the input
+        self.beforeWalk(i, this.input);
+        self.walk(i, this.input, endsWithEOF);
+        self.afterWalk(i, this.input);
+
+        // TODO: endsWithEOF handling
+        listeners.postWalk && this.emit(listeners.postWalk, [lastState, self.state, i, endsWithEOF]);
+    }
+};
+
 /**
  * @function FastParser#beforeWalk
  *
  * @param {integer} i - the location of the head pointer.
  * @param {string} input - the input stream
  *
- * @return {integer} the new location of the head pointer.
- *
  * @description
  * Interface function for subclass to implement logics before parsing the character.
  *
  */
-FastParser.prototype.beforeWalk = function( i, input ) {
-    return i;
-};
+FastParser.prototype.beforeWalk = function (i, input) {};
 
 /**
  * @function FastParser#afterWalk
  *
- * @param {string} ch - The character consumed.
- * @param {integer} i - the head pointer location of this character
+ * @param {integer} i - the location of the head pointer.
+ * @param {string} input - the input stream
  *
  * @description
  * Interface function for subclass to implement logics after parsing the character.
  *
  */
-FastParser.prototype.afterWalk = function( ch, i ) {
+FastParser.prototype.afterWalk = function (i, input) {};
+
+/**
+ * @function FastParser#getStartTagName
+ * @depreciated Replace it by getCurrentTagIndex and getCurrentTag
+ *
+ * @returns {string} The current handling start tag name
+ *
+ */
+FastParser.prototype.getStartTagName = function() {
+    return this.tags[0] !== undefined? this.tags[0].toLowerCase() : undefined;
 };
 
+/**
+ * @function FastParser#getCurrentTagIndex
+ *
+ * @returns {integer} The current handling tag Idx
+ *
+ */
+FastParser.prototype.getCurrentTagIndex = function() {
+    return this.tagIdx;
+};
 
-function Parser () {
-    FastParser.call(this);
-    this.bytes = [];  /* Save the processed bytes */
-    this.states = [stateMachine.State.STATE_DATA]; /* Save the processed status */
-    this.contexts = [];
-    this.buffer = []; /* Save the processed character into the internal buffer */
-    this.symbols = []; /* Save the processed symbols */
+/**
+ * @function FastParser#getCurrentTag
+ *
+ * @params {integer} The tag Idx
+ *
+ * @returns {string} The current tag name indexed by tag Idx
+ *
+ */
+FastParser.prototype.getCurrentTag = function(tagIdx) {
+    return tagIdx === 0 || tagIdx === 1? (this.tags[tagIdx] !== undefined? this.tags[tagIdx].toLowerCase():undefined) : undefined;
+};
 
+/**
+ * @function FastParser#getAttributeName
+ *
+ * @returns {string} The current handling attribute name.
+ *
+ * @description
+ * Get the current handling attribute name of HTML tag.
+ *
+ */
+FastParser.prototype.getAttributeName = function() {
+    return this.attrName.toLowerCase();
+};
+
+/**
+ * @function FastParser#getAttributeValue
+ *
+ * @returns {string} The current handling attribute name's value.
+ *
+ * @description
+ * Get the current handling attribute name's value of HTML tag.
+ *
+ */
+FastParser.prototype.getAttributeValue = function(htmlDecoded) {
+    // TODO: html decode the attribute value
+    return this.attributeValue;
+};
+
+/**
+* @module Parser
+*/
+function Parser (config, listeners) {
+    var self = this, k;
+
+    config = config || {};
+
+    // config defaulted to false
+    config.enableCanonicalization = (config.enableCanonicalization === true);
+    config.enableVoidingIEConditionalComments = (config.enableVoidingIEConditionalComments === true);
+
+    // config defaulted to true
+    config.enableStateTracking = (config.enableStateTracking !== false);
+
+    // super constructor, reset() is called here
+    FastParser.call(self, config);
+
+    // deep copy the provided listeners, if any
+    if (typeof listeners === 'object') {
+        for (k in listeners) {
+            self.listeners[k] = listeners[k].slice();
+        }
+        return;
+    }
+
+    // ### DO NOT CHANGE THE ORDER OF THE FOLLOWING COMPONENTS ###
+    // fix parse errors before they're encountered in walk()
+    config.enableCanonicalization && self.on('preWalk', Canonicalize).on('reWalk', Canonicalize);
+    // enable IE conditional comments
+    config.enableVoidingIEConditionalComments && self.on('preWalk', DisableIEConditionalComments);
+    // TODO: rewrite IE <comment> tags
+    // TODO: When a start tag token is emitted with its self-closing flag set, if the flag is not acknowledged when it is processed by the tree construction stage, that is a parse error.
+    // TODO: When an end tag token is emitted with attributes, that is a parse error.
+    // TODO: When an end tag token is emitted with its self-closing flag set, that is a parse error.
+
+    // for bookkeeping the processed inputs and states
+    if (config.enableStateTracking) {
+        self.on('postWalk', function (lastState, state, i, endsWithEOF) {
+            this.buffer.push(this.input[i]);
+            this.states.push(state);
+            this.symbol.push(this._getSymbol(i));
+        }).on('reWalk', self.setCurrentState);
+    }
 }
 
 // as in https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Object/prototype 
 Parser.prototype = Object.create(FastParser.prototype);
-Parser.prototype.constructor = FastParser;
-
-Parser.prototype.walk = function(i, input) {
-    i = FastParser.prototype.walk.call(this, i, input);
-    var ch = input[i];
-    this.bytes[i + 1] = ch;
-    this.states[i + 1] = this.state;
-    this.symbols[i + 1] = this.lookupChar(ch);
-    return i;
+Parser.prototype.constructor = Parser;
+
+
+
+/**
+ * @function Parser#reset
+ *
+ * @description
+ * Reset all internal states, as if being created with the new operator
+ */
+Parser.prototype.reset = function () {
+    var self = this;
+
+    FastParser.prototype.reset.call(self);
+
+    if (self.config.enableStateTracking) {
+        self.states = [this.state];
+        self.buffer = [];
+        self.symbol = [];
+    }
+
+    // delete any pending corrections (e.g., to close bogus comment)
+    delete self.listeners.preCanonicalize;
+
+    return self;
+};
+
+
+/**
+* @function Parser._getSymbol
+* @param {integer} i - the index of input stream
+*
+* @description
+* Get the html symbol mapping for the character located in the given index of input stream
+*/
+Parser.prototype._getSymbol = function (i) {
+    return i < this.inputLen ? this.lookupChar(this.input[i]) : -1;
+};
+
+/**
+* @function Parser._getNextState
+* @param {integer} state - the current state
+* @param {integer} i - the index of input stream
+* @returns {integer} the potential state about to transition into, given the current state and an index of input stream
+*
+* @description
+* Get the potential html state about to transition into
+*/
+Parser.prototype._getNextState = function (state, i, endsWithEOF) {
+    return i < this.inputLen ? stateMachine.lookupStateFromSymbol[this._getSymbol(i)][state] : -1;
 };
 
+/**
+* @function Parser._convertString2Array
+*
+* @description
+* Convert the immutable this.input to array type for Strict Context Parser processing (lazy conversion).
+*
+*/
+Parser.prototype._convertString2Array = function () {
+    if (typeof this.input === "string") this.input = this.input.split('');
+};
+
+/**
+* @function Parser.fork
+* @returns {object} a new parser with all internal states inherited
+*
+* @description
+* create a new parser with all internal states inherited
+*/
+Parser.prototype.fork = function() {
+    var parser = new this.constructor(this.config, this.listeners);
+
+    parser.state = this.state;
+    parser.tags = this.tags.slice();
+    parser.tagIdx = this.tagIdx;
+    parser.attrName = this.attrName;
+    parser.attributeValue = this.attributeValue;
 
+    if (this.config.enableStateTracking) {
+        parser.buffer = this.buffer.slice();
+        parser.states = this.states.slice();
+        parser.symbol = this.symbol.slice();
+    }
+    return parser;
+};
+
+/**
+ * @function Parser#contextualize
+ * @param {string} input - the input stream
+ *
+ * @description
+ * It is the same as the original contextualize() except that this method returns the internal input stream.
+ */
+Parser.prototype.contextualize = function (input, endsWithEOF) {
+    FastParser.prototype.contextualize.call(this, input, endsWithEOF);
+    return this.getModifiedInput();
+};
+
+/**
+ * @function Parser#getModifiedInput
+ *
+ * @description
+ * Get the modified input due to Strict Context Parser processing.
+ *
+ */
+Parser.prototype.getModifiedInput = function() {
+    // TODO: it is not defensive enough, should use Array.isArray, but need polyfill
+    return (typeof this.input === "string")? this.input:this.input.join('');
+};
 
 /**
  * @function Parser#setCurrentState
@@ -274,7 +591,22 @@ Parser.prototype.walk = function(i, input) {
  *
  */
 Parser.prototype.setCurrentState = function(state) {
-    this.state = state;
+    this.states.pop();
+    this.states.push(this.state = state);
+    return this;
+};
+
+/**
+ * @function Parser#getCurrentState
+ *
+ * @returns {integer} The last state of the HTML5 Context Parser.
+ *
+ * @description
+ * Get the last state of HTML5 Context Parser.
+ *
+ */
+Parser.prototype.getCurrentState = function() {
+    return this.state;
 };
 
 /**
@@ -287,7 +619,7 @@ Parser.prototype.setCurrentState = function(state) {
  *
  */
 Parser.prototype.getStates = function() {
-    return this.states;
+    return this.states.slice();
 };
 
 /**
@@ -300,7 +632,8 @@ Parser.prototype.getStates = function() {
  *
  */
 Parser.prototype.setInitState = function(state) {
-    this.states[0] = state;
+    this.states = [state];
+    return this;
 };
 
 /**
@@ -331,40 +664,314 @@ Parser.prototype.getLastState = function() {
 };
 
 /**
- * @function Parser#getAttributeName
- *
- * @returns {string} The current handling attribute name.
- *
- * @description
- * Get the current handling attribute name of HTML tag.
- *
- */
-Parser.prototype.getAttributeName = function() {
-    return this.attributeName;
-};
+* The implementation of Strict Context Parser functions
+* 
+* - ConvertBogusCommentToComment
+* - PreCanonicalizeConvertBogusCommentEndTag
+* - Canonicalize
+* - DisableIEConditionalComments
+*
+*/
 
-/**
- * @function Parser#getAttributeValue
- *
- * @returns {string} The current handling attribute name's value.
- *
- * @description
- * Get the current handling attribute name's value of HTML tag.
- *
- */
-Parser.prototype.getAttributeValue = function() {
-    return this.attributeValue;
-};
+function ConvertBogusCommentToComment(i) {
+    // for lazy conversion
+    this._convertString2Array();
 
-/**
- * @function Parser#getStartTagName
- *
- * @returns {string} The current handling start tag name
- *
- */
-Parser.prototype.getStartTagName = function() {
-    return this.tagNames[0];
-};
+    // convert !--. i.e., from <* to <!--*
+    this.input.splice(i, 0, '!', '-', '-');
+    this.inputLen += 3;
+
+    // convert the next > to -->
+    this.on('preCanonicalize', PreCanonicalizeConvertBogusCommentEndTag);
+}
+
+function PreCanonicalizeConvertBogusCommentEndTag(state, i, endsWithEOF) {
+    if (this.input[i] === '>') {
+        // remove itself from the listener list
+        this.off('preCanonicalize', PreCanonicalizeConvertBogusCommentEndTag);
+
+        // for lazy conversion
+        this._convertString2Array();
+
+        // convert [>] to [-]->
+        this.input.splice(i, 0, '-', '-');
+        this.inputLen += 2;
+
+        this.emit(this.listeners.bogusCommentCoverted, [state, i, endsWithEOF]);
+    }
+}
+
+// those doctype states (52-67) are initially treated as bogus comment state, but they are further converted to comment state
+// Canonicalize() will create no more bogus comment state except the fake (context-parser treats <!doctype as bogus) one hardcoded as <!doctype html> that has no NULL inside
+var statesRequiringNullReplacement = [
+//    0, 1, 2, 3, 4, 5, 6, 7, 8, 9
+/*0*/ 0, 0, 0, 1, 0, 1, 1, 1, 0, 0,
+/*1*/ 1, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+/*2*/ 0, 0, 1, 1, 1, 0, 0, 0, 0, 1,
+/*3*/ 1, 1, 0, 0, 1, 1, 1, 1, 1, 1,
+/*4*/ 1, 0, 0, 0, 1, 0, 1, 1, 1, 1,
+/*5*/ 1, 1
+];
+// \uFFFD replacement is not required by the spec for DATA state
+statesRequiringNullReplacement[htmlState.STATE_DATA] = 1;
+
+function Canonicalize(state, i, endsWithEOF) {
+
+    this.emit(this.listeners.preCanonicalize, [state, i, endsWithEOF]);
+
+    var reCanonicalizeNeeded = true,
+        chr = this.input[i], nextChr = this.input[i+1],
+        potentialState = this._getNextState(state, i, endsWithEOF),
+        nextPotentialState = this._getNextState(potentialState, i + 1, endsWithEOF);
+
+    // console.log(i, state, potentialState, nextPotentialState, this.input.slice(i).join(''));
+
+    // batch replacement of NULL with \uFFFD would violate the spec
+    //  - for example, NULL is untouched in CDATA section state
+    if (chr === '\x00' && statesRequiringNullReplacement[state]) {
+        // for lazy conversion
+        this._convertString2Array();
+        this.input[i] = '\uFFFD';
+    }
+    // encode < into &lt; for [<]* (* is non-alpha) in STATE_DATA, [<]% and [<]! in STATE_RCDATA and STATE_RAWTEXT
+    else if ((potentialState === htmlState.STATE_TAG_OPEN && nextPotentialState === htmlState.STATE_DATA) ||  // [<]*, where * is non-alpha
+             ((state === htmlState.STATE_RCDATA || state === htmlState.STATE_RAWTEXT) &&                            // in STATE_RCDATA and STATE_RAWTEXT
+            chr === '<' && (nextChr === '%' || nextChr === '!'))) {   // [<]% or [<]!
+        // for lazy conversion
+        this._convertString2Array();
+
+        // [<]*, [<]%, [<]!
+        this.input.splice(i, 1, '&', 'l', 't', ';');
+        this.inputLen += 3;
+    }
+    // enforce <!doctype html>
+    // + convert bogus comment or unknown doctype to the standard html comment
+    else if (potentialState === htmlState.STATE_MARKUP_DECLARATION_OPEN) {            // <[!]***
+        reCanonicalizeNeeded = false;
+
+        // for lazy conversion
+        this._convertString2Array();
+
+        // context-parser treats the doctype and [CDATA[ as resulting into STATE_BOGUS_COMMENT
+        // so, we need our algorithm here to extract and check the next 7 characters
+        var commentKey = this.input.slice(i + 1, i + 8).join('');
+
+        // enforce <!doctype html>
+        if (commentKey.toLowerCase() === 'doctype') {               // <![d]octype
+            // extract 6 chars immediately after <![d]octype and check if it's equal to ' html>'
+            if (this.input.slice(i + 8, i + 14).join('').toLowerCase() !== ' html>') {
+
+                // replace <[!]doctype xxxx> with <[!]--!doctype xxxx--><doctype html>
+                ConvertBogusCommentToComment.call(this, i);
+
+                this.once('bogusCommentCoverted', function (state, i) {
+                    [].splice.apply(this.input, [i + 3, 0].concat('<!doctype html>'.split('')));
+                    this.inputLen += 15;
+                });
+
+                reCanonicalizeNeeded = true;
+            }
+        }
+        // do not touch <![CDATA[ and <[!]--
+        else if (commentKey === '[CDATA[' ||
+                    (nextChr === '-' && this.input[i+2] === '-')) {
+            // noop
+        }
+        // ends up in bogus comment
+        else {
+            // replace <[!]*** with <[!]--***
+            // will replace the next > to -->
+            ConvertBogusCommentToComment.call(this, i);
+            reCanonicalizeNeeded = true;
+        }
+    }
+    // convert bogus comment to the standard html comment
+    else if ((state === htmlState.STATE_TAG_OPEN &&
+             potentialState === htmlState.STATE_BOGUS_COMMENT) ||           // <[?] only from STATE_TAG_OPEN
+            (potentialState === htmlState.STATE_END_TAG_OPEN &&             // <[/]* or <[/]> from STATE_END_TAG_OPEN
+             nextPotentialState !== htmlState.STATE_TAG_NAME &&
+             nextPotentialState !== -1)) {                                  // TODO: double check if there're any other cases requiring -1 check
+        // replace <? and </* respectively with <!--? and <!--/*
+        // will replace the next > to -->
+        ConvertBogusCommentToComment.call(this, i);
+    }
+    // remove the unnecessary SOLIDUS
+    else if (potentialState === htmlState.STATE_SELF_CLOSING_START_TAG &&             // <***[/]*
+            nextPotentialState === htmlState.STATE_BEFORE_ATTRIBUTE_NAME) {           // this.input[i+1] is ANYTHING_ELSE (i.e., not EOF nor >)
+        // if ([htmlState.STATE_TAG_NAME,                                             // <a[/]* replaced with <a[ ]*
+        //     /* following is unknown to CP
+        //     htmlState.STATE_RCDATA_END_TAG_NAME,
+        //     htmlState.STATE_RAWTEXT_END_TAG_NAME,
+        //     htmlState.STATE_SCRIPT_DATA_END_TAG_NAME,
+        //     htmlState.STATE_SCRIPT_DATA_ESCAPED_END_TAG_NAME,
+        //     */
+        //     htmlState.STATE_BEFORE_ATTRIBUTE_NAME,                                 // <a [/]* replaced with <a [ ]*
+        //     htmlState.STATE_AFTER_ATTRIBUTE_VALUE_QUOTED].indexOf(state) !== -1)   // <a abc=""[/]* replaced with <a abc=""[ ]*
+   
+        // for lazy conversion
+        this._convertString2Array();
+
+        this.input[i] = ' ';
+
+        // given this.input[i] was    '/', nextPotentialState was htmlState.STATE_BEFORE_ATTRIBUTE_NAME
+        // given this.input[i] is now ' ', nextPotentialState becomes STATE_BEFORE_ATTRIBUTE_NAME if current state is STATE_ATTRIBUTE_NAME or STATE_AFTER_ATTRIBUTE_NAME
+        // to preserve state, remove future EQUAL SIGNs (=)s to force STATE_AFTER_ATTRIBUTE_NAME behave as if it is STATE_BEFORE_ATTRIBUTE_NAME
+        // this is okay since EQUAL SIGNs (=)s will be stripped anyway in the STATE_BEFORE_ATTRIBUTE_NAME cleanup handling
+        if (state === htmlState.STATE_ATTRIBUTE_NAME ||                               // <a abc[/]=abc  replaced with <a abc[ ]*
+                state === htmlState.STATE_AFTER_ATTRIBUTE_NAME) {                     // <a abc [/]=abc replaced with <a abc [ ]*
+            for (var j = i + 1; j < this.inputLen && this.input[j] === '='; j++) {
+                this.input.splice(j, 1);
+                this.inputLen--;
+            }
+        }
+    }
+    // remove unnecessary equal signs, hence <input checked[=]> become <input checked[>], or <input checked [=]> become <input checked [>]
+    else if (potentialState === htmlState.STATE_BEFORE_ATTRIBUTE_VALUE &&   // only from STATE_ATTRIBUTE_NAME or STATE_AFTER_ATTRIBUTE_NAME
+            nextPotentialState === htmlState.STATE_DATA) {                  // <a abc[=]> or <a abc [=]>
+        // for lazy conversion
+        this._convertString2Array();
+
+        this.input.splice(i, 1);
+        this.inputLen--;
+    }
+    // insert a space for <a abc="***["]* or <a abc='***[']* after quoted attribute value (i.e., <a abc="***["] * or <a abc='***['] *)
+    else if (potentialState === htmlState.STATE_AFTER_ATTRIBUTE_VALUE_QUOTED &&        // <a abc=""[*] where * is not SPACE (\t,\n,\f,' ')
+            nextPotentialState === htmlState.STATE_BEFORE_ATTRIBUTE_NAME &&
+            this._getSymbol(i + 1) !== stateMachine.Symbol.SPACE) {
+        // for lazy conversion
+        this._convertString2Array();
+
+        this.input.splice(i + 1, 0, ' ');
+        this.inputLen++;
+    }
+    // else here means no special pattern was found requiring rewriting
+    else {
+        reCanonicalizeNeeded = false;
+    }
+
+    // remove " ' < = from being treated as part of attribute name (not as the spec recommends though)
+    switch (potentialState) {
+        case htmlState.STATE_BEFORE_ATTRIBUTE_NAME:     // remove ambigious symbols in <a [*]href where * is ", ', <, or =
+            if (nextChr === "=") {
+                // for lazy conversion
+                this._convertString2Array();
+
+                this.input.splice(i + 1, 1);
+                this.inputLen--;
+                reCanonicalizeNeeded = true;
+                break;
+            }
+            /* falls through */
+        case htmlState.STATE_ATTRIBUTE_NAME:            // remove ambigious symbols in <a href[*] where * is ", ', or <
+        case htmlState.STATE_AFTER_ATTRIBUTE_NAME:      // remove ambigious symbols in <a href [*] where * is ", ', or <
+            if (nextChr === '"' || nextChr === "'" || nextChr === '<') {
+                // for lazy conversion
+                this._convertString2Array();
+
+                this.input.splice(i + 1, 1);
+                this.inputLen--;
+                reCanonicalizeNeeded = true;
+            }
+            break;
+    }
+
+    if (reCanonicalizeNeeded) {
+        return Canonicalize.call(this, state, i, endsWithEOF);
+    }
+
+    switch (state) {
+    // escape " ' < = ` to avoid raising parse errors for unquoted value
+        case htmlState.STATE_ATTRIBUTE_VALUE_UNQUOTED:
+            if (chr === '"') {
+                // for lazy conversion
+                this._convertString2Array();
+
+                this.input.splice(i, 1, '&', 'q', 'u', 'o', 't', ';');
+                this.inputLen += 5;
+                break;
+            } else if (chr === "'") {
+                // for lazy conversion
+                this._convertString2Array();
+
+                this.input.splice(i, 1, '&', '#', '3', '9', ';');
+                this.inputLen += 4;
+                break;
+            }
+            /* falls through */
+        case htmlState.STATE_BEFORE_ATTRIBUTE_VALUE:     // treat < = ` as if they are in STATE_ATTRIBUTE_VALUE_UNQUOTED
+            if (chr === '<') {
+                // for lazy conversion
+                this._convertString2Array();
+
+                this.input.splice(i, 1, '&', 'l', 't', ';');
+                this.inputLen += 3;
+            } else if (chr === '=') {
+                // for lazy conversion
+                this._convertString2Array();
+
+                this.input.splice(i, 1, '&', '#', '6', '1', ';');
+                this.inputLen += 4;
+            } else if (chr === '`') {
+                // for lazy conversion
+                this._convertString2Array();
+
+                this.input.splice(i, 1, '&', '#', '9', '6', ';');
+                this.inputLen += 4;
+            }
+            break;
+
+    // add hyphens to complete <!----> to avoid raising parsing errors
+        // replace <!--[>] with <!--[-]->
+        case htmlState.STATE_COMMENT_START:
+            if (chr === '>') {                          // <!--[>]
+                // for lazy conversion
+                this._convertString2Array();
+
+                this.input.splice(i, 0, '-', '-');
+                this.inputLen += 2;
+                // reCanonicalizeNeeded = true;  // not need due to no where to treat its potential states
+            }
+            break;
+        // replace <!---[>] with <!---[-]>
+        case htmlState.STATE_COMMENT_START_DASH:
+            if (chr === '>') {                          // <!---[>]
+                // for lazy conversion
+                this._convertString2Array();
+
+                this.input.splice(i, 0, '-');
+                this.inputLen++;
+                // reCanonicalizeNeeded = true;  // not need due to no where to treat its potential states
+            }
+            break;
+    // replace --[!]> with --[>]
+        case htmlState.STATE_COMMENT_END:
+            if (chr === '!' && nextChr === '>') {
+                // for lazy conversion
+                this._convertString2Array();
+
+                this.input.splice(i, 1);
+                this.inputLen--;
+                // reCanonicalizeNeeded = true;  // not need due to no where to treat its potential states
+            }
+            // if (chr === '-'), ignored this parse error. TODO: consider stripping n-2 hyphens for ------>
+            break;
+    }
+
+    if (reCanonicalizeNeeded) {
+        return Canonicalize.call(this, state, i, endsWithEOF);
+    }
+}
+
+// remove IE conditional comments
+function DisableIEConditionalComments(state, i){
+    if (state === htmlState.STATE_COMMENT && this.input[i] === ']' && this.input[i+1] === '>') {
+        // for lazy conversion
+        this._convertString2Array();
+
+        this.input.splice(i + 1, 0, ' ');
+        this.inputLen++;
+    }
+}
 
 module.exports = {
     Parser: Parser,
@@ -510,9 +1117,6 @@ StateMachine.lookupSymbolFromChar = [
     11,11,11,12
 ];
 
-
-// 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10, 1, 2, 3, 4, 5, 6, 7, 8, 9,20, 1, 2, 3, 4, 5, 6, 7, 8, 9,30, 1, 2, 3, 4, 5, 6, 7, 8, 9,40, 1, 2, 3, 4, 5, 6, 7, 8, 9,50, 1, 2, 3, 4, 5, 6, 7, 8, 9,
-
 StateMachine.lookupStateFromSymbol = [
  [ 0, 1, 0, 3, 0, 5, 6, 7, 1,44,34, 3, 3, 3, 5, 5, 5, 6, 6, 6, 6, 6,22,22,22,22,22,22,22,29,29,29,29,29,34,36,36,37,38,39,34, 0,34,34,44,44,48,48,48,48,48,48, 0,44],
  [ 0, 1, 0, 3, 0, 5, 6, 7,45,44,10, 3, 3, 3, 5, 5, 5,20, 6, 6, 6, 6,22,22,22,22,22,22,22,29,29,29,29,29,35,35,35,40,38,39,40, 0,34,34,44,44,48,48,48,48,51,48, 0,44],
@@ -627,4 +1231,4 @@ StateMachine.lookupContext = [
 
 module.exports = StateMachine;
 },{}]},{},[1])(1)
-});
+});
\ No newline at end of file
diff --git a/package.json b/package.json
index 72e5a36..5ee13fc 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "context-parser",
-  "version": "2.0.0",
+  "version": "2.0.1",
   "licenses": [
     {
       "type": "BSD",