/*
 * [HTMLState.java]
 *
 * Summary: State machine for HTMLTokenizer to parse HTML for colourising.
 *
 * Copyright: (c) 2004-2017 Roedy Green, Canadian Mind Products, http://mindprod.com
 *
 * Licence: This software may be copied and used freely for any purpose but military.
 *          http://mindprod.com/contact/nonmil.html
 *
 * Requires: JDK 1.8+
 *
 * Created with: JetBrains IntelliJ IDEA IDE http://www.jetbrains.com/idea/
 *
 * Version History:
 *  3.1 2009-04-12 shorter style names, improved highlighting.
 *  3.2 2010-02-08 handle attributes and CDATA with special attributes.
 *                 add markHandled method.
 */
/**
 * State machine for HTMLTokenizer to parse HTML for colourising.
 *
 * @author Roedy Green, Canadian Mind Products
 * @version 3.2 2010-02-08 handle attributes and CDATA with special attributes.
 * add markHandled method. mark start and end of comments specially.
 * Use simpler look-ahead logic with mark handled.
 * @since 2004-05-15
 */
package com.mindprod.jprep;

import com.mindprod.common18.ST;
import com.mindprod.entities.DeEntifyStrings;
import com.mindprod.jtokens.Gibberish;
import com.mindprod.jtokens.NL;
import com.mindprod.jtokens.Operator;
import com.mindprod.jtokens.Space;
import com.mindprod.jtokens.Start;
import com.mindprod.jtokens.Stop;
import com.mindprod.jtokens.Token;
import com.mindprod.jtokens.html.CDATAContents;
import com.mindprod.jtokens.html.CDATATag;
import com.mindprod.jtokens.html.HTMLAttribute;
import com.mindprod.jtokens.html.HTMLCommentContents;
import com.mindprod.jtokens.html.HTMLCommentTag;
import com.mindprod.jtokens.html.HTMLEntity;
import com.mindprod.jtokens.html.HTMLStringLiteral;
import com.mindprod.jtokens.html.HTMLTag;
import com.mindprod.jtokens.html.HTMLText;

import java.util.ArrayList;

import static java.lang.System.*;

/**
 * State machine for HTMLTokenizer to parse HTML for colourising.
 * <p/>
 * This parser is complicated by the nesting of HTML. You can have a tag
 * inside body text, a string inside a tag and an entity inside a string. You can also have entities inside body text.
 * Comments may be embedded in body text or tags, but not strings or entities. So, for example, an entity state must
 * what what state to return to when the entity is over.<br> NL is not as significant as it is in other languages. You
 * can have a NL in the middle of a tag or a string, but not an entity. The tags and strings token don't have embedded
 * NLs, though they do during accumulation. <br> String tokens may be missing lead or trail quotes. We don't have
 * END_OF_LINE or WHITESPACE states. We split out SPACE and NL tokens LATER <br> The notion of kosher = false means just
 * that one char is not kosher but we let it slide and don't necessarily change state. However the split that naughty
 * character off into a Gibberish token by itself for special highlighting. <br> The notion of balanced is similar. In
 * that case the entire unbalanced entity, string etc is put into a Gibberish token.
 *
 * @author Roedy Green, Canadian Mind Products
 * @version 3.2 2010-02-08 handle attributes and CDATA with special attributes.
 * add markHandled method. mark start and end of comments specially.
 * Use simpler look-ahead logic with mark handled.
 * @see com.mindprod.compactor.HTMLState
 * @see com.mindprod.htmlreflow.HTMLState
 * @since 2004-07-17
 */
@SuppressWarnings( { "NestedAssignment", "UnnecessaryContinue", "EnumeratedConstantNamingConvention" } )
public enum HTMLState
    {
        /**
         * in an entity &...; inside quotes
         */
        @SuppressWarnings( { "WeakerAccess" } ) IN_QUOTE_ENTITY
                    {
                    private boolean balanced;

                    /**
                     * Consume one character. It has been predecided that you can
                     * and will consume it.
                     * @param c        char to consume
                     */
                    void consume( char c )
                        {
                        accumulated.append( c );
                        }

                    /**
                     * what to do on leaving state, after last char is consumed.
                     */
                    void leaving()
                        {
                        String entity = accumulated.toString();
                        accumulated.setLength( 0 );
                        // no embedded spaces or nls to deal with.
                        if ( balanced )
                            {
                            // does not have lead & or trailing ;
                            // Known includes hex and numeric forms.
                            // case-sensitive
                            boolean known = DeEntifyStrings.bareHTMLEntityToChar( entity, ' ' ) != 0;
                            // add even if chars 0 long. Could erroneously have written &;
                            tokens.add( new HTMLEntity( entity
                                    /* &; stripped */,
                                    known ) );
                            }
                        else
                            {
                            // screwed up entity that was not really one.
                            // should treat it really part of a quotation. It is an error..
                            tokens.add( new Gibberish( "&" + entity ) );
                            // note that we have handled this anomaly
                            balanced = true;
                            }
                        } // end leaving

                    /**
                     * Figure out what state we should go in after we process this
                     * character and whether we can consume it. We must set consume
                     * = false if we cannot consume the character and have to
                     * postpone processing to the next state.
                     * @param category category of character
                     * @param nextChar character in the stream we are processing
                     * @param first true if we just entered this state.
                     * @return next state to go in
                     */
                    HTMLState next( HTMLCharCategory category,
                                    char nextChar,
                                    boolean first )
                        {
                        switch ( category )
                            {
                            case AMP:// entities not magic in comments
                                if ( first )
                                    {
                                    balanced = false;
                                    // discard lead &
                                    how = HowToProcess.DISCARD;
                                    return IN_QUOTE_ENTITY;
                                    }
                                else
                                    {
                                    // malformed entity
                                    balanced = false;
                                    how = HowToProcess.FORWARD;
                                    return IN_QUOTES;
                                    }
                            case DASH:
                            case END_TAG:
                            case EOL:
                            case OTHER:
                            case QUOTE:
                            case SPACE:
                            case START_TAG:
                                // oops. Should not see these inside an entity.
                                balanced = false;
                                how = HowToProcess.FORWARD;
                                return IN_QUOTES;
                            case EQUALS:
                            case PLAIN:
                            case RIGHT_BRACKET:
                                if ( SAFEENTITYCHARS.indexOf( nextChar ) >= 0 )
                                    {
                                    how = HowToProcess.CONSUME;
                                    return IN_QUOTE_ENTITY;
                                    }
                                else
                                    {
                                    balanced = false;
                                    how = HowToProcess.FORWARD;
                                    return IN_QUOTES;
                                    }
                            case SEMICOLON:
                                balanced = true;
                                how = HowToProcess.DISCARD;
                                return IN_QUOTES;
                            case IGNORE:
                            default:
                                assert false :
                                        "bad state "
                                        + category
                                        + " "
                                        + nextChar;
                                return null;
                            } // end switch
                        } // end next
                    // end IN_QUOTE_ENTITY
                    },
        /**
         * inside " ... " Use entities inside quotes to embed quote. Quotes only exist inside tags. We don't allow
         * quotes to span an EOL.
         */
        @SuppressWarnings( { "WeakerAccess" } ) IN_QUOTES
                    {
                    /**
                     * Consume one character. It has been predecided that you can
                     * and will consume it.
                     * @param c        char to consume
                     */
                    void consume( char c )
                        {
                        accumulated.append( c );
                        }

                    /**
                     * what to do on leaving state, after last char is consumed.
                     */
                    void leaving()
                        {
                        // either lead or trail quotes could be missing.. May even have embedded
                        // nl. or strings of spaces
                        addMultilineTokens( new HTMLStringLiteral( "" ),
                                accumulated.toString() );
                        accumulated.setLength( 0 );
                        // end leaving
                        }

                    /**
                     * Figure out what state we should go in after we process this
                     * character and whether we can consume it. We must set consume
                     * = false if we cannot consume the character and have to
                     * postpone processing to the next state.
                     * @param category category of character
                     * @param nextChar character in the stream we are processing
                     * @param first true if we just entered this state.
                     * @return next state to go in
                     */
                    HTMLState next( HTMLCharCategory category,
                                    char nextChar,
                                    boolean first )
                        {
                        switch ( category )
                            {
                            case AMP:
                                // entity inside quotes
                                how = HowToProcess.FORWARD;
                                return IN_QUOTE_ENTITY;
                            case DASH:
                            case EOL:// treat as part of quote, split it out later
                            case END_TAG:
                            case EQUALS:
                            case OTHER:
                            case PLAIN:
                            case RIGHT_BRACKET:
                            case SEMICOLON:
                            case SPACE:
                            case START_TAG:
                                how = HowToProcess.CONSUME;
                                return IN_QUOTES;
                            case QUOTE:
                                // handle the first " differently
                                if ( first )
                                    {
                                    how = HowToProcess.CONSUME;
                                    return IN_QUOTES;
                                    }
                                else
                                    {
                                    // last
                                    how = HowToProcess.CONSUME;
                                    return IN_TAG;
                                    }
                            case IGNORE:
                            default:
                                assert false :
                                        "bad state "
                                        + category
                                        + " "
                                        + nextChar;
                                return null;
                            } // end switch
                        // end next
                        }
                    // end IN_QUOTES
                    },
        /**
         * inside tag &lt;xxx  something="abc"... &gt; inside attribute name, not value.
         */
        @SuppressWarnings( { "WeakerAccess" } ) IN_ATTRIBUTE
                    {
                    /**
                     * a single char was detected that was erroneous. We mark it
                     * with a special token, but otherwise carry on as if it were a
                     * correct character.
                     */
                    private boolean gibberish;

                    /**
                     * Consume one character. It has been predecided that you can
                     * and will consume it.
                     * @param c        char to consume
                     */
                    void consume( char c )
                        {
                        if ( gibberish )
                            {
                            // we must split off this single char into its own token, may be consolidated later.
                            addMultilineTokens( new HTMLAttribute( "" ),
                                    accumulated.toString() );
                            accumulated.setLength( 0 );
                            addToken( new Gibberish( c ) );
                            gibberish = false;
                            }
                        else
                            {
                            accumulated.append( c );
                            }
                        }

                    /**
                     * what to do on leaving state, after last char is consumed.
                     */
                    void leaving()
                        {
                        addMultilineTokens( new HTMLAttribute( "" ),
                                accumulated.toString() );
                        accumulated.setLength( 0 );
                        } // end leaving

                    /**
                     * Figure out what state we should go in after we process this
                     * character and whether we can consume it. We must set consume
                     * = false if we cannot consume the character and have to
                     * postpone processing to the next state.
                     * @param category category of character
                     * @param nextChar character in the stream we are processing
                     * @param first true if we just entered this state.
                     * @return next state to go in
                     */
                    HTMLState next( HTMLCharCategory category,
                                    char nextChar,
                                    boolean first )
                        {
                        switch ( category )
                            {
                            case AMP:// entity inside tag, oops
                            case OTHER:
                            case QUOTE:
                            case START_TAG:
                                gibberish = true;
                                how = HowToProcess.CONSUME;
                                return IN_TAG;
                            case END_TAG:
                            case EQUALS:
                            case EOL:
                            case SPACE:
                                // attribute is complete
                                how = HowToProcess.FORWARD;
                                return IN_TAG;
                            case DASH:
                            case PLAIN:
                            case RIGHT_BRACKET:
                            case SEMICOLON:
                                // another char in attribute name.
                                how = HowToProcess.CONSUME;
                                return IN_ATTRIBUTE;
                            case IGNORE:
                            default:
                                assert false :
                                        "bad state "
                                        + category
                                        + " "
                                        + nextChar;
                                return null;
                            } // end switch
                        // end next
                        }
                    // end IN_ATTRIBUTE
                    },
        /**
         * inside tag &lt;xxx  something="abc"... &gt; inside space inside a tag not inside quote
         */
        @SuppressWarnings( { "WeakerAccess" } ) IN_TAG_SPACE
                    {
                    /**
                     * a single char was detected that was erroneous. We mark it
                     * with a special token, but otherwise carry on as if it were a
                     * correct character.
                     */
                    private boolean gibberish;

                    /**
                     * Consume one character. It has been predecided that you can
                     * and will consume it.
                     * @param c        char to consume
                     */
                    void consume( char c )
                        {
                        if ( gibberish )
                            {
                            // we must split off this single char into its own token, may be consolidated later.
                            addMultilineTokens( new HTMLAttribute( "" ), accumulated.toString() );
                            accumulated.setLength( 0 );
                            addToken( new Gibberish( c ) );
                            gibberish = false;
                            }
                        else
                            {
                            accumulated.append( c );
                            }
                        }

                    /**
                     * what to do on leaving state, after last char is consumed.
                     */
                    void leaving()
                        {
                        addMultilineTokens( new Space(), accumulated.toString() );
                        accumulated.setLength( 0 );
                        } // end leaving

                    /**
                     * Figure out what state we should go in after we process this
                     * character and whether we can consume it. We must set consume
                     * = false if we cannot consume the character and have to
                     * postpone processing to the next state.
                     * @param category category of character
                     * @param nextChar character in the stream we are processing
                     * @param first true if we just entered this state.
                     * @return next state to go in
                     */
                    HTMLState next( HTMLCharCategory category,
                                    char nextChar,
                                    boolean first )
                        {
                        switch ( category )
                            {
                            case AMP:
                            case EQUALS:
                            case OTHER:
                            case QUOTE:
                            case SEMICOLON:
                            case START_TAG:
                                gibberish = true;
                                how = HowToProcess.CONSUME;
                                return IN_TAG_SPACE;
                            case END_TAG:
                                how = HowToProcess.FORWARD;
                                return IN_TAG;
                            case EOL:
                            case SPACE:
                                // still is space between attributes
                                how = HowToProcess.CONSUME;
                                return IN_TAG_SPACE;
                            case DASH:
                            case PLAIN:
                            case RIGHT_BRACKET:
                                // another char in attribute name.
                                how = HowToProcess.FORWARD;
                                return IN_ATTRIBUTE;
                            case IGNORE:
                            default:
                                assert false :
                                        "bad state "
                                        + category
                                        + " "
                                        + nextChar;
                                return null;
                            } // end switch
                        // end next
                        }
                    // end IN_ATTRIBUTE
                    },
        /**
         * inside tag &lt; ... &gt; not inside tagged body text.
         */
        @SuppressWarnings( { "WeakerAccess" } ) IN_TAG
                    {
                    /**
                     * a single char was detected that was erroneous. We mark it
                     * with a special token, but otherwise carry on as if it were a
                     * correct character.
                     */
                    private boolean gibberish;

                    /**
                     * Consume one character. It has been predecided that you can
                     * and will consume it.
                     * @param c        char to consume
                     */
                    void consume( char c )
                        {
                        if ( gibberish )
                            {
                            // we must split off this single char into its own token
                            addMultilineTokens( new HTMLTag( "" ), accumulated.toString() );
                            accumulated.setLength( 0 );
                            addToken( new Gibberish( c ) );
                            gibberish = false;
                            }
                        else
                            {
                            accumulated.append( c );
                            }
                        }

                    /**
                     * what to do on leaving state, after last char is consumed.
                     */
                    void leaving()
                        {
                        // each chunk of the <tag ... > is a separate token.
                        // We have no mechanism to nest styles or tokens.
                        addMultilineTokens( new HTMLTag( "" ),
                                accumulated.toString() );
                        accumulated.setLength( 0 );
                        } // end leaving

                    /**
                     * Figure out what state we should go in after we process this
                     * character and whether we can consume it. We must set consume
                     * = false if we cannot consume the character and have to
                     * postpone processing to the next state.
                     * @param category category of character
                     * @param nextChar character in the stream we are processing
                     * @param first true if we just entered this state.
                     * @return next state to go in
                     */
                    HTMLState next( HTMLCharCategory category,
                                    char nextChar,
                                    boolean first )
                        {
                        switch ( category )
                            {
                            case AMP:// entity inside tag, oops
                            case OTHER:
                            case SEMICOLON:
                                gibberish = true;
                                how = HowToProcess.CONSUME;
                                return IN_TAG;
                            case START_TAG:
                                if ( first )
                                    {
                                    how = HowToProcess.CONSUME;
                                    return IN_TAG;
                                    }
                                else
                                    {
                                    // starting a new tag without finishing the previous one
                                    gibberish = true;
                                    how = HowToProcess.CONSUME;
                                    return IN_TAG;
                                    }
                            case END_TAG:
                                how = HowToProcess.CONSUME;
                                return IN_TEXT;
                            case DASH:
                            case PLAIN:
                            case RIGHT_BRACKET:
                                how = HowToProcess.CONSUME;
                                return IN_TAG;
                            case EQUALS:
                                addMultilineTokens( new HTMLTag( "" ), accumulated.toString() );
                                accumulated.setLength( 0 );
                                addToken( new Operator( '=' ) );
                                how = HowToProcess.DISCARD;
                                return IN_TAG;
                            case SPACE:
                            case EOL:// consume EOLs in side tags as part of tag, split them
                                // out
                                // later
                                how = HowToProcess.FORWARD;
                                return IN_TAG_SPACE;
                            case QUOTE:
                                how = HowToProcess.FORWARD;
                                return IN_QUOTES;
                            case IGNORE:
                            default:
                                assert false :
                                        "bad state "
                                        + category
                                        + " "
                                        + nextChar;
                                return null;
                            } // end switch
                        // end next
                        }
                    // end IN_TAG
                    },
        /**
         * in comment &lt;!-- ... --&gt; in text, inside tag, same as IN_TEXT_COMMENT but returns to IN_TAG
         */
        @SuppressWarnings( { "WeakerAccess" } ) IN_TAG_COMMENT
                    {
                    /**
                     * Consume one character. It has been predecided that you can
                     * and will consume it.
                     * @param c        char to consume
                     */
                    void consume( char c )
                        {
                        accumulated.append( c );
                        }

                    /**
                     * what to do on leaving state, after last char is consumed.
                     */
                    void leaving()
                        {
                        // token added includes the begin and end markers, with possibly
                        // either end marker missing.
                        addToken( new HTMLCommentTag( "<!--" ) );
                        addMultilineTokens( new HTMLCommentContents( "" ), accumulated.toString() );
                        accumulated.setLength( 0 );
                        addToken( new HTMLCommentTag( "-->" ) );
                        } // end leaving

                    /**
                     * Figure out what state we should go in after we process this
                     * character and whether we can consume it. We must set consume
                     * = false if we cannot consume the character and have to
                     * postpone processing to the next state.
                     * @param category category of character
                     * @param nextChar character in the stream we are processing
                     * @param first true if we just entered this state.
                     * @return next state to go in
                     */
                    HTMLState next( HTMLCharCategory category,
                                    char nextChar,
                                    boolean first )
                        {
                        switch ( category )
                            {
                            case AMP:// entities not magic in comments
                            case EOL:// just accumulate them for now
                            case END_TAG: // without lead --, just an ordinary char
                            case EQUALS:
                            case OTHER:// don't sweat high ascii in comments.
                            case PLAIN:
                            case QUOTE:// quotes are nothing special in comments
                            case RIGHT_BRACKET:
                            case SEMICOLON:
                            case SPACE:// just accumulate them for now
                                how = HowToProcess.CONSUME;
                                return IN_TAG_COMMENT;
                            case DASH:
                                if ( isComing( "-->" ) )
                                    {
                                    markHandled( 3 );
                                    // don't accumulate
                                    how = HowToProcess.DISCARD;
                                    return IN_TAG;
                                    }
                                else
                                    {
                                    // just an ordinary char
                                    how = HowToProcess.CONSUME;
                                    return IN_TAG_COMMENT;
                                    }
                            case START_TAG:
                                if ( first && isComing( "<!--" ) )
                                    {
                                    markHandled( 4 );
                                    // don't accumulate
                                    how = HowToProcess.DISCARD;
                                    return IN_TAG_COMMENT;
                                    }
                                else
                                    {
                                    // just an ordinary char . We don't treat nested comment as an error.
                                    how = HowToProcess.CONSUME;
                                    return IN_TAG_COMMENT;
                                    }
                            case IGNORE:
                            default:
                                assert false :
                                        "bad state "
                                        + category
                                        + " "
                                        + nextChar;
                                return null;
                            // end switch
                            }
                        // end next
                        }
                    },
        /**
         * in html body text, not inside a tag.
         */
        @SuppressWarnings( { "WeakerAccess" } ) IN_TEXT
                    {
                    /**
                     * a single char was detected that was erroneous. We mark it
                     * with a special token, but otherwise carry on as if it were a
                     * correct character.
                     */
                    private boolean gibberish;

                    /**
                     * Consume one character. It has been predecided that you can
                     * and will consume it.
                     * @param c        char to consume
                     */
                    void consume( char c )
                        {
                        if ( gibberish )
                            {
                            // we must split off this single char into its own token
                            addMultilineTokens( new HTMLText( "" ), accumulated.toString() );
                            accumulated.setLength( 0 );
                            addToken( new Gibberish( c ) );
                            gibberish = false;
                            }
                        else
                            {
                            accumulated.append( c );
                            }
                        }

                    /**
                     * what to do on leaving state, after last char is consumed.
                     */
                    void leaving()
                        {
                        addMultilineTokens( new HTMLText( "" ),
                                accumulated.toString() );
                        accumulated.setLength( 0 );
                        // end leaving
                        }

                    /**
                     * Figure out what state we should go in after we process this
                     * character and whether we can consume it. We must set consume
                     * = false if we cannot consume the character and have to
                     * postpone processing to the next state.
                     * @param category category of character
                     * @param nextChar character in the stream we are processing
                     * @param first true if we just entered this state.
                     * @return next state to go in
                     */
                    HTMLState next( HTMLCharCategory category,
                                    char nextChar,
                                    boolean first )
                        {
                        switch ( category )
                            {
                            case AMP:
                                how = HowToProcess.FORWARD;
                                return IN_TEXT_ENTITY;
                            case END_TAG:// should be entity
                            case OTHER:// squirrelly high ASCII
                            case QUOTE:// should be entity
                                how = HowToProcess.CONSUME;
                                gibberish = true;
                                return IN_TEXT;
                            case START_TAG:
                                if ( isComing( "<!--" ) )
                                    {
                                    how = HowToProcess.FORWARD;
                                    return IN_TEXT_COMMENT;
                                    }
                                else if ( isComing( "<![CDATA[" ) )
                                    {
                                    how = HowToProcess.FORWARD;
                                    return IN_CDATA;
                                    }
                                else
                                    {
                                    how = HowToProcess.FORWARD;
                                    return IN_TAG;
                                    }
                            case DASH:
                            case EOL:// treat as ordinary char for now, split out later
                            case EQUALS:
                            case PLAIN:
                            case RIGHT_BRACKET:
                            case SEMICOLON:
                            case SPACE:// treat as ordinary char for now, split out later
                                how = HowToProcess.CONSUME;
                                return IN_TEXT;
                            case IGNORE:
                            default:
                                assert false :
                                        "bad state "
                                        + category
                                        + " "
                                        + nextChar;
                                return null;
                            // end switch
                            }
                        // end next
                        }
                    // end IN_TEXT
                    },
        /**
         * in comment &lt;!-- ... --&gt; in text, not inside tag, same at IN_TAG_COMMENT but returns to IN_TEXT
         */
        @SuppressWarnings( { "WeakerAccess" } ) IN_CDATA
                    {
                    /**
                     * Consume one character. It has been predecided that you can
                     * and will consume it.
                     * @param c        char to consume
                     */
                    void consume( char c )
                        {
                        accumulated.append( c );
                        }

                    /**
                     * what to do on leaving state, after last char is consumed.
                     */
                    void leaving()
                        {
                        addToken( new CDATATag( "<![CDATA[" ) );
                        addMultilineTokens( new CDATAContents( "" ),
                                accumulated.toString() );
                        accumulated.setLength( 0 );
                        addToken( new CDATATag( "]]>" ) );
                        }

                    /**
                     * Figure out what state we should go in after we process this
                     * character and whether we can consume it. We must set consume
                     * = false if we cannot consume the character and have to
                     * postpone processing to the next state.
                     * @param category category of character
                     * @param nextChar character in the stream we are processing
                     * @param first true if we just entered this state.
                     * @return next state to go in
                     */
                    HTMLState next( HTMLCharCategory category,
                                    char nextChar,
                                    boolean first )
                        {
                        switch ( category )
                            {
                            case AMP:// entities not magic in comments
                            case DASH:
                            case EOL:// just accumulate them for now
                            case END_TAG:
                            case EQUALS:
                            case OTHER:// don't sweat high ascii in comments.
                            case PLAIN:
                            case QUOTE:// quotes are nothing special in comments
                            case SEMICOLON:
                            case SPACE:// just accumulate them for now
                                how = HowToProcess.CONSUME;
                                return IN_CDATA;
                            case RIGHT_BRACKET:
                                if ( isComing( "]]>" ) )
                                    {
                                    markHandled( 3 );
                                    how = HowToProcess.DISCARD;
                                    return IN_TEXT;
                                    }
                                else
                                    {
                                    // just an ordinary char
                                    how = HowToProcess.CONSUME;
                                    return IN_CDATA;
                                    }
                            case START_TAG:
                                if ( first && isComing( "<![CDATA[" ) )
                                    {
                                    markHandled( 9 );
                                    how = HowToProcess.DISCARD;
                                    // we don't bother to accumulate the open and close markers.
                                    return IN_CDATA;
                                    }
                                else
                                    {
                                    // just an ordinary character
                                    how = HowToProcess.CONSUME;
                                    return IN_CDATA;
                                    }
                            case IGNORE:
                            default:
                                assert false :
                                        "bad state "
                                        + category
                                        + " "
                                        + nextChar;
                                return null;
                            // end switch
                            }
                        // end next
                        }
                    },
        /**
         * in comment &lt;!-- ... --&gt; in text, not inside tag, same at IN_TAG_COMMENT but returns to IN_TEXT
         */
        @SuppressWarnings( { "WeakerAccess" } ) IN_TEXT_COMMENT
                    {
                    /**
                     * Consume one character. It has been predecided that you can
                     * and will consume it.
                     * @param c        char to consume
                     */
                    void consume( char c )
                        {
                        accumulated.append( c );
                        }

                    /**
                     * what to do on leaving state, after last char is consumed.
                     */
                    void leaving()
                        {
                        // token added includes the begin and end markers, with possibly
                        // either end marker missing.
                        addToken( new HTMLCommentTag( "<!--" ) );
                        addMultilineTokens( new HTMLCommentContents( "" ), accumulated.toString() );
                        accumulated.setLength( 0 );
                        addToken( new HTMLCommentTag( "-->" ) );
                        } // end leaving

                    /**
                     * Figure out what state we should go in after we process this
                     * character and whether we can consume it. We must set consume
                     * = false if we cannot consume the character and have to
                     * postpone processing to the next state.
                     * @param category category of character
                     * @param nextChar character in the stream we are processing
                     * @param first true if we just entered this state.
                     * @return next state to go in
                     */
                    HTMLState next( HTMLCharCategory category,
                                    char nextChar,
                                    boolean first )
                        {
                        switch ( category )
                            {
                            case AMP:// entities not magic in comments
                            case EOL:// just accumulate them for now
                            case END_TAG: // without lead --, just an ordinary char
                            case EQUALS:
                            case OTHER:// don't sweat high ascii in comments.
                            case PLAIN:
                            case QUOTE:// quotes are nothing special in comments
                            case RIGHT_BRACKET:
                            case SEMICOLON:
                            case SPACE:// just accumulate them for now
                                how = HowToProcess.CONSUME;
                                return IN_TEXT_COMMENT;
                            case DASH:
                                if ( isComing( "-->" ) )
                                    {
                                    markHandled( 3 );
                                    // don't accumulate
                                    how = HowToProcess.DISCARD;
                                    return IN_TEXT;
                                    }
                                else
                                    {
                                    // just an ordinary char
                                    how = HowToProcess.CONSUME;
                                    return IN_TEXT_COMMENT;
                                    }
                            case START_TAG:
                                if ( first && isComing( "<!--" ) )
                                    {
                                    markHandled( 4 );
                                    // don't accumulate
                                    how = HowToProcess.DISCARD;
                                    return IN_TEXT_COMMENT;
                                    }
                                else
                                    {
                                    // just an ordinary char
                                    how = HowToProcess.CONSUME;
                                    return IN_TEXT_COMMENT;
                                    }
                            case IGNORE:
                            default:
                                assert false :
                                        "bad state "
                                        + category
                                        + " "
                                        + nextChar;
                                return null;
                            // end switch
                            }
                        // end next
                        }
                    },
        /**
         * in an entity &...; inside body text
         */
        @SuppressWarnings( { "WeakerAccess" } ) IN_TEXT_ENTITY
                    {
                    /*
                     * true if this entity is properly formed with : ... ;
                     */
                    private boolean balanced;

                    /**
                     * Consume one character. It has been predecided that you can
                     * and will consume it.
                     * @param c        char to consume
                     */
                    void consume( char c )
                        {
                        accumulated.append( c );
                        }

                    /**
                     * what to do on leaving state, after last char is consumed.
                     */
                    void leaving()
                        {
                        String entity = accumulated.toString();
                        accumulated.setLength( 0 );
                        if ( balanced )
                            {
                            // does not have lead & or trailing ;
                            // Known includes hex and numeric forms.
                            // case-sensitive
                            boolean known = DeEntifyStrings.bareHTMLEntityToChar( entity, ' ' ) != 0;
                            // add even if 0 long. Could erroneously have written &;
                            tokens.add( new HTMLEntity( entity
                                    /* &; stripped */,
                                    known ) );
                            }
                        else
                            {
                            // screwed up entity that was not really one.
                            // should treat it is either text or tag or quote. It is an error..
                            tokens.add( new Gibberish( "&" + entity ) );
                            // note that we have handled this anomaly
                            balanced = true;
                            }
                        // end leaving
                        }

                    /**
                     * Figure out what state we should go in after we process this
                     * character and whether we can consume it. We must set consume
                     * = false if we cannot consume the character and have to
                     * postpone processing to the next state.
                     * @param category category of character
                     * @param nextChar character in the stream we are processing
                     *   @param first true if we just entered this state.
                     * @return next state to go in
                     */
                    HTMLState next( HTMLCharCategory category,
                                    char nextChar,
                                    boolean first )
                        {
                        switch ( category )
                            {
                            case AMP:// entities not magic in comments
                                if ( first )
                                    {
                                    how = HowToProcess.DISCARD;
                                    balanced = false;
                                    return IN_TEXT_ENTITY;
                                    }
                                else
                                    {
                                    // malformed entity
                                    balanced = false;
                                    how = HowToProcess.FORWARD;
                                    return IN_TEXT;
                                    }
                            case DASH:
                            case END_TAG:
                            case EOL:
                            case OTHER:
                            case QUOTE:
                            case SPACE:
                            case START_TAG:
                                // oops. Should not see these inside an entity.
                                balanced = false;
                                how = HowToProcess.FORWARD;
                                return IN_TEXT;
                            case EQUALS:
                            case PLAIN:
                            case RIGHT_BRACKET:
                                if ( SAFEENTITYCHARS.indexOf( nextChar ) >= 0 )
                                    {
                                    how = HowToProcess.CONSUME;
                                    return IN_TEXT_ENTITY;
                                    }
                                else
                                    {
                                    balanced = false;
                                    how = HowToProcess.FORWARD;
                                    return IN_TEXT;
                                    }
                            case SEMICOLON:
                                balanced = true;
                                how = HowToProcess.DISCARD;
                                return IN_TEXT;
                            case IGNORE:
                            default:
                                assert false :
                                        "bad state "
                                        + category
                                        + " "
                                        + nextChar;
                                return null;
                            // end switch
                            }
                        // end next
                        }
                    // end IN_TEXT_ENTITY
                    };
    // declarations

    /**
     * true if want voluminous debugging output
     */
    private static final boolean DEBUGGING = false;

    /**
     * list of tokens we have parsed out.
     */
    private static final ArrayList<Token> tokens =
            new ArrayList<>( 10000 );

    /**
     * used to accumulate comments
     */
    private static final StringBuilder accumulated =
            new StringBuilder( 80 );

    // common to all enum constants
    private static final String SAFEENTITYCHARS =
            "#abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ";

    /**
     * how far we are through parsing the program
     */
    private static int charIndex;

    /**
     * the length of the program fragment we are parsing.
     */
    private static int size;

    /**
     * the program or program fragment we are parsing
     */
    private static String program;

    /**
     * how we plan to process this character, consume, procrastinate to next state, or discard
     */
    private static HowToProcess how;
    // /declarations
    // methods

    /**
     * add a token to the end of the list to be rendered.
     *
     * @param t a token. Useless tokens will be not be added.
     */
    private static void addToken( Token t )
        {
        // check out token for validity, if pointless, don't bother adding it.
        if ( !t.isUseless() )
            {
            tokens.add( t );
            }
        }// /method

    /**
     * Replace junk chars with something that won't cause trouble
     *
     * @param category category of this char
     * @param c        the char
     *
     * @return c if char is clean, a replacement if it were dirty e.g. tab
     */
    private static char clean( HTMLCharCategory category, char c )
        {
        if ( category == HTMLCharCategory.SPACE )
            {
            return ' ';
            }
        else
            {
            return c;
            }
        }// /method

    /**
     * crunch multiple tokens into a single token where feasible.
     */
    private static void crunch()
        {
        int size;
        do
            {
            size = tokens.size();
            for ( int i = size - 1; i >= 1; i-- )
                {
                Token current = tokens.get( i );
                Token prev = tokens.get( i - 1 );
                if ( current instanceof NL && prev instanceof Space )
                    {
                    // trim trailing blanks on line
                    tokens.remove( i - 1 );
                    // don't i--. same NL token will be repeatedly compared
                    // against prev.
                    }
                else if ( current instanceof Space )
                    {
                    Space sp = ( Space ) ( current );
                    if ( sp.length() <= 10 && prev.isCollapsible() )
                        {
                        // combine space into previous token
                        prev.setText( prev.getText() + sp.getText() );
                        tokens.remove( i );
                        // don't i--, combined token will be compared against
                        // its predecessor
                        }
                    }
                else if ( prev.isCollapsible( current ) )
                    {
                    // combine two tokens into one
                    prev.setText( prev.getText() + current.getText() );
                    tokens.remove( i );
                    // don't i--. combined token will be compared with its
                    // predecessor
                    }
                } // end for
            // keep going while it is still finding something to crunch
            }
        while ( tokens.size() < size );
        }// /method

    /**
     * debugging dump system state
     *
     * @param theChar  char we are processing
     * @param category category of the char
     * @param first    did we just enter this state
     * @param oldState old state
     * @param state    current state
     * @param newState next state
     * @param how      do we consume, forward or discard this character.
     */
    private static void dumpState( char theChar,
                                   HTMLCharCategory category,
                                   boolean first,
                                   HTMLState oldState,
                                   HTMLState state,
                                   HTMLState newState,
                                   HowToProcess how )
        {
        if ( how == HowToProcess.CONSUME )
            {
            /* use slightly abbreviated form, black */
            out.println( theChar
                         + " "
                         + category
                         + " "
                         + first
                         + " o:"
                         + oldState
                         + " s:"
                         + state
                         + " n:"
                         + newState );
            }
        else
            {/* in red */
            err.println( theChar
                         + " "
                         + category
                         + " "
                         + first
                         + " o:"
                         + oldState
                         + " s:"
                         + state
                         + " n:"
                         + newState
                         + " "
                         + how );
            }
        }// /method

    /**
     * Make sure nothing there is left accumulated in buffers from parsing not yet converted to tokens
     */
    private static void ensureNoLeftovers()
        {
        assert accumulated.length() == 0 : " residual : "
                                           + accumulated.toString();
        accumulated.setLength( 0 );
        }// /method

    /**
     * Is a given string isComing up in the stream starting with this character. Compares ignoring case.
     *
     * @param expected string to TEST if isComing in the stream
     *
     * @return true if this string isComing up, case-insensitive
     */
    private static boolean isComing( String expected )
        {
        // check if there are enough characters left for a match.
        return charIndex + expected.length() < size && program.substring(
                charIndex,
                charIndex + expected.length() )
                .equalsIgnoreCase( expected );
        }// /method

    /**
     * normally we handle one character at a time each iteration of the loop.  If we handle more,
     * we mark the extra characters as handled. If you handle just one character, there is no need to
     * call markHandled( 1 );
     *
     * @param numberOfChars how many characters we have processed, including the usual 1.
     */
    private static void markHandled( int numberOfChars )
        {
        charIndex += numberOfChars - 1;
        }// /method

    /**
     * clear out the state machine ready to parse a new program
     */
    private static void reset()
        {
        accumulated.setLength( 0 );
        tokens.clear();
        how = null;
        charIndex = 0;
        // keeping certain variables local or global is crucial.
        // Don't mess with them without thinking carefully and
        // changing the docs:
        // local: category, first, oldState, state, theChar
        // global: charIndex, how
        // we make these local to discourage accidental snooping or
        // accidentally picking up the static version instead of the parm.
        }// /method

    /**
     * get rid of leading and trailing NLs tokens. It is easier to handle it later than during parsing.
     */
    private static void trimNLs()
        {
        // remove leading NLs.
        while ( tokens.size() > 0 && ( tokens.get( 0 ) instanceof NL ) )
            {
            tokens.remove( 0 );
            }
        // remove trailing NLs
        int count;
        while ( ( count = tokens.size() ) > 0 && ( tokens.get( count
                                                               - 1 ) instanceof NL ) )
            {
            tokens.remove( count - 1 );
            }
        // We don't need an NL at either beginning or end.
        // insert at the beginning
        tokens.add( 0, new Start( "<pre class=\"html\">" ) );
        // add to end
        addToken( new Stop( "</pre>" ) );
        }// /method

    /**
     * Add a block of text as tokens.
     *
     * @param example   sample of the sort of token we are creating.
     * @param blockText Text may contain embedded newlines and spaces which need to be made into separate tokens.
     */
    void addMultilineTokens( Token example, String blockText )
        {
        int nlPlace;
        // we don't trim the trailing spaces. More tokens may be isComing on
        // this line.
        String remainingBlockText = blockText;
        while ( ( nlPlace = remainingBlockText.indexOf( '\n' ) ) >= 0 )
            {
            // Possibly empty line prior to the found \n.
            // It now has no trailing \n, possibly embedded/leading/trailing
            // spaces
            // ok to trim trailing.. We know an NL immediately follows.
            String line = ST.trimTrailing( remainingBlockText
                    .substring( 0, nlPlace ) );
            if ( line.length() > 0 )
                {
                addTokenLineByExample( example, line );
                }
            // strip off the first line, up to the nl, leaving the remaining
            // starting with \n
            remainingBlockText = remainingBlockText.substring( nlPlace );
            // guaranteed we have at least one \n
            int embeddedNLCount =
                    ST.countLeading( remainingBlockText, '\n' );
            addToken( new NL( embeddedNLCount ) );
            // strip off leading nls
            remainingBlockText =
                    remainingBlockText.substring( embeddedNLCount );
            } // end while
        // when we fall out the bottom there is one last chunk to do without
        // any nls anywhere.
        // we don't trim it, since there may be tokens following on the same
        // line.
        if ( remainingBlockText.length() > 0 )
            {
            addTokenLineByExample( example, remainingBlockText );
            }
        }// /method

    /**
     * Add one token, whose text contains no nl or space characters to be separately tokenized.
     *
     * @param example sample of teh sort of token we are creating.
     * @param text    Text man contain embedded newlines and spaces which need to be made into separate tokens.
     */
    void addTokenByExample( Token example, String text )
        {
        if ( text.length() > 0 )
            {
            try
                {
                Token t = ( Token ) example.clone();
                t.setText( text );
                addToken( t );
                }
            catch ( CloneNotSupportedException e )
                {
                err.println( "clone of Token failed "
                             + example.toString() );
                }
            }
        }// /method

    /**
     * Add one line of identical tokens, interrupted with embedded spaces which must be separately tokenized, but no NL
     * characters.
     *
     * @param example  sample of the sort of token we are creating.
     * @param lineText Text may contain embedded spaces which need to be made into separate tokens. There are no nls.
     */
    void addTokenLineByExample( Token example, String lineText )
        {
        int spPlace;
        // we don't trim. Caller does that. Trailing spaces may be
        // significant.
        String remainingLineText = lineText;
        while ( ( spPlace = remainingLineText.indexOf( ' ' ) ) >= 0 )
            {
            // possibly empty word on line before first space, "" if first thing
            // on line is ' '
            String wordText = remainingLineText.substring( 0, spPlace );
            if ( wordText.length() > 0 )
                {
                addTokenByExample( example, wordText );
                // strip off the first word, prior to the first space.
                remainingLineText = remainingLineText.substring( spPlace );
                }
            // remainingText guaranteed has leading spaces that were just
            // after the word we processed.
            int embeddedSpaceCount =
                    ST.countLeading( remainingLineText, ' ' );
            addToken( new Space( embeddedSpaceCount ) );
            // strip off leading spaces
            remainingLineText = remainingLineText
                    .substring( embeddedSpaceCount );
            // we leave ourself positioned at the start of the next word on the
            // line.
            } // end while
        // when we fall out he bottom there is one last word on the line,
        // without any spaces..
        if ( remainingLineText.length() > 0 )
            {
            addTokenByExample( example, remainingLineText );
            }
        }// /method

    /**
     * Default Consume one character. It has been pre-decided that you can and will consume it.
     *
     * @param c char to consume
     */
    abstract void consume( char c );// /method

    /**
     * default what to do on leaving state, after last char is consumed.
     */
    abstract void leaving();// /method

    /**
     * default next method determines the next state based on current state, and next char
     *
     * @param category class of next character
     * @param nextChar next character to process
     * @param first    true if this is the first character after we entered this state.
     *
     * @return next HTMLState
     */
    abstract HTMLState next( HTMLCharCategory category,
                             char nextChar,
                             boolean first );// /method

    /**
     * Parse program and leave a list of Tokens in tokens ArrayList.
     *
     * @param program the text we are going parse and eventually render.
     *
     * @return an array of tokens representing the text and how it will be rendered.
     */
    @SuppressWarnings( { "UnusedAssignment" } )
    public static Token[] parse( String program )
        {
        reset();
        HTMLState.program = program;
        size = program.length();
        // keeping certain variables local or global is crucial.
        // Don't mess with them without thinking carefully and
        // changing the docs:
        // local: category, first, oldState, state, theChar
        // global: charIndex, how
        // we make these local to discourage accidental snooping or
        // accidentally picking up the static version instead of the parm.
        // where we were
        HTMLState oldState = IN_TEXT;
        // where we are
        HTMLState state = IN_TEXT;
        // were we will be next
        HTMLState newState;
        // how is global however, so next can return both a state and how.
        how = null;
        // Note, NO int charIndex !! Don't "repair that".
        // charIndex is a static variable globally known so "isComing" can use it.
        for ( charIndex = 0; charIndex < size; charIndex++ )
            {
            // next char to process
            char theChar = program.charAt( charIndex );
            // decide which general category the char falls in
            final HTMLCharCategory category = HTMLCharCategory
                    .categorise( theChar );
            theChar = clean( category, theChar );
            if ( category != HTMLCharCategory.IGNORE )
                {
                // keep going till some state consumes/discards the character.
                // Allow up to three forwarding attempts to deal with the
                // character.
                // Usually we should succeed on the first or second attempt.
                // We always make at least on trip through.
                attempts:
                for ( int times = 0; times < 3; times++ )
                    {
                    // first is deliberately local
                    // first true if we just entered this state.
                    final boolean first = state != oldState;
                    // crank the state machine one cycle,
                    // State should modify how in addition to returning the new
                    // state.
                    // A little ugly but simplest way to return a pair of
                    // values:
                    // state and how
                    how = null;// setting to null ensures not setting it will
                    // be caught.
                    // This is the guts of the finite state automaton
                    // decide the next state
                    // vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv
                    newState = state.next( category, theChar, first );
                    // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
                    if ( DEBUGGING )
                        {
                        dumpState( theChar,
                                category,
                                first,
                                oldState,
                                state,
                                newState,
                                how );
                        }
                    // kick over to the next generation,
                    // we are now in the newState. We make the transition here.
                    oldState = state;
                    state = newState;
                    newState = null;
                    if ( how == null )
                        {
                        throw new NullPointerException( "HTMLSTate bug: how not set. OldState=" + oldState + " " +
                                                        "newState=" + state + " next() must not be private." );
                        }
                    switch ( how )
                        {
                        case CONSUME:
                            oldState.consume( theChar );
                            if ( state != oldState )
                                {
                                oldState.leaving();
                                }
                            // and on to the next char
                            break attempts;
                        case DISCARD:
                            if ( state != oldState )
                                {
                                oldState.leaving();
                                }
                            // and on to the next char
                            break attempts;
                        default:
                            // should never get here
                            assert false :
                                    "HTMLTokenizer state machine failed to set how variable.";
                            break attempts;
                        case FORWARD:
                            assert state
                                   != oldState :
                                    "HTMLTokenizer state machine attempted to forward a char to the same state.";
                            oldState.leaving();
                            // we give that character another try with the new
                            // state
                        } // end switch
                    /*
                     * we will only ever get here if we are forwarding. The
                     * other cases all leave the loop early.
                     */
                    }
                // end attempts loop
                // we fall out the bottom and land here no matter what
                assert how == HowToProcess.CONSUME || how == HowToProcess
                        .DISCARD :
                        "HTMLTokenizer state machine failed to consume char in three state forwarding attempts.";
                } // end if ignore
            } // end for each character
        // we must leave the last state if we have not already:
        if ( state == oldState )
            {
            oldState.leaving();
            }
        // make sure nothing still sitting in accumulation buffer after we have
        // finished parsing the entire program.
        ensureNoLeftovers();
        crunch();
        trimNLs();
        // covert to vanilla array for even more efficient use in the final
        // Applet.
        return tokens.toArray( new Token[ tokens.size() ] );
        }// /method
    // /methods
    }