/*
 * [Replacer.java]
 *
 * Summary: Expands HTML boilerplate refresher macros.
 *
 * Copyright: (c) 2002-2017 Roedy Green, Canadian Mind Products, http://mindprod.com
 *
 * Licence: This software may be copied and used freely for any purpose but military.
 *          http://mindprod.com/contact/nonmil.html
 *
 * Requires: JDK 1.8+
 *
 * Created with: JetBrains IntelliJ IDEA IDE http://www.jetbrains.com/idea/
 *
 * Version History:
 *  1.3 2009-01-01 rewrite to use StringBuilder and fell-swoop I/O for both in and out, and also avoid writing if files
 *                 are the same.
 *  1.4 2013-01-13 permit missing spaces in <!--cseignore-->
 */
package com.mindprod.htmlmacros;

import com.mindprod.commandline.CommandLine;
import com.mindprod.common18.EIO;
import com.mindprod.common18.Misc;
import com.mindprod.common18.ST;
import com.mindprod.common18.Twirler;
import com.mindprod.compactor.Compactor;
import com.mindprod.fastcat.FastCat;
import com.mindprod.filter.AllButFootDirectoriesFilter;
import com.mindprod.filter.ExtensionListFilter;
import com.mindprod.htmlmacros.macro.Global;
import com.mindprod.htmlmacros.macro.Macro;
import com.mindprod.htmlmacros.support.Embellishment;
import com.mindprod.htmlmacros.support.Fireup;
import com.mindprod.htmlmacros.support.InlineFooter;
import com.mindprod.htmlmacros.support.Lazy;
import com.mindprod.htmlmacros.support.LoadCodeToProcessMacro;
import com.mindprod.htmlmacros.support.ParmParser;
import com.mindprod.htmlmacros.support.Shutdown;
import com.mindprod.htmlmacros.support.Tools;
import com.mindprod.hunkio.HunkIO;

import java.awt.Toolkit;
import java.io.File;
import java.io.IOException;
import java.util.regex.Pattern;

import static com.mindprod.htmlmacros.macro.Global.configuration;
import static java.lang.System.*;
/*
 * TODO:
 * Futures: <ul> <li>Optionally remove generations to make code easier to maintain.</li> <li>Optionally remove original
 * macros leaving only the generated code for compactness.</li> </ul>
 */

/**
 * Expands HTML boilerplate refresher macros.
 * <p/>
 * e.g. expands [!-- macro XXXXX parms -]  with  Allows arbitrary expander macro code.
 * <p/>
 * WARNING: Replacer will remove all previous [!-- generated ... [/generated] even ones without matching macro commands.
 * <p/>
 * We read the entire file into RAM at once. We accumulate the expansion is a StringBuilder then write the whole thing
 * to a temporary file, if there were any changes. Then we delete the original and rename the temporary. ParmParser does
 * the tricky work of extracting the parms from the macro with the various kinds of delimiter.
 * <p/>
 * main calls expandMacrosInFile calls parseAndExpandOneMacro calls processorDelegate.expandNoRef<br>
 * expandMacrosInFile catches
 * the exceptions and does error reporting.
 *
 * @author Roedy Green, Canadian Mind Products
 * @version 1.4 2013-01-13 permit missing spaces in <!--cseignore-->
 * @since 2002-01-01
 */
public class Replacer
    {
    // declarations

    /**
     * do we want extra debugging output, assigned in main. Assigned mby configuration. Not known at compile time.
     */
    private static final boolean DEBUGGING = false;

    /**
     * we may bypass processing using caching if there are at least this many files.
     */
    private static final int BYPASS_LIMIT = 10;

    /**
     * minimum size we expect the file we are expanded to me.  Smallest will be a Moved macro that has been
     * StripGenerated
     */
    private static final int MIN_FILE_SIZE = 30;

    /**
     * easily visible marker for end of error message group
     */
    private static final String END_OF_ERROR_MARKER =
            "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^";

    /**
     * easily visible marker for start of error message group
     */
    private static final String START_OF_ERROR_MARKER =
            "vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv";

    /**
     * <!-- /generated ... --><br> pattern includes initial <!-- <br> \\s* optional whitespace<br> \\s++ mandatory
     * whitespace <br>.* anything
     */
    private static final Pattern endGenerated = Pattern
            .compile( "<!\\-\\-\\s*/\\s*generated\\s*.*\\-\\->" );

    /**
     * to detect end of line in generated text, always \n in generated macros
     */
    private static final Pattern endOfLine = Pattern.compile( "\\n" );

    /**
     * <!-- generated -->
     */
    private static final Pattern startGenerated = Pattern
            .compile( "<!\\-\\-\\s*generated\\s*\\-\\->" );

    private static final Toolkit toolkit = Toolkit.getDefaultToolkit();

    /**
     * rotation state of the twirling animation
     */
    private static final Twirler twirler = new Twirler();

    /**
     * true we are copying the input to the output . false we deleting the input from the output stream, e.g. between
     * generate-/generate
     */
    private static boolean copy = true;

    /**
     * true if expanding masters in /master/ subdir to distributable, compacting out macros and spaces.
     */
    private static boolean separateMasters;

    /**
     * offset in chars we have so far processed the file image.
     */
    private static int offset;

    /**
     * length of the original input string before any processing.
     */
    private static int originalLength;

    /**
     * files older than this are not expected to change expansions.
     * They might change because of coding change or because of include changes.
     * Differences are reported in case they are caused by bugs.
     * Most changes are caused by recent edits or bulk search/replace changes.
     */
    private static long ageOfUnexpectedChanges;

    /**
     * String representing to text just prior to the marker scanned for.
     */
    private static String chunk;

    /**
     * contents expaneded macros.
     */
    private static String contents;

    /**
     * what this particular document uses for a line separator
     */
    private static String documentLineSeparator;

    /**
     * name of the macro we are about to execute e.g. Head Include Image
     */
    private static String macroName;

    /**
     * The string representing the original file image we are processing.
     */
    private static String original;

    /**
     * array of parms for latest macro
     */
    private static String[] parms;

    /**
     * previous contents of the distributed file.
     * For old style in the original.
     * For separateMasters in the previous expansion without macros.
     */
    private static String prevContents;

    /**
     * where we emit the expanded file. We generate a new StringBuilder for each file. Otherwise one giant file could
     * pump it up big and leave it there needlessly.
     */
    private static FastCat emit;

    /**
     * track which files we can bypass processing
     */
    private static Lazy trackAlreadyDone;
    // /declarations
    // methods

    /**
     * file where we write the expanded macros, old style back on fileBeingProcessed,
     * new style is separate file one level up.
     */
    private static File fileBeingDistributed;

    /**
     * file where the macros are
     */
    private static File fileBeingProcessed;

    /**
     * Adjust line separator \n to document convention
     *
     * @param text line of text to have embedded new lines fixed.
     *
     * @return text with \n replaced with \r\n or whatever.
     */
    private static String adjustLineSeparator( String text )
        {
        if ( documentLineSeparator.equals( "\n" ) )
            {
            return text;
            }
        else
            {
            // macros use \n. Expand to something else.
            return endOfLine.matcher( text ).replaceAll( documentLineSeparator );
            }
        }// /method

    /**
     * announce we are starting to expand the macros in a file.
     *
     * @param quiet true if just twirler. false if want name of file displayed.
     *
     * @see com.mindprod.htmlmacros.macro.Include#
     */
    private static void announceStartOfExpandingOneFile( final boolean quiet )
        {
        if ( !quiet )
            // don't println till finished whole file. Macros can print single letters to
            // status line
            {
            out.print( "  expanding "
                       + EIO.getCanOrAbsPath( fileBeingProcessed )
                       + " " );
            }
        }// /method

    /**
     * beep a to alert to an error.<br> This is a polite little DOS speaker beep, not the full system error tone.<br> It
     * sounds quite different from ordinary sound-card sounds,<br> so it catches the user's attention.<br> It also can't
     * be shut off with the volume control.
     */
    private static void beep()
        {
        toolkit.beep();
        }// /method

    /**
     * calculate fileBeingDistributed and fileBeingProcessed.
     */
    private static void calcFileBeingDistributed()
        {
        if ( separateMasters )
            {
            if ( Tools.isFileAMaster( fileBeingProcessed ) )
                {
                fileBeingDistributed = Tools.correspondingDistributable( fileBeingProcessed );
                }
            else
                {
                // swap roles.
                fileBeingDistributed = fileBeingProcessed;
                fileBeingProcessed = Tools.correspondingMaster( fileBeingDistributed );
                }
            }
        else
            {
            // old style, write expanded back on original.
            fileBeingDistributed = fileBeingProcessed;
            }
        }// /method

    /**
     * display start and end of a bit of offending text.
     * Pruned to one line.
     *
     * @param text raw HTML text to display.
     */
    private static void displayOffendingText( String text )
        {
        if ( text.length() <= 100 )
            {
            // display whole thing
            err.println( "\n " + text );
            }
        else
            {
            // chop out the middle
            err.println( "\n " + text.substring( 0, 45 ) + " ... " + text.substring( text.length() - 45 ) );
            }
        }// /method

    /**
     * show what changed by he macro expansions
     */
    private static void displayWhatChanged()
        {
        // prev in old expanion if this page.
        if ( prevContents == null )
            {
            try
                {
                prevContents = HunkIO.readEntireFile( fileBeingDistributed, HunkIO.UTF8 );
                }
            catch ( IOException e )
                {
                // if had trouble reading, that is fine.  It may not exist yet.
                }
            }
        if ( prevContents == null )
            {
            return;
            }
        // dump out the change.
        out.println( "\n  previous expansion length: " + prevContents.length() + " new expansion length: " + contents.length() );
        final int length = Math.min( prevContents.length(), contents.length() );
        for ( int i = 0; i < length; i++ )
            {
            if ( contents.charAt( i ) != prevContents.charAt( i ) )
                {
                final int start = Math.max( 0, i - 30 );
                final int end = Math.min( i + 30, length );
                final FastCat sb = new FastCat( 10 );
                sb.append( "  info: file " );
                sb.append( fileBeingDistributed.toString() );
                sb.append( " changed at offset " );
                sb.append( i );
                sb.append( "\n  old[" );
                sb.append( makeVisible( prevContents.substring( start, end ) ) );
                sb.append( "]\n  new[" );
                sb.append( makeVisible( contents.substring( start, end ) ) );
                sb.append( "]\n" );
                out.print( sb.toString() );   // suppress blank line
                // name of file changed will follow.
                break;
                }
            }
        }// /method

    /**
     * parse the big file, replacing boiler plate, and copying ordinary text.
     * Handles <!-- macro <!-- generated <!-- /generated.
     *
     * @param fileBeingProcessed file to process, the file whose macros we expand.
     * @param quiet              true if want progress messages suppressed
     * @param verbose            true if want extra output
     * @param forceRegenerate    true if regenerate even if file has not changed recently.
     *
     * @return true if stop
     * @throws IOException if some I/O trouble reading/writing files.
     */
    static boolean expandMacrosInFile( final File fileBeingProcessed, final boolean quiet, final boolean verbose, final boolean forceRegenerate ) throws IOException
        {
        if ( Global.gracefulStop )
            {
            out.println( "graceful stop requested" );
            return true;
            }
                    /* even if already done, we redo the footer.
           It contains random elements, and its own mechanism to avoid too frequent changes.
         */
        Replacer.fileBeingProcessed = fileBeingProcessed;
        InlineFooter.updateInlineFooterFile( fileBeingProcessed );
        // even if we are bypassing, we still twirl.  Otherwise appears togo dead as bypasses nearly all of site.
        if ( quiet )
            {
            twirler.twirl( out );
            }
        // bypass if unchanged since last done
        if ( trackAlreadyDone.isAlreadyDone( fileBeingProcessed )
             && !forceRegenerate )
            {
            return false;
            }
        separateMasters = configuration.getCompacting() == '*';
        announceStartOfExpandingOneFile( quiet );
        // start processing at the first character.
        offset = 0;
        boolean abort = false;
        try
            {
            calcFileBeingDistributed();
            original = null;
            original = HunkIO.readEntireFile( fileBeingProcessed, HunkIO.UTF8 );
            Macro.resetUniqueOnPage();
            originalLength = original.length();
            if ( originalLength < MIN_FILE_SIZE )
                {
                err.println();
                err.println( "File " + EIO.getCanOrAbsPath( fileBeingProcessed ) + " only " + originalLength
                             + " " +
                             "characters long." );
                beep();
                err.println();
                }
            // find out line separator document is using. we match it in
            // expansions.
            documentLineSeparator = guessLineSeparator();
            // Generate output into a StringBuilder than later write it out to
            // a temporary file, and replace the original
            // Some exceptional files get extra buffer space.
            if ( !quiet )
                {
                out.print( "V" );
                }
            verifyGeneratedTags( original, fileBeingProcessed );
            if ( !quiet )
                {
                out.print( "E" );
                }
            emit = new FastCat( estimateFastCatSlots( original ) );
            if ( !quiet )
                {
                out.print( "e" );
                }
            emit.setWhereTextCameFrom( "file " + Tools.uPathName( fileBeingDistributed ) );
            // make sure <!-- --> and <!-- generated --> <!-- /generated -->
            // balance properly. will throw IllegalArgumentException if problems.
            // We don't remove them! We later just leave them behind as we copy over other stuff to emit buffer.
            copy = true;
            // loop for each fragment
            while ( true )
                {
                // scan for beginning of next comment
                if ( scan( "<!--", false ) )
                    {
                    // we found the start of a comment.
                    if ( copy )
                        {
                        // copy over non-comment text prior to comment
                        emit.append( chunk );
                        }
                    // bypass the initial <!--
                    scan( "<!--", true );
                    String command = nextWord();
                    if ( command.equals( "macro" ) )
                        {
                        // expansion of a macro, might expandNoRef differently,
                        // will TEST later if it really changed.
                        // handles scanning to -->   generates: <!-- macro Xxx ... </generated>
                        emit.append( parseAndExpandOneMacro( quiet, verbose ) );
                        }
                    // can't use stripGenerated before hand or we would lose
                    // track of offsets.
                    else if ( command.equals( "generated" ) )
                        {
                        // delete generated through /generated, for any macro
                        if ( scan( "-->", true ) )
                            {
                            copy = false;
                            }
                        else
                            {
                            throw new IllegalArgumentException(
                                    "syntax error <generated missing -->" );
                            }
                        }
                    else if ( command.equals( "/generated" ) )
                        {
                        if ( scan( "-->", true ) )
                            {
                            copy = true;
                            }
                        else
                            {
                            throw new IllegalArgumentException(
                                    "syntax error </generated missing -->" );
                            }
                        }
                    else
                        {
                        // just ordinary comment, treat it like other HTML text.
                        if ( scan( "-->", true ) )
                            {
                            if ( copy )
                                {
                                // comment might have been inside <generated, in which case would bypass it.
                                // emit entire ordinary comment in two pieces
                                emit.append( "<!--", chunk );   // chunk is from end of <!-- to tend of -->
                                }
                            }
                        else
                            {
                            throw new IllegalArgumentException(
                                    "syntax error missing -->" );
                            }
                        }
                    }
                else
                    {
                    // no more comments, just copy tail and we are done.
                    if ( copy )
                        {
                        // tail end
                        assert 0 <= offset && offset <= originalLength;
                        emit.append( original.substring( offset ) );
                        }
                    break;
                    }
                } // end while
            // all done
            }
        catch ( IllegalArgumentException e )
            {
            // garden variety error, e.g. wrong number of parms or wrong types.
            err.println();
            err.println();
            err.println( e.getMessage() );
            err.println( START_OF_ERROR_MARKER );
            e.printStackTrace( err );
            err.println( e.getClass() + " " + e.getMessage() + where() );
            showParms();
            beep();
            err.println();
            abort = true;
            }
        catch ( NoClassDefFoundError e )
            {
            err.println();
            err.println();
            err.println( START_OF_ERROR_MARKER );
            err.println( "Unrecognised macro command "
                         + macroName
                         + where() );
            showParms();
            beep();
            err.println();
            abort = true;
            }
        catch ( IndexOutOfBoundsException e )
            {
            err.println();
            err.println();
            err.println( START_OF_ERROR_MARKER );
            e.printStackTrace( err );
            err.println( "Index out of bounds, probable missing parm or FastCat overflow"
                         + "\n"
                         + e.getMessage()
                         + where() );
            showParms();
            beep();
            err.println();
            abort = true;
            }
        catch ( Throwable e )
            {
            err.println();
            err.println();
            err.println( START_OF_ERROR_MARKER );
            e.printStackTrace( err );
            err.println( "Unusual Error: " + e.getClass() + " " + e.getMessage() + where() );
            showParms();
            beep();
            err.println();
            abort = true;
            }
        if ( abort )
            {
            // mark it bad. Even though unchanged, even though may have passed previously,
            // is not acceptable to leave this untouched in future.
            trackAlreadyDone.markStatus( fileBeingProcessed, false );
            err.println( "file "
                         + EIO.getCanOrAbsPath( fileBeingProcessed )
                         + " left unchanged." );
            err.println( END_OF_ERROR_MARKER );
            err.println();
            err.println();
            }
        else
            {
            // we must avoid saving file if was error in expansion, we have only a partial file.
            // generate possibly compacted output for new file as a String
            // We need to compact file even if nothing changed.
            contents = emit.toString();
            contents = Compactor.compactStringAsNeeded( contents,
                    fileBeingProcessed.getPath(),
                    configuration.getCompacting() );
            if ( DEBUGGING )
                {
                emit.dump();
                }
            emit.checkEstimate();
            emit = null;
            // whether it changed or not, it is good.
            trackAlreadyDone.markStatus( fileBeingProcessed, true );
            final boolean different = isExpansionDifferentFromBefore();
            if ( different )
                {
                // It is unusual for expansions to change if it has not been recently edited.
                // it can happen if new code is being generated for the same macros.
                // We suppress change display for edits done in the last 45 minutes.
                if ( verbose || DEBUGGING || !forceRegenerate && fileBeingProcessed.lastModified() < System.currentTimeMillis() - ageOfUnexpectedChanges )
                    {
                    // don't bother showing change if expected
                    displayWhatChanged();
                    }
                // really changed, save it back
                try
                    {
                    saveProcessedContents();
                    }
                catch ( IOException e )
                    {
                    err.println( e.getMessage() );
                    System.exit( 2 );
                    }
                }
            // notify the user of the change.
            showPostChangeProgress( quiet, different );
            }
        return false;
        } // end processFile// /method

    /**
     * Guess what line separator is being used in the current file.
     *
     * @return guessed line separator.
     */
    private static String guessLineSeparator()
        {
        String test =
                ( originalLength < 400 )
                ? original
                : original.substring( 80, 400 );
        if ( test.contains( "\r\n" ) )
            {
            return "\r\n";
            }
        if ( test.indexOf( '\r' ) >= 0 )
            {
            return "\r";
            }
        if ( test.indexOf( '\n' ) >= 0 )
            {
            return "\n";
            }
        return configuration.getPreferredLineSeparator();
        }// /method

    /**
     * have the expanded macros actually changed so we need to write the out?
     *
     * @return true if we need to write
     * @throws java.io.IOException
     */
    private static boolean isExpansionDifferentFromBefore() throws IOException
        {
        // prevConents is previouzs expansion of this page, i.e. old distributed or orginial
        if ( separateMasters )
            {
            // this looks like useless extra work it saves on untouch later.
            if ( fileBeingDistributed.exists() && contents.length() == fileBeingDistributed.length() )
                {
                // prev contents look like a match to what we just generated. Check to be sure.
                prevContents = HunkIO.readEntireFile( fileBeingDistributed, HunkIO.UTF8 );
                return prevContents == null || !prevContents.equals( contents );
                }
            else
                {
                // fileBeingBeingDistributed  is not the same as what we just generated.
                // we might need to read it later if want verbose display of what changed.
                prevContents = null;
                return true;
                }
            }
        else
            {
            // about to write on top of original.  See if can avoid the work.
            prevContents = original;
            return !contents.equals( original );
            }
        }// /method

    /**
     * make the \r and \n in a string visible
     *
     * @param s string to make more visiblu
     *
     * @return visible string.
     */
    private static String makeVisible( final String s )
        {
        return s.replace( '\n', '|' ).replace( '\r', '~' );
        }// /method

    /**
     * mark the HTMLMacros data files as busy.
     */
    private static void markAsBusy()
        {
        final File webrootDir = new File( configuration.getLocalWebrootWithSlashes() );
        // There are three different mechanisms.
        // 1. running.dat managed by the htmlmacros program
        // 2. busy.dat managed by btm files
        // 3. stop.dat managed by user when wants a graceful stop.
        final File busy = new File( webrootDir, "busy/running.dat" );
        if ( busy.exists() )
            {
            err.println( "Another copy of HTMLMacros already running. Aborting. Clear manually with reset.btm." );
            System.exit( 2 );
            }
        // create the busy marker file with dummy contents.
        try
            {
            HunkIO.writeEntireFile( busy, "HTMLMacros busy", EIO.UTF8 );
            }
        catch ( IOException e )
            {
            err.println( "Unable to write busy marker file: busy/running.dat" );
            err.println( e.getMessage() );
            System.exit( 2 );
            }
        }// /method

    /**
     * mark the HTMLMacros data files as no longer busy.
     */
    private static void markAsUnBusy()
        {
        final File webrootDir = new File( configuration.getLocalWebrootWithSlashes() );
        final File busy = new File( webrootDir, "busy/running.dat" );
        busy.delete();
        }// /method

    /**
     * Scan from the current offset looking for the next word (delimited by white space) It does not modify chunk or
     * offset.
     *
     * @return next word in original starting at offset.
     */
    private static String nextWord()
        {
        // we want to avoid creating needless substrings that could
        // be very long, so we process char by char.
        boolean inLeading = true;
        int start = offset;
        for ( int i = offset; i < originalLength; i++ )
            {
            if ( Character.isWhitespace( original.charAt( i ) ) )
                {
                if ( inLeading )
                    {
                    start = i + 1;
                    }
                else
                    {
                    return original.substring( start, i/* one past end */ );
                    }
                }
            else
                {
                inLeading = false;
                }
            } // end for
        return ( original.substring( start ) );
        }// /method

    /**
     * process a macro of the form: <!-- macro Include dogs.txt -->
     *
     * @param quiet   true if want progress messages suppressed
     * @param verbose true if want extra output
     */
    private static String parseAndExpandOneMacro( final boolean quiet, final boolean verbose )
        {
        // we found something of the form:
        // <!-- macro Include dogs.txt -->
        // we have parsed up to but not including the word macro
        // parse out the various sections.
        // has to be ok, since we already found it.
        // We don't yet have a macroName or parms, set null it case get
        // exception.
        // FastCat slots: macro lead,  macroname,  parmarea,  macro tail,
        // generated, expansion, /generated.
        final FastCat emit = new FastCat( 10 );
        macroName = null;
        parms = null;
        scan( "macro", true );
        // name of macro to invoke.
        macroName = nextWord();
        // at worst would be empty name
        scan( macroName, true );
        if ( !scan( "-->", false ) )
            {
            throw new IllegalArgumentException(
                    "syntax error missing --> to close macro"
                    + where()
            );
            }
        final String parmArea = chunk.trim();
        /* get array of parms */
        parms = ParmParser.parseParms( parmArea );
        // has to be ok, since already found it earlier
        scan( "-->", true );
        // leave the original macro in place, slightly tidied.
        // don't println or append an extra \n. We have not parsed that yet.
        emit.append( "<!-- macro " );
        emit.append( macroName );
        if ( parmArea.length() != 0 )
            {
            emit.append( " " );
            emit.append( parmArea );
            }
        emit.append( " -->" );
        // don't add nl before or after or they will accumulate each time macro
        // is run.
        // we replace only stuff strictly inside.
        // No extra \n before or after to avoid introducing spurious white space.
        emit.append( "<!-- generated -->" );
        // invoke the specific macro processor, getting it to expandNoRef
        // tell it the filename, no so that it can read it, but so that it
        // can use its names
        // in the generation of the macro.
        try
            {
            // get the Macro-implementing class instance that will process the macro.
            // It implement the Macro interface with the expandNoRef method.
            // Hopefully already loaded from previous use.  Will throw exception on trouble. Should not return null.
            final Macro macroDelegate = LoadCodeToProcessMacro.getMacroProcessorInstance( macroName );
            assert macroDelegate != null : "null delegate to process macro " + macroName;
            // Tell the macro the source and target files.  It does not need to read/write them,
            // but it may need to know them.
            macroDelegate.setFileBeingProcessedAndFileBeingDistributed( fileBeingProcessed, fileBeingDistributed );
            // F I N A L L Y ! _ E X P A N D _ T H E _ M A C R O !
            String expansion = macroDelegate.expandMacro( parms, quiet, verbose );
            // compacting individual macro expansions
            // This saves RAM since we get rid of fat early
            // Compactor might be able to avoid work at the end if all is already compact.
            expansion = Compactor.compactStringAsNeeded( expansion,
                    fileBeingProcessed.getPath(),
                    configuration.getCompacting() );
            expansion = adjustLineSeparator( expansion );
            // tack the macro expansion just after the macro command
            emit.append( expansion );
            }
        catch ( ClassNotFoundException e )
            {
            throw new IllegalArgumentException( "No such macro as " + macroName + " or possibly missing code to " +
                                                "handle it." );
            }
        catch ( InstantiationException e )
            {
            throw new IllegalArgumentException( "No such macro as " + macroName + " or possibly missing code to " +
                                                "handle it." );
            }
        catch ( IllegalAccessException e )
            {
            throw new IllegalArgumentException( "No such macro as " + macroName + " or possibly missing code to " +
                                                "handle it." );
            } // we don't catch other exceptions here.
        // don't add \n before or after or they will accumulate each elapsedTime macro is
        // run. Further we want to avoid adding spurious white space.
        emit.append( "<!-- /generated by ", macroName, " -->" );
        copy = true;
        return emit.toString();
        }// /method

    /**
     * save expanded/compacted contents of the file.
     *
     * @throws IOException if trouble doing the io.
     */
    private static void saveProcessedContents() throws IOException
        {
        // we REALLY changed.
        // If we were to crash in the middle of writing to the original,
        // it would be corrupted. So instead we write to temp, then
        // after it is safely written, we delete the original and
        // rename.
        final File tempFile = HunkIO.createTempFile( "temp_",
                ".tmp",
                fileBeingProcessed );
        HunkIO.writeEntireFile( tempFile, contents, HunkIO.UTF8 );
        HunkIO.deleteAndRename( tempFile, fileBeingProcessed );
        }// /method

    /**
     * Scan original looking for the lookFor string, starting at offset.<br> It leaves the material prior to what was
     * sought in chunk. It updates the offset to point either before or after the lookFor string.
     *
     * @param lookFor String to search for. Not a regex.
     * @param after   true if scan to char after the lookFor String <br> false, leave offset pointing to beginning of
     *                the lookFor string.
     *
     * @return true if successful.
     */
    @SuppressWarnings( { "BooleanMethodNameMustStartWithQuestion" } )
    private static boolean scan( String lookFor, boolean after )
        {
        int where = original.indexOf( lookFor, offset );
        if ( where < 0 )
            {
            // did not find it
            chunk = null;
            return false;
            }
        if ( after )
            {
            where += lookFor.length();
            }
        chunk = original.substring( offset, where );
        offset = where;
        return true;
        } // end scan// /method

    /**
     * Display macro name and parms on error
     */
    private static void showParms()
        {
        // macroName and parms may be null.
        if ( macroName == null )
            {
            macroName = "?";
            }
        err.println( "macro where problem was detected:" );
        err.println( "    <!-- macro " + macroName );
        if ( parms != null )
            {
            for ( String parm : parms )
                {
                err.println( "      {" + parm + "}" );
                }
            }
        err.println( "    -->" );
        }// /method

    /**
     * show progress of how we are processing files.
     *
     * @param quiet     true to suppress progress messages
     * @param different true if this file has changed
     */
    private static void showPostChangeProgress( boolean quiet, boolean different )
        {
        if ( different )
            {
            if ( quiet )
                {
                // quiet
                // only print names of files that actually changed
                out.println( "* " + EIO.getCanOrAbsPath( fileBeingProcessed ) );
                }
            else
                {
                // verbose
                // append * onto list of all the macros expanded in that
                // file.
                out.println( "*" );
                }
            }
        else
            {
            // leave original file untouched, same date and elapsedTime.
            // Notify user we left the file unchanged.
            if ( quiet )
                {
                // quiet, say nothing at all
                // out.print( "-" );
                }
            else
                {
                // verbose, tack onto end of letters for macro expansion.
                out.println( "-" );
                }
            }
        }// /method

    /**
     * warn user of problem.
     *
     * @param message text of error message.
     */
    private static void warn( String message )
        {
        err.println();
        err.println( "------------------------------------------------------" );
        err.println( "W A R N I N G: " + message );
        // This is a polite little DOS speaker beep, not the full system error
        // tone. It sounds quite different from ordinary sounds, so catches
        // attention.
        // It also can't be shut off with the volume control.
        beep();
        }// /method

    /**
     * display text near current given offset
     *
     * @return text near that offset, line number etc to help user find the error
     * @see com.mindprod.compactor.HTMLState#where
     */
    private static String where()
        {
        final FastCat sb = new FastCat( 11 );
        sb.append( "\n>>> in file: " );
        sb.append( EIO.getCanOrAbsPath( fileBeingProcessed ) );
        sb.append( " near offset: " );
        sb.append( offset );
        sb.append( " near line: " );
        sb.append( ST.countInstances( original.substring( 0, offset ), '\n' ) + 1 /* one based */ );
        sb.append( "\n  [" );
        // get at least 100 chars before and after, break at even line.
        int start = Math.max( 0, offset - 100 );
        // cannot compute in one line
        int startBreak = original.lastIndexOf( '\n', start );
        if ( startBreak >= 0 )
            {
            start = startBreak;
            }
        int end = Math.min( offset + 100, original.length() );
        // cannot compute in one line
        final int endBreak = original.indexOf( '\n', end );
        if ( endBreak >= 0 )
            {
            end = endBreak;
            }
        sb.append( original.substring( start, offset ).trim() );
        sb.append( "  <><>||||<><>  " );
        sb.append( original.substring( offset, end ).trim() );
        sb.append( "]\n\n" );
        return sb.toString();
        }// /method

    /**
     * estimate how many FastCat slots will be needed to process this page
     *
     * @param original contents of the page
     *
     * @return number of slots needed
     * @see com.mindprod.htmlmacros.macro.Include
     * @see com.mindprod.htmlmacros.macro.Insert
     */
    public static int estimateFastCatSlots( String original )
        {
        // macros may been bang up against each other with text
        // count 1 for each block of text length
        //       2 for each comment
        //       1 for each macro
        //       0 for each <generated
        //       0 for text between <generated </generated
        //       0 for </generated
        // --> might start a line so cannot search for space-->
        // <-- might end a line, so cannot search for <--space
        int slots = 0;
        int start = 0;
        if ( DEBUGGING )
            {
            out.println( "estimateFastCatSlots" );
            }
        loop:
        while ( start < original.length() )
            {
            int place = original.indexOf( "<!--", start );
            if ( place < 0 )
                {
                slots++; // tail
                break loop;
                }
            if ( place > start )
                {
                if ( DEBUGGING )
                    {
                    out.println( slots + ": text " + original.substring( start, Math.min( start + 60, Math.min( place, original.length() - 1 ) ) ) );
                    }
                slots++;   // some plain text
                }
            final String kindOfComment = ST.firstWord( original.substring( place + "<!--".length() ) );
            switch ( kindOfComment )
                {
                case "macro":
                    if ( DEBUGGING )
                        {
                        out.println( slots + ": " + kindOfComment + " " + original.substring( place, Math.min( place + 60, original.length() - 1 ) ) );
                        }
                    slots++;
                    start = original.indexOf( "-->", place + "<!-- macro ".length() );
                    if ( start < 0 )
                        {
                        break loop;
                        }
                    start += "-->".length();
                    break;
                case "generated":
                    /* skip over all generated text */
                    start = place + "<!-- generated ".length();
                    place = original.indexOf( "<!-- /generated ", start );
                    if ( place < 0 )
                        {
                        break loop;
                        }
                    start = original.indexOf( "-->", place + "<!-- /generated ".length() );
                    if ( start < 0 )
                        {
                        break loop;
                        }
                    start += "-->".length();
                    break;
                case "/generated":
                    start = original.indexOf( "-->", place + "<!-- /generated ".length() );
                    if ( start < 0 )
                        {
                        break loop;
                        }
                    start += "-->".length();
                    break;
                default:
                    // ordinary comment
                    slots++; /* for <!--  */
                    if ( DEBUGGING )
                        {
                        out.println( slots + ": " + kindOfComment + " " + original.substring( place, Math.min( place + 60, original.length() - 1 ) ) );
                        }
                    slots++; /* comment in two pieces */
                    start = original.indexOf( "-->", place + "<!--".length() );
                    if ( start < 0 )
                        {
                        break loop;
                        }
                    start += "-->".length();
                } // /switch
            }  // /loop
        slots++; /* why? don't know */
        if ( DEBUGGING )
            {
            out.println( "final slots: " + slots );
            }
        return slots;
        }

    /**
     * what do do when replacer starts up
     */
    public static void fireup()
        {
        markAsBusy();
        trackAlreadyDone = new Lazy();
        trackAlreadyDone.open( new File( "E:\\mindprod\\busy\\cache.bin" ), "E:\\mindprod\\", ".html", 10_000 /* estimated files */ );
        }// /method

    /**
     * Expands macros written in Java in HTML files files.
     *
     * @param args name of configuration class, e.g. ConfigurationForMindprod.com then names of files to process,
     *             no wildcards.
     */
    public static void main( String[] args )
        {
        if ( args.length < 2 )
            {
            throw new IllegalArgumentException( "Must have Configuration and at least one directory/file on the command line." );
            }
        Global.installConfiguration( args );
        final String absParm = EIO.getCanOrAbsPath( new File( args[ args.length - 1 ] ) ).toLowerCase();
        if ( !absParm.startsWith( Global.configuration.getLocalWebrootWithBackslashes().toLowerCase() ) )
            {
            throw new IllegalArgumentException(
                    Global.configuration.getClass().toString() + " incompatible with  " + absParm );
            }
        // When old files chuange, we display the change details
        ageOfUnexpectedChanges = Global.configuration.getAgeOfUnexpectedChanges();
        Fireup.fireup();
        // gather all the files mentioned on the command line.
        // either directories, files, *.*, with -s and subdirs option.
        out.println( "Gathering html files to expand macros..." );
        // command line, estimated files, dir filter, file filter.
        // These are fully qualified names.
        final CommandLine commandLine = new CommandLine( args,
                new AllButFootDirectoriesFilter(),
                new ExtensionListFilter( ExtensionListFilter.COMMON_HTML_EXTENSIONS ) );
        final boolean quiet = commandLine.isQuiet();
        if ( quiet )
            {
            out.println( "-q quiet" );
            }
        final boolean verbose = commandLine.isVerbose();
        // command line displays it automatically
        // this mechanism does not modify the cache.
        // if just expanding a few files, obviously want them regenerated.
        final boolean smallSet = commandLine.size() < BYPASS_LIMIT;
        for ( File file : commandLine )
            {
            try
                {
                // replace any non-canonical files.
                // ensure added files are canonical.
                try
                    {
                    // don't use getCanOrAbsPath
                    final File fileBeingProcessed = new File( file.getCanonicalPath() );
                    final boolean alwaysRegenerate = Global.configuration.alwaysRegenerate( fileBeingProcessed );
                    final boolean containsRandomQuotation = Embellishment.QUOTATION.isFileEmbellishedWithThisLetter( fileBeingProcessed );
                    final boolean forceRegenerate = smallSet || alwaysRegenerate || containsRandomQuotation;
                    if ( expandMacrosInFile( fileBeingProcessed, quiet, verbose, forceRegenerate ) )
                        {
                        break;
                        }
                    }
                catch ( IOException e )
                    {
                    err.println( e.getMessage() );
                    // usually problem is can't make file canonical.
                    System.exit( 1 );
                    }
                }
            catch ( Exception e )
                {
                err.println();
                err.println( e.getMessage() + "\n" + "in file " + EIO.getCanOrAbsPath( file ) );
                err.println();
                }
            }
        Shutdown.shutdown();
        Misc.trackLastThread();
        out.println( "done" );
        System.exit( 0 );
        } // end main// /method

    /**
     * what do do when replacer quits
     */
    public static void shutdown()
        {
        trackAlreadyDone.close();
        trackAlreadyDone = null;
        markAsUnBusy();
        }// /method

    /**
     * Ensures <!-- --> and <!-- generated ...---> and <!-- /generated...--> tags balance, and do not nest. <br> Also
     * used by Include.java
     * * @param fileBeingProcessed file being processed. Just needed for error messages.
     *
     * @param segment            html text in which to check the <generated tags
     * @param fileBeingProcessed the file currently being processed.
     *
     * @throws IllegalArgumentException on bad syntax
     * @see com.mindprod.htmlmacros.StripGenerated#stripGenerated(String)
     */
    public static void verifyGeneratedTags( String segment, File fileBeingProcessed )
        {
        // We use a hand-coded parser to find <!-- .. --> and regex to recognize
        // <!-- generated <!-- /generated -->
        int length = segment.length();
        boolean insideGenerated = false;
        // indexes which character we are working on
        int i = 0;
        while ( i < length )
            {
            // search for start of comment
            int startCommentPlace = segment.indexOf( "<!--", i );
            if ( startCommentPlace < 0 )
                {
                // no more comments to eof
                break;
                }
            // we found the start of a comment
            // find the end of comment
            int endCommentPlace =
                    segment.indexOf( "-->",
                            startCommentPlace + "<!--".length() );
            if ( endCommentPlace < 0 )
                {
                offset = startCommentPlace;
                throw new IllegalArgumentException(
                        "missing --> on a <!-- comment <!-- macro or <!--generated" );
                }
            endCommentPlace += "-->".length();
            // comment includes <!-- and -->
            String comment =
                    segment.substring( startCommentPlace, endCommentPlace );
            // make sure the comments not malformed. Should be no embedded start
            // comment marker
            String commentGuts = comment.substring( "<!--".length(), comment.length() - "-->".length() );
            if ( commentGuts.contains( "<!--" ) )
                {
                offset = startCommentPlace;
                throw new IllegalArgumentException( "<!-- ... --> not balanced" );
                }
            // three special kinds of comment
            // <!-- macro ... --> treat like ordinary comments here.
            // <!-- generated .. --> start stripping
            // <!-- /generated .. --> stop stripping
            // we treat macro like ordinary comments. We don't try to enforce
            // macro generated /generated or similar order. The whole point of
            // this is
            // to clean up mess where you may have got dups etc.
            if ( startGenerated.matcher( comment ).matches() )
                {
                // was <!-- generated
                if ( insideGenerated )
                    {
                    offset = startCommentPlace;
                    throw new IllegalArgumentException(
                            "<!-- generated --> without previous <!-- /generated -->" );
                    }
                insideGenerated = true;
                }
            else if ( endGenerated.matcher( comment ).matches() )
                {
                // was <!-- /generated
                if ( !insideGenerated )
                    {
                    offset = startCommentPlace;
                    throw new IllegalArgumentException(
                            "<!-- /generated --> without previous <!-- generated -->" );
                    }
                insideGenerated = false;
                }
            else
                {
                // was <!-- ordinary comment or macro
                final int afterCommentStart = startCommentPlace + "<!--".length();
                final char charAfterCommentStart = segment.charAt( afterCommentStart );
                if ( charAfterCommentStart != '#'
                     && !segment.substring( afterCommentStart ).startsWith( "cseignore" )
                     && !segment.substring( afterCommentStart ).startsWith( "/cseignore" ) )
                    {
                    // not <!--#....--> SSI comment which does not need  whitespace, e.g. ordinary comment or macro
                    // not <!--cseignore-->
                    // not <!--/cseignore-->
                    if ( !( charAfterCommentStart == ' ' || Character.isWhitespace( charAfterCommentStart ) ) )
                        {
                        warn( "Missing whitespace after <!-- at offset "
                              + startCommentPlace
                              + " in file "
                              + fileBeingProcessed.toString() );
                        displayOffendingText( segment.substring( startCommentPlace, endCommentPlace ) );
                        }
                    final char charBeforeCommentEnd = segment.charAt( endCommentPlace - "-->".length() - 1 );
                    if ( !( charBeforeCommentEnd == ' ' || Character.isWhitespace( charBeforeCommentEnd ) ) )
                        {
                        warn( "Missing whitespace before --> at offset "
                              + ( endCommentPlace - "-->".length() - 1 )
                              + " in file "
                              + fileBeingProcessed.toString() );
                        displayOffendingText( segment.substring( startCommentPlace, endCommentPlace ) );
                        }
                    }
                }
            i = endCommentPlace;
            } // end while
        // make sure last <!-- generated had a match
        if ( insideGenerated )
            {
            throw new IllegalArgumentException(
                    "Missing <!-- /generated --> at end of file." );
            }
        }// /method
    // /methods
    } // end Replacer