/*
 * [SeeSort.java]
 *
 * Summary: Sorts and repairs HTML references.
 *
 * Copyright: (c) 1997-2017 Roedy Green, Canadian Mind Products, http://mindprod.com
 *
 * Licence: This software may be copied and used freely for any purpose but military.
 *          http://mindprod.com/contact/nonmil.html
 *
 * Requires: JDK 1.8+
 *
 * Created with: JetBrains IntelliJ IDEA IDE http://www.jetbrains.com/idea/
 *
 * Version History:
 *  2.0 2005-06-16 split off from Compactor
 *  2.1 2005-06-16 use
 *                 com.mindprod.entities.stripHTMLTags and stripEntities instead of an a
 *                 crude TidyKey to fix sort problem. futures: catch dup when it is
 *                 created. warning if have a dup in the alpha part of a link even if
 *                 link differs.
 *  2.2 2006-03-05 reformat with IntelliJ and add Javadoc.
 *  2.3 2009-04-14 automatically correct missing <br> between links. treat <li> and </li> and separators too
 *  2.4 2014-04-16 change marker to <div class="see" id="SEE">
 *  2.5 2014-05-29 new way of doing dup detection that ignores class.
 *  2.6 2016-06-04 now use Configuration instead of command line
 */
package com.mindprod.seesort;

import com.mindprod.common18.EIO;
import com.mindprod.common18.ST;
import com.mindprod.entities.DeEntifyStrings;
import com.mindprod.fastcat.FastCat;
import com.mindprod.filter.ExtensionListFilter;
import com.mindprod.go.GoList;
import com.mindprod.htmlmacros.macro.Global;
import com.mindprod.hunkio.HunkIO;
import org.jetbrains.annotations.NotNull;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.regex.Pattern;

import static com.mindprod.entities.DeEntifyStrings.flattenHTML;
import static java.lang.System.*;

/**
 * Sorts and repairs HTML references.
 * <p/>
 * sort links in alphabetical order. Only works on directories for which there is an index(or a dummy empty index).
 * Removes duplicates. Must match url and name, ignore case.
 *
 * @author Roedy Green, Canadian Mind Products
 * @version 2.6 2016-06-04 now use Configuration instead of command line
 * @see com.mindprod.sortaliases.SortAliases
 * @since 1997
 */
public final class SeeSort
    {
    // declarations
    private static final int FIRST_COPYRIGHT_YEAR = 1997;

    private static final boolean DEBUGGING = false;

    private static final int endMarkerLength = "</div>".length();

    /**
     * ideal  splitter, one we tidy to. <br> will end on end of line, not on a line by itself
     */
    private static final String CANONICAL_BR = "<br>\n";

    /**
     * undisplayed copyright notice
     */
    @SuppressWarnings( { "UnusedDeclaration" } )
    private static final String EMBEDDED_COPYRIGHT =
            "Copyright: (c) 1997-2017 Roedy Green, Canadian Mind Products, http://mindprod.com";

    @SuppressWarnings( { "UnusedDeclaration" } )
    private static final String RELEASE_DATE = "2016-06-04";

    /**
     * how to use the command line
     */
    private static final String USAGE = "\nSeeSort needs the name of a configuration on the command line.";

    /**
     * embedded version string.
     */
    @SuppressWarnings( { "UnusedDeclaration" } )
    private static final String VERSION_STRING = "2.6";

    private static final String endMarker = "</div>";

    private static final String startMarker = "<div class=\"see\"";

    private static final String startMarker1 = " id=\"SEE\">";

    private static final String startMarker2 = ">";

    /**
     * used to sort see items in order.
     */
    private static final Comparator<String> tidyComparator = new SeeLine();

    private static final Pattern LINE_SPLITTER = Pattern
            .compile( "<br>|<br/>|<br />|<p>|<li>|</br>|</p>|</li>", Pattern.CASE_INSENSITIVE );

    /**
     * list of known words and links
     */
    private static GoList goList;

    /**
     * keyed lookup back to the GoList
     */
    private static HashMap<String, Integer> lookup;
    // /declarations

    /**
     * Constructor
     */
    private SeeSort()
        {
        }
    // methods

    /**
     * informational message
     *
     * @param message            text of error message.
     * @param fileBeingProcessed file were trouble was
     * @param near               word in file where trouble was near.
     */
    private static void info( final String message, final File fileBeingProcessed, final String near )
        {
        final FastCat sb = new FastCat( 7 );
        sb.append( "\n    Note: ", message, " in file: ", EIO.getCanOrAbsPath( fileBeingProcessed ) );
        if ( near != null )
            {
            sb.append( " near: ", near );
            }
        sb.append( "\n" );
        err.println( sb.toString() );
        }// /method

    private static void readForSeeSort( File serFile )
        {
        // read  forseesort.ser. Use to repair links without URLs.
        if ( serFile.exists() )
            {
            try
                {
                final InputStream is = new FileInputStream( serFile );
                goList = GoList.getData( is );
                // index the golist
                int size = goList.size();
                assert size < 20000 : "giant index " + size;
                lookup = new HashMap<>( size * 3 / 2 );
                // add human names from index to hashMap so we can see if they are
                // fixable. Lookup takes name of link and gets you index into goList for details.
                for ( int i = 0; i < size; i++ )
                    {
                    lookup.put( goList.getHumanname( i ).toLowerCase() /* key */, i /* value */ );
                    }
                }
            catch ( Exception e )
                {
                err.println( "Exception fetching  " + EIO.getCanOrAbsPath( serFile ) );
                err.println( e.getMessage() );
                exit( 1 );
                }
            }
        else
            {
            goList = null;
            lookup = null;
            }
        }

    /**
     * find see-style references, sort each group, and repair missing links on one entire file.
     *
     * @param fileBeingProcessed the file currently being processed.
     *
     * @throws IOException if cannot read file
     */
    @SuppressWarnings( { "JavaDoc" } )
    private static void sortLinksInFile( File fileBeingProcessed ) throws IOException
        {
        String big = HunkIO.readEntireFile( fileBeingProcessed );
        String result = sortLinksInString( big, fileBeingProcessed );
        if ( result.equals( big ) )
            {
            // nothing changed. No need to write results.
            return;
            }
        // generate output into a temporary file until we are sure all is ok.
        // create a temp file in the same directory as filename
        // create a tempFile in the same directory as
        // the input file we have just processed.
        final File tempFile =
                HunkIO.createTempFile( "temp_", ".tmp", fileBeingProcessed );
        FileWriter emit = new FileWriter( tempFile );
        emit.write( result );
        emit.close();
        // successfully created output in same directory as input,
        // Now make it replace the input file.
        HunkIO.deleteAndRename( tempFile, fileBeingProcessed );
        }// /method

    /**
     * sort and fix groups of see references in this string representing an entire file
     *
     * @param big                entire file contents containing <div class="see"> links
     * @param fileBeingProcessed the file currently being processed.
     *
     * @return same string with links repaired, sorted, and tidied.
     */
    private static String sortLinksInString( String big, File fileBeingProcessed )
        {
        final String fileBeingProcessedName = fileBeingProcessed.getName();
        final int originalLength = big.length();
        final StringBuilder sb = new StringBuilder( originalLength + 1000 );
        int i = 0;
        // i indexes char in the big string, goes in hops.
        // We might find several <div class="see" a page
        while ( i < originalLength )
            {
            // wo don't use Regex. This runs so often on all files, it must be fast.
            final int start = big.indexOf( startMarker, i );
            if ( start < 0 )
                {
                // did not find a start,
                // just copy over tail end
                sb.append( big.substring( i ) );
                break;
                }
            else
                {
                // find out which variant it was:
                // starts with either <div class="see"> or <div class="see" id="SEE">
                final int startMarkerLength;
                String middle = big.substring( start + startMarker.length() );
                if ( middle.startsWith( startMarker1 ) )
                    {
                    startMarkerLength = startMarker.length() + startMarker1.length();
                    }
                else if ( middle.startsWith( startMarker2 ) )
                    {
                    startMarkerLength = startMarker.length() + startMarker2.length();
                    }
                else
                    {
                    throw new IllegalArgumentException( "Invalid syntax "
                                                        + big.substring( start, start + 30 )
                                                        + "\n Should be <div class=\"see\"> or <div class=\"see\" id=\"SEE\">" );
                    }
                // copy over stuff in front of the <div, and the marker up the >.  From then an are <a... lines.
                sb.append( big.substring( i, start + startMarkerLength ) );
                // we found start of <div. Find the </div>.
                final int end = big.indexOf( endMarker, start + startMarkerLength );
                if ( end < 0 )
                    {
                    // we don't just print a warning. We must abort processing
                    // this file. We could
                    // do it serious damage.
                    throw new IllegalArgumentException( "Unmatched "
                                                        + startMarker
                                                        + " ... "
                                                        + endMarker );
                    }
                // we will soon handle all between, so mark it handled now.
                i = end + endMarkerLength;
                // contents of <div ...</div>
                final String see = big.substring( start + startMarkerLength, end );
                // the Start marker has already been appended.
                // Split contents of see division into lines at <br>
                // Then slit lines with two entries into multiple lines.
                final String[] lines = splitMoreFinely( LINE_SPLITTER.split( see ) );
                // put into alpha order by lines so can detect dups
                Arrays.sort( lines, tidyComparator );
                // used to detect duplicates
                Comparator<String> compareDupSeeLines = new CompareDupSeeLines();
                String prev = "";
                boolean needSeparator = false;
                for ( String line : lines )
                    {
                    if ( line == null || line.length() == 0 )
                        {
                        // ignore
                        continue;
                        }
                    // line might contain embedded \n which we leave intact.
                    // trim lead trail spaces, \n, Collapse double spaces,
                    // control chars.
                    // 'line' is a line, a phrase, or a link to somewhere,
                    // -- the contents between &lt;br&gt;
                    line = ST.condense( line );
                    // remove trailing dots.
                    while ( line.endsWith( "." ) )
                        {
                        line = line.substring( 0, line.length() - 1 ).trim();
                        }
                    // ignore empty lines.
                    if ( line.length() == 0 )
                        {
                        continue;
                        }
                    if ( compareDupSeeLines.compare( line, prev ) == 0 )
                        {
                        // Eliminate dup, ignore this one.
                        // Both link and text describing link must match to be considered a dup
                        info( line + " duplicate removed", fileBeingProcessed, null );
                        // toss the second one.
                        continue;
                        }
                    if ( needSeparator )
                        {
                        // put each on own line, easier to proofread
                        // with &lt;br&gt; separator
                        sb.append( CANONICAL_BR );
                        }
                    else
                        {
                        // on all but the first, we need a separator in front.
                        needSeparator = true;
                        }
                    // if has double <a ... <a we are in trouble
                    // but if has macro, all bets are off.
                    if ( line.startsWith( "<a " ) && line.substring( 2 ).contains( "<a " )
                         && !( line.contains( "<!-- macro " ) || line.length() < 3 ) )
                        {
                        throw new IllegalArgumentException( "Failed to autocorrect correct missing <br> or double <a" );
                        }
                    if ( line.startsWith( "<a" ) )
                        {
                        // already done. starts with <a, leave as is
                        // normally the description should not end in .html, it should be a description of the link,
                        // not its name.
                        if ( line.contains( ".html</a>" ) )
                            {
                            warn( "extraneous \".html\"", fileBeingProcessed, line );
                            }
                        if ( line.contains( "</a>." ) )
                            {
                            // Dot is ok, just not right after </a>. We are enforcing consistent style here
                            warn( "extraneous trailing dot after </a>", fileBeingProcessed, line );
                            }
                        if ( line.contains( "</a>," ) )
                            {
                            // comma is ok, just not right after </a>. We are enforcing consistent style here
                            warn( "extraneous trailing comma after </a>. Use </a>: instead", fileBeingProcessed, line );
                            }
                        // TODO: should put code in here to deal with class="xxx"
                        if ( line.startsWith( "<a href=\"" + fileBeingProcessedName + "\"" ) )
                            {
                            info( "removed self reference", fileBeingProcessed, line );
                            // ignore it
                            // have an extraneous <br> suppress a second
                            // one, still might get one extra
                            needSeparator = false;
                            }
                        else
                            {
                            // append everything <a. > ... </a>
                            sb.append( line );
                            }
                        }
                    else if ( line.startsWith( "<!-- macro " ) )
                        {
                        // already done. trust was done correctly
                        // append everything up to <br>
                        sb.append( line );
                        }
                    else
                        {
                        // line is not a link already. See if it
                        // can be turned into one.
                        String wordlc = line.toLowerCase();
                        boolean fixable;
                        Integer elt;
                        if ( lookup == null )
                            {
                            fixable = false;
                            elt = null;
                            }
                        // Repair link if possible
                        else if ( ( elt = lookup.get( wordlc ) ) != null )
                            {
                            // as is
                            fixable = true;
                            }
                        else if ( ( elt = lookup.get( DeEntifyStrings.deEntifyHTML( wordlc, ' ' ) ) ) != null )
                            {
                            // ic can be found if entities converted to chars
                            fixable = true;
                            }
                        else if ( ( elt = lookup.get( wordlc + 's' ) ) != null )
                            {
                            // pluralised add s
                            fixable = true;
                            }
                        else if ( wordlc.endsWith( "s" )
                                  && ( elt = lookup.get( wordlc + "es" ) )
                                     != null )
                            {
                            // pluralised add es
                            fixable = true;
                            }
                        else //noinspection SimplifiableIfStatement
                            if ( wordlc.endsWith( "s" )
                                 && ( elt =
                                    lookup.get( wordlc.substring( 0, wordlc.length() - 1 ) ) ) != null )
                                {
                                // made singular, chop s
                                fixable = true;
                                }
                            else
                                {
                                fixable = wordlc.endsWith( "es" )
                                          && ( elt =
                                        lookup.get( wordlc.substring( 0, wordlc
                                                                                 .length() - 2 ) ) ) != null
                                          || ( elt = lookup.get( wordlc + "ing" ) )
                                             != null
                                          || wordlc.endsWith( "e" )
                                             && ( elt =
                                        lookup.get( wordlc.substring( 0, wordlc
                                                                                 .length() - 1 ) + "ing" ) ) != null
                                          || wordlc.endsWith( "ing" )
                                             && ( elt =
                                        lookup.get( wordlc.substring( 0, wordlc
                                                                                 .length() - 3 ) ) ) != null
                                          || wordlc.endsWith( "ing" )
                                             && ( elt =
                                        lookup.get( wordlc.substring( 0,
                                                wordlc.length()
                                                - 3
                                        ) + "e" ) ) != null;
                                }
                        // at this point we have decided if the link is fixable
                        // or not
                        // and elt points to the index entry that can be used to
                        // fix it.
                        if ( fixable )
                            {
                            // fixable
                            final int k = elt;
                            String linkFile = goList.getFilename( k );
                            String target = goList.getTarget( k );
                            if ( fileBeingProcessedName.equals( linkFile ) )
                                {
                                info( "Removed self reference", fileBeingProcessed, line );
                                // ignore it
                                // have an extraneous &curren; suppress a second
                                // one, still might get one extra
                                needSeparator = false;// we have already done
                                // the separator.
                                }
                            else
                                {
                                // fixing
                                info( "Repaired missing " + line + " link", fileBeingProcessed, null );
                                sb.append( "<a href=\"" ); // class patched later
                                sb.append( linkFile );
                                if ( target != null && target.length() > 0 )
                                    {
                                    sb.append( '#' );
                                    sb.append( target );
                                    }
                                sb.append( "\">" );
                                sb.append( line );
                                sb.append( "</a>" );
                                // We may have created a dup with this. This is
                                // a bug.
                                // It will get eliminated on the next round.
                                }
                            }
                        else
                            {
                            // not fixable, leave as is.
                            warn( "Cannot repair " + line + " link", fileBeingProcessed, null );
                            sb.append( line );
                            }
                        } // end not a link already
                    prev = line;
                    }
                // no <br> after the last one.
                sb.append( "</div>" );
                } // end else
            } // end for
        return sb.toString();
        }// /method

    /**
     * check lines to see if any have an embedded <a, if so split there, leaving the <a intact
     * This is not recursive..
     *
     * @param lines one line per link usually of form <a ..>xxx</a>
     *
     * @return new list of lines, more finely split if necessary.
     */
    private static String[] splitMoreFinely( String[] lines )
        {
        // determine if any line needs a finer split because of entries without improperly without a separator
        boolean needFinerSplit = false;
        // do a rough pass to see if we need any finer splitting, usually not.
        for ( String line : lines )
            {
            line = line.trim();
            // assume macros done correctly. Don't mess with them.
            if ( line.startsWith( "<!-- macro" ) )
                {
                continue;
                }
            if ( line.length() >= 3 && line.indexOf( "<a ", 3 ) >= 0 )
                {
                // line with multiple entries on it.  We need to do  even finer splitting
                // This should not happen often.
                needFinerSplit = true;
                break;
                }
            } // end for
        if ( !needFinerSplit )
            {
            // nothing to do, return list unmodified.
            return lines;
            }
        // split all the lines, sometimes a single line may have multiple chunks.
        final ArrayList<String> growth = new ArrayList<>( lines.length + 5 );
        for ( String line : lines )
            {
            line = line.trim();
            // assume macros done correctly. Don't mess with them.
            if ( line.startsWith( "<!-- macro" ) )
                {
                growth.add( line );
                continue;
                }
            int place;
            // extract all chunks on the line, but the last
            while ( line.length() >= 3 && ( place = line.indexOf( "<a ", 3 ) ) >= 0 )
                {
                // get chunk prior to <a
                final String chunk = line.substring( 0, place ).trim();
                if ( chunk.length() != 0 )
                    {
                    assert growth.size() < 1000 : "size explosion: " + chunk;
                    growth.add( chunk );
                    }
                line = line.substring( place ).trim();
                } // end while
            // extract last chunk on line, possibly the only chunk
            final String chunk = line.trim();
            if ( chunk.length() != 0 )
                {
                assert growth.size() < 1000 : "size explosion: " + chunk;
                growth.add( chunk );
                }
            } // end for
        // return the expanding list with chunks not taking a whole line.
        return growth.toArray( new String[ growth.size() ] );
        }// /method

    /**
     * warn user of problem
     *
     * @param message            text of error message.
     * @param fileBeingProcessed file were trouble was
     * @param near               word in file where trouble was near.
     */
    private static void warn( String message, final File fileBeingProcessed, String near )
        {
        // this is a polite little DOS speaker eep, not the full system error
        // tone.
        final FastCat sb = new FastCat( 7 );
        sb.append( "\n\007Warning: ", message, " in file: ", EIO.getCanOrAbsPath( fileBeingProcessed ) );
        if ( near != null )
            {
            sb.append( " near ", near );
            }
        sb.append( "\n" );
        }// /method

    /**
     * corrects SEE links in HTML files.
     *
     * @param args name of configuration. Rest of info comes from config.
     */
    public static void main( String[] args )
        {
        Global.installConfiguration( args );
        final String webroot = Global.configuration.getLocalWebrootWithSlashes();
        List<String> dirsToSeeSort = Global.configuration.getDirsToSeeSort();
        for ( String dirToSeeSort : dirsToSeeSort )
            {
            final File absDir = new File( webroot, dirToSeeSort );
            readForSeeSort( new File( absDir, "forseesort.ser" ) );
            // collect all this files in dir
            final String[] allFilesInDir = absDir.list( new ExtensionListFilter( "html" ) );
            if ( DEBUGGING )
                {
                err.println( "SeeSorting " + allFilesInDir.length + " files in " + EIO.getCanOrAbsPath( absDir ) + "..." );
                }
            // empty dir is fine.
            for ( String filename : allFilesInDir )
                {
                final File file = new File( absDir, filename );
                try
                    {
                    sortLinksInFile( file );
                    }
                catch ( FileNotFoundException e )
                    {
                    err.println( "\nError: "
                                 + EIO.getCanOrAbsPath( file )
                                 + " not found.\n" );
                    }
                catch ( Exception e )
                    {
                    err.println( "\nError: "
                                 + e.getMessage()
                                 + " in file "
                                 + EIO.getCanOrAbsPath( file )
                                 + "\n" );
                    }
                catch ( Error e )
                    {
                    e.printStackTrace( err );
                    err.println( "\nCatastrophic Error: "
                                 + e.getMessage()
                                 + " in file "
                                 + EIO.getCanOrAbsPath( file )
                                 + "\n" );
                    err.println();
                    }
                } // end for  firs
            } // end for dirs
        err.println( "SeeSort done." );
        }// /method
    // /methods

    /**
     * Sort by flattened keys.
     * <p/>
     * Defines an alternate sort order for String.
     */
    private static class SeeLine implements Comparator<String>
        {
        /**
         * trim lead JUNK off the sort key, e.g. <a href ...><strong>' ideally we would do this once and store in the
         * matrix.
         * Convert to pure lower case.
         *
         * @param key sort key
         *
         * @return sortable version of key
         */
        private static String flatten( String key )
            {
            return flattenHTML( key, ' ' ).trim().toLowerCase();
            } // end flatten

        /**
         * Sort by flattened keys.
         * Defines an alternate sort order for String.
         * Compare two String Objects.
         * Informally, returns (a-b), or +ve if a is more positive than b.
         *
         * @param a first String to compare
         * @param b second String to compare
         *
         * @return +ve if a&gt;b, 0 if a==b, -ve if a&lt;b
         */
        public final int compare( String a, String b )
            {
            // markup, link, spaces and class is not relevant
            // we don't have to remove class. It will go along with <a tag.
            // strip
            a = flatten( a );  // strip all html including <a link , entify
            b = flatten( b );  // flatten produces lower case
            return a.compareTo( b );
            }  // /method
        }  // end class

    /**
     * compare lines with link, description.
     */
    private static class CompareDupSeeLines implements Comparator<String>
        {
        /**
         * Compare lines with link, description.
         * Defines an alternate sort order for String  * Compare two String Objects.
         * Compares whole object case sensitively.
         * Informally, returns (a-b), or +ve if a sorts after b.
         *
         * @param a first String to compare
         * @param b second String to compare
         *
         * @return +ve if a&gt;b, 0 if a==b, -ve if a&lt;b
         */
        public final int compare( @NotNull String a, @NotNull String b )
            {
            // class is not relevant , link and desc are, case is not.
            a = a.replaceAll( "class=\"[\\w]+\"", "" );
            b = b.replaceAll( "class=\"[\\w]+\"", "" );
            a = ST.squish( a );  // no spaces are relevant.
            b = ST.squish( b );
            // strip
            // we don't flatten.  If link is different, but description is the same, we keep.
            return a.compareToIgnoreCase( b );
            }    // /method
        }// end class
    }