/*
 * [ReplicatorSender.java]
 *
 * Summary: Prepares for zips for sending, and arranges to upload them to the website.
 *
 * Copyright: (c) 2003-2017 Roedy Green, Canadian Mind Products, http://mindprod.com
 *
 * Licence: This software may be copied and used freely for any purpose but military.
 *          http://mindprod.com/contact/nonmil.html
 *
 * Requires: JDK 1.8+
 *
 * Created with: JetBrains IntelliJ IDEA IDE http://www.jetbrains.com/idea/
 *
 * Version History:
 *  7.2 2006-02-11 add RETIREMENT property to delay deleting old zips.
 *  7.3 2006-03-05 reformat with IntelliJ and add Javadoc
 *  7.4 2007-05-30
 *  7.5 2007-07-16 IntelliJ inspector.
 *  7.6 2007-07-17 refactor to use use enums, for:each, pass IntelliJ Inspector.
 *  7.7 2007-07-20 compress manifests
 *  7.8 2007-07-23 debugging logs, extra validations.
 *  7.9 2007-07-23 new way to do zip retires to avoid out-of-order bug.
 *  8.0 2007-07-28 fix log so works on virgin machines.
 *  8.1 2007-07-29 Remove Windows centric code.
 *  8.2 2007-07-30 properties file specified on the command line.
 *  8.3 2007-08-04 Use new version of FileTransfer with timeouts, more efficient CPU use, etc.
 *  8.4 2007-08-18 merged clump, move fileLength to FD for verify.
 *                 sender and receiver to make it work properly on Ubuntu. More
 *                 efficient unpacking of zips, bypassing work it has done
 *                 previously.
 *  8.5 2007-09-02 better stats, can hit verify during download, more spacious layout.
 *  8.6 2007-09-13 reduced frequency of zip repacking
 *  8.7 2007-09-18 Fix zip order when deletions
 *  8.8 2007-09-20 flip to JDK 1.6
 *  8.9 2007-09-22 now handles files that change length without changing date.
 *  9.0 2007-09-23 retirement time now specified in minutes rather than hours.
 *  9.1 2007-10-28 fix bug to ensure gui state persisted.
 *  9.2 2007-12-24 display range of times for retired.
 *  9.3 2008-01-07 emaciate and delete files in the same replicatorSender run.
 *  9.4 2008-01-28 calculate percentage of deadwood more conservatively
 *  9.5 2008-02-08 better error message if can't create dirs, tool tip help.
 *  9.6 2008-02-22 display computed lag and retirement timestamps
 *  9.7 2008-03-20 only unpack new files, not the entire zip.
 *  9.8 2008-03-28 fine tune generated JNLP
 *  9.9 2008-04-09 compact zips under tighter rules immediately after LAG days.
 * 10.0 2008-08-10 extend timeouts to be more patient with network congestion.
 * 10.1 2009-01-23 fix harmless bug in saving the internal state
 * 10.2 2009-04-03 tidy up code to check presence of necessary files to make it more WORA.
 * 10.3 2009-05-03 localise log output file with Localise and PrintWriterPortable
 *                 use import static on ConfigForSender to make code terser
 *                 qualifyInSenderPersist to encapsulate the File.separator logic.
 * 10.4 2009-08-08 add RECOVER, HIGHEST_ACCEPTABLE_EXIT_CODE, MAX_UPLOAD_TRIES config parms.
 * 10.5 2011-04-22 fully automate StartOver.bat, so no additional copies are needed.
 * 10.6 2011-07-30 add support for Java 1.7
 * 10.7 2011-08-05 add packing statistics
 * 10.8 2011-09-08 sort packing statistics, avoiding voids.
 * 10.9 2011-09-14 base stats on clumping, and stats on newest repackaged.
 * 11.0 2011-12-22 update the JNLP.
 * 11.1 2012-03-15 serialise arrays instead of ArrayLists. Correct docs on how LAG works.
 * 11.2 2012-05-12 more consistent use of empty timestamp markers. Should make more compact uploads.
 * 11.3 2012-05-13 further more consistent use of empty markers.
 * 11.4 2014-04-06 bypass Oracle dropping support for JNLP properties.
 * 11.5 2014-08-19 modify SortedArrayList to account for JDK 1.8.0_20s fobbing Collections.sort on ArrayList.sort.
 * 11.6 2015-03-10 sign the jar.a
 */
package com.mindprod.replicatorsender;

import com.mindprod.common18.Build;
import com.mindprod.fastcat.FastCat;
import com.mindprod.replicatorcommon.Config;
import com.mindprod.replicatorcommon.IO;
import com.mindprod.replicatorcommon.ReplicatorCommon;
import com.mindprod.sorted.SortedArrayList;
import com.mindprod.untouch.Untouch;

import java.awt.Toolkit;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;

import static com.mindprod.replicatorsender.ConfigForSender.*;
import static java.lang.System.*;

/**
 * Prepares for zips for sending, and arranges to upload them to the website.
 * <p/>
 * Creates record of what we have previously distributed and deleted in E:\com\mindprod\replicatorsender\sender.ser.
 * We don't give this to the clients.
 * <p/>
 * It works by maintaining the newest file in each zip in replicator\zipmanifest.ser which we do distribute to the
 * clients.
 * <p/>
 * When we distribute, we create an in-ram list of all files we plan to distribute, remove those have already
 * distributed and are already up to date. We prepare a parallel list of files to be deleted. Then we break the entries
 * up into clumps assigning them to zip file we are about to create.
 * <p/>
 * To retire old zips, we simply mark its files as undistributed. Then they get added to newly created zips. They are
 * added from the disk original, not from the old zip.
 *
 * @author Roedy Green, Canadian Mind Products
 * @version 11.6 2015-03-10 sign the jar.
 * @since 2003-09-15
 */
public final class ReplicatorSender
    {
    private static final int FIRST_COPYRIGHT_YEAR = 2003;

    /**
     * undisplayed copyright notice
     */
    @SuppressWarnings( { "UnusedDeclaration" } )
    private static final String EMBEDDED_COPYRIGHT =
            "Copyright: (c) 2003-2017 Roedy Green, Canadian Mind Products, http://mindprod.com";

    /**
     * when version released.
     */
    @SuppressWarnings( { "UnusedDeclaration" } )
    private static final String RELEASE_DATE = ReplicatorCommon.RELEASE_DATE;

    /**
     * name of this application.
     */
    private static final String TITLE_STRING = "T h e   R e p l i c a t o r   S e n d e r";

    /**
     * embedded version string.
     */
    @SuppressWarnings( { "UnusedDeclaration" } )
    private static final String VERSION_STRING = ReplicatorCommon.VERSION_STRING;

    /**
     * all zip descriptors, including unused, emaciated, but not deleted zips, kept in order by zip number. We never
     * sort this an any other other.
     */
    static ArrayList<MaxiZD> allZips;

    /**
     * Combined list of filesToDistribute and filesToDelete. Not persisted to sender.ser. Persisted in filemanifest.ser
     * sent to receiver for verify. Recreated from filesToDelete and filesToDistribute.
     */
    static SortedArrayList<MaxiFD> combinedFilesAndDeletions;

    /**
     * Files with timestamps prior to this are too late to fit in their natural zips. Timestamp just after last file we
     * distributed previously.
     */
    static long cutOffTimestamp;

    /**
     * Estimates of how many files fall in the category of trees wanted, unwanted, dirs wanted, unwanted and files
     * wanted, unwanted. Enables us to make more accurate estimates on the next run. 0=wanted tree 1=unwanted tree
     * 2=wanted dirs 3=unwanted dirs 4=wanted files 5=unwanted files
     */
    static int[] estimates = new int[ 6 ];

    /**
     * file descriptors of all files marked deleted in the current active distribution, Including ones in zips
     * previously distributed. kept in order by filename.
     */
    static SortedArrayList<MaxiFD> filesToDelete;

    /**
     * file descriptors of all files in the current active distribution, Including ones in zips previously distributed.
     * kept in order by clumping. All these SortedArrayLists reference some subset of the same objects, so the RAM needs
     * are not as bad as they look.
     */
    static SortedArrayList<MaxiFD> filesToDistribute;

    /**
     * timestamp of last file to be repacked, prior to new files and newly changed files,
     * Long.MIN_VALUE if no repackings.
     * repacked deletions count as files.
     */
    static long newestRepackagedTimestamp;

    /**
     * Next unused zip number. starts at 1. The first file is z1.zip
     */
    static int nextZipNumber;

    /**
     * file descriptors of all the files were marked deleted ina nay zip on the previous run. Kept in order by filename.
     */
    static SortedArrayList<MaxiFD> oldFilesDeleted;

    /**
     * file descriptors of all the files we distributed the previous run. kept in order by clumping.
     */
    static SortedArrayList<MaxiFD> oldFilesDistributed;

    /*
     * Even though we maintain counts for many other collections of files, there is no corresponding SortedArrayList.
     * The counts in StatsForSender are generated by arithmetic on the sizes of the basic sets.
     */
    static
        {
        final FastCat sb = new FastCat( 4 );
        sb.append( "Programming error: Mismatched versions: " );
        sb.append( Config.VERSION );
        sb.append( " vs " );
        sb.append( VERSION_STRING );
        assert Config.VERSION.equals( VERSION_STRING ) : sb.toString();
        }

    /**
     * Calculate cutoff timestamp. Files with timestamps prior to this are too late to fit in their natural zips.
     * Such files
     * appear if they are old files copied into the distribution, or back dated.
     *
     * @return the cutoff timestamp.
     */
    static long calcCutOffTimestamp()
        {
        long cutoff = Config.CUTOFF_TIMESTAMP;
        final int size = allZips.size();
        if ( size != 0 )
            {
            final MaxiZD lastZip = allZips.get( size - 1 );
            // one second since last file of previous distribution.
            cutoff = lastZip.getHighestClumpingInZip() + 1000;
            }
        return cutoff;
        }

    /**
     * Get estimate of number of files in a given category so we can use better ArrayList initialisation size.
     *
     * @param index 0=trees wanted 1=trees withheld 2=dirs wanted 3=dirs withheld 4=files wanted 5=files withheld
     *
     * @return estimate of how many files will be in that category. Includes a 10% growth factor.
     */
    static int getEstimate( int index )
        {
        return estimates[ index ];
        }

    /**
     * Set estimate of number of files in a given category so we can use better ArrayList initialisations next
     * elapsedTime.
     *
     * @param index         0=trees wanted 1=trees withheld 2=dirs wanted 3=dirs withheld 4=files wanted 5=files
     *                      withheld
     * @param numberOfFiles Number of files that fell in that category this elapsedTime.
     */
    static void setEstimate( int index, int numberOfFiles )
        {
        // bump estimate up for next elapsedTime by 10% of this elapsedTime's actual.
        numberOfFiles = numberOfFiles * 110 / 100;
        if ( numberOfFiles < 16 )
            {
            numberOfFiles = 16;
            }
        estimates[ index ] = numberOfFiles;
        }

    /**
     * Warning on send side
     *
     * @param warnMessage what went wrong
     */
    static void warn( String warnMessage )
        {
        err.println( "W A R N I N G" );
        Toolkit.getDefaultToolkit().beep();
        err.println( warnMessage );
        }

    /**
     * Main method to create archive files.
     *
     * @param args arg[0] is fully qualified absolute or relative name of properties file e.g.
     *             xxxreplicator.properties.
     */
    public static void main( String[] args )
        {
        try
            {
            out.println();
            out.println( TITLE_STRING
                         + "  "
                         + VERSION_STRING
                         + " build "
                         + Build
                                 .BUILD_NUMBER
            );
            // get config info from sender's xxxreplicator.properties file
            getConfigFromPropertiesFile( args[ 0 ] );
            // display configuration.
            out.println( dumpConfig() );
            // roll call sanity check all crucial files present.
            DistributeAuxAndJNLP.auxRollCall();
            // make sure directories we need exist
            IO.ensureDirectoryExists( SENDER_BASE_DIR );
            IO.ensureDirectoryExists( SENDER_ZIP_STAGING_DIR );
            if ( UNTOUCH )
                {
                out.println( "untouching " + ConfigForSender
                        .SENDER_BASE_DIR + " ..." );
                new Untouch( true
                        /* quiet */, false
                        /* wantLogs */ ).untouchAllFilesInTree( new File(
                        SENDER_BASE_DIR ) );
                }
            // prepare files for distribution other than zip manifest and zips
            DistributeAuxAndJNLP.distribute();
            // restore previous system state
            PersistSenderState.restoreStateFromSenderSer();
            // gather list of files to distribute, all files existing now
            Gather.process();
            // find files deleted since last distribution. Do deadwood accounting.
            Deletions.process();
            // we don't need oldFilesDeleted any more since complete list of files
            // already deleted plus those to delete are now
            // safely in filesToDelete
            ReplicatorSender.oldFilesDeleted = null;
            // detect changes since last distribution. Compute list of files to distribute both old and new mixed.
            Changes.process();
            // we don't need oldFilesDistributed list any more since complete list of files
            // already distributed plus those to distribute are now
            // safely in filesToDistribute
            ReplicatorSender.oldFilesDistributed = null;
            // retire zips with excess deadwood.
            Retire.process();
            // clump files, repack files previously distributed in retired zips, and pack new files and newly changed
            // files  into zips
            Clump.process();
            // now is a good Time for gc when we have pruned down to the minimum.
            System.gc();
            // prepare zips, this is where we compose the nnnn.zip files we
            // distribute
            ProcessZips.processZipsPass1();
            // Computer stats for display.
            StatsForSender.computeZipsStats();
            // display stats on how many files processed
            out.println( StatsForSender.dump() );
            // save state so we can pick up where we left off
            PersistSenderState.saveStateToSenderSer();
            // also in human-readable form to figure out problems.
            PersistSenderState.saveStateToSenderLog();
            // upload files have prepared so far to the website, will spawn netload
            // to do it.  Upload zips for pass 1
            Upload.uploadZips();
            // no need to save state. Upload does not modify it.
            // delete emaciated files.
            ProcessZips.processZipsPass2();
            // prepare a manifest of just the active zips and files for the client.
            WriteManifests.prepareZipManifestSer();
            // save list of files for client to verify.
            WriteManifests.prepareFileManifestSer();
            // save modified state so we can pick up where we left off
            PersistSenderState.saveStateToSenderSer();
            // also in human-readable form to figure out problems.
            PersistSenderState.saveStateToSenderLog();
            // upload the zip and file manifests separately after zips safely uploaded.
            // zipmanifest.ser will naturally upload after the z999.zip files, but
            // having two passes makes sure it will.
            // Also does to final delete stage for emaciated zips
            Upload.uploadManifests();
            // no need to resave state. Upload does not modify it.
            out.println( "Replicator sender done" );
            out.println();
            }
        catch ( IOException e )
            {
            e.printStackTrace( err );
            err.println();
            }
        }
    }