/*
 * [BulkFTP.java]
 *
 * Summary: Upload a large set of files with FTP avoiding those already uploaded.
 *
 * Copyright: (c) 2011-2017 Roedy Green, Canadian Mind Products, http://mindprod.com
 *
 * Licence: This software may be copied and used freely for any purpose but military.
 *          http://mindprod.com/contact/nonmil.html
 *
 * Requires: JDK 1.8+
 *
 * Created with: JetBrains IntelliJ IDEA IDE http://www.jetbrains.com/idea/
 *
 * Version History:
 *  1.0 2011-12-03 initial release
 */
package com.mindprod.bulkftp;

import com.enterprisedt.net.ftp.AdvancedFTPSettings;
import com.enterprisedt.net.ftp.FTPConnectMode;
import com.enterprisedt.net.ftp.FTPException;
import com.enterprisedt.net.ftp.FTPFile;
import com.enterprisedt.net.ftp.FTPTransferType;
import com.enterprisedt.net.ftp.FileTransferClient;
import com.enterprisedt.net.ftp.WriteMode;
import com.mindprod.common18.EIO;
import com.mindprod.filter.ExtensionListFilter;
import org.jetbrains.annotations.NotNull;

import java.io.File;
import java.io.FilenameFilter;
import java.io.IOException;
import java.text.ParseException;
import java.util.Arrays;
import java.util.Iterator;

import static java.lang.System.*;

/**
 * Upload a large set of files with FTP avoiding those already uploaded.
 * <p/>
 * requires  edtftpj.jar both to compile and to run. from http://enterprisedt.com/products/edtftpj/
 * needs to be installed in ext directory.
 *
 * @author Roedy Green, Canadian Mind Products
 * @version 1.0 2011-12-03 initial release
 * @since 2011-12-03
 */
public class BulkFTP
    {
    /**
     * when this version was released
     */
    @SuppressWarnings( { "UnusedDeclaration" } )
    static final String RELEASE_DATE = "2011-12-11";

    /**
     * embedded version string
     */
    @SuppressWarnings( { "UnusedDeclaration" } )
    static final String VERSION_STRING = "1.0";

    private static final int FIRST_COPYRIGHT_YEAR = 2011;

    /**
     * how frequently to save, in case we crash
     */
    private static final int HOW_OFTEN_TO_SAVE = 50;

    /**
     * which directory we are processing, webroot relative
     */
    private static final String DIR = "politics";

    /**
     * undisplayed copyright notice
     */
    @SuppressWarnings( { "UnusedDeclaration" } )
    private static final String EMBEDDED_COPYRIGHT =
            "Copyright: (c) 2011-2017 Roedy Green, Canadian Mind Products, http://mindprod.com";

    /**
     * temporary filter
     */
    static FilenameFilter filter = new ExtensionListFilter( "html", "zip", "xml" );

    /**
     * count of files deleted
     */
    private static int countOfFilesDeleted;

    /**
     * count of files uploaded
     */
    private static int countOfFilesUploaded;

    /**
     * connection to server, control and date sockets
     */
    private static FileTransferClient ftc;

    /**
     * connect to the server
     *
     * @throws FTPException if trouble connecting
     * @throws IOException
     */
    private static void connect() throws FTPException, IOException
        {
        ftc = new FileTransferClient();  // caonnot recycle
        final AdvancedFTPSettings advancedFTPSettings = ftc.getAdvancedFTPSettings();
        advancedFTPSettings.setConnectMode( Config.PASV ? FTPConnectMode.PASV : FTPConnectMode.ACTIVE );
        advancedFTPSettings.setStrictReturnCodes( false );
        advancedFTPSettings.setActivePortRange( Config.FTP_PORT_LOW, Config.FTP_PORT_HIGH );
        ftc.setRemoteHost( Config.HOST );
        ftc.setUserName( Config.USER_NAME );
        ftc.setPassword( Config.PASSWORD );
        ftc.setContentType( FTPTransferType.BINARY );  // So lengths on client and server will match exactly
        ftc.connect();
        ftc.changeDirectory( Config.SERVER_ROOT_DIR );
        }

    /**
     * delete files from server that no longer exists on client.
     *
     * @param filesToDelete files to delete from server
     *
     * @throws java.io.IOException                   if trouble with local file
     * @throws com.enterprisedt.net.ftp.FTPException if trouble with server file
     */
    private static void deleteFilesFromServer( @NotNull FD[] filesToDelete ) throws FTPException, IOException
        {
        for ( FD fd : filesToDelete )
            {
            out.println( "Deleting " + fd.getFilenameWithSlashes() );
            ftc.deleteFile( fd.getFilenameWithSlashes() );
            // values of last scan, might be out of date. Could retest.  Will be retested on next run.
            fd.setLastModifiedOnServer( System.currentTimeMillis() );
            fd.setSizeOnClient( -1 );
            fd.setSizeOnServer( -1 );
            countOfFilesDeleted++;
            }
        }

    /**
     * find files that need to be deleted from the server
     *
     * @return array of FDs to delete.
     */
    @NotNull
    private static FD[] findFilesToDeleteFromServer()
        {
        // we have to scan twice, but we avoid ArrayList and memory estimate.
        int countOfFilesToDelete = 0;
        for ( FD fd : FD.values() )
            {
            if ( fd.getSizeOnClient() < 0 && fd.getSizeOnServer() >= 0 )
                {
                countOfFilesToDelete++;
                }
            }
        final FD[] filesToDelete = new FD[ countOfFilesToDelete ];
        int j = 0;
        for ( FD fd : FD.values() )
            {
            if ( fd.getSizeOnClient() < 0 && fd.getSizeOnServer() >= 0 )
                {
                filesToDelete[ j++ ] = fd;
                }
            }
        Arrays.sort( filesToDelete );
        return filesToDelete;
        }

    /**
     * find files with different size or newer date that need to be uploaded
     *
     * @return array of FDs to upload.
     */
    @NotNull
    private static FD[] findFilesToUpload()
        {
        // we have to scan twice, but we avoid ArrayList and memory estimate.
        int countOfFilesToUpload = 0;
        for ( FD fd : FD.values() )
            {
            if ( fd.getSizeOnClient() >= 0
                 && ( fd.getLastModifiedOnClient() > fd.getLastModifiedOnServer()
                      || fd.getSizeOnClient() != fd.getSizeOnServer() ) )
                {
                countOfFilesToUpload++;
                }
            }
        final FD[] filesToUpload = new FD[ countOfFilesToUpload ];
        int j = 0;
        for ( FD fd : FD.values() )
            {
            if ( fd.getSizeOnClient() >= 0
                 && ( fd.getLastModifiedOnClient() > fd.getLastModifiedOnServer()
                      || fd.getSizeOnClient() != fd.getSizeOnServer() ) )
                {
                filesToUpload[ j++ ] = fd;
                }
            }
        Arrays.sort( filesToUpload );
        return filesToUpload;
        }

    /**
     * prune files already deleted from local disk and server
     */
    private static void pruneDeadwood()
        {
        for ( Iterator<FD> iter = FD.iterator(); iter.hasNext(); )
            {
            FD fd = iter.next();
            if ( fd.getSizeOnClient() < 0 && fd.getSizeOnServer() < 0 )
                {
                out.println( "dropping deleted file " + fd.getFilenameWithSlashes() );
                iter.remove();
                }
            }
        }

    /**
     * find out which files are on the server that a9re in dirs of interest and exts of interest
     *
     * @throws java.io.IOException                   if trouble with local file
     * @throws com.enterprisedt.net.ftp.FTPException if trouble with server file
     * @throws java.text.ParseException              if trouble analying list back from server
     */
    private static void scanServerFiles() throws FTPException, IOException, ParseException
        {
        FD.clearAllFlags();
        final FTPFile[] descriptions = ftc.directoryList( DIR );
        for ( FTPFile s : descriptions )
            {
            final String filenameWithSlashes = DIR + "/" + s.getName();
            if ( s.isFile() && filenameWithSlashes.endsWith( ".html" ) )
                {
                FD match = FD.get( filenameWithSlashes );
                if ( match != null )
                    {
                    match.setLastModifiedOnServer( s.lastModified().getTime() );
                    match.setSizeOnServer( s.size() );
                    match.setFlag( true );
                    }
                else
                    {
                    // out.println( "server: " + filenameWithSlashes );
                    final FD fd = new FD( filenameWithSlashes, -1, s.lastModified().getTime(), -1, s.size() );
                    fd.setFlag( true );
                    FD.put( filenameWithSlashes, fd );
                    }
                }
            }
        // anything without a flag no longer exists on server. This may trigger an upload or a delete from server.
        for ( FD fd : FD.values() )
            {
            if ( !fd.isFlag() )
                {
                fd.setSizeOnServer( -1 );
                }
            }
        }

    /**
     * find files with different size or newer date that need to be uploaded
     *
     * @param filesToUpload arry of files to upload
     *
     * @throws java.io.IOException                   if trouble with local file
     * @throws com.enterprisedt.net.ftp.FTPException if trouble with server file
     */
    private static void uploadFiles( @NotNull FD[] filesToUpload ) throws FTPException, IOException
        {
        for ( FD fd : filesToUpload )
            {
            final File localFile = new File( Config.CLIENT_ROOT_DIR, fd.getFilenameWithSlashes() );
            out.println( "uploading " + fd.getFilenameWithSlashes() );
            ftc.uploadFile( EIO.getCanOrAbsPath( localFile ), fd.getFilenameWithSlashes(),
                    WriteMode.OVERWRITE );
            // retest the local file at the last possible moment to ensure accuracy.
            fd.setLastModifiedOnClient( localFile.lastModified() );
            fd.setLastModifiedOnServer( System.currentTimeMillis() );
            final long size = localFile.length();
            fd.setSizeOnClient( size );
            fd.setSizeOnServer( size );
            countOfFilesUploaded++;
            // intermediate saves in case we crash, we will not lone track of what we have done.
            if ( countOfFilesUploaded % HOW_OFTEN_TO_SAVE == 0 )
                {
                FD.save();
                }
            }
        }

    /**
     * mainline
     *
     * @param args -verify if you want to rescan the server to find out exectly what has been uploaded.
     */
    public static void main( @NotNull String[] args )
        {
        Config.analyseCommandLine( args );
        Config.getConfigFromPropertiesFile();
        Config.ensureConfigValid();
        out.println( Config.dumpConfig() );
        try
            {
            final boolean restoreFailed = FD.restore();
            if ( restoreFailed || Config.VERIFY )
                {
                out.println( "verifying..." );
                connect();
                scanServerFiles();
                ftc.disconnect();
                }
            ScanClient.scanClientFiles();
            final FD[] filesToUpload = findFilesToUpload();
            final FD[] filesToDelete = findFilesToDeleteFromServer();
            if ( filesToUpload.length > 0 || filesToDelete.length > 0 )
                {
                connect();
                uploadFiles( filesToUpload );
                deleteFilesFromServer( filesToDelete );
                ftc.disconnect();
                }
            pruneDeadwood();
            FD.save();
            out.println( countOfFilesUploaded + " files uploaded and " + countOfFilesDeleted + " files deleted." );
            }
        catch ( ParseException e )
            {
            err.println( e.getMessage() + " Unable to proceed" );
            e.printStackTrace( err );
            System.exit( 1 );
            }
        catch ( FTPException e )
            {
            err.println( e.getMessage() + " Unable to proceed" );
            e.printStackTrace( err );
            System.exit( 1 );
            }
        catch ( IOException e )
            {
            err.println( e.getMessage() + " Unable to proceed" );
            e.printStackTrace( err );
            System.exit( 1 );
            }
        catch ( Exception e )
            {
            err.println( e.getMessage() + " Fatal Error: inable to proceed" );
            e.printStackTrace( err );
            System.exit( 2 );
            }
        }
    }