/*
 * [CurrConAux.java]
 *
 * Summary: Prepare serialised binary file of today's exchange rates.
 *
 * Copyright: (c) 2002-2017 Roedy Green, Canadian Mind Products, http://mindprod.com
 *
 * Licence: This software may be copied and used freely for any purpose but military.
 *          http://mindprod.com/contact/nonmil.html
 *
 * Requires: JDK 1.8+
 *
 * Created with: JetBrains IntelliJ IDEA IDE http://www.jetbrains.com/idea/
 *
 * Version History:
 *  2.7 2006-09-13 new names in boccodes to match Bank of Canada
 *  2.8 2008-01-29 get version number is sync with currCon
 *  2.9 2008-01-29
 *  3.0 2008-01-31 locks and other means to deal with interApplet fibrillation.
 *  3.1 2008-01-31 volatile and yield to help smooth interApplet fibrillation.
 *  3.2 2008-01-31 avoid shared variable to smooth interApplet fibrillation.
 *  3.3 2008-02-01 use threads and invokeLater to avoid freezes on Linux
 *  3.4 2008-02-15 lower price.
 *  3.5 2009-01-10 improved error messages
 *  3.6 2010-02-04 new colours to match mindprod style sheet. wider currency code selector to hold KRW
 *  3.7 2010-05-31 add P code that acts like A, but gives precise value.
 *  3.8 2010-06-02 ensure P code prices update when currency changed elsewhere on the page.
 *  3.9 2010-12-04 compress resource. use pure arrays to avoid generics trouble.
 *  4.0 2011-01-21 Boc Changed format of file, now has Currency abbreviations.
 *  4.1 2011-01-26 switch to iso currency codes, allow multi-char currency symbols. Allow accents in currency names.
 *  4.2 2011-02-19 now supports Windows/IE variable resolution.
 *  4.3 2011-05-07 adjust to new Boc Format, URLs and filenames.
 *  4.4 2011-05-18 adjust to new format of Boc exchange rates. Simplify by eliminating currconaux directory.
 *  4.5 2011-12-29 adjust to new format from Bank of Canada.
 */
package com.mindprod.currcon;

import com.mindprod.csv.CSVReader;
import com.mindprod.entities.DeEntifyStrings;

import java.io.BufferedReader;
import java.io.EOFException;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.ObjectOutputStream;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Locale;
import java.util.zip.GZIPOutputStream;

import static java.lang.System.*;

/**
 * Prepare serialised binary file of today's exchange rates.
 * <p/>
 * Create a serialised list of exchange rates merging information from several sources:
 * <p/>
 * <ol><li>http://www.bankofcanada.ca/stats/assets/csv/fx-seven-day.csv: Bankof Canada daily exch rates from website.
 * Names of currencies change often, but we use more stable codes. BoC makes them
 * up ad hoc.</li>
 * <p/>
 * <li>boccodes.csv : used to help interpret BoC's currency codes. The three letter codes used here come from ISO</li>
 * <p/>
 * <li>countrytocur.csv : country to currency lookup. The country codes and currency codes are ISO standard. We
 * ignore the country and currency names..</li>
 * <p/>
 * <li>currencydetails.csv : details about each currency. The currency codes came
 * from Oanda.</li></ol>
 * <p/>
 * exchs.ser: all this information is combined and squeezed into a single file in Serialised
 * Object FormatPadSites. It eventually in packaged in the jar along with the Applet which compresses it further.
 * <p/>
 * You may prune the countrytocur.csv and currencydetails.csv of unwanted currencies. If you leave in extra currencies
 * for which there are no daily values, no harm is done. They will simply be left out.
 * <p/>
 * The program will automatically prune fx-seven-day.csv to match..
 *
 * @author Roedy Green, Canadian Mind Products
 * @version 4.5 2011-12-29 adjust to new format from Bank of Canada.
 * @since 2002-03-27
 */
@SuppressWarnings( { "InfiniteLoopStatement" } )
public final class CurrConAux
    {
    private static final int FIRST_COPYRIGHT_YEAR = 2002;

    /**
     * undisplayed copyright notice
     */
    @SuppressWarnings( { "UnusedDeclaration" } )
    private static final String EMBEDDED_COPYRIGHT =
            "Copyright: (c) 2002-2017 Roedy Green, Canadian Mind Products, http://mindprod.com";

    @SuppressWarnings( { "UnusedDeclaration" } )
    private static final String RELEASE_DATE = "2011-12-29";

    /**
     * embedded version string.
     */
    private static final String VERSION_STRING = "4.5";

    /**
     * Lets us look up 3-letter currency code given 2-letter country code.
     */
    private static HashMap<String, String> countryToCurr;

    /**
     * list of the daily exchange rates for various currencies. shared by all Applets
     */
    private static Exch[] exchs;
    /**
     * <pre>
     *    futures
     *    SELLING IDEAS
     *        - free version has moose logo
     *        - expires based on elapsedTime of exchange rates
     *        - pay version tied to a single website
     *        - keep renaming free version, so have to redo website.
     *        - pay for update service.
     *        - free version has max of $1000.00
     *        - limit number of instances per page.
     *        - use JWS to distribute daily updates,
     *        - JWS composes jar file by putting in customisation.
     * <p/>
     *        IMPROVEMENT IDEAS
     *        - alternate shorter list of currencies
     *        - stomper to calculate Applet width etc.
     *        - automate getting exchange rates
     *        - automate getting fresh version from me.
     *        - buy/sell exchange rates relative to base
     *        - export to Javascript.
     *        - load from my website or some central host rather than having
     * copy on
     *        cust website.
     * </pre>
     */
    /**
     * display list of currencies we do track, which may be a subset of the entire BoC set.
     */
    private static void displayTracked()
        {
        out.println( "Currencies tracked:" );
        for ( Exch exch : exchs )
            {
            out.println( exch.getCurrAbbr() + " " + exch.getExchangeRate() );
            }
        }

    /**
     * drop any countries for which we had no data today
     */
    private static void dropNulls()
        {
        ArrayList<Exch> a = new ArrayList<>( Arrays.asList( exchs ) );
        int size = a.size();
        for ( int i = size - 1; i >= 0; i-- )
            {
            Exch ex = a.get( i );
            if ( ex.getExchangeRate() == 0 )
                {
                a.remove( i );
                }
            }
        exchs = a.toArray( new Exch[ a.size() ] );
        }

    /**
     * Reads list of countries to track from a CSV-formatted stream.
     *
     * @param track Where to find the descriptions of the countries. country code, currency code, county name,
     *
     * @throws java.io.IOException if cannot read countries file
     */
    private static void getCountriesToTrack( BufferedReader track ) throws IOException
        {
        countryToCurr = new HashMap<>( 257/* prime */ );
        CSVReader t = new CSVReader( track );
        try
            {
            while ( true )
                {
                String countryCode = t.get().toUpperCase().intern();
                t.skip( 1 ); // skip country name
                String currCode = t.get().toUpperCase().intern();
                /* countryName */
                t.skip( 1 );// skip currency name.
                // instead.
                countryToCurr.put( countryCode, currCode );
                t.skipToNextLine();
                } // end while
            }
        catch ( EOFException e )
            {
            t.close();
            }
        // Verify list is complete.
        boolean die = false;
        String[] countries = Locale.getISOCountries();
        for ( String country : countries )
            {
            if ( countryToCurr.get( country ) == null )
                {
                err.println( "Fatal Error: currency for country "
                             + country
                             + " is undefined." );
                die = true;
                }
            }
        if ( die )
            {
            System.exit( 2 );
            }
        } // end getCountriesToTrack

    /**
     * Reads list of countries to track from a CSV-formatted stream.
     *
     * @param track Where to find the descriptions of the currencies. currency code, decimal places, symbol, currency
     *              name.
     *
     * @throws java.io.IOException if cant read local file
     */
    private static void getCurrenciesToTrack( BufferedReader track ) throws IOException
        {
        // this code does not go on the website, so ArrayList is ok.
        final ArrayList<Exch> a = new ArrayList<>( 100 );
        final CSVReader t = new CSVReader( track );
        try
            {
            while ( true )
                {
                final String currCode = t.get().toUpperCase().intern();
                final int decPlaces = Integer.parseInt( t.get() );
                String symbol = DeEntifyStrings.deEntifyHTML( t.get(), ' ' ).intern();
                if ( !( symbol.endsWith( "$" )
                        || symbol.equals( "\u00a3" /* pound */ )
                        || symbol.equals( "\u00a4" /* curren */ )
                        || symbol.equals( "\u00a5" /* yen */ ) ) )
                    {
                    // append a thin space after currency symbols except $ and symbols that have enough space built-in..
                    symbol += '\u2009';
                    }
                final String currName = DeEntifyStrings.deEntifyHTML( t.get(), ' ' ).intern();
                a.add( new Exch( currCode,
                        currName,
                        symbol,
                        decPlaces,
                        0.00 ) );
                t.skipToNextLine();
                } // end while
            }
        catch (
                EOFException e
                )
            {
            t.close();
            }
        // clean up to pure array exactly the right size, also hide ArrayList
        // from
        // the Applet JDK 1.1..
        exchs = a.toArray( new Exch[ a.size() ] );
        } // end getCurrenciesToTrack

    /**
     * is this rate available
     *
     * @param rate as a fraction string
     *
     * @return true if rate is available
     */
    private static boolean isAvailable( String rate )
        {
        return !( rate == null
                  || rate.length() == 0
                  || rate.equalsIgnoreCase( "Bank holiday" )
                  || rate.equalsIgnoreCase( "NA" )
                  || rate.equalsIgnoreCase( "N/A" )
                  || rate.equalsIgnoreCase( "Not available" ) );
        }

    /**
     * Refresh exchange rates from http://www.bankofcanada.ca/stats/assets/csv/fx-seven-day.csv in CSV format.
     * presumes exchs[] has the currencies of interest to us in it.
     *
     * @param rates CSV-formatted stream, captured from the bank of Canada website. field are: currency name currency
     *              code exchange rate (value in US$ usually < 1 ) inverse exchange rate
     *
     * @throws java.io.IOException if cant read CSV file
     */
    private static void refreshExchangeRatesFromBoC( BufferedReader rates ) throws IOException
        {
        // update USD exchange rate to 1.0 by definition.
        // could easily be missing from published list since too obvious to mention.
        // It won't be in a list
        for ( Exch exch : exchs )
            {
            if ( exch.getCurrAbbr().equals( "USD" ) )
                {
                /* set value of USD in US$ */
                exch.setExchangeRate( 1 );
                break;
                }
            }
        final CSVReader r = new CSVReader( rates );
        // value of canadian dollar in US $.
        double cadValue = 0;
        try
            {
            // search till find USA
            while ( true )
                {
                // BoC rates are relative to CAD, we convert to relative to USD
                // opening lines look like this:
                //   # The daily noon exchange rates for major foreign currencies are published every business day at
                // about 12:30
                //   # p.m. EST. They are obtained from market or official sources around noon and show the rates for
                // the
                //   # various currencies in Canadian dollars converted from US dollars. The rates are nominal
                // quotations -
                //   # neither buying nor selling rates - and are intended for statistical or analytical purposes. Rates
                //   # available from financial institutions will differ.
                //   #
                //   Date (<y>-<m>-<d>), ISO4217, 2011-12-28, 2011-12-29, 2011-12-30, 2012-01-02, 2012-01-03,
                // 2012-01-04, 2012-01-05
                //   U.S. dollar , USD, 1.0234, 1.0210, 1.0170, Bank holiday, 1.0090, 1.0135, 1.0197
                //   U.S. dollar (close), USD, 1.0242, 1.0208, 1.0170, Bank holiday, 1.0110, 1.0123, 1.0191
                //   US/Canada noon 3-month forward points spread, IEXE0124, 0.20, 0.20, 0.20, Bank holiday, 0.21,
                // 0.21, 0.21
                //   US/Canada noon 6-month forward points spread , IEXE0125, 0.36, 0.37, 0.37, Bank holiday, 0.38,
                // 0.39, 0.39
                //   Argentine peso, ARS, 0.2161, 0.2163, 0.2148, Bank holiday, 0.2133, 0.2118, 0.2127
                //   Australian dollar, AUD, 1.0327, 1.0347, 1.0424, Bank holiday, 1.0472, 1.0487, 1.0455
                // Then by stripping out all but cols 1 and 8 with CSVReshape it is converted to:
                //# The daily noon exchange rates for major foreign currencies are published every business day at
                // about 12:30
                //# p.m. EST. They are obtained from market or official sources around noon and show the rates for the
                //# various currencies in Canadian dollars converted from US dollars. The rates are nominal quotations -
                //# neither buying nor selling rates - and are intended for statistical or analytical purposes. Rates
                //# available from financial institutions will differ.
                //#
                // ISO4217,2012-01-05
                // USD,1.0197
                // USD,1.0191
                // IEXE0124,0.21
                // IEXE0125,0.39
                // ARS,0.2127
                // AUD,1.0455
                // Then using CleanBoC, we clean it to:
                // USD, 1.0197
                // ARS, 0.2127
                // AUD, 1.0455
                if ( r.get().equalsIgnoreCase( "USD" ) )
                    {
                    // Value read will be about 0.9945.
                    // cadValue is value of CAD in USD, about 1.02
                    final String rate = r.get();
                    if ( isAvailable( rate ) )
                        {
                        double rateFrac = Double.parseDouble( rate );
                        if ( rateFrac != 0 )
                            {
                            cadValue = 1 / rateFrac;
                            r.skipToNextLine();
                            r.skipToNextLine(); // skip over second USD (close)
                            break; // out of the while loop
                            }
                        }
                    }
                r.skipToNextLine();
                }
            }
        catch ( EOFException e )
            {
            // should not happen
            }
        if ( cadValue == 0 )
            {
            err.println( "Fatal Error: missing USD<->CAD exchange rate in file from Bank of Canada." );
            System.exit( 1 );
            }
        // update CAD exchange rate
        for ( Exch exch1 : exchs )
            {
            if ( exch1.getCurrAbbr().equals( "CAD" ) )
                {
                /* set value of CAD in US$ */
                exch1.setExchangeRate( cadValue );
                break;
                }
            }
        try
            {
            // search till process each country.  USA and Canada already done.
            while ( true )
                {
                // start reading after USD line.
                // each line of the CAD-based rates like this:
                // ARS, 0.2392, 8
                final String currCode = r.get().toUpperCase();
                final String rate = r.get();
                if ( !isAvailable( rate ) )
                    {
                    r.skipToNextLine();
                    err.println( "Warning: exchange rate for: " + currCode + " is not available today from the Bank " +
                                 "of Canada." );
                    // treat as though the line were missing entirely
                    continue;
                    }
                // adjust exchange rates to USD from Boc CAD numbers.
                // cadValue will usually be about 10.2
                final double exchangeRate = Double.parseDouble( rate ) * cadValue;
                r.skipToNextLine();
                // search list to see if there is any interest in this exchange
                // rate.
                boolean used = false;
                for ( Exch exch : exchs )
                    {
                    if ( exch.getCurrAbbr().equals( currCode ) )
                        {
                        if ( used )
                            {
                            err.println( "Warning: duplicate " + currCode + " in the Bank of Canada file." );
                            }
                        /* got a match. update exchange rate */
                        exch.setExchangeRate( exchangeRate );
                        used = true;
                        /* no break, keep going in case there are duplicates */
                        }
                    }
                if ( !used )
                    {
                    err.println( "Warning: currency ignored " + currCode + " in the Bank of Canada file." );
                    }
                } // end while
            }
        catch (
                EOFException e
                )
            {
            r.close();
            }
        } // end refreshExchangeRates

    /**
     * Refresh exchange rates from http://www.oanda.com/convert/fxdaily in CSV format. presumes exchs[] has the
     * currencies of interest to us in it.
     *
     * @param rates CSV-formatted stream, captured from the oanda website. field are: currency name currency code
     *              exchange rate (value in US$ usually < 1 ) inverse exchange rate
     *
     * @throws java.io.IOException if cannot read local oanda file
     * @deprecated Oanda will not let me use their data any more.
     */
    private static void refreshExchangeRatesFromOanda( BufferedReader rates ) throws IOException
        {
        final CSVReader r = new CSVReader( rates );
        try
            {
            while ( true )
                {
                // each line of the rates like this:
                // Canadian Dollar,CAD,0.628,1.5926
                // only the second and third fields are useful.
                r.skip( 1 );
                final String currCode = r.get();
                final double exchangeRate = Double.parseDouble( r.get() );
                r.skipToNextLine();
                // search list to see if there is any interest in this exchange
                // rate.
                boolean used = false;
                for ( Exch exch : exchs )
                    {
                    if ( exch.getCurrAbbr().equals( currCode ) )
                        {
                        if ( used )
                            {
                            err.println( "Warning: duplicate " + currCode );
                            }
                        /* got a match. update exchange rate */
                        exch.setExchangeRate( exchangeRate );
                        used = true;
                        /* no break, keep going in case there are duplicates */
                        }
                    }
                if ( !used )
                    {
                    err.println( "Warning: currency ignored " + currCode );
                    }
                } // end while
            }
        catch ( EOFException e )
            {
            r.close();
            }
        }

    /**
     * Refresh exchange rates from manual manual.csv in CSV format. presumes exchs[] has the currencies of interest to
     * us in it.
     *
     * @param rates CSV-formatted stream, captured from the oanda website. field are: currency code exchange rate (value
     *              in US$ usually < 1 )
     *
     * @throws java.io.IOException if cant read manual.csv
     */
    private static void refreshExchangeRatesManually( BufferedReader rates ) throws IOException
        {
        final CSVReader r = new CSVReader( rates );
        try
            {
            while ( true )
                {
                // each line of the rates like this:
                // CAD,0.628
                final String currCode = r.get();
                final double exchangeRate = Double.parseDouble( r.get() );
                r.skipToNextLine();
                // search list to see if there is any interest in this exchange
                // rate.
                boolean used = false;
                for ( Exch exch : exchs )
                    {
                    if ( exch.getCurrAbbr().equals( currCode ) )
                        {
                        if ( used )
                            {
                            err.println( "Warning: duplicate " + currCode );
                            }
                        /* got a match. update exchange rate */
                        exch.setExchangeRate( exchangeRate );
                        used = true;
                        /* no break, keep going in case there are duplicates */
                        }
                    }
                if ( !used )
                    {
                    err.println( "Warning: currency ignored " + currCode );
                    }
                } // end while
            }
        catch ( EOFException e )
            {
            r.close();
            }
        } // end refreshExchangeRatesManually

    /**
     * Save the list of countries and updated exchange rates in serialised form.
     *
     * @param fos Stream where the serialised output is to go.
     *
     * @throws java.io.IOException if cannot write resource
     */
    private static void save( OutputStream fos ) throws IOException
        {
        // take HashMap apart for serializing
        final int countryCount = countryToCurr.size();
        final String[] countries = countryToCurr.keySet().toArray( new String[ countryCount ] );
        final String[] currencies = countryToCurr.values().toArray( new String[ countryCount ] );
        // O P E N
        final GZIPOutputStream gzos = new GZIPOutputStream( fos, 4 * 1024 /* buffsize in bytes */ );
        final ObjectOutputStream oos = new ObjectOutputStream( gzos );
        // W R I T E
        oos.writeObject( Exch.getSerialVersionUID() );
        oos.writeObject( countries );  // for country to currency lookup
        oos.writeObject( currencies );
        oos.writeObject( exchs ); // currency details and exchange rate for supported currencies
        // C L O S E
        oos.close();
        }

    /**
     * Create an exchs.ser for cmp, ready to include in the jar file.
     *
     * @param args boc manual oanda (source of exchange rates).
     */
    public static void main( String[] args )
        {
        try
            {
            getCountriesToTrack( new BufferedReader( new FileReader( "countrytocur.csv" ) ) );
            getCurrenciesToTrack( new BufferedReader( new FileReader( "currencydetails.csv" ) ) );
            char source = 'b';
            if ( args.length > 0 )
                {
                source = args[ 0 ].toLowerCase().charAt( 0 );
                }
            switch ( source )
                {
                case 'o':/* oanda */
                    // not used any more.
                    /*   refreshExchangeRatesFromOanda( new BufferedReader( new FileReader( "oanda.csv" ) ) ); */
                    break;
                case 'm':/* manual */
                    // not normally used.
                    refreshExchangeRatesManually( new BufferedReader( new FileReader( "manual.csv" ) ) );
                    break;
                default:
                case 'b':/* bank of Canada */
                    out.println( "analysing Bank of Canada exchange rates..." );
                    refreshExchangeRatesFromBoC( new BufferedReader( new FileReader( "boc-clean.csv" ) ) );
                    break;
                }
            out.println( "analysing daily exchanges rates to be tracked..." );
            dropNulls();
            displayTracked();
            save( new FileOutputStream( "exchs.ser" ) );
            }
        catch ( IOException e )
            {
            err.println();
            e.printStackTrace( err );
            err.println();
            }
        }
    }