/*
 * [Stock.java]
 *
 * Summary: Track which products are in stock in which store.
 *
 * Copyright: (c) 2014-2017 Roedy Green, Canadian Mind Products, http://mindprod.com
 *
 * Licence: This software may be copied and used freely for any purpose but military.
 *          http://mindprod.com/contact/nonmil.html
 *
 * Requires: JDK 1.8+
 *
 * Created with: JetBrains IntelliJ IDEA IDE http://www.jetbrains.com/idea/
 *
 * Version History:
 *  1.0 2014-07-19 initial version
 *  1.1 2016-06-21 add countdownToSave intermediate saves.
 */
package com.mindprod.stores;

import com.mindprod.common18.EIO;
import com.mindprod.common18.Misc;
import com.mindprod.common18.ST;
import com.mindprod.htmlmacros.macro.Global;

import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.File;
import java.io.IOException;
import java.text.DecimalFormat;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;

import static com.mindprod.stores.StockStatus.INSTOCK;
import static com.mindprod.stores.StockStatus.NOTCARRIED;
import static com.mindprod.stores.StockStatus.OUTOFSTOCK;
import static java.lang.System.*;

/**
 * Track which products are in stock in which store.
 *
 * @author Roedy Green, Canadian Mind Products
 * @version 1.1 2016-06-21 add countdownToSave intermediate saves.
 * @since 2014-07-19
 */
public class Stock
    {
    // On Stock.fireup, all the Stocks are allocated, and the inventories loaded from disk.
    // On BulkProbe.fireup all the stocks are probed (depending on set parms).
    // In Book, we look up the status for the ISBN and build a link to it for each store,
    // If the book in not in the stock file, we add it to the stock, and as a side effect of adding it, we probe with Stock.getProductStatus
    // Every time we do a lookup in a stock, that product is marked as referenced.
    // We can then get rid of deadwood by removing anything unreferenced after a full probe.
    // The weakness of the Stock system is cannot add or delete a store without deleting the corresponding stock file and
    // rebuilding it with a long slow probe.
    // There in no "new" bit maintained in the stock file.  A product is no longer considered new after it has
    // been encountered, but has not been probed yet because checknewproducts=no. Unprobed new products turn into
    // not carried, until a manual probe is done.
    // Remove deadwood by deleting stock file, It will regenerate, albeit slowly.    // or by set deadwood=yes.

    /**
     * 0=no=leave deadwood 1=yes=remove deadwood 2=find=find deadwood;
     */
    public static final int DEADWOOD;

    /**
     * true  if should force requested probes, even if they have been done in the last 4 days
     */
    private static final boolean FORCE_PROBES = Misc.parseBoolean( System.getenv( "forceprobes" ), false );

    /**
     * avoid reprobing a product we have already reprobed in the last 1 days
     */
    private static final int AVOID_REPROBE_DAYS = 1;

    /**
     * true if want extra output
     */
    private static final boolean DEBUGGING = true;

    /**
     * how much extra space to give HashMap above size needed to hold eltsmarkAsUnreferenced
     */
    private static final int BREATHING_ROOM_PERCENT = 50;

    /**
     * initial capacity for lookupBundleByProduct HashMap
     */
    private static final int INITIAL_BUNDLE_BY_PRODUCT_CAPACITY = 100;

    /**
     * Defining a layout version for a class.
     */
    private static final long serialVersionUID = 3L;

    /**
     * format for display of a percentage one decimal places and a %
     */
    private static final DecimalFormat COMMAS = new DecimalFormat( "#,##0" );

    /**
     * format for display of a percentage one decimal places and a %
     */
    private static final DecimalFormat PERCENT = new DecimalFormat( "##0.0'%'" );

    /**
     * immediately probe for any new products discovered in Book Electronic or DVD macro
     **/
    private static final boolean checknewproducts = Misc.parseBoolean( System.getenv( "checknewproducts" ), true );

    /**
     * where website files are kept
     */
    private static final File webrootDir = new File( Global.configuration.getLocalWebrootWithSlashes() );

    /**
     * how often we do intermediate saves, after every 50 probes, possibly to the same product with different stores.
     */
    private static final int SAVE_FREQUENCY = 50;

    /**
     * keep track of all stocks for final save
     */
    private static final ArrayList<Stock> allStocks = new ArrayList<>( 20 );

    /**
     * HashMap wrapper to track which books are in stock for Amazon stores.  lookup by ASIN
     * Includes kindle, and all amazon stores that use AWS.
     */
    static Stock amazonBookStock;

    private static int inStockCount = 0;

    private static int notCarriedCount = 0;

    private static int outOfStockCount = 0;

    private static int productCount = 0;

    /**
     * HashMap wrapper to track which books are in stock for Amazon stores.  lookup by ASIN
     */
    private static Stock amazonDVDStock;

    /**
     * HashMap wrapper to track which electronic are in stock for Amazon stores.  lookup by ASIN
     */
    private static Stock amazonElectronicsStock;

    /**
     * HashMap wrapper to track which electronics are in stock for BestBuy.ca.
     */
    private static Stock bestBuyCaStock;

    /**
     * HashMap wrapper to track which electronics are in stock for BestBuy.com.
     */
    private static Stock bestBuyComStock;

    /**
     * HashMap wrapper to track which electronics are in stock for CanadaComputers.
     */
    private static Stock canadaComputersStock;

    /**
     * HashMap wrapper to track which dvds are in stock in stores that track by upc.
     */
    private static Stock dvdByUPCStock;

    /**
     * HashMap wrapper to track which dvds are in stock in stores that track by upc.
     */
    private static Stock dvdByISBNStock;

    /**
     * HashMap wrapper to track which electronics are in stock for staples.caa
     */
    private static Stock staplesCaStock;

    /**
     * HashMap wrapper to track which electronics are in stock for staples.com
     */
    private static Stock staplesStock;

    /**
     * HashMap wrapper to track which electronics are in stock for ncix.ca
     */
    private static Stock ncixCaStock;

    /**
     * HashMap wrapper to track which electronics are in stock for ncix us
     */
    private static Stock ncixUsStock;

    /**
     * HashMap wrapper to track which electronics are in stock for newEgg.ca
     */
    private static Stock newEggCaStock;

    /**
     * HashMap wrapper to track which electronics are in stock for newEgg.com.
     */
    private static Stock newEggComStock;

    /**
     * HashMap wrapper to track which books are is stock for bookstores that track by ISBN
     */
    private static Stock otherBStoreStock;

    /**
     * HashMap wrapper to track which books are is stock for ebookstores , including Nook and Kobo, but not Kindle
     */
    private static Stock ebookStoreStock;

    /**
     * HashMap wrapper to track which electronics are in stock for tiger.ca
     */
    private static Stock tigerCaStock;
    // /declarations

    /**
     * HashMap wrapper to track which electronics are in stock for tiger.com
     */
    private static Stock tigerComStock;

    /**
     * is it safe to process deadwood?
     */
    private static boolean deadwoodCheckSafe = false;

    // methods
    static
        {
        // note getenv not getEnv
        final String deadwood = System.getenv( "deadwood" );
        if ( deadwood == null )
            {
            DEADWOOD = 0;
            }
        else
            {
            switch ( deadwood )
                {
                case "no":
                    DEADWOOD = 0;
                    break;
                case "yes":
                case "remove":
                    DEADWOOD = 1;
                    break;
                case "find":
                case "check":
                    DEADWOOD = 2;
                    break;
                default:
                    throw new IllegalArgumentException( "invalid value for set deadwood=no/yes/find" );
                }
            }
        }

    /**
     * name of this stock (products + part numbering scheme using by one or more stores )
     */
    private final String name;

    /**
     * stores that use this stock numbering scheme
     */
    private final OnlineStore[] stores;

    /**
     * track last product probed for each store, so that we can pick up where we left off after a stall
     */
    private final String[] lastProductForStore;

    private final int storeCount;

    /**
     * time when this stock was last saved
     **/
    private long lastSavedTimestamp;

    /**
     * used to speed lookup of same product over and over
     */
    private String prevProduct = "NULL";

    /**
     * used to speed lookup of same product over and over
     */
    private Bundle prevBundle = null;

    /**
     * lookup product number to bundle containing in-stock bits.  One per stock
     */
    private HashMap<String, Bundle> lookupBundleByProduct;

    /**
     * We save after every SAVE_FREQUENCY changes to the stock to save work in case of stall or crash
     */
    private int countdownToSave = SAVE_FREQUENCY;

    /**
     * constructor for Stock
     */
    private Stock( final String name, final OnlineStore... stores )
        {
        this.name = name;
        this.stores = stores;
        // notify stores they are attached to this stock.
        storeCount = stores.length;
        // ordinal  0 2 4 ...
        int ordinal = 0;
        for ( OnlineStore store : stores )
            {
            store.configureStock( this, ordinal );
            ordinal += 2;
            }
        lastProductForStore = new String[ storeCount ];
        for ( int i = 0; i < storeCount; i++ )
            {
            lastProductForStore[ i ] = "";
            }
        // load up inventory from disk
        load();
        allStocks.add( this );
        }

    /**
     * get all the stocks for isbns, ASINS, DVDs, electronics
     */
    private static ArrayList<Stock> allStocks()
        {
        return allStocks;
        }

    /**
     * count products by various instock categories.
     */
    private static void countProductCategories( OnlineStore store )
        {
        final String[] all = store.getAllSortedProducts();
        productCount = all.length;
        inStockCount = 0;
        outOfStockCount = 0;
        notCarriedCount = 0;
        for ( String product : all )
            {
            switch ( store.getProductStatus( product, false ) )
                {
                case INSTOCK:
                    inStockCount++;
                    break;
                case OUTOFSTOCK:
                    outOfStockCount++;
                    break;
                case NOTCARRIED:
                    notCarriedCount++;
                    break;
                default:
                    throw new IllegalArgumentException( "Unexpected ProductStock" );
                }
            }
        }// /method

    /**
     * find but do not remove deadwood in all stocks
     *
     * @see removeDeadwood()
     */
    private static void findDeadwood()
        {
        final ArrayList<Stock> allStocks = Stock.allStocks();
        for ( Stock stock : allStocks )
            {
            final Set<String> allProducts = stock.allProducts();
            final ArrayList<String> toKillLater = new ArrayList<String>( 100 );
            for ( String product : allProducts )
                {
                boolean referenced = stock.wasProductReferenced( product );
                if ( !referenced )
                    {
                    out.println( "Deadwood found: " + stock.getName() + " " + product );
                    }
                }
            }
        }

    /**
     * print count of how many books each store has in stock.  This helps detect faulty hints
     *
     * @param stores stores to analyse
     */
    static void printCountsOfItemsInStock( OnlineStore... stores )
        {
        out.println( "  total   stock      %      out      % unlisted      %  Store" );
        for ( OnlineStore store : stores )
            {
            countProductCategories( store );
            double inStockPercent = ( double ) inStockCount * 100.d / productCount;
            double outOfStockPercent = ( double ) outOfStockCount * 100.d / productCount;
            double notCarriedPercent = ( double ) notCarriedCount * 100.d / productCount;
            out.println(
                    ST.leftPad( COMMAS.format( productCount ), 7, false )
                    + " "
                    + ST.leftPad( COMMAS.format( inStockCount ), 7, false )
                    + " "
                    + ST.leftPad( PERCENT.format( inStockPercent ), 6, false )
                    + "  "
                    + ST.leftPad( COMMAS.format( outOfStockCount ), 7, false )
                    + " "
                    + ST.leftPad( PERCENT.format( outOfStockPercent ), 6, false )
                    + "  "
                    + ST.leftPad( COMMAS.format( notCarriedCount ), 7, false )
                    + " "
                    + ST.leftPad( PERCENT.format( notCarriedPercent ), 6, false )
                    + "  "
                    + store.getUndecoratedStoreName() );
            }
        out.println( "  total   stock      %      out      % unlisted      %  Store" );
        out.println();
        }// /method

    /**
     * find and remove deadwood in all stocks
     *
     * @see findDeadwood()
     */
    private static void removeDeadwood()
        {
        final ArrayList<Stock> allStocks = Stock.allStocks();
        for ( Stock stock : allStocks )
            {
            final Set<String> allProducts = stock.allProducts();
            final ArrayList<String> toKillLater = new ArrayList<String>( 100 );
            for ( String product : allProducts )
                {
                boolean referenced = stock.wasProductReferenced( product );
                if ( !referenced )
                    {
                    out.println( "Deadwood removed: " + stock.getName() + " " + product );
                    toKillLater.add( product ); // can't remove elts while iterating.
                    }
                }
            for ( String product : toKillLater )
                {
                stock.removeProduct( product );
                }
            }
        }

    /**
     * Report unused clues.  This helps detect faulty hints
     *
     * @param stores stores to analyse
     */
    static void reportUnusedClues( OnlineStore... stores )
        {
        for ( OnlineStore store : stores )
            {
            for ( Clue clue : store.getClues() )
                {
                if ( !clue.isUsed() )
                    {
                    out.println( "unused Clue " + store.getEnumName() + " " + clue.getHint().name() + " " + clue.getMarker() );
                    }
                if ( clue.isConflicted() )
                    {
                    out.println( "conflicted Clue " + store.getEnumName() + " " + clue.getHint().name() + " " + clue.getMarker() );
                    }
                }
            }
        out.println();
        }// /method

    /**
     * invoke deadwood handling on shutdown
     */
    private static void shutdownDeadwood()
        {
        if ( deadwoodCheckSafe )
            {
            switch ( DEADWOOD )
                {
                case 0: /* no */
                    break;
                case 1: /* yes */
                    removeDeadwood();
                    break;
                case 2: /* find */
                    findDeadwood();
                    break;
                default:
                    throw new IllegalArgumentException( "invalid switch value in Stock.shutdownDeadwood" );
                }
            }
        }

    /**
     * get all  products potentially  in stock.
     *
     * @return Set of  all products
     */
    Set<String> allProducts()
        {
        // returns same set for all stores in the stock.
        return Collections.unmodifiableSet( lookupBundleByProduct.keySet() );
        }// /method

    /**
     * we have to start the lookupBundleByProduct HashMap over.
     * Just the one for this stock.
     */
    private void emptyLookupBundleByProduct()
        {
        lookupBundleByProduct = new HashMap<>( Stock.INITIAL_BUNDLE_BY_PRODUCT_CAPACITY );
        // we will save this later.  We don't correct the flawed *.dat file right away.
        }// /method

    /**
     * lookup bundle
     *
     * @param product product number
     * @param probe   true if should probe to get a Bundle, false should just return an empty bundle.
     *
     * @return corresponding bundle. never null.
     */
    private Bundle getBundle( String product, final boolean probe )
        {
        assert product != null : "null product fed to Stock.getBundle";
        assert !product.contains( "," ) : "comma list fed to Stock.getBundle";
        final boolean isNewProduct; // not permanently stored in the Bundle.
        Bundle bundle;
        synchronized ( lookupBundleByProduct )
            {
            if ( product.equals( prevProduct ) )
                {
                return prevBundle;
                }
            bundle = lookupBundleByProduct.get( product );
            if ( bundle == null )
                {
                // we have never seen this product before.
                bundle = new Bundle();
                lookupBundleByProduct.put( product, bundle );
                isNewProduct = true;
                }
            else
                {
                isNewProduct = false;
                }
            prevProduct = product;
            prevBundle = bundle;
            }
        // do not call markAsReferenced().  This code is used in too many places.
        // we have to be out of synchronised block, since bulkProbe1 will recursively call getBundle
        // if checknewproducts is false, we will probe later, more efficiently and a bulk probe.
        if ( probe && isNewProduct && checknewproducts )
            {
            // probe just one products, without threads.
            BulkProber.bulkProbe1OneProduct( stores, product );
            // will update bundle.
            }
        return bundle;
        }// /method

    /**
     * get the name of this stock set
     */
    private String getName()
        {
        return name;
        }

    /**
     * is product defined with a Bundle in this stock?
     * true if this product is defined in this stock for all stores, any status, referenced or unreferenced.
     */
    boolean isProductDefined( String product )
        {
        return lookupBundleByProduct.get( product ) != null;
        }

    /**
     * load up binary disk file into HashMap
     *
     * @see com.mindprod.stores.Stock#save(boolean)
     */
    private void load()
        {
        final File file = new File( webrootDir, "embellishment/" + name + ".dat" );
        if ( !file.canRead() )
            {
            err.println( EIO.getCanOrAbsPath( file ) + " missing. Starting from scratch." );
            emptyLookupBundleByProduct();
            return;
            }
        try
            {
            final DataInputStream dis = EIO.getDataInputStream( file, 64 * 1024 );
            final long expectedSerialVersionUID = dis.readLong();
            if ( expectedSerialVersionUID != serialVersionUID )
                {
                err.println( EIO.getCanOrAbsPath( file ) + " wrong version. Starting over." );
                emptyLookupBundleByProduct();
                dis.close();
                return;
                }
            lastSavedTimestamp = dis.readLong();
            final int expectedStoreCount = dis.readInt();
            if ( expectedStoreCount != storeCount )
                {
                err.println( EIO.getCanOrAbsPath( file ) + " wrong version. Starting over." );
                emptyLookupBundleByProduct();
                dis.close();
                return;
                }
            for ( int i = 0; i < storeCount; i++ )
                {
                lastProductForStore[ i ] = dis.readUTF();
                }
            final int size = dis.readInt();
            lookupBundleByProduct = new HashMap<>( ( int ) ( size * ( 100d + BREATHING_ROOM_PERCENT ) / 100d ) );
            for ( int i = 0; i < size; i++ )
                {
                String product = dis.readUTF();
                final int p = product.indexOf( ',' );
                if ( p >= 0 )
                    {
                    err.println( "corrupt stock contains comma " + name + " for " + product + " corrected." );
                    product = product.substring( 0, p );
                    }
                long bits = dis.readLong();   // bit pairs
                if ( !Bundle.isValid( bits, storeCount ) )  // does not fix referenced/unreferenced.
                    {
                    err.println( "corrupt stock Bundle bits " + name + " for " + product + " corrected." );
                    bits = Bundle.repairMinorDamage( bits, storeCount );
                    }
                if ( name.startsWith( "amazon" ) && product.length() != 10 )
                    {
                    out.println( "dropping product " + product + " from " + name + ".dat" );
                    }
                else
                    {
                    final Bundle bundle = new Bundle( bits );
                    // don't need to do a markAsUnreferenced because Bundle constructor does it.
                    lookupBundleByProduct.put( product, bundle );
                    }
                }
            dis.close();
            }
        catch ( IOException e )
            {
            err.println( EIO.getCanOrAbsPath( file ) + " trouble reading. Starting over. " + e.getMessage() );
            emptyLookupBundleByProduct();
            }
        if ( FORCE_PROBES )
            {
            // erase history of past incomplete probe runs
            lastSavedTimestamp = 0;
            for ( int i = 0; i < storeCount; i++ )
                {
                lastProductForStore[ i ] = "";
                }
            }
        }// /method

    /**
     * remove a product.  Must not be iterating over products
     */
    private void removeProduct( String product )
        {
        lookupBundleByProduct.remove( product );
        }

    /**
     * save HashMap to binary disk file
     *
     * @param finalSave true if this is the last save before exit.
     *
     * @see com.mindprod.stores.Stock#load()
     */
    private void save( final boolean finalSave )
        {
        // must save even if no stock changed so will avoid recomputing.
        // even if no stats changes, the  lastProductForStore and lastSavedTimestamp will have changed.
        // out.println( "      saving: " + this.name );
        final File file = new File( webrootDir, "embellishment/" + name + ".dat" );
        try
            {
            // this is a binary file, not a serialised one.
            final DataOutputStream dos = EIO.getDataOutputStream( file, 64 * 1024 );
            dos.writeLong( serialVersionUID );
            lastSavedTimestamp = System.currentTimeMillis();
            dos.writeLong( lastSavedTimestamp );
            dos.writeInt( storeCount );
            // do not clear listProducForStore. Do not want to rescan for 3 days, even if complete.
            for ( int i = 0; i < storeCount; i++ )
                {
                assert lastProductForStore[ i ] != null : "Bug: lastProductForStore may not be null.";
                dos.writeUTF( lastProductForStore[ i ] );
                }
            dos.writeInt( lookupBundleByProduct.size() );
            // we output all the bundles, including ones already probed. Not in order.
            for ( Map.Entry<String, Bundle> entry : lookupBundleByProduct.entrySet() )
                {
                final String product = entry.getKey();
                assert !product.contains( "," ) :
                        "corrupt stock contains comma product for " + name + " for " + product + " dropped.";
                dos.writeUTF( product );
                final Bundle bundle = entry.getValue();
                assert bundle != null : "Corrupt " + name + " file, contains null.";
                if ( !Bundle.isValid( bundle.bits, storeCount ) )
                    {
                    err.println( "corrupt stock bundle " + name + " for " + product );
                    bundle.bits = Bundle.repairMinorDamage( bundle.bits, storeCount );
                    }
                dos.writeLong( bundle.bits );
                }
            dos.close();
            }
        catch ( IOException e )
            {
            err.println( EIO.getCanOrAbsPath( file ) + " trouble writing. Starting over." );
            file.delete();
            }
        }// /method

    /**
     * check if a product was referenced in this stock at any store.
     * We can't look in the Bundle directly from the outside because it is private.
     *
     * @param product product id.
     *
     * @return true if this product was referenced.
     */
    private boolean wasProductReferenced( String product )
        {
        final Bundle b = getBundle( product, false );
        return b.isReferenced();
        }

    /**
     * note that it is safe to process deadwood
     */
    public static void DeadwoodCheckSafe()
        {
        deadwoodCheckSafe = true;
        }

    /**
     * run at start to get in-ram HashMaps going
     * If you delete any of these .dat files, they will recreate, and reprobe on the next run.
     */
    public static void fireup()
        {
        // define various stocks products numbering scheme/inventories.
        // indexed by asin
        // if you remove or reorder, you must regenerate the corresponding *.dat files.
        amazonBookStock = new Stock( "amazonbookstock",  /* lookup by ASIN, indirectly  by ISBN */
                BStore.AMAZONCA,
                // BStore.AMAZONCN,
                BStore.AMAZONCOM,
                BStore.AMAZONDE,
                BStore.AMAZONES,
                BStore.AMAZONFR,
                BStore.AMAZONIT,
                BStore.AMAZONUK,
                BStore.JUNGLEE );
        // indexed by asin
        amazonDVDStock = new Stock( "amazondvdstock",  /* lookup directly by ASIN */
                DStore.AMAZONCA,
                // DStore.AMAZONCN,
                DStore.AMAZONCOM,
                DStore.AMAZONDE,
                DStore.AMAZONES,
                DStore.AMAZONFR,
                DStore.AMAZONIT,
                DStore.AMAZONUK,
                DStore.JUNGLEE );
        // indexed by asin
        amazonElectronicsStock = new Stock( "amazonelectronicsstock",  /* lookup directly by ASIN */
                EStore.AMAZONCA,
                // EStore.AMAZONCN,
                EStore.AMAZONCOM,
                EStore.AMAZONDE,
                EStore.AMAZONES,
                EStore.AMAZONFR,
                EStore.AMAZONIT,
                EStore.AMAZONUK,
                EStore.JUNGLEE );
        // constructor does a load
        bestBuyCaStock = new Stock( "bestbuycastock", EStore.BESTBUYCA );    /* lookup by webcode */
        bestBuyComStock = new Stock( "bestbuycomstock", EStore.BESTBUY );    /* lookup by sku */
        canadaComputersStock = new Stock( "canadacomputersstock", EStore.CANADACOMPUTERS );  /* lookup by itemid */
        dvdByUPCStock = new Stock( "dvdbyupcstock", DStore.BN );  /* lookup by UPC */
        dvdByISBNStock = new Stock( "dvdbyisbnstock", DStore.POWELLS ); /* lookup by Powells ISBN */
        ncixCaStock = new Stock( "ncixcastock", EStore.NCIXCA );   /* lookup by sku */
        ncixUsStock = new Stock( "ncixusstock", EStore.NCIXUS );   /* lookup by sku */
        newEggCaStock = new Stock( "neweggcastock", EStore.NEWEGGCA ); /* lookup by item */
        newEggComStock = new Stock( "neweggcomstock", EStore.NEWEGG ); /* lookup by item */
        // indexed by isbn13
        otherBStoreStock = new Stock( "otherbookstorestock",  /* lookup done by ISBN13 */
                BStore.ABEANZ,
                BStore.ABECA,
                BStore.ABECOM,
                BStore.ABEDE,
                BStore.ABEES,
                BStore.ABEFR,
                BStore.ABEIT,
                BStore.ABEUK,
                BStore.BN,
                BStore.CHAPTERS,
                BStore.GOOGLE,
                BStore.POWELLS,
                BStore.SAFARI );
        ebookStoreStock = new Stock( "ebookstock", BStore.NOOK, BStore.KOBO, BStore.CHAPTERSEBOOKS );   /* lookup by ISBN13 */
        staplesCaStock = new Stock( "staplescastock", EStore.STAPLESCA ); /* lookup by item */
        staplesStock = new Stock( "staplesstock", EStore.STAPLES ); /* lookup by item */
        tigerCaStock = new Stock( "tigercastock", EStore.TIGERCA );   /* look up by edpno */
        tigerComStock = new Stock( "tigercomstock", EStore.TIGER ); /* look up by edpno */
        }// /method

    /**
     * run at shutdown to persist hashmaps
     */
    public static void shutdown()
        {
        shutdownDeadwood();
        for ( Stock stock : allStocks )
            {
            stock.save( true /* final */ );
            }
        }// /method

    /**
     * is item in stock at ith store using this stock file.
     *
     * @param product      product number, may be null or empty.
     * @param storeOrdinal index to select bookstore, assigned by this stock 0..30
     * @param probe        true if want a probe to get stock status if one not on file
     *
     * @return Stock Status
     */
    public StockStatus getProductStatus( final String product, final int storeOrdinal, final boolean probe )
        {
        if ( ST.isEmpty( product ) )
            {
            return NOTCARRIED;
            }
        // create new bundle and probe if product not in stock yet
        final Bundle bundle = getBundle( product, probe );  // will not be null.  All stores will be probed.
        /* mark in use for deadwood processing */
        bundle.markAsReferenced();
        return bundle.getStockStatus( storeOrdinal );
        }

    /**
     * Record the highest product number probed for this store
     *
     * @param product      product number, may be null or empty.
     * @param stockOrdinal index to select bookstore, assigned by this stock  0 2 4 ...
     */
    public void noteHighestProbe( final String product, int stockOrdinal )
        {
        stockOrdinal >>>= 1;  // convert to 0-based
        assert product.compareTo( lastProductForStore[ stockOrdinal ] ) > 0 :
                "noteHighestProbe: " + this.name + " products out of order " + product + " should be > " + lastProductForStore[ stockOrdinal ];
        lastProductForStore[ stockOrdinal ] = product;
        }// /method

    /**
     * set single stockItem value
     *
     * @param ordinal index to select bookstore, assigned by this stock  0 2 4 ...
     * @param product product number
     * @param status  0=not in stock 1=in stock 2=unknown 3=store is refusing probes.
     */
    public synchronized void setProductStockStatus( final String product, final int ordinal,
                                                    final StockStatus status )
        {
        Bundle bundle = getBundle( product, false );
        bundle.setItemStockStatus( ordinal, status );
        // we save based on setting product status, even if it is the same as before, each store, each product counts down 1 to next save.
        countdownToSave--;
        if ( countdownToSave <= 0 )
            {
            save( false /* just an intermediate save */ );
            countdownToSave = SAVE_FREQUENCY;
            }
        }// /method

    /**
     * Was this item successfully probed on a recent run? so we can bypass probing.
     *
     * @param product      product number, may be null or empty.
     * @param stockOrdinal index to select bookstore, assigned by this stock 0 2 4 ...
     *
     * @return true if recently probed already.
     */
    public boolean wasRecentlyProbed( final String product, int stockOrdinal )
        {
        // if three days have passed since last probe, we reprobe anyway
        long elapsedInMillis = System.currentTimeMillis() - lastSavedTimestamp;
        if ( TimeUnit.MILLISECONDS.toDays( elapsedInMillis ) >= AVOID_REPROBE_DAYS )
            {
            return false;
            }
        stockOrdinal >>>= 1;  // convert to 0-based
        // lastProductForStore is a separate array from the Bundles, conventionally indexed
        // FORCE_PROBES clears lastProductForStore
        // This will work even if lastProductForStore is no longer in the list of products.
        return product.compareTo( lastProductForStore[ stockOrdinal ] ) <= 0;
        }// /method

    /**
     * the bundles we lookup in the HashMap.  Just an int of 32 bits. we manipulate at a low level
     */
    private static class Bundle  /* nested in Stock */
        {
        /**
         * pair 0 in LSB.
         * 31 pairs of bits, one pair per store, 00 = not carried 10 = carried but not in stock 11 = instock
         * first bit is carried. second is instock.
         * bit 62 true if product has been referenced this session.
         * bit 63 sign bit is not used.
         * store ordinal is bit number for that store.  0 2 4 ...
         */
        private long bits;

        /**
         * constructor
         */
        private Bundle()
            {
            bits = 0;
            }

        /**
         * constructor
         */
        private Bundle( long bits )
            {
            // turn off the referenced bit
            this.bits = bits;
            this.markAsUnreferenced();
            }

        /**
         * Is this product is stock?
         *
         * @param ordinal store ordinal  0 2 4 ...
         *
         * @return StockStatus enum for product stock status
         */
        synchronized StockStatus getStockStatus( int ordinal )
            {
            // we don't know the max allowable ordinal
            // >>> done before &
            switch ( ( int ) ( ( bits >>> ordinal ) & 0b11L ) )
                {
                case 0b00:
                    return NOTCARRIED;
                case 0b01:
                    throw new IllegalArgumentException( "Stock.getStockStatus corrupt Bundle b01" );
                case 0b10:
                    return OUTOFSTOCK; // not in stock
                case 0b11:
                    return INSTOCK; // in stock
                default:
                    throw new IllegalArgumentException( "Stock.getStockStatus corrupt Bundle" );
                }
            } // /method

        /**
         * mark this product as unreferenced
         */
        private synchronized void markAsUnreferenced()
            {
            // store 0 in bit 62, msb
            bits &= ~( 0b1L << 62 );
            }

        /**
         * set single stockItem status for this Bundle.
         *
         * @param ordinal index to select bookstore, assigned by this stock  0 2 4 ...
         * @param status  enum for status of stock
         */
        private synchronized void setItemStockStatus( int ordinal, StockStatus status )
            {
            // we set it, even if the effect is unchanged
            final long mask = 0b11L << ordinal;
            switch ( status )
                {
                case NOTCARRIED:
                    // zero out bit pair 0b00
                    bits &= ~mask;
                    break;
                case OUTOFSTOCK:
                    // set to 0b10
                    // done in order ~ << & |
                    bits = bits & ~mask | 0b10L << ordinal;
                    break;
                case INSTOCK:
                    // set to 11
                    bits |= mask;
                    break;
                case UNKNOWN:
                case REFUSINGPROBES:
                default:
                    // ignore, leave as is.
                }
            }

        /**
         * check that bits entry in a bundle has not been corrupted.
         *
         * @param bits the  bits field from a bundle
         *
         * @return true if these bits are valid
         */
        public synchronized static boolean isValid( long bits, int storeCount )
            {
            // bit 63 must be 0
            // bit 62 (referenced) can be anything
            // middle bits must be 0  . up to 60-61
            // low bits must be anything but 01
            // check unused sign bit
            if ( ( bits & ( 0b1L << 63 ) ) != 0 )
                {
                err.println( "bad sign bit" );
                return false;
                }
            //  0 2 4 ...
            for ( int ordinal = 0; ordinal < storeCount * 2; ordinal += 2 )
                {
                int status = ( int ) ( ( bits >>> ordinal ) & 0b11L );
                if ( status == 0b01 )
                    {
                    err.println( "bad status bit pair" );
                    return false;
                    }
                }
            // check the unused bit pairs, top bits
            for ( int ordinal = storeCount * 2; ordinal < 60; ordinal += 2 )
                {
                int status = ( int ) ( ( bits >>> ordinal ) & 0b11L );
                if ( status != 0 )
                    {
                    err.println( "bad unused bits" );
                    return false;
                    }
                }
            return true;
            }

        /**
         * repair minor damage to a bundle.
         *
         * @param bits the long bits field from a bundle
         *
         * @return repaired bits
         */
        public synchronized static long repairMinorDamage( long bits, int storeCount )
            {
            // clear high order bits including sign, ref, unused, leave low bit pairs
            for ( int i = storeCount * 2; i < 64; i++ )
                {
                bits &= ~( 0b1L << i );
                }
            return bits;
            }

        /**
         * is this product referenced? e.g. mentioned in some Book, Ebook or DVD macro.
         */
        public synchronized boolean isReferenced()
            {
            // stored in bit 62
            return ( bits & ( 0b1L << 62 ) ) != 0;
            }

        /**
         * mark this product as referenced.
         */
        public synchronized void markAsReferenced()
            {
            // store 1 in bit 62
            bits |= 0b1L << 62;
            }
        }
    }