/*
* [EStore.java]
*
* Summary: Facts about various electronic bookstores that sell with ASIN product numbers.
*
* Copyright: (c) 2011-2017 Roedy Green, Canadian Mind Products, http://mindprod.com
*
* Licence: This software may be copied and used freely for any purpose but military.
* http://mindprod.com/contact/nonmil.html
*
* Requires: JDK 1.8+
*
* Created with: JetBrains IntelliJ IDEA IDE http://www.jetbrains.com/idea/
*
* Version History:
* 1.0 2011-10-04 initial version
*/
package com.mindprod.stores;
import com.mindprod.aws.jax.AWSECommerceService;
import com.mindprod.aws.jax.AWSECommerceServicePortType;
import com.mindprod.common18.Misc;
import com.mindprod.common18.ST;
import com.mindprod.entities.DeEntifyStrings;
import com.mindprod.fastcat.FastCat;
import com.mindprod.htmlmacros.macro.CountryFlag;
import com.mindprod.http.Get;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.Arrays;
import java.util.EnumSet;
import java.util.HashSet;
import java.util.Set;
import static com.mindprod.stores.BStoreConstants.DOLLAR_PRICE;
import static com.mindprod.stores.EStoreConstants.CONFIGURATION;
import static com.mindprod.stores.EStoreConstants.INTERPROBE_DELAY_FOR_AMAZON_GENERIC;
import static com.mindprod.stores.EStoreConstants.INTERPROBE_DELAY_FOR_CANADACOMPUTERS;
import static com.mindprod.stores.EStoreConstants.INTERPROBE_DELAY_GENERIC;
import static com.mindprod.stores.Hint.*;
import static java.lang.System.*;
/**
* Facts about various electronic bookstores that sell with ASIN product numbers.
*
* All amazon stores now use AWS.
*
* @author Roedy Green, Canadian Mind Products
* @version 1.0 2011-10-04 initial version
* @see Electronic
* @see Stock#reportUnusedClues
* @since 2011-09-28
*/
public enum EStore implements OnlineStore
{
AMAZONCA( true, "amazon.ca", "Amazon Canada", "https://", true, "www.amazon.ca", false, CountryFlag.Canada, CONFIGURATION.getAccountForAmazonCa(),
INTERPROBE_DELAY_FOR_AMAZON_GENERIC, null, 0, 15121, 330641 )
{
public AWSECommerceServicePortType getPort( AWSECommerceService service )
{
return service.getAWSECommerceServicePortCA();
}
public String buildProductLink( final String asin )
{
return buildAmazonProductLink( asin );
}
public String buildQueryLink( final String description, final String author )
{
// author ignored for electronic
return buildAmazonQueryLink( description );
}
public StockStatus probe( final String asin )
{
return BulkProber.awsProbe( this, asin );
}
},
AMAZONCOM( true, "amazon.com", "Amazon USA", "https://", true, "www.amazon.com", true, CountryFlag.USA, CONFIGURATION.getAccountForAmazonCom(),
INTERPROBE_DELAY_FOR_AMAZON_GENERIC, null, 0, 1789, 9325 )
{
public AWSECommerceServicePortType getPort( AWSECommerceService service )
{
return service.getAWSECommerceServicePort();
}
public String buildProductLink( final String asin )
{
return buildAmazonProductLink( asin );
}
public String buildQueryLink( final String description, final String author )
{
return buildAmazonQueryLink( description );
}
public StockStatus probe( final String asin )
{
return BulkProber.awsProbe( this, asin );
}
},
AMAZONDE( true, "amazon.de", "Amazon Germany", "https://", false, "www.amazon.de", true, CountryFlag.Germany, CONFIGURATION.getAccountForAmazonDe(),
INTERPROBE_DELAY_FOR_AMAZON_GENERIC, null, 0, 1638, 6742 )
{
public AWSECommerceServicePortType getPort( AWSECommerceService service )
{
return service.getAWSECommerceServicePortDE();
}
public String buildProductLink( final String asin )
{
return buildAmazonProductLink( asin );
}
public String buildQueryLink( final String description, final String author )
{
return buildAmazonQueryLink( description );
}
public StockStatus probe( final String asin )
{
return BulkProber.awsProbe( this, asin );
}
},
AMAZONES( true, "amazon.es", "Amazon Spain", "https://", true, "www.amazon.es", true, CountryFlag.Spain, CONFIGURATION.getAccountForAmazonEs(),
INTERPROBE_DELAY_FOR_AMAZON_GENERIC, null, 0, 624, 6746 )
{
public AWSECommerceServicePortType getPort( AWSECommerceService service )
{
return service.getAWSECommerceServicePortES();
}
public String buildProductLink( final String asin )
{
return buildAmazonProductLink( asin );
}
public String buildQueryLink( final String description, final String author )
{
return buildAmazonQueryLink( description );
}
public StockStatus probe( final String asin )
{
return BulkProber.awsProbe( this, asin );
}
},
AMAZONFR( true, "amazon.fr", "Amazon France", "https://", true, "www.amazon.fr", true, CountryFlag.France, CONFIGURATION.getAccountForAmazonFr(),
INTERPROBE_DELAY_FOR_AMAZON_GENERIC, null, 0, 1642, 6746 )
{
public AWSECommerceServicePortType getPort( AWSECommerceService service )
{
return service.getAWSECommerceServicePortFR();
}
public String buildProductLink( final String asin )
{
return buildAmazonProductLink( asin );
}
public String buildQueryLink( final String description, final String author )
{
return buildAmazonQueryLink( description );
}
public StockStatus probe( final String asin )
{
return BulkProber.awsProbe( this, asin );
}
},
AMAZONIT( true, "amazon.it", "Amazon Italy", "https://", true, "www.amazon.it", true, CountryFlag.Italy, CONFIGURATION.getAccountForAmazonIt(),
INTERPROBE_DELAY_FOR_AMAZON_GENERIC, null, 0, 3370, 23322 )
{
public AWSECommerceServicePortType getPort( AWSECommerceService service )
{
return service.getAWSECommerceServicePortIT();
}
public String buildProductLink( final String asin )
{
return buildAmazonProductLink( asin );
}
public String buildQueryLink( final String description, final String author )
{
return buildAmazonQueryLink( description );
}
public StockStatus probe( final String asin )
{
return BulkProber.awsProbe( this, asin );
}
},
AMAZONUK( true, "amazon.co.uk", "Amazon UK", "https://", true, "www.amazon.co.uk", true, CountryFlag.UK, CONFIGURATION.getAccountForAmazonUk(),
INTERPROBE_DELAY_FOR_AMAZON_GENERIC, null, 0, 1634, 6738 )
{
public AWSECommerceServicePortType getPort( AWSECommerceService service )
{
return service.getAWSECommerceServicePortUK();
}
public String buildProductLink( final String asin )
{
return buildAmazonProductLink( asin );
}
public String buildQueryLink( final String description, final String author )
{
return buildAmazonQueryLink( description );
}
public StockStatus probe( final String asin )
{
return BulkProber.awsProbe( this, asin );
}
},
JUNGLEE( true, /* amazon india, no affiliate program */ "junglee.com", "Junglee (Amazon in India)", "https://", false, "www.junglee.com", false, CountryFlag.India,
CONFIGURATION.getAccountForJungleeCom(), // included in amazonelectronicsstock
INTERPROBE_DELAY_FOR_AMAZON_GENERIC,
/* does not support SOAP AWS */
new Clue[]
{
// no AWS support.
new Clue( CHOP_TAIL, "Explore More Items" ),
new Clue( IN, "regex:[0-9,]+" ),
new Clue( CHOP_LEAD, "" ),
new Clue( MUST, "class=\"whole-price\">" ),
}, 55_000, 0, 0 )
{
public String buildProbeLink( final String asin )
{
return buildAmazonManualProbeLink( asin );
}
public AWSECommerceServicePortType getPort( AWSECommerceService service )
{
return service.getAWSECommerceServicePortIN();
}
public String buildProductLink( final String asin )
{
return buildAmazonProductLink( asin );
}
public String buildQueryLink( final String description, final String author )
{
return buildAmazonQueryLink( description );
}
public StockStatus probe( final String asin )
{
validateAsin( asin );
return BulkProber.screenScrapeProbe( this, asin, buildProbeLink( asin ) );
}
},
BESTBUYCA( true, "bestbuy.ca", "Best Buy Canada", "https://", true, "www.bestbuy.ca", true, CountryFlag.Canada,
null, INTERPROBE_DELAY_GENERIC, new Clue[]
{
new Clue( IN, "http://schema.org/Offer" ),
new Clue( IN, "regex:class=\"amount\">\\$[0-9,.]+<" ),
new Clue( UNLISTED, "Sorry, we couldn\u2019t find any results" ),
}, 198_000, 0, 0 )
{
public String buildProductLink( final String webCode )
{
return buildBestBuyCaProductLink( webCode );
}
public String buildQueryLink( final String description, final String author )
{
return buildBestBuyCaQueryLink( description );
}
public StockStatus probe( final String webCode )
{
return BulkProber.screenScrapeProbe( this, webCode, buildBestBuyCaProductLink( webCode ) );
}
},
BESTBUY( true, "bestbuy.com", "Best Buy USA", "https://", true, "www.bestbuy.com", true, CountryFlag.USA,
null, INTERPROBE_DELAY_GENERIC, new Clue[]
{
new Clue( IN, "regex:<span class=\"denominator\">\\$</span>[\\d.,]+<" ),
new Clue( IN, "regex:data-price=\"[1-9][\\d\\.]+\"" ),
new Clue( IN, "regex:regularPrice":[1-9][\\d\\.]+,"" ),
new Clue( IN, "id=\"sku-value-xxx\"" ),
new Clue( UNLISTED, "We couldn't find anything matching" ),
}, 109_000, 0, 0 )
{
public String buildProductLink( final String webCode )
{
return buildBestBuyComProductLink( webCode );
}
public String buildQueryLink( final String description, final String author )
{
return buildBestBuyComQueryLink( description );
}
public StockStatus probe( final String sku )
{
return BulkProber.screenScrapeProbe( this, sku, buildBestBuyComProbeLink( sku ) );
}
},
CANADACOMPUTERS( true, "canadacomputers.com", "Canada Computers", "http://", false, "www.canadacomputers.com", true, CountryFlag.Canada,
null, INTERPROBE_DELAY_FOR_CANADACOMPUTERS, new Clue[]
{
// actual price only present after JavaScript has run
new Clue( IN, "regex:class=\"discounted-price\">$[1-9][\\d\\.]+</span>" ),
new Clue( IN, ">Our Price<" ),
new Clue( IN, "regex:<span id=\"SalePrice\" >\\$[0-9,.]+" ),
new Clue( UNLISTED, ">The page can't be found!<" ),
}, 128_000, 0, 0 )
{
public String buildProductLink( final String itemId )
{
return buildCanadaComputersProductLink( itemId );
}
public String buildQueryLink( final String description, final String author )
{
return buildCanadaComputersQueryLink( description );
}
public StockStatus probe( final String itemId )
{
return BulkProber.screenScrapeProbe( this, itemId, buildCanadaComputersProductLink( itemId ) );
}
},
NCIXCA( true, "ncix.ca", "NCIX Canada", "http://", false, "www.ncix.com", true, CountryFlag.Canada, null, INTERPROBE_DELAY_GENERIC, new Clue[]
{
new Clue( CHOP_TAIL, ">May We Also Suggest<" ),
new Clue( IN, ">In Stock<" ),
new Clue( IN, "regex:<span itemprop=\"price\" content=\"[0-9,.]+\">" ),
new Clue( CHOP_LEAD, "<form name=\"frmaddtocart\"" ),
new Clue( OUT2, ">This Product is Not Available<" ),
new Clue( OUT2, "This item is no longer available." ),
}, 190_000, 0, 0 )
{
public String buildProductLink( final String sku )
{
// http://www.ncix.com/products/?sku=100650
final FastCat sb = new FastCat( 4 );
sb.append( getProtocol(), getHost(), "/products/?sku=", sku );
return sb.toString();
} // /method
public String buildQueryLink( final String description, final String author )
{
return buildNCIXQueryLink( description );
}
// buildProbe link would be same as buildProductLink
public StockStatus probe( final String sku )
{
return BulkProber.screenScrapeProbe( this, sku, buildProductLink( sku ) );
}
},
NCIXUS( true, "ncixus.com", "NCIX USA", "http://", false, "www.ncixus.com", true, CountryFlag.USA, null, INTERPROBE_DELAY_GENERIC, new Clue[]
{
new Clue( IN, ">In Stock<" ),
new Clue( IN, "regex:<span itemprop=\"price\">" + DOLLAR_PRICE + "</span>" ),
new Clue( IN2, "regex:http://schema.org/(?:InStock|LimitedAvailability|OnlineOnly|InStoreOnly)" ),
}, 126_000, 0, 0 )
{
public String buildProductLink( final String sku )
{
// http://www.ncixus.com/products/?sku=80748
final FastCat sb = new FastCat( 4 );
sb.append( getProtocol(), getHost(), "/products/?sku=", sku ); /* leave off "?affiliateid=", CONFIGURATION.getAccountNCIXUsCom() ); */
return sb.toString();
} // /method
public String buildQueryLink( final String description, final String author )
{
return buildNCIXQueryLink( description );
}
public StockStatus probe( final String sku )
{
return BulkProber.screenScrapeProbe( this, sku, buildProductLink( sku ) );
}
},
NEWEGGCA( true, "newegg.ca", "NewEgg Canada", "https://", false, "www.newegg.ca", true, CountryFlag.Canada, null, INTERPROBE_DELAY_GENERIC, new Clue[]
{
new Clue( IN, "product_instock:['1']" ),
new Clue( OUT, "product_instock:['0']" ),
}, 72_000, 0, 0 )
{
public String buildProductLink( final String neweggcaitem )
{
return buildNeweggProductLink( neweggcaitem );
}
public String buildQueryLink( final String description, final String author )
{
return buildNeweggQueryLink( description );
}
public StockStatus probe( final String neweggcaitem )
{
return BulkProber.screenScrapeProbe( this, neweggcaitem, buildNeweggProbeLink( neweggcaitem ) );
}
},
NEWEGG( true, "newegg.com", "NewEgg USA", "https://", false, "www.newegg.com", true, CountryFlag.USA, null, INTERPROBE_DELAY_GENERIC, new Clue[]
{
new Clue( IN, " product_instock:['1']" ),
new Clue( IN, "regex:product_instock:\\['[1-9][0-9]?'\\]" ),
new Clue( CHOP_LEAD, "<!--header area begin-->" ),
new Clue( OUT, "product_instock:['0']" ),
}, 138_000, 0, 0 )
{
public String buildProductLink( final String newegg )
{
return buildNeweggProductLink( newegg );
}
public String buildQueryLink( final String description, final String author )
{
return buildNeweggQueryLink( description );
}
public StockStatus probe( final String newegg )
{
return BulkProber.screenScrapeProbe( this, newegg, buildNeweggProbeLink( newegg ) );
}
},
STAPLESCA( true /* alive */, "www.staples.ca", "Staples Canada", "https://", true, "www.staples.ca", false /* no paypal */, CountryFlag.Canada, null /* no affiliate account */, INTERPROBE_DELAY_GENERIC, new Clue[]
{ // item number not present until JavaScript run
new Clue( CHOP_LEAD, "<!-- Individual Product -->" ),
new Clue( CHOP_TAIL, "<!-- Product Pricing Template -->" ),
new Clue( CHOP_TAIL, "Customers who viewed this also viewed" ),
new Clue( MUST, "<strong>Item :</strong> xxx" ),
new Clue( IN, "Expected Shipping:" ),
new Clue( IN, "man_out_stock_flag\":0" ),
new Clue( IN, "outofstock_flag\":0" ),
new Clue( CHOP_TAIL, "<button id=\"addToCart_" ),
new Clue( IN, "regex:\"price\":[\\d.]+[<,]" ),
new Clue( IN2, "regex:http://schema.org/(?:InStock|LimitedAvailability|OnlineOnly|InStoreOnly)" ),
new Clue( UNLISTED, "Sorry. Good thing this is not permanent." ),
new Clue( UNLISTED, "The product or category you were looking for is no longer available." ),
}, 160_000 /* min size */, 0, 0 )
{
public String buildProductLink( final String staplescaitem )
{
return buildProbeLink( staplescaitem );
}// /method
public String buildProbeLink( final String staplescaitem )
{
final FastCat sb = new FastCat( 5 );
sb.append( getProtocol(), getHost(), "/product_", staplescaitem, "_2-CA_1_20001" );
return sb.toString();
}// /method
public String buildQueryLink( final String description, final String author )
{
// http://www.staples.ca/directory_ssd+solid+state+drive_20051_1_20001?
final FastCat sb = new FastCat( 5 );
sb.append( getProtocol(), getHost() );
sb.append( "/directory_" );
sb.append( prepareQueryField( description ) );
sb.append( "_20051_1_20001" );
return sb.toString();
}
public StockStatus probe( final String staplescaitem )
{
return BulkProber.screenScrapeProbe( this, staplescaitem, buildProbeLink( staplescaitem ) );
}
},
STAPLES( true /* alive */, "www.staples.com", "Staples USA", "https://", true, "www.staples.com", false /* no paypal */, CountryFlag.USA, null /* no affiliate account */, INTERPROBE_DELAY_GENERIC, new Clue[]
{
new Clue( CHOP_TAIL, "Related Products" ),
new Clue( IN, ">Expected Delivery By:<" ),
new Clue( IN, "This Item Ships Free" ),
new Clue( IN, "regex:class=\"SEOFinalPrice\">[\\d.]+</span>" ),
new Clue( IN2, "regex:http://schema.org/(?:InStock|LimitedAvailability|OnlineOnly|InStoreOnly)" ),
new Clue( CHOP_LEAD, "v:title\">Home</a>" ),
}, 200_000 /* min size */, 0, 0 )
{
public String buildProductLink( final String staplesitem )
{
return buildProbeLink( staplesitem );
}// /method
public String buildProbeLink( final String staplesitem )
{
// http://www.staples.com/product_IM1QM6571
final FastCat sb = new FastCat( 4 );
sb.append( getProtocol(), getHost(), "/product_", staplesitem );
return sb.toString();
}// /method
public String buildQueryLink( final String description, final String author )
{
// http://www.staples.com/directory_SSD+solid+state+drive?
final FastCat sb = new FastCat( 4 );
sb.append( getProtocol(), getHost(), "/directory_" );
sb.append( prepareQueryField( description ) );
return sb.toString();
}
public StockStatus probe( final String staplesitem )
{
return BulkProber.screenScrapeProbe( this, staplesitem, buildProbeLink( staplesitem ) );
}
},
TIGERCA( true, "tigerdirect.ca", "Tiger Direct Canada", "http://", false, "www.tigerdirect.ca", true, CountryFlag.Canada,
CONFIGURATION.getAccountForTigerDirectCa(), INTERPROBE_DELAY_GENERIC, new Clue[]
{
new Clue( CHOP_TAIL, "<div class=\"arrow\"></div>" ),
new Clue( IN, "regex:<dd class=\"priceList\">\\$[0-9,.]+</dd>" ),
new Clue( IN, "regex:<dt class=\"priceFinal\">\\s*Price:\\s*</dt>" ),
new Clue( IN, "regex:<span class=\"salePrice\"><sup>\\$</sup>[,\\d]+<sup><span class=\"priceDecimalMark\">\\.</span>\\d+<span class=\"priceFlag\"></span>" ),
new Clue( IN2, "regex:<meta itemprop=\"availability\" content=\"(?:InStock|LimitedAvailability|OnlineOnly|InStoreOnly)" ),
new Clue( LISTED, "regex:<meta itemprop=\"price\" content=\"[0-9.]+\"/>" ),
new Clue( LISTED, "regex:Price:\\s*</span><span class=\"st_finalpr\">\\s*\\$[0-9]+<sup class=\"stic\">[0-9]+<" ),
new Clue( OUT2, "regex:<meta itemprop=\"availability\" content=\"(?:OutOfStock|PreOrder|SoldOut)" ),
new Clue( OUT2, "<h2 class=\"outofStock\">Currently Out Of Stock!</h2>" ),
new Clue( UNLISTED, "<input class=\"inputSearch\" id=\"tigerca_input\" type=\"text\" name=\"keywords\" autocomplete=\"Off\" value=\"Search by Keyword or Item #\" />" ),
}, 181_000, 0, 0 )
{
public String buildProductLink( final String tigerca )
{
return buildTigerProductLink( tigerca );
}
public String buildQueryLink( final String description, final String author )
{
return buildTigerQueryLink( description );
}
/**
* Probe tiger store to see if product in stock
*
* @param tigerca edpno of product
* @return does bookstore carry this book 0=no 1=yes 2=don't know
*/
public StockStatus probe( final String tigerca )
{
return BulkProber.screenScrapeProbe( this, tigerca, buildTigerProbeLink( tigerca ) );
}
},
TIGER( true, "tigerdirect.com", "Tiger Direct USA", "http://", false, "www.tigerdirect.com", true, CountryFlag.USA,
CONFIGURATION.getAccountForTigerDirectCom(), INTERPROBE_DELAY_GENERIC, new Clue[]
{
new Clue( CHOP_LEAD, "<!--[/masthead]-->" ),
new Clue( CHOP_TAIL, "<span>Overview</span>" ),
new Clue( IN, "To see details, proceed to checkout page." ),
new Clue( IN, "regex:<dt class=\"priceFinal\">\\s*Price:\\s*</dt>" ),
new Clue( IN, "regex:<dt class=\"pricemapa\">List Price:</dt>\\s+<dd class=\"pricemapa\"><span><del>" + DOLLAR_PRICE + "</del></span></dd>" ),
new Clue( LISTED, "EdpNo=xxx&" ),
new Clue( OUT2, "Currently Out Of Stock!" ),
}, 205_000, 0, 0 )
{
public String buildProductLink( final String tiger )
{
return buildTigerProductLink( tiger );
}
public String buildQueryLink( final String description, final String author )
{
return buildTigerQueryLink( description );
}
/**
* Probe tiger store to see if product in stock
*
* @param tiger edpno of product
*
* @return does bookstore carry this book 0=no 1=yes 2=don't know
*/
public StockStatus probe( final String tiger )
{
return BulkProber.screenScrapeProbe( this, tiger, buildTigerProbeLink( tiger ) );
}
};
/**
* all the amazon stores including , and Amazon.cn
* Must be here after enums defined.
*/
static final EnumSet<EStore> AMAZON_ESTORES = EnumSet.range( EStore.AMAZONCA, EStore.JUNGLEE );
/**
* max number of refused probes in a row from a store before we give up on it for this session
*/
private static final int MAX_ALLOWABLE_PROBES_REFUSED = 20;
/**
* max number stalls for a store before we give up restarting it.
*/
private static final int MAX_ALLOWABLE_STALLS = 9;
// declarations
/**
* only applies to amazon stores. Just a field needed. Who knows why.
*/
private final int camp;
/**
* only applies to amazon stores. Just a field needed. Who knows why.
*/
private final int creative;
/**
* how much time we must allow between probes to this bookstore, in ms
*/
private final long interProbeTime;
/**
* our account id with this bookstore used to mark our sales
*/
private final String account;
/**
* true if store takes PayPal
*/
private final boolean takesPayPal;
/**
* bookstoreName of website with entities
*/
private final String decoratedEStoreName;
/**
* bookstoreName of website witout entities
*/
private final String undecoratedEStoreName;
/**
* domain to go to with http://
*/
private final String website;
/**
* text appearing on a page that carries this book for this bookstore
*/
private final Clue[] clues;
/**
* flat to display beside bookstore
*/
private final CountryFlag flag;
/**
* electronic products we will probe for this store to see if they are instock
*/
private final HashSet<String> productsYetToProbe = new HashSet<>( 400 );
/**
* protocol for this store: http:// or https://. Defined in constructor
*/
private final String protocol;
private final int minimumResponseLength;
private final boolean isAlive;
private final boolean sni;
/**
* how many products were added to this store recently
*/
private int productsAdded = 0;
/**
* how many products were dropped from this store recently
*/
private int productsDropped = 0;
/**
* count of refused probes
*/
private volatile int refused = 0;
/**
* 1=should check this store, 0=should not check, -1= don't know yet
*/
private int shouldCheck = -1;
// /declarations
// methods
/**
* number 1..31 assigned by stock to indicate this store.
*/
private int stockOrdinal;
/**
* time stamp of when last probe completed.
*/
private long lastProbeCompleted = 0;
/**
* which stock product/numbering-inventory to use
*/
private Stock stock;
/**
* Get connection in progress, null if none
*/
private Get getInProgress;
/**
* countdown of how many stalls/restarts. When hit 0, we give up
*/
private int stallsToGo = MAX_ALLOWABLE_STALLS;
/**
* last product probed, not necessary completed.
*/
private String lastProductProbed;
/**
* track whether probe completed
*/
private int probeCompletionStage = 0;
/**
* constructor
*
* @param isAlive true if store is functioning
* @param decoratedEstoreName store name with entities
* @param undecoratedEstoreName store name without entities
* @param protocol either http:// or https://
* @param sni server name indication true/false
* @param website URL of website we link to
* @param takesPayPal true if takes PayPal
* @param flag which flag tco display
* @param account account used to track sales.
* @param interProbeTime minimum time between hitting same store in ms.
* @param clues text on page indicating it carries or does not carry book
* @param minimumResponseLength minimum response we bother to parse
* @param camp mysterious Amazon link code
* @param creative mysterious Amazon link code
*/
EStore( final boolean isAlive,
final String decoratedEstoreName,
final String undecoratedEstoreName,
final String protocol,
final boolean sni, final String website,
final boolean takesPayPal,
final CountryFlag flag,
final String account,
final long interProbeTime,
final Clue[] clues,
final int minimumResponseLength,
final int camp,
final int creative )
{
this.isAlive = isAlive;
this.decoratedEStoreName = decoratedEstoreName;
this.protocol = protocol;
this.sni = sni;
this.website = website;
this.account = account;
this.camp = camp;
this.clues = clues == null ? new Clue[ 0 ] : clues;
this.minimumResponseLength = minimumResponseLength;
this.creative = creative;
this.takesPayPal = takesPayPal;
this.flag = flag;
this.interProbeTime = interProbeTime;
this.undecoratedEStoreName = undecoratedEstoreName;
}
/**
* prepare title, author, description for use in a general query
*
* @param description description of the item
*
* @return endocoded field
*/
private static String prepareQueryField( final String description )
{
try
{
return URLEncoder.encode( DeEntifyStrings.deEntifyHTML( description, ' ' ), "UTF-8" );
}
catch ( UnsupportedEncodingException e )
{
err.println( "UTF-8 not supported" );
System.exit( 2 );
return null;
}
}
/**
* print count of how many books each store has in stock. This helps detect faulty hints
*/
private static void printCountsOfItemsInStock()
{
if ( !Misc.parseBoolean( System.getenv( "showcountsofelectronicsinstock" ), false ) )
{
return;
}
out.println( "E L E C T R O N I C S I N S T O C K" );
Stock.printCountsOfItemsInStock( EStore.values() );
}// /method
/**
* Report unused clues. This helps detect faulty hints()
*/
private static void reportUnusedClues()
{
if ( Misc.parseBoolean( System.getenv( "reportunusedclues" ), false )
|| Misc.parseBoolean( System.getenv( "checkstores" ), false )
|| Misc.parseBoolean( System.getenv( "checkelectronicstores" ), false ) )
{
out.println( "U N U S E D E L E C T R O N I C C L U E S" );
Stock.reportUnusedClues( EStore.values() );
}
}// /method
/**
* check that a string is a asin
*
* @param asin number
*/
private static void validateAsin( String asin )
{
assert !ST.isEmpty( asin ) && asin.length() == 10 && asin.startsWith( "B" ) : "bad asin [" + asin + "]";
}// /method
/**
* flag link for a referral to Amazon store, manual probe
*
* @param asin asin,.
*
* @return code for link without <td> </td> or flag.
*/
String buildAmazonManualProbeLink( final String asin )
{
validateAsin( asin );
final FastCat sb = new FastCat( 4 );
sb.append( getProtocol(), getHost() );
sb.append( "/dp/", asin );
return sb.toString();
}// /method
/**
* flag link for a referral to Amazon store, default.
*
* @param asin asin,.
*
* @return code for link without <td> </td> or flag.
*/
String buildAmazonProductLink( final String asin )
{
validateAsin( asin );
// we link using asin,
if ( asin == null )
{
return "."; // dummy link
}
final String account = this.getAccount();
if ( account == null )
{
final FastCat sb = new FastCat( 4 );
sb.append( getProtocol(), getHost() );
sb.append( "/dp/", asin );
return sb.toString();
}
else
{
// rechecked 2016-04-29
final int camp = this.getCamp();
final int creative = this.getCreative();
final FastCat sb = new FastCat( 14 );
sb.append( getProtocol(), getHost() );
sb.append( "/gp/product/", asin );
sb.append( "/ref=as_li_qf_br_asin_tl?ie=UTF8" );
sb.append( "&tag=", account );
sb.append( "&linkCode=as2" );
sb.append( "&camp=", camp );
sb.append( "&creative=", creative );
sb.append( "&creativeASIN=", asin );
return sb.toString();
}
}// /method
/**
* general query to an Amazon EStore. Used when product not is stock.
*
* @param description prodduct description. Cannot be private.
*
* @return general link.
*/
String buildAmazonQueryLink( final String description )
{
final FastCat sb = new FastCat( 6 );
// http://www.amazon.ca/s/?field-keywords=the+God+Delusion+by+richard+Dawkins&tag=canadianmin07-20
sb.append( getProtocol(), getHost() );
sb.append( "/s?field-keywords=" );
sb.append( prepareQueryField( description ) );
if ( account != null )
{
sb.append( "&tag=", account );
}
return sb.toString();
}// /method
/**
* link to BestBuy.ca product
*
* @param webCode product number
*
* @return URLString to link to that product
*/
String buildBestBuyCaProductLink( final String webCode )
{
final FastCat sb = new FastCat( 4 );
sb.append( getProtocol(), getHost(), "/Search/SearchResults.aspx?query=", webCode );
return sb.toString();
}// /method
/**
* link to BestBuy.ca query
*
* @param description, description of the product
*
* @return URLString to link to that product
*/
String buildBestBuyCaQueryLink( final String description )
{
final FastCat sb = new FastCat( 4 );
sb.append( getProtocol(), getHost() );
sb.append( "/Search/SearchResults.aspx?query=" );
sb.append( prepareQueryField( description ) );
return sb.toString();
}// /method
/**
* Affiliate link to BestBuy.com product
*
* @param sku product number
*
* @return URLString to link to that product
*/
String buildBestBuyComProbeLink( final String sku )
{
final FastCat sb = new FastCat( 4 );
// this just gets us to the small intro page, not the main page.
// http://www.bestbuy.com/site/searchpage.jsp?qp=&sp=&ks=960&sc=Global&list=n&_dyncharset=ISO-8859-1&usc=All+Categories&type=page&id=pcat17071&iht=y&cp=1&nrp=15&st=1307707104
sb.append( getProtocol(), getHost() );
sb.append( "/site/searchpage.jsp?qp=&sp=&ks=960&sc=Global&list=n&_dyncharset=ISO-8859-1"
+ "&usc=All+Categories&type=page&id=pcat17071&iht=y&cp=1&nrp=15&st=" );
sb.append( sku );
return sb.toString();
}// /method
/**
* Affiliate link to BestBuy.com product
*
* @param sku product number
*
* @return URLString to link to that product
*/
String buildBestBuyComProductLink( final String sku )
{
final FastCat sb = new FastCat( 4 );
// this just gets us to the small intro page, not the main page.
// http://www.bestbuy.com/site/searchpage.jsp?qp=&sp=&ks=960&sc=Global&list=n&_dyncharset=ISO-8859-1&usc=All+Categories&type=page&id=pcat17071&iht=y&cp=1&nrp=15&st=1307707104
sb.append( getProtocol(), getHost() );
sb.append( "/site/searchpage.jsp?qp=&sp=&ks=960&sc=Global&list=n&_dyncharset=ISO-8859-1"
+ "&usc=All+Categories&type=page&id=pcat17071&iht=y&cp=1&nrp=15&st=" );
sb.append( sku );
return sb.toString();
}// /method
/**
* link to BestBuy.com query
*
* @param description, description of the product
*
* @return URLString to link to that product
*/
String buildBestBuyComQueryLink( final String description )
{
final FastCat sb = new FastCat( 20 );
sb.append( getProtocol(), getHost() );
sb.append( "/site/searchpage.jsp" );
sb.append( "?_dyncharset=UTF-8" );
sb.append( "&_dynSessConf=" );
sb.append( "&id=pcat17071" );
sb.append( "&type=page" );
sb.append( "&sc=Global" );
sb.append( "&cp=1" );
sb.append( "&nrp=15" );
sb.append( "&sp=" );
sb.append( "&qp=" );
sb.append( "&list=n" );
sb.append( "&iht=y" );
sb.append( "&usc=All+Categories" );
sb.append( "&ks=960" );
sb.append( "&fs=saas" );
sb.append( "&keys=keys" );
sb.append( "&st=" );
sb.append( prepareQueryField( description ) );
return sb.toString();
}// /method
/**
* build a product link to CanadaComputers product
*
* @param itemId product id.
*
* @return link starting with http:
*/
String buildCanadaComputersProductLink( final String itemId )
{
// take user to page http://www.canadacomputers.com/product_info.php?item_id=070757
final FastCat sb = new FastCat( 4 );
sb.append( getProtocol(), getHost() );
sb.append( "/product_info.php?item_id=", itemId );
return sb.toString();
}// /method
/**
* build an general product areas link to CanadaComputers product
*
* @param description description of product area
*
* @return link starting with http:
*/
String buildCanadaComputersQueryLink( final String description )
{
// take user to page http://www.canadacomputers.com/advanced_search_result.php?keywords=ssd
final FastCat sb = new FastCat( 4 );
sb.append( getProtocol(), getHost() );
sb.append( "/advanced_search_result.php?keywords=" );
try
{
sb.append( URLEncoder.encode( description, "UTF-8" ) ); // space -> +
}
catch ( UnsupportedEncodingException e )
{
err.println( "UTF-8 not supported" );
System.exit( 2 );
}
return sb.toString();
}// /method
/**
* build a general query for NCIX
*
* @param description description of product
*
* @return http:// search query link
*/
String buildNCIXQueryLink( final String description )
{
// http://search.ncix.com/search/q=xxx&qcatid=0
final FastCat sb = new FastCat( 5 );
sb.append( getProtocol(), getHost() );
sb.append( "/search/?q=" );
sb.append( prepareQueryField( description ) );
sb.append( "&qcatid=0" );
return sb.toString();
}// /method
/**
* build an probe link to newegg product, no affiliate agreement
*
* @param newegg productID
*
* @return link starting with http:
*/
String buildNeweggProbeLink( final String newegg )
{
// e.g. http://www.newegg.com/Product/Product.aspx?Item=N82E16826336001
final FastCat sb = new FastCat( 4 );
sb.append( getProtocol(), getHost() );
sb.append( "/Product/Product.aspx?Item=", newegg );
return sb.toString();
}// /method
/**
* build an link to newegg product, no affiliate agreement
*
* @param newegg productID
*
* @return link starting with http:
*/
String buildNeweggProductLink( final String newegg )
{
// e.g. http://www.newegg.com/Product/Product.aspx?Item=N82E16826336001
final FastCat sb = new FastCat( 4 );
sb.append( getProtocol(), getHost() );
sb.append( "/Product/Product.aspx?Item=", newegg );
return sb.toString();
}// /method
/**
* build an general product areas link to newegg product
*
* @param description description of product area
*
* @return link starting with http:
*/
String buildNeweggQueryLink( final String description )
{
// http://www.newegg.ca/Product/ProductList.aspx?Submit=ENE&DEPA=0&Order=BESTMATCH&Description=Intel+320+Series+80+GB+SATA-2+3+Gb%2Fs
final FastCat sb = new FastCat( 4 );
sb.append( getProtocol(), getHost() );
sb.append( "/Product/ProductList.aspx?Submit=ENE&DEPA=0&Order=BESTMATCH&Description=" );
sb.append( prepareQueryField( description ) );
return sb.toString();
}// /method
/**
* build an probe link to tiger/tigerca product
*
* @param tiger productID edpno
*
* @return link starting with http:
*/
String buildTigerProbeLink( final String tiger )
{
final FastCat sb = new FastCat( 4 );
// http://www.tigerdirect.ca/applications/SearchTools/item-details.asp?EdpNo=8466640
sb.append( getProtocol(), getHost(), "/applications/SearchTools/item-details.asp?EdpNo=", tiger );
return sb.toString();
}// /method
/**
* build an affiliate link to tiger/tigerca product
*
* @param tiger productID
*
* @return link starting with http:
*/
String buildTigerProductLink( final String tiger )
{
return buildTigerProbeLink( tiger );
}
/**
* build an general link to tiger/tigerca product area
*
* @param description description of product area.
*
* @return link starting with http:
*/
String buildTigerQueryLink( final String description )
{
// http://www.tigerdirect.com/applications/SearchTools/search.asp?
// keywords=xxx
final FastCat sb = new FastCat( 4 );
sb.append( getProtocol(), getHost() );
sb.append( "/applications/SearchTools/search.asp?keywords=" );
sb.append( prepareQueryField( description ) );
return sb.toString();
}// /method
/**
* getter of Amazon camp number
*
* @return camp
*/
int getCamp()
{
return camp;
}// /method
/**
* getter for Amazon creative magic number
*
* @return creative number.
*/
int getCreative()
{
return creative;
}// /method
/**
* summary reports
*/
public static void summary()
{
printCountsOfItemsInStock();
reportUnusedClues();
}// /method
/**
* clear counts of products added and dropped
*/
public void clearStats()
{
productsAdded = 0;
productsDropped = 0;
}// /method
/**
* configure this estore to use a particular stock (numbering system, inventory)
*
* @param stock Stack that tracks products for this store.
* @param ordinal number 1..31 assigned by Stack to indicate this store.
*/
public void configureStock( Stock stock, int ordinal )
{
this.stock = stock;
this.stockOrdinal = ordinal;
}// /method
/**
* enque a probe for a given book for this bookstore
*
* @param product asin
*/
public void enqueueProductToProbe( final String product )
{
productsYetToProbe.add( product );
}// /method
/**
* forgive past string of refusals because we had a success
*/
public void forgiveRefusals()
{
refused = 0;
}
public String getAccount()
{
return account;
}// /method
/**
* get all products this store potentially has in stock.
*
* @return Set of all products
*/
public String[] getAllSortedProducts()
{
// returns same set for all stores in the stock.
final Set<String> allProducts = stock.allProducts();
final String[] products = allProducts.toArray( new String[ allProducts.size() ] );
Arrays.sort( products );
return products;
}// /method
/**
* get strings that indicate item is in or out of stock.
*
* @return clues array [0] if no clues.
*/
public Clue[] getClues()
{
return clues;
}// /method
/**
* how many products we still have to probe
*/
public int getCountOfProductsYetToProbe()
{
synchronized ( productsYetToProbe )
{
return productsYetToProbe.size();
}
}
public String getDecoratedStoreName()
{
return decoratedEStoreName;
}// /method
/**
* make enum.name() visible. Don't change to name.
*
* @return enum name
*/
public String getEnumName()
{
return this.name();
}// /method
public CountryFlag getFlag()
{
return flag;
}// /method
/**
* get the Get connection in progress
*
* @return Get connection in progress, null if none
*/
public Get getGetInProgress()
{
return this.getInProgress;
}
public String getHost()
{
return website;
}// /method
/**
* milliseconds min between probing the same store.
*/
public long getInterProbeTime()
{
return interProbeTime;
}// /method
/**
* record last product probed for this store
*
* @return last product probed, not necessary completed.
*/
public String getLastProductProbed()
{
return this.lastProductProbed;
}
/**
* record last product probed for this store
*
* @param product product id
*/
public void setLastProductProbed( String product )
{
this.lastProductProbed = product;
}
/**
* miminum length of response from store to parse
*/
public int getMinimumResponseLength()
{
return this.minimumResponseLength;
}
/**
* get eStore ordinal
*/
public int getOrdinal()
{
return ordinal();
}
/**
* get the port for the server for that country
*
* @param service the soap service
*
* @return the port for that country.
*/
public AWSECommerceServicePortType getPort( AWSECommerceService service )
{
return null;
}// /method
@Override
/** and what stage of completion in the most recent probe?
*
* @return 0 .. 7
*/
public int getProbeCompletionStage()
{
return probeCompletionStage;
}
@Override
/**
* record whether probe completed
*
* @param completionStage 0..7
*/
public void setProbeCompletionStage( final int completionStage )
{
this.probeCompletionStage = completionStage;
}
/**
* is a given product in stock at this store? Used when expanding Electronic macro.
*
* @param product product number
* @param probe true if should probe to get status if one not on file.
* @param isEBook true if this is an electronic book (out of stock treated as instock)
*
* @return StockStatus enum.
*/
public StockStatus getProductStatus( final String product, final boolean probe )
{
// Override errors in product database
final StockStatus override = CONFIGURATION.stockStatusOverride( this /* store */, product );
if ( override != null )
{
return override;
}
synchronized ( stock )
{
// normal case
return stock.getProductStatus( product, stockOrdinal, probe );
}
}// /method
// // split out account:offerid does not seem to work anymore.
// final String combi = this.getAccount();
// final int place = combi.indexOf( ':' );
// final String account = combi.substring( 0, place );
// final String offerid = combi.substring( place + 1 );
// final FastCat sb = new FastCat( 10 );
// sb.append( "http://click.linksynergy.com/link?id=", account );
// sb.append( "&offerid=", offerid, "." ); // code for tigerdirect.com or tigerdirect.ca
// sb.append( tiger );
// sb.append( "&type=2&murl=http%3A%2F%2F", getHost() );
// sb.append( "%2Fapplications%2Fsearchtools%2Fitem-details.asp%3FEdpNo%3D" );
// sb.append( tiger );
// return sb.toString()
// }
/**
* get count of products recently added
*
* @return count of products recently added
*/
public int getProductsAddedCount()
{
return productsAdded;
}// /method
/**
* get count of products recently dropped
*
* @return count of products recently dropped
*/
public int getProductsDroppedCount()
{
return productsDropped;
}// /method
/**
* get protocol for this store: http:// or https://
*/
public String getProtocol()
{
return this.protocol;
}
/**
* get HashSet of all the products we plan to probe for this store .
*/
public String[] getSortedProductsToProbe()
{
synchronized ( productsYetToProbe )
{
final String[] products = productsYetToProbe.toArray( new String[ productsYetToProbe.size() ] );
Arrays.sort( products );
return products;
}
}// /method
/**
* get timestamp of when previous probe was completed
*
* @return timestamp in millis since 1970
*/
public long getTimeLastProbeCompleted()
{
return lastProbeCompleted == 0 ? System.currentTimeMillis() : lastProbeCompleted;
}// /method
public String getUndecoratedStoreName()
{
return undecoratedEStoreName;
}// /method
/**
* true if still probing this store, it has not refused too many probes
*/
public synchronized boolean isAcceptingProbes()
{
return refused <= MAX_ALLOWABLE_PROBES_REFUSED;
}// /method
/**
* is this store functioning.
* If false, show with under construction icon, without any links
*
* @return true if store working
*/
public boolean isAlive()
{
return isAlive;
}
/**
* Should we give up reviving this stall store?
*
* @return true if this store has stalled many times and should not be revived,
* false if store should be revived from stall yet again;
*/
public boolean isHopelesslyStalled()
{
return stallsToGo > 0;
}
/**
* restart the counter of refused probes, if not already hit the limit
*
* @param product isbn, asin, upc just probed successfully.
* @param straggler true if we are processing random stragglers. false if processing bulk sorted products.
*/
public void noteCompletedProbe( final String product, final boolean straggler )
{
// out.println( "completed probe: " + this.name() + " " + product + " " + straggler );
synchronized ( productsYetToProbe )
{
productsYetToProbe.remove( product );
}
lastProbeCompleted = System.currentTimeMillis();
if ( !straggler )
{
stock.noteHighestProbe( product, stockOrdinal );
}
}// /method
/**
* increment count of products added on this probe
*/
public void noteProductAdded()
{
productsAdded++;
}// /method
/**
* increment count of products dropped on this probe
*/
public void noteProductDropped()
{
productsDropped++;
}// /method
/**
* increment the count of refused probes
*/
public void noteRefusedProbe( final String product )
{
synchronized ( productsYetToProbe )
{
productsYetToProbe.remove( product );
}
refused++;
if ( refused <= MAX_ALLOWABLE_PROBES_REFUSED )
{
err.println( this + " refusing probes " + refused );
}
else
{
err.println( this + " refusing probes " + refused + " giving up probing" );
}
lastProbeCompleted = System.currentTimeMillis();
}// /method
/**
* record fact this store has stalled again
*/
public void noteStall()
{
stallsToGo--;
}
/**
* what this store sells
*
* @return "electronics"
*/
public String productCategory()
{
return "electronics";
}// /method
/**
* record the Get connection in progress
*
* @param get Get connection
*/
public void recordGetInProgress( Get getInProgress )
{
this.getInProgress = getInProgress;
}
/**
* set single stock product status
*
* @param product product number
* @param status 0=not in stock 1=in stock 2=unknown 3=store is refusing probes.
*/
public void setProductStockStatus( String product, StockStatus status )
{
synchronized ( stock )
{
stock.setProductStockStatus( product, stockOrdinal, status );
}
}
/**
* true if this store's inventory should be checked, according to env
*
* @return true if should check this store. e.g. set checkstaplescaelectronics=yes
*/
public boolean shouldCheck()
{
// cache the result of probing the environment. Only checks th eenvironment on the first time
switch ( shouldCheck )
{
case -1:
// look for set checktigerelectronics checkamazoncaelectronics
if ( Misc.parseBoolean( System.getenv( "check" + this.name().toLowerCase() + "electronics" ), false ) )
{
shouldCheck = 1;
return true;
}
else
{
shouldCheck = 0;
return false;
}
case 0:
return false;
case 1:
return true;
default:
throw new IllegalArgumentException( "bug in shouldCheck case" );
}
}// /method
/**
* do we need to suppress cookie errors
*
* @return true if need to suppress cookie errors
*/
public boolean suppressCookieErrors()
{
return false;
}// /method
/**
* @return true if this store takes PayPal
*/
public boolean takesPayPal()
{
return takesPayPal;
}
/**
* Was this item successfully probed on a recent run?
*
* @param product product number, may be null or empty.
*
* @return true if recently probed already.
*/
public boolean wasRecentlyProbed( String product )
{
return stock.wasRecentlyProbed( product, stockOrdinal );
}
}