/*
 * [ImportBiostar.java]
 *
 * Summary: Import all data about motherboards from the BioStar Website.
 *
 * Copyright: (c) 2011-2017 Roedy Green, Canadian Mind Products, http://mindprod.com
 *
 * Licence: This software may be copied and used freely for any purpose but military.
 *          http://mindprod.com/contact/nonmil.html
 *
 * Requires: JDK 1.7+
 *
 * Created with: JetBrains IntelliJ IDEA IDE http://www.jetbrains.com/idea/
 *
 * Version History:
 *  1.0 2011-02-14 initial version
 */
package com.mindprod.mother;
import com.mindprod.common17.BigDate;
import com.mindprod.common17.ST;
import com.mindprod.csv.CSVReader;
import com.mindprod.htmlmacros.support.Configuration;
import com.mindprod.htmlmacros.support.ConfigurationForMindprod;
import com.mindprod.http.Get;
import com.mindprod.hunkio.HunkIO;
import java.io.BufferedReader;
import java.io.EOFException;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.net.URL;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import static java.lang.System.*;
/**
 * Import all data about motherboards from the BioStar Website.
 *
 * @author Roedy Green, Canadian Mind Products
 * @version 1.0 2011-02-14 initial version
 * @since 2011-02-14
 */
public class ImportBiostar extends ImportManufacturer
    {
    /**
     * where master list of all Biostar motherboards is.
     */
    private static final String[] INDEX_URLS = {
            "http://www.biostar-usa.com/mbmain.asp",
            "http://www.biostar-usa.com/app/en-us/mb/index.php?S_ID=2",
            "http://www.biostar-usa.com/app/en-us/mb/index.php?S_ID=3",
            "http://www.biostar-usa.com/app/en-us/mb/index.php?S_ID=4",
            "http://www.biostar-usa.com/app/en-us/mb/index.php?S_ID=5",
            "http://www.biostar-usa.com/app/en-us/mb/index.php?S_ID=6",
            "http://www.biostar-usa.com/app/en-us/mb/index.php?S_ID=7",
            "http://www.biostar-usa.com/app/en-us/mb/index.php?S_ID=8",
            "http://www.biostar-usa.com/app/en-us/mb/index.php?S_ID=9",
            "http://www.biostar-usa.com/app/en-us/mb/index.php?S_ID=10",
            "http://www.biostar-usa.com/app/en-us/mb/index.php?S_ID=11",
            "http://www.biostar-usa.com/app/en-us/mb/index.php?S_ID=12",
            "http://www.biostar-usa.com/app/en-us/mb/index.php?S_ID=13" };
    /**
     * mindprod configuration
     */
    private static final Configuration configuration;
    private static final File sourceDir;
    /**
     * looks for Audio channels
     */
    private static final Pattern AUDIO_CHANNEL_FINDER = Pattern.compile(
            "([\\.\\d]+)[\\- ]ch", Pattern.CASE_INSENSITIVE );
    /**
     * looks for info about the form factor
     * >Micro ATX Form Factor Dimension: 23.5cm X 18.2cm
     */
    private static final Pattern FORM_FACTOR_FINDER_L = Pattern.compile(
            "[\\s>]+([\\p{Alnum}\\- ]+?)\\s*Form Factor\\s*(?:<br>)?(?:Dimension: )?([\\d\\.]+)\\s*(?:cm)?\\s*X\\s*" +
            "([\\d\\.]+)",
            Pattern.CASE_INSENSITIVE
    );
    // >ATX Form Factor Dimension: 30.5cm x 24.4cm ( W x L )
    // Micro ATX form factor<br>24.5cm x 24.5cm ( W x L )
    /**
     * looks for info about the form factor
     * >Micro ATX Form Factor Dimension: 23.5cm X 18.2cm
     */
    private static final Pattern FORM_FACTOR_FINDER_S = Pattern.compile(
            ">([\\p{Alnum}\\- ]+?) Form Factor Dimension: ([\\d\\.]+)\\s*(?:cm)*\\s*X\\s*([\\d\\.]+)\\s*(?:cm)*",
            Pattern.CASE_INSENSITIVE );
    /**
     * looks for count of IDE ports
     */
    private static final Pattern IDE_FINDER = Pattern.compile(
            "(\\d{1,2}) x [E]*IDE", Pattern.CASE_INSENSITIVE );
    /**
     * looks for max ram capacity
     */
    private static final Pattern MAX_GIG_FINDER_L1 = Pattern.compile(
            "([0-9\\.]{1,4})\\s*GB[\\p{Print}&&[^/]]", Pattern.CASE_INSENSITIVE );
    /**
     * looks for max ram capacity each slot is 1 GB.
     */
    private static final Pattern MAX_GIG_FINDER_L2 = Pattern.compile(
            "(\\d{1,3})[ x]+184-pin DDR slots", Pattern.CASE_INSENSITIVE );
    /**
     * looks for max ram capacity
     */
    private static final Pattern MAX_GIG_FINDER_S = Pattern.compile(
            "(?:Supports up to |DDR2:)(\\d{1,3})\\s*GB", Pattern.CASE_INSENSITIVE );
    /**
     * look for links like this:  <a href="mbdetails.asp?model=n4sie-a7" class=mbtable>N4SIE-A7</a>
     */
    private static final Pattern MB_FINDER_L0 = Pattern.compile(
            "<a href=\"mbdetails\\.asp\\?model=([\\p{Alnum}\\-\\+\\. ]+)\" class=mbtable>([\\p{Alnum}\\-\\+\\s\\. ]+)" +
            "</a>",
            Pattern.CASE_INSENSITIVE
    );
    /**
     * look for links like this:  <a href='introduction.php?S_ID=397'>TPower X58A</a>
     */
    private static final Pattern MB_FINDER_L1 = Pattern.compile(
            "<a href='introduction.php\\?S_ID=(\\d+{1,4})'>([\\p{Alnum}\\-\\+\\s\\. ]+?)</a>",
            Pattern.CASE_INSENSITIVE );
    /**
     * gets same of motherboard
     */
    private static final Pattern MB_FINDER_S = Pattern.compile(
            "\"name\": \"([\\p{Alnum} \\-]+)&nbsp;\\(rev\\.&nbsp;([\\p{Digit}\\.]+)\\)\"," +
            "\\s+\"value\": \"(\\p{Digit}+)\""
    );
    /**
     * looks for info about RAM
     */
    private static final Pattern MEMORY_TYPE_FINDER_L = Pattern.compile(
            "(DDR2|DDR3|DDR|SDRAM)", Pattern.CASE_INSENSITIVE );
    //  Supports up to 2GB Memory
    /**
     * looks for info about RAM
     */
    private static final Pattern MEMORY_TYPE_FINDER_S = Pattern.compile(
            ">(?:Support Dual Channel|Support Triple Channel|supported|Support|DIMM| x) (DDR2|DDR3|DDR|SDRAM)",
            Pattern.CASE_INSENSITIVE );
    /**
     * looks for ram speed
     */
    private static final Pattern RAM_SPEED_MHZ_FINDER_1 = Pattern.compile(
            "(?:<b>)?DDR[23]*(?:</b>)?\\s*(?:up to )?((\\s*/\\s*|\\d{3,4}\\s*\\(OC\\)|\\d{3,4}\\s*\\(XMP\\)|\\d{3," +
            "4}\\s*O\\.C\\.|OC\\d{3,4}+|\\d{3,4})+)",
            Pattern.CASE_INSENSITIVE
    );
    /**
     * looks for ram speed
     */
    private static final Pattern RAM_SPEED_MHZ_FINDER_2 = Pattern.compile(
            "PC((\\s*/\\s*|\\d{3,4})+)", Pattern.CASE_INSENSITIVE );
    // PC2100/2700
    private static final Pattern RAM_SPEED_MHZ_SPLITTER = Pattern.compile(
            "\\s*/\\s*", Pattern.CASE_INSENSITIVE );
    /**
     * looks for count of sata2 ports
     */
    private static final Pattern SATA2_FINDER_L1 = Pattern.compile(
            "(\\d{1,2}) SATA[2]* (?:3Gb/s )?connector", Pattern.CASE_INSENSITIVE );
    /**
     * looks for count of sata2 ports
     */
    private static final Pattern SATA2_FINDER_L2 = Pattern.compile(
            "SATA[2]* RAID \\(up to (\\d{1,2}) devices", Pattern.CASE_INSENSITIVE );
    /**
     * looks for count of sata2 ports
     */
    private static final Pattern SATA2_FINDER_L3 = Pattern.compile(
            "(\\d{1,2}) x SATA[2]* (?:3Gb/s )?connector", Pattern.CASE_INSENSITIVE );
    /**
     * looks for count of sata2 ports
     */
    private static final Pattern SATA2_FINDER_L4 = Pattern.compile(
            "up to (\\d{1,2}) SATA[2]* devices", Pattern.CASE_INSENSITIVE );
    /**
     * looks for count of sata2 ports
     */
    private static final Pattern SATA2_FINDER_L5 = Pattern.compile(
            "(\\d{1,2}) x Serial ATA Connectors", Pattern.CASE_INSENSITIVE );
    // 4 x Serial ATA Connectors
    /**
     * looks for count of sata2 ports
     */
    private static final Pattern SATA2_FINDER_S1 = Pattern.compile(
            "(\\d{1,2}) SATA[2]* (connectors|devices)", Pattern.CASE_INSENSITIVE );
    /**
     * looks for count of sata2 ports
     */
    private static final Pattern SATA2_FINDER_S2 = Pattern.compile(
            "(\\d{1,2}) x SATA[2]* (?:3Gb/s )?connector", Pattern.CASE_INSENSITIVE );
    /**
     * looks for count of sata3 ports
     */
    private static final Pattern SATA3_FINDER_L1 = Pattern.compile(
            "(\\d{1,2}) SATA3 (?:6Gb/s )?connector", Pattern.CASE_INSENSITIVE );
    /**
     * looks for count of sata3 ports
     */
    private static final Pattern SATA3_FINDER_L2 = Pattern.compile(
            "SATA3 RAID \\(up to (\\d{1,2}) devices", Pattern.CASE_INSENSITIVE );
    /**
     * looks for count of sata3 ports
     */
    private static final Pattern SATA3_FINDER_L3 = Pattern.compile(
            "(\\d{1,2}) x SATA3 (?:6Gb/s )?connector", Pattern.CASE_INSENSITIVE );
    /**
     * looks for count of sata3 ports
     */
    private static final Pattern SATA3_FINDER_L4 = Pattern.compile(
            "up to (\\d{1,2}) SATA3 devices", Pattern.CASE_INSENSITIVE );
    /**
     * looks for count of sata3 ports
     */
    private static final Pattern SATA3_FINDER_S1 = Pattern.compile(
            "(\\d{1,2}) SATA3 (connectors|devices)", Pattern.CASE_INSENSITIVE );
    /**
     * looks for count of sata3 ports
     */
    private static final Pattern SATA3_FINDER_S2 = Pattern.compile(
            "(\\d{1,2}) x SATA3 (?:6Gb/s )?connector", Pattern.CASE_INSENSITIVE );
    /**
     * looks for socket
     */
    private static final Pattern SOCKET_FINDER_L = Pattern.compile(
            "socket\\s*[=:]\\s*([\\+ \\p{Alnum}]+)[\"&;<]", Pattern.CASE_INSENSITIVE );
    /**
     * looks for socket
     */
    private static final Pattern SOCKET_FINDER_S = Pattern.compile(
            "socket\\s*:\\s*([\\+ \\p{Alnum}]+)[\"&;<]", Pattern.CASE_INSENSITIVE );
    /**
     * looks for USB2Internal ports capacity
     */
    private static final Pattern USB2_INTERNAL_FINDER_L1 = Pattern.compile(
            "USB 2.0\\p{Print}+?(\\d+)[x ]*front", Pattern.CASE_INSENSITIVE );
    /**
     * looks for USB2Internal ports capacity
     */
    private static final Pattern USB2_INTERNAL_FINDER_L2 = Pattern.compile(
            "(\\d{1,2})[ x]*Front USB", Pattern.CASE_INSENSITIVE );
    /**
     * looks for USB2Rear ports capacity
     */
    private static final Pattern USB2_INTERNAL_FINDER_L3 = Pattern.compile(
            "(\\d{1,2})[ x]* USB 2.0 Header", Pattern.CASE_INSENSITIVE );
    /**
     * looks for USB2Internal ports capacity
     */
    private static final Pattern USB2_INTERNAL_FINDER_S = Pattern.compile(
            "(\\d{1,2})[ x]*USB 2.0 Header", Pattern.CASE_INSENSITIVE );
    /**
     * looks for USB2Rear ports capacity
     */
    private static final Pattern USB2_REAR_FINDER_L1 = Pattern.compile(
            "USB 2.0\\p{Print}+?\\((\\d{1,2})[x ]*rear", Pattern.CASE_INSENSITIVE );
    /**
     * looks for USB2Rear ports capacity
     */
    private static final Pattern USB2_REAR_FINDER_L2 = Pattern.compile(
            "(\\d{1,2})[ x]*Rear USB", Pattern.CASE_INSENSITIVE );
    /**
     * looks for USB2Rear ports capacity
     */
    private static final Pattern USB2_REAR_FINDER_L3 = Pattern.compile(
            "(\\d{1,2})[ x]* USB 2.0 Port", Pattern.CASE_INSENSITIVE );
    /**
     * looks for USB2Rear ports capacity
     */
    private static final Pattern USB2_REAR_FINDER_S = Pattern.compile(
            "(\\d{1,2})[ x]*USB 2\\.0 Port", Pattern.CASE_INSENSITIVE );
    /**
     * looks for USB3Internal ports capacity
     */
    private static final Pattern USB3_INTERNAL_FINDER_L = Pattern.compile(
            "(\\d{1,2}) USB3.0 ports on the front panel", Pattern.CASE_INSENSITIVE );
    /**
     * looks for USB3Internal ports capacity
     */
    private static final Pattern USB3_INTERNAL_FINDER_S = Pattern.compile(
            "(\\d{1,2})[ x]*USB 3.0 Header", Pattern.CASE_INSENSITIVE );
    /**
     * looks for USB3Rear ports capacity
     */
    private static final Pattern USB3_REAR_FINDER_L = Pattern.compile(
            "(\\d{1,2}) USB3.0 ports on the back panel", Pattern.CASE_INSENSITIVE );
    /**
     * looks for USB3Rear ports capacity
     */
    private static final Pattern USB3_REAR_FINDER_S = Pattern.compile(
            "(\\d{1,2})[ x]*USB 3\\.0 Port", Pattern.CASE_INSENSITIVE );
    /**
     * looks for integrated Video
     */
    private static final Pattern VIDEO_FINDER = Pattern.compile(
            "<b>GRAPHICS</b><td[\\s\"\\+\\./=\\p{Alnum}\\_]+>&#186;&nbsp;(?:<font color=\"red\">|On Chip|Integrated|)" +
            "*[\\( ]*(\\p{Alnum}+)",
            Pattern.CASE_INSENSITIVE
    );
    /**
     * looks for Integrated Video
     */
    private static final Pattern VIDEO_FINDER_L1 = Pattern.compile(
            "Video:</span><br>\\s+([/\\p{Alnum}]+)", Pattern.CASE_INSENSITIVE );
    /**
     * looks for Integrated Video
     */
    private static final Pattern VIDEO_FINDER_L2 = Pattern.compile(
            "INTEGRATED VIDEO</div>[\\s&'/;->\\p{Alnum}]+<li class='SpecList'>([/\\p{Alnum}]+)",
            Pattern.CASE_INSENSITIVE );
    /**
     * looks for Integrated Video
     */
    private static final Pattern VIDEO_FINDER_S = Pattern.compile(
            "INTEGRATED VIDEO</div>[\\s&'/;->\\p{Alnum}]+<li class='SpecList'>([/\\p{Alnum}]+)",
            Pattern.CASE_INSENSITIVE );
    /**
     * catch missing video regexes missed.
     */
    private static final Pattern VIDEO_VALIDATOR = Pattern.compile(
            "(?!video card)video", Pattern.CASE_INSENSITIVE );
    /**
     * looks for wattage
     */
    private static final Pattern WATTS_FINDER_S = Pattern.compile(
            "\\(Thermal Design Power\\)\\s*:\\s*(\\d{2,4})\\s*Watt", Pattern.CASE_INSENSITIVE );
    // (Thermal Design Power) : 125Watt
    static
        {
        configuration = new ConfigurationForMindprod();
        // combine dirsWithMacros and dirsWithIncludes into dirsToProcess;
        sourceDir = new File( configuration.getSourceDirWithSlashes() );
        }
    /**
     * Collect all the names of motherboard models, keeping specs on local hard disk..
     *
     * @throws IOException           if urls malformed or I/O trouble
     * @throws java.sql.SQLException
     */
    public static void fetchAllMBs() throws IOException, SQLException
        {
        manufacturer = Manufacturer.BIOSTAR;
        lastUpdated = BigDate.localToday();
        conn = connect();
        final PreparedStatement inserter = conn.prepareStatement(
                "INSERT INTO mboards( manufacturer, model, manufacturerPartNo, lastUpdated) VALUES(?,?,?,?);" );
        for ( String indexURL : INDEX_URLS )
            {
            final Get get = new Get();
            // no parms needed
            final String htmlListOfBiostarMBs = get.send( new URL( indexURL ), Get.UTF8 );
            final int responseCode = get.getResponseCode();
            if ( responseCode >= 300 || htmlListOfBiostarMBs == null )
                {
                err.println( "Could not fetch master Biostar page " + indexURL );
                System.exit( 1 );
                }
            Pattern mbFinder = indexURL.endsWith( ".asp" ) ? MB_FINDER_L0 : MB_FINDER_L1;
            final Matcher m0 = mbFinder.matcher( htmlListOfBiostarMBs );  // Matchers are used both for matching and
            // finding.
            while ( m0.find() )
                {
                manufacturerPartNo = m0.group( 1 );
                model = ST.condense( m0.group( 2 ).replace( '\n', ' ' ).replace( '\r', ' ' ) );
                out.println( manufacturer.ordinal() + ", " + model + ",, " + manufacturerPartNo + ", false" );
                final Get oneMB = new Get();
                final String rawMBSpecs = oneMB.send( new URL( Manufacturer.BIOSTAR.miningURL( manufacturerPartNo ) ),
                        Get.UTF8 );
                final int responseCode2 = oneMB.getResponseCode();
                if ( responseCode2 >= 300 || rawMBSpecs == null )
                    {
                    err.println( "Could not fetch Biostar motherboard page " + model );
                    System.exit( 1 );
                    }
                HunkIO.writeEntireFile( new File( "E:/mb/biostar/" + model + ".html" ), rawMBSpecs,
                        HunkIO.UTF8 );
                inserter.setInt( 1, manufacturer.ordinal() );
                inserter.setString( 2, model );
                inserter.setString( 3, manufacturerPartNo );
                inserter.setInt( 4, lastUpdated.ordinal() );
                inserter.executeUpdate();
                }
            }
        conn.close();
        }
    /**
     * extract specs from previously downloaded raw specs
     *
     * @throws java.sql.SQLException
     * @throws java.io.IOException
     */
    private static void extractSpecs() throws SQLException, IOException
        {
        manufacturer = Manufacturer.BIOSTAR;
        lastUpdated = BigDate.localToday();
        initDatabase();
        final File dir = new File( "E:/mb/biostar/" );
        CSVReader r = new CSVReader( new BufferedReader( new FileReader( new File( sourceDir,
                "mother/biostar.csv" ) ) ) );
        try
            {
            while ( true )
                {
                r.skip( 1 );  // mfr
                model = r.get();
                revision = r.get();
                manufacturerPartNo = r.get();
                r.skipToNextLine();
                final char group = manufacturerPartNo.length() > 3 ? 'l' : 's';
                String rawMBSpecs = HunkIO.readEntireFile( new File( dir, group + "/" + model + ".html" ),
                        HunkIO.UTF8 );
                // set default values so won't propagate from previous MB
                clearMBSpecs();
                switch ( group )
                    {
                    case 'l':
                    {
                    // new style pages
                    extractAudioChannels( rawMBSpecs, new Pattern[] { AUDIO_CHANNEL_FINDER } );
                    extractFormFactor( rawMBSpecs, FORM_FACTOR_FINDER_L, 1 );
                    extractMemoryType( rawMBSpecs, MEMORY_TYPE_FINDER_L );
                    extractMaxGig( rawMBSpecs, MAX_GIG_FINDER_L1 );
                    final int maxGig1 = maxGig;
                    extractMaxGig( rawMBSpecs, MAX_GIG_FINDER_L2 );
                    extractIde( rawMBSpecs, new Pattern[] { IDE_FINDER } );
                    maxGig = Math.max( maxGig1, maxGig );
                    extractRamSpeedMHz( rawMBSpecs, RAM_SPEED_MHZ_FINDER_1, RAM_SPEED_MHZ_SPLITTER );
                    int ramSpeedMHz1 = ramSpeedMHz;
                    extractRamSpeedMHz( rawMBSpecs, RAM_SPEED_MHZ_FINDER_2, RAM_SPEED_MHZ_SPLITTER );
                    ramSpeedMHz = Math.max( ramSpeedMHz1, pcToMHz( ramSpeedMHz ) );
                    // no extractSocket, get info from surrounding pages;
                    extractSata2( rawMBSpecs,
                            new Pattern[] {
                                    SATA2_FINDER_L1,
                                    SATA2_FINDER_L2,
                                    SATA2_FINDER_L3,
                                    SATA2_FINDER_L4,
                                    SATA2_FINDER_L5 }
                    );
                    extractSata3( rawMBSpecs,
                            new Pattern[] { SATA3_FINDER_L1, SATA3_FINDER_L2, SATA3_FINDER_L3, SATA3_FINDER_L4 } );
                    extractSocket( rawMBSpecs, new Pattern[] { SOCKET_FINDER_L } );
                    extractUSB( rawMBSpecs,
                            new Pattern[ 0 ],
                            new Pattern[] { USB2_REAR_FINDER_L1, USB2_REAR_FINDER_L2, USB2_REAR_FINDER_L3 },
                            new Pattern[] { USB2_INTERNAL_FINDER_L1, USB2_INTERNAL_FINDER_L2, USB2_INTERNAL_FINDER_L3 },
                            new Pattern[ 0 ],
                            new Pattern[] { USB3_REAR_FINDER_L },
                            new Pattern[] { USB3_INTERNAL_FINDER_L } );
                    extractVideo( rawMBSpecs, new Pattern[] { VIDEO_FINDER_L1, VIDEO_FINDER_L2 } );
                    validateVideo( rawMBSpecs, VIDEO_VALIDATOR );
                    // no extractWattsL, no info
                    break;
                    }
                    case 's':
                    {
                    // old style pages
                    extractAudioChannels( rawMBSpecs, new Pattern[] { AUDIO_CHANNEL_FINDER } );
                    extractFormFactor( rawMBSpecs, FORM_FACTOR_FINDER_S, 1 );
                    extractMemoryType( rawMBSpecs, MEMORY_TYPE_FINDER_S );
                    extractMaxGig( rawMBSpecs, MAX_GIG_FINDER_S );
                    extractIde( rawMBSpecs, new Pattern[] { IDE_FINDER } );
                    extractRamSpeedMHz( rawMBSpecs, RAM_SPEED_MHZ_FINDER_1, RAM_SPEED_MHZ_SPLITTER );
                    int ramSpeedMHz1 = ramSpeedMHz;
                    extractRamSpeedMHz( rawMBSpecs, RAM_SPEED_MHZ_FINDER_2, RAM_SPEED_MHZ_SPLITTER );
                    ramSpeedMHz = Math.max( ramSpeedMHz1, pcToMHz( ramSpeedMHz ) );
                    // no extractSocket, get info from surrounding pages;
                    extractSata2( rawMBSpecs, new Pattern[] { SATA2_FINDER_S1, SATA2_FINDER_S2 } );
                    extractSata3( rawMBSpecs, new Pattern[] { SATA3_FINDER_S1, SATA3_FINDER_S2 } );
                    extractSocket( rawMBSpecs, new Pattern[] { SOCKET_FINDER_S } );
                    extractUSB( rawMBSpecs,
                            new Pattern[ 0 ],
                            new Pattern[] { USB2_REAR_FINDER_S },
                            new Pattern[] { USB2_INTERNAL_FINDER_S },
                            new Pattern[ 0 ],
                            new Pattern[] { USB3_REAR_FINDER_S },
                            new Pattern[] { USB3_INTERNAL_FINDER_S } );
                    extractVideo( rawMBSpecs, new Pattern[] { VIDEO_FINDER_S } );
                    validateVideo( rawMBSpecs, VIDEO_VALIDATOR );
                    extractWatts( rawMBSpecs, WATTS_FINDER_S );
                    break;
                    }
                    default:
                        err.println( "invalid group" );
                    }
                dumpExtracts();
                updateMBFields();
                }
            }
        catch ( EOFException e )
            {
            out.println( incomplete + " incomplete records" );
            r.close();
            closeDatabase();
            }
        }
    /**
     * extract wattage from Biostar motherboard page
     *
     * @param mbData motherboard page data from BioStar website.
     * @param mbName name of motherboard model we are mining.
     */
    private static void extractWattsS( String mbData, String mbName )
        {
        Matcher m = WATTS_FINDER_S.matcher( mbData );
        if ( m.find() )
            {
            watts = Integer.parseInt( m.group( 1 ) );
            }
        }
    /**
     * handle oddities, typos on website etc.
     *
     * @throws java.sql.SQLException
     */
    private static void oddDucks() throws SQLException
        {
        // rename 6VLQ to M6VLQ manually
        manufacturer = Manufacturer.BIOSTAR;
        lastUpdated = BigDate.localToday();
        conn = connect();
        final Statement updater = conn.createStatement();
        // will not trigger exception even if records already deleted.
        updater.executeUpdate( "UPDATE mboards SET maxGig=2 WHERE manufacturer="
                               + manufacturer.ordinal()
                               + " AND model='M7VIG';" );
        updater.executeUpdate( "DELETE FROM mboards WHERE manufacturer="
                               + manufacturer.ordinal()
                               + " AND model='I945G-M7';" );
        updater.executeUpdate( "DELETE FROM mboards WHERE manufacturer="
                               + manufacturer.ordinal()
                               + " AND model='I945P-A7';" );
        updater.executeUpdate( "DELETE FROM mboards WHERE manufacturer="
                               + manufacturer.ordinal()
                               + " AND model='K8M800-M7A';" );
        updater.executeUpdate( "UPDATE mboards SET maxGig=8 WHERE manufacturer="
                               + manufacturer.ordinal()
                               + " AND model='TP43E Combo';" );
        updater.executeUpdate( "UPDATE mboards SET formFactor=" + FormFactor.ATX.ordinal() + " WHERE manufacturer="
                               + manufacturer.ordinal()
                               + " AND model='U8568 Pro';" );
        // missing info taken from http://www.biostar-usa.com/mbdetails.asp?model=P4TAW%20EXTREME
        updater.executeUpdate( "UPDATE mboards SET formFactor=" + FormFactor.ATX.ordinal()
                               + ",widthInCm=24.5,heightInCm=29.3 WHERE manufacturer="
                               + manufacturer.ordinal()
                               + " AND model='P4TAW Extreme';" );
        // missing info taken from http://www.biostar-usa.com/mbdetails.asp?model=p4tdq+pro
        updater.executeUpdate( "UPDATE mboards SET maxGig=2,ramSpeedMHz=2700,memoryType="
                               + MemoryType.DDR.ordinal()
                               + "  WHERE manufacturer="
                               + manufacturer.ordinal()
                               + " AND model='P4TDQ Pro';" );
        // missing info taken from http://www.biostar-usa.com/mbdetails.asp?model=m7sxg
        // usually http://www.biostar-usa.com/app/en-us/mb/content.php?S_ID=334
        updater.executeUpdate( "UPDATE mboards SET formFactor=" + FormFactor.ATX.ordinal()
                               + ",widthInCm=24.4,heightInCm=24.4"
                               + "  WHERE manufacturer="
                               + manufacturer.ordinal()
                               + " AND model='M7SXG';" );
        conn.close();
        }
    /**
     * Put the skeleton info into the DB.
     *
     * @throws IOException           if urls malformed or I/O trouble
     * @throws java.sql.SQLException
     */
    private static void skeleton() throws IOException, SQLException
        {
        manufacturer = Manufacturer.BIOSTAR;
        lastUpdated = BigDate.localToday();
        conn = connect();
        final PreparedStatement inserter = conn.prepareStatement(
                "INSERT INTO mboards( manufacturer, model, revision, manufacturerPartNo, lastUpdated) VALUES(?,?,?,?," +
                "?);"
        );
        CSVReader r = new CSVReader( new BufferedReader( new FileReader( new File( sourceDir,
                "mother/biostar.csv" ) ) ) );
        try
            {
            while ( true )
                {
                r.skip( 1 );
                model = r.get();
                revision = r.get();
                manufacturerPartNo = r.get();
                r.skipToNextLine();
                out.println( manufacturer.ordinal() + ", " + model + ", " + revision + ", " + manufacturerPartNo + "," +
                             " false, " + lastUpdated );
                // record our findings in SQL
                inserter.setInt( 1, manufacturer.ordinal() );
                inserter.setString( 2, model );
                inserter.setString( 3, revision );
                inserter.setString( 4, manufacturerPartNo );
                inserter.setInt( 5, lastUpdated.ordinal() );
                inserter.executeUpdate();
                }
            }
        catch ( EOFException e )
            {
            r.close();
            conn.close();
            }
        }
    private static void test()
        {
        String s2 = " Micro ATX form factor<br>24.5cm x 24.5cm ( W x L )  ";
        Matcher m = FORM_FACTOR_FINDER_L.matcher( s2 );
        if ( m.find() )
            {
            out.println( "[" + m.group( 1 ) + "]" );
            }
        else
            {
            out.println( "no find" );
            }
        }
    /**
     * collect all motherboard data from the Biostar website
     *
     * @param args not used
     *
     * @throws IOException           if urls malformed or I/O trouble
     * @throws java.sql.SQLException
     */
    public static void main( final String[] args ) throws IOException, SQLException
        {
        // test();
        // fetchAllMBs();
        // skeleton();
        extractSpecs();
        oddDucks();
        }
    //todo screenscrape the following fields. Export to database.
    //video
    // socket
    }