URL|
*/
private static final Pattern INDIRECT_HOME_URL = Pattern.compile(
" | URL | Submission URL |
*/
private static final Pattern INDIRECT_SUBMIT_URL = Pattern.compile(
" | Submission URL | Modified | 2012-01-19 13:02:17 |
* on main page is shown as 15-01-2014 | Modified(\\p{Digit}{4}\\-\\p{Digit}{2}\\-\\p{Digit}{2}) ",
Pattern.CASE_INSENSITIVE );
/**
* extracts site name from siteinfo page
* CoolScreens
*/
private static final Pattern SITENAME = Pattern.compile(
"([ !%'\\(\\)" +
"\\-\\._\\p{Alnum}]+)",
Pattern.CASE_INSENSITIVE
);
/**
* number of pages on the ASP website http://padsites.org. Change comment on grabasp.btm
*/
private static int pages;
/**
* get url on ASP site for given page 1 .. PAGES
*
* @param pageNumber page number
*
* @return URL on asp site of that page of database.
* @throws java.net.MalformedURLException if bad URL generated
*/
private static URL URLForPage( int pageNumber ) throws MalformedURLException
{
return new URL( ASP_HOST + "/page" + ST.toLZ( pageNumber, 3 ) + "/" );
}
/**
* get local file for given page 1 .. PAGES
*
* @param pageNumber page number
*
* @return local file where that page's html is stored..
*/
private static File fileForPage( int pageNumber )
{
return new File( Build.MINDPROD_SOURCE + "/submitter/" + ST.toLZ( pageNumber, 3 ) + ".html" );
}
/**
* fetch all ASP web pages from their site
*
* @throws java.io.IOException if cannot save page.
*/
private static void getASPWebPages() throws IOException
{
Get g = new Get();
for ( int pageNumber = 1; pageNumber <= pages; pageNumber++ )
{
final String page = g.send( URLForPage( pageNumber ), Get.UTF8 );
HunkIO.writeEntireFile( fileForPage( pageNumber ), page );
}
}
/**
* dump out regexes
*/
private static void proofreadRegexes()
{
out.println( SITENAME );
out.println( INDIRECT_HOME_URL );
out.println( INDIRECT_SUBMIT_URL );
}
/**
* scan one page of the ASP website, extracting padsites. Slow because it chases each URL to resolve redirects.
*
* @param page HTML from one page from ASP site
* @param w CSVWrite to export found PADSites to
*
* @throws MalformedURLException if bad URL generated
*/
private static void scanPage( String page, CSVWriter w ) throws MalformedURLException
{
int from = 0;
rowLoop:
while ( true )
{
final int start = page.indexOf( "", start + "".length();
for ( String junkWord : rowsToIgnore )
{
if ( row.contains( junkWord ) )
{
// ignore
continue rowLoop;
}
}
final String siteName;
final String infoURLCore;
// extract sitename
// we want only two things from the row, name of site, and indirect URL to site.
final Matcher s = SITENAME.matcher( row );
if ( s.find() )
{
infoURLCore = s.group( 1 );
siteName = s.group( 2 );
}
else
{
out.println( ">>>failed to find sitename : " + row );
continue;
}
final Get g = new Get();
final String infoPage = g.send( new URL( ASP_HOST + "/siteinfo/" + infoURLCore +
".html" ), Get.UTF8 );
final String indirectHomeURLCore;
final Matcher h = INDIRECT_HOME_URL.matcher( infoPage );
if ( h.find() )
{
indirectHomeURLCore = h.group( 1 );
}
else
{
out.println( ">>>failed to find home URL on info page : " + infoPage );
continue;
}
final String indirectSubmitURLCore;
final Matcher b = INDIRECT_SUBMIT_URL.matcher( infoPage );
if ( b.find() )
{
indirectSubmitURLCore = b.group( 1 );
}
else
{
out.println( ">>>failed to find Submit URL on info page : " + infoPage );
continue;
}
final String lastModified;
final Matcher m = MODIFIED.matcher( infoPage );
if ( m.find() )
{
lastModified = m.group( 1 );
}
else
{
out.println( ">>>failed to find last modified date on info page : " + infoPage );
continue;
}
final Chase hh = new Chase();
final String homeURL = hh.send( new URL( ASP_HOST + "/go_site.php?uin=site-" +
indirectHomeURLCore ) );
final Chase bb = new Chase();
final String submitURL = bb.send( new URL( ASP_HOST + "/go_site.php?uin=site-" +
indirectSubmitURLCore + "&type=1" ) );
// echo to screen and well as log to file
out.println( siteName + ", " + homeURL + ", " + submitURL + ", " + lastModified );
// name, homeUrl, submitUrl , date
w.put( siteName );
w.put( homeURL );
w.put( submitURL );
w.put( lastModified );
w.nl();
} // end while
}
/**
* Refresh PAGES first.
* Results end up in E:\com\mindprod\submitter\grabasp.csv and console.
* sort, then prune old PADSites before analysing new ones as candidates.
* use FilterASP.btm to sort and clean up.
*
* @param args args[0]="-fetch" to refetch raw data from ASP website, otherwise reprocessesses previously fetched
* data
*
* @throws IOException if problems reading/writing files.
*/
public static void main( String args[] ) throws IOException
{
pages = Integer.parseInt( args[ 0 ] );
// proofreadRegexes();
if ( args.length >= 2 && args[ 1 ].equalsIgnoreCase( "-fetch" ) )
{
getASPWebPages();
}
final CSVWriter w = new CSVWriter( EIO.getPrintWriter( new File( PADSITE_CSV_NAME ), 32 * 1024, EIO.UTF8 ) );
for ( int pageNumber = 1; pageNumber <= pages; pageNumber++ )
{
final String page = HunkIO.readEntireFile( fileForPage( pageNumber ) );
out.println( "-----------scanning page " + pageNumber + "------------------" );
w.nl( ">>>page " + pageNumber );
// takes a while, since has to read matching info pages.
scanPage( page, w );
}
w.close();
}
}
|