/*
* [JDisplayOrphans.java]
*
* Summary: finds Finds orphaned JDisplay snippets, i.e. snippets no longer used by website.
*
* Copyright: (c) 2007-2017 Roedy Green, Canadian Mind Products, http://mindprod.com
*
* Licence: This software may be copied and used freely for any purpose but military.
* http://mindprod.com/contact/nonmil.html
*
* Requires: JDK 1.8+
*
* Created with: JetBrains IntelliJ IDEA IDE http://www.jetbrains.com/idea/
*
* Version History:
* 1.0 2007-06-04 first version.
* 1.1 2007-07-26 add annotations
* 1.2 2007-08-20 tidy
* 1.3 2008-01-01
* 1.4 2008-01-01
* 1.5 2008-01-01
* 1.6 2008-01-11 add support for hex and octal numerics.
* 1.7 2008-04-18 get JDisplay and CSS font renderings in closer sync
* 1.8 2009-08-30 tone down colour for keywords.
* 1.9 2011-11-22 delete potentially tampered .html files.
* 2.0 2012-04-28 put in checks to deal with dirs with no snippets.
* 2.1 2014-08-01 switch from Adler32 to FNV1a64 checksums
* 2.2 2014-08-01 change *.html to *.htm, *.adler to *.checksum, use 64bit FNV1a64 checksums.
*/
package com.mindprod.jdisplayorphans;
import com.mindprod.common18.EIO;
import com.mindprod.common18.FNV1a64Digester;
import com.mindprod.common18.Misc;
import com.mindprod.common18.ST;
import com.mindprod.filter.EndsWithFilter;
import com.mindprod.filter.OnlyFilesFilter;
import com.mindprod.htmlmacros.macro.Global;
import com.mindprod.htmlmacros.support.ConfigurationForMindprod;
import com.mindprod.hunkio.HunkIO;
import java.io.BufferedReader;
import java.io.DataInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import static java.lang.System.*;
/**
* finds Finds orphaned JDisplay snippets, i.e. snippets no longer used by website.
*
* Must run e.bat first to prepare list of all used snippets.
*
* @author Roedy Green, Canadian Mind Products
* @version 2.2 2014-08-01 change *.html to *.htm, *.adler to *.checksum, use 64bit FNV1a64 checksums.
* @since 2007-06-04
*/
public final class JDisplayOrphans
{
// declarations
/**
* true if want extra debug output
*/
private static final boolean DEBUGGING = false;
private static final int FIRST_COPYRIGHT_YEAR = 2007;
/**
* undisplayed copyright notice
*
* @noinspection UnusedDeclaration
*/
private static final String EMBEDDED_COPYRIGHT =
"Copyright: (c) 2007-2017 Roedy Green, Canadian Mind Products, http://mindprod.com";
/**
* @noinspection UnusedDeclaration
*/
private static final String RELEASE_DATE = "2014-08-01";
/**
* Version
*/
@SuppressWarnings( { "UnusedDeclaration" } )
private static final String VERSION_STRING = "2.2";
/**
* list of exception files which are not really orphans
*/
private static final HashSet exceptions =
new HashSet<>( Arrays.asList( "untouch.digest",
"index.html",
"DESCRIPT.ION" ) );
/**
* name of file e.g. book/9789506710118.html referenced by the insert
*/
private static final HashSet insertsFound = new HashSet<>( 1000 );
private static String sourceDirName;
/**
* directories that have snippet, use /. Converted to \ on display. Webroot relative.
*/
private static List dirsWithSnippets;
/**
* root directory of website
*/
private static File webRootDir;
/**
* all snippets actually used in the HTML
*/
private static HashSet usedSnippets;
// methods
/**
* .
* delete snippets *.ser and *.ifram ef dates strange or checksums fail
*/
private static void deleteCorruptSers() throws IOException
{
// find orphan iframe/*.html. Delete.
// webroot relative with /
for ( String dir : dirsWithSnippets )
{
final File snippetDir = new File( webRootDir, dir + "/snippet" );
final File serDir = new File( webRootDir, dir + "/snippet/ser" );
final File iframeDir = new File( webRootDir, dir + "/snippet/iframe" );
final String[] extantIframes = iframeDir.list( new EndsWithFilter( ".htm" ) );
if ( extantIframes != null )
{
// just filenames without any dir.
for ( String extantIframe : extantIframes )
{
final File iframeFile = new File( iframeDir, extantIframe );
// chop off .html
final String snippet = EIO.getCoreName( iframeFile );
final File snippetFile = new File( snippetDir, snippet );
final File serFile = new File( serDir, snippet + ".ser" );
final File checksumFile = new File( serDir, snippet + ".checksum" );
if ( usedSnippets.contains( dir + "/snippet/" + snippet )
|| exceptions.contains( snippet ) )
{
// check for tampered .html
if ( iframeDir.exists() && serFile.exists() && iframeFile.lastModified() > serFile
.lastModified() + 1000 )
{
// like a search replace wrecked the html
// force regeneration
err.println( iframeFile.toString() + " corrupted. Deleting to force regeneration." );
serFile.delete();
checksumFile.delete();
iframeFile.delete();
}
// use checksums to make sure no tampering
else if ( snippetFile.exists() && serFile.exists() && iframeFile.exists() && checksumFile.exists
() )
{
// no need for buffer
final FileInputStream fis = new FileInputStream( checksumFile );
final DataInputStream dis = new DataInputStream( fis );
final long expectedChecksumForSnippet = dis.readLong();
final long expectedChecksumForSer = dis.readLong();
final long expectedChecksumForIframe = dis.readLong();
dis.close();
// the checksum file includes three checksums.
final FNV1a64Digester digester = new FNV1a64Digester();
digester.update( HunkIO.readEntireFileAsBytes( snippetFile ) );
final long checksumForSnippet = digester.getValue();
digester.reset();
digester.update( HunkIO.readEntireFileAsBytes( serFile ) );
final long checksumForSer = digester.getValue();
digester.reset();
digester.update( HunkIO.readEntireFileAsBytes( iframeFile ) );
final long checksumForIframe = digester.getValue();
if ( DEBUGGING && ( expectedChecksumForSnippet != checksumForSnippet || expectedChecksumForSer !=
checksumForSer ||
expectedChecksumForIframe != checksumForIframe ) )
{
out.println( "was : " + expectedChecksumForSnippet + " : " + expectedChecksumForSer + " : "
+ expectedChecksumForIframe
+ " now: " + checksumForSnippet + " : " + checksumForSer + " : " +
checksumForIframe );
}
if ( expectedChecksumForSnippet != checksumForSnippet )
{
err.println( "Snippets checksum changed, deleting " + serFile + " and " + iframeFile
+ " to force regeneration." );
serFile.delete();
checksumFile.delete();
iframeFile.delete();
}
else if ( expectedChecksumForSer != checksumForSer )
{
err.println( "*.ser checksum changed. *.ser file corrupted " + serFile + " and " +
iframeFile );
serFile.delete();
checksumFile.delete();
iframeFile.delete();
}
else if ( expectedChecksumForIframe != checksumForIframe )
{
err.println( "*.iframe file checksum changed. *.iframe file corrupted , " +
"deleting " + serFile + " and " + iframeFile + " to force regeneration." );
serFile.delete();
checksumFile.delete();
iframeFile.delete();
}
}
}
else
{
err.println( "Orphan iframe html deleted " + iframeFile.toString() );
serFile.delete();
checksumFile.delete();
iframeFile.delete();
}
}
}
}
} // /method
/**
* delete ser if no matching Iframe or iframe without matching ser.
*/
private static void deleteMismatches()
{
for ( String dir : dirsWithSnippets )
{
final File serDir = new File( webRootDir, dir + "/snippet/ser" );
final String[] extantSers = serDir.list( new EndsWithFilter( ".ser" ) );
final HashSet unmatchedSers;
if ( extantSers != null )
{
unmatchedSers = new HashSet<>( extantSers.length * 130 / 100 );
for ( String extantSer : extantSers )
{
final String snippet = ST.chopTrailingString( extantSer, ".ser" );
unmatchedSers.add( snippet );
}
}
else
{
unmatchedSers = new HashSet<>();
}
final File iFrameDir = new File( webRootDir, dir + "/snippet/iframe" );
final String[] extantIframes = iFrameDir.list( new EndsWithFilter( ".htm" ) );
final HashSet unmatchedIframes;
if ( extantIframes != null )
{
unmatchedIframes = new HashSet<>( extantIframes.length * 130 / 100 );
for ( String extantIframe : extantIframes )
{
final String snippet = ST.chopTrailingString( extantIframe, ".htm" );
unmatchedIframes.add( snippet );
}
}
else
{
unmatchedIframes = new HashSet<>();
}
final HashSet common = new HashSet<>( unmatchedSers );
common.retainAll( unmatchedIframes );
unmatchedSers.removeAll( common );
unmatchedIframes.removeAll( common );
// at this point unmatchedSers and unmatchedIframes just have unmatched.
for ( String snippet : unmatchedSers )
{
// missing iframe
final File serFile = new File( serDir, snippet + ".ser" );
final File checksumFile = new File( serDir, snippet + ".checksum" );
final File iframeFile = new File( iFrameDir, snippet + ".htm" );
err.println( iframeFile + " missing, deleting " + serFile + " to force regeneration." );
serFile.delete();
checksumFile.delete();
iframeFile.delete();
}
for ( String snippet : unmatchedIframes )
{
// missing ser
final File serFile = new File( serDir, snippet + ".ser" );
final File checksumFile = new File( serDir, snippet + ".checksum" );
final File iframeFile = new File( iFrameDir, snippet + ".htm" );
err.println( serFile + " missing, deleting " + iframeFile + " to force regeneration." );
serFile.delete();
checksumFile.delete();
iframeFile.delete();
}
}
} // /method
/**
* delete *.ser and *.iframe for unused snippets. We leave the snippet itself alone.
*/
private static void deleteOrphanSers()
{
// find orphan *.ser. Delete.
for ( String dir : dirsWithSnippets )
{
final String[] extantSers =
new File( webRootDir, dir + "/snippet/ser" ).list( new EndsWithFilter(
".ser" ) );
if ( extantSers != null )
{
for ( String extantSer : extantSers )
{
// chop off .ser
final String snippet =
extantSer.substring( 0,
extantSer.length()
- ".ser".length()
);
if ( usedSnippets.contains( dir + "/snippet/" + snippet )
|| exceptions.contains( snippet ) )
{
}
else
{
err.println( "Orphan ser deleted " + ( dir
+ "/snippet/ser/"
+ extantSer ) );
new File( webRootDir, dir + "/snippet/ser/" + extantSer ).delete();
new File( webRootDir, dir + "/snippet/ser/" + snippet + ".checksum" ).delete();
new File( webRootDir, dir + "/snippet/iframe/" + snippet + ".htm" ).delete();
}
}
}
}
} // /method
/**
* just display orphaned (unused) snippets without deleting them
*/
private static void displayOrphanSnippets()
{
// find orphan snippets. Just notify. Don't delete.
for ( String dir : dirsWithSnippets )
{
final String[] extantSnippets =
new File( webRootDir, dir + "/snippet" ).list( new OnlyFilesFilter() );
if ( extantSnippets != null )
{
for ( String extantSnippet : extantSnippets )
{
if ( usedSnippets.contains( dir + "/snippet/" + extantSnippet )
|| exceptions.contains( extantSnippet ) )
{
}
else
{
err.println( "Orphan snippet " + ( dir + "/snippet/" + extantSnippet ) );
}
}
}
}
} // /method
/**
* load up set of all snippets actually referenced in the HTML
* Generated by last ReplaceMindprod run.
*/
private static void loadUsedSnippets()
{
usedSnippets = new HashSet<>( 1500 );
// read list of used snippets into a HashSet
try
{
FileReader ir = new FileReader( sourceDirName + "/jdisplayorphans/usedsnippets.log" );
BufferedReader br = new BufferedReader( ir );
while ( true )
{
final String uSnippetName = br.readLine();
if ( uSnippetName == null )
{
break;
}
usedSnippets.add( uSnippetName ); // fully qualified, webroot-relative snippet name with /.
}
// no eof
br.close();
}
catch ( FileNotFoundException e )
{
err.println();
err.println( e.getMessage() );
err.println( "could not open " + sourceDirName + "/jdisplayorphans/usedsnippets.log" );
err.println();
System.exit( 1 );
}
catch ( IOException e )
{
err.println();
err.println( e.getMessage() );
err.println( "could not read " + sourceDirName + "/jdisplayorphans/usedsnippets.log" );
err.println();
System.exit( 1 );
}
out.println( usedSnippets.size() + " snippets used" );
} // /method
/**
* detect orphan snippets
*
* @param args not used
*/
public static void main( String[] args ) throws IOException
{
Global.installConfiguration( new ConfigurationForMindprod() );
// combine dirsWithMacros and dirsWithIncludes into dirsToProcess;
webRootDir = new File( Global.configuration.getLocalWebrootWithSlashes() );
sourceDirName = Global.configuration.getSourceDirWithSlashes();
dirsWithSnippets = Global.configuration.getDirsContainingSnippets();
if ( DEBUGGING )
{
out.println( "webRootDir:" + webRootDir );
out.println( "sourceDirName:" + sourceDirName );
}
out.println( "loading log of used snippets..." );
loadUsedSnippets();
out.println( "\n---Orphan Snippets---" );
displayOrphanSnippets();
out.println( "\ndeleting mismatched snippets..." );
deleteMismatches();
out.println( "\ndeleting orphan snippets..." );
deleteOrphanSers();
out.println( "\ndeleting corrupt snippets..." );
deleteCorruptSers();
out.println( "done" );
Misc.trackLastThread();
System.exit( 0 );
} // /method
// /methods
}