/* * [JPrep.java] * * Summary: Pre-parse snippets to prepare tokenized serialised versions for JDisplay. * * Copyright: (c) 2004-2017 Roedy Green, Canadian Mind Products, http://mindprod.com * * Licence: This software may be copied and used freely for any purpose but military. * http://mindprod.com/contact/nonmil.html * * Requires: JDK 1.8+ * * Created with: JetBrains IntelliJ IDEA IDE http://www.jetbrains.com/idea/ * * Version History: * 1.7 2005-07-27 major rewrite using new tokenizer interface * 1.8 2005-12-25 adds support for *.properties files. * 1.9 2005-12-25 add parser for *.csv files * 2.0 2005-12-25 add parser for *.ini files * 2.1 2006-01-22 correct bug in handling of entities in HTMLState * 2.2 2006-03-06 reformat with IntelliJ, add Javadoc * 2.3 2007-05-05 add iformat renderering, use of snippet/ser and snippet/iformat * 2.4 2007-07-26 add support for annotations. * 2.5 2007-08-20 IntelliJ inspector cleanup of code. * 2.6 2008-01-11 add support for hex and octal numerics. * 2.7 2008-02-23 fix Java parser to bold variable definitions. * 2.8 2008-04-18 get JDisplay and CSS font renderings in closer sync * 2.9 2008-04-30 improve way numeric literals are rendered in Java. * 3.0 2008-08-08 add parser for vanilla text * 3.1 2009-04-12 shorter style names, improved highlighting. * 3.2 2009-08-30 tone down colour for keywords. * 3.3 2010-02-08 highlight begin and ends of comments and CDATAs specially. * 3.4 2010-02-10 add manifest tokenizer. * 3.5 2014-08-01 change *.html to *.htm, *.adler to *.checksum, use 64bit FNV1a64 checksums. */ package com.mindprod.jprep; import com.mindprod.common18.EIO; import com.mindprod.common18.FNV1a64Digester; import com.mindprod.common18.Misc; import com.mindprod.compactor.Compactor; import com.mindprod.fastcat.FastCat; import com.mindprod.filter.ExtensionListFilter; import com.mindprod.htmlmacros.support.JPrepConfiguration; import com.mindprod.hunkio.HunkIO; import com.mindprod.jdisplay.Footprint; import com.mindprod.jtokens.Token; import java.awt.Panel; import java.io.BufferedOutputStream; import java.io.ByteArrayOutputStream; import java.io.DataOutputStream; import java.io.File; import java.io.FileOutputStream; import java.io.FileWriter; import java.io.FilenameFilter; import java.io.IOException; import java.io.ObjectOutputStream; import java.io.OutputStream; import java.util.List; import java.util.zip.GZIPOutputStream; import static java.lang.System.*; /** * Pre-parse snippets to prepare tokenized serialised versions for JDisplay. *
* JDisplay is an Applet to render * large snippets. JDisplayAux handles inserting code into the HTML for htmlmacros. JPrep parses the * snippets. * Each Tokenizer decides on its own which extensions it will handle. If two tokenizers handle the same extension, * the one that looks at it first will process it. Subsequent ones will see the *.ser file already done and leave it * be. * That is determined by the order of invoking * Tokenizers in prepareAllSnippetsForOneDirectory. * Currently there are specific Tokenizers for: bat, sv, http (headers), html, ini, java, mft, properties, sql, text. * Some tokenizer handle multiple extensions. * TODO: tokenizers for css, C source, ASM source * * @author Roedy Green, Canadian Mind Products * @version 3.5 2014-08-01 change *.html to *.htm, *.adler to *.checksum, use 64bit FNV1a64 checksums. * @since 2004-06-06 */ public final class JPrep { // declarations /** * normally true. Might set false to have a look at the binary stream during debugging. */ private static final boolean COMPRESS_SER = true; /** * DOCTYPE_DTD for iframe snippet, case-sensitive, * using HTML5 */ private static final String DOCTYPE_FOR_IFRAME = ""; /** * undisplayed copyright notice */ @SuppressWarnings( { "UnusedDeclaration" } ) private static final String EMBEDDED_COPYRIGHT = "Copyright: (c) 2004-2017 Roedy Green, Canadian Mind Products, http://mindprod.com"; @SuppressWarnings( { "UnusedDeclaration" } ) private static final String RELEASE_DATE = "2014-08-01"; /** * embedded version string. */ @SuppressWarnings( { "UnusedDeclaration" } ) private static final String VERSION_STRING = "3.5"; /** * Needed to get at FontMetrics */ private static final Panel dummyPanelForFontMetrics = new Panel(); /* * R E S P O N S I B I L I T I E S : JPrep : preparing the tokens, * calculating approximate footprint, the approximate lineNumber footprint, * preparing *.ser file. Knows names of various possible JTokenizers. * PrepTokenizer : analysing text and producing an array of Tokens to render * it in pretty form. Knows Token classes using in rendering that class of * document. Token : decides on the text, the font, the colour for the * token, and how it would be rendered inline as prettified HTML, normally * using as CSS style. Knows TokenColourScheme, TokenFonts. JDisplay macro : * deciding how much screen real estate to use, whether to inline, whether a * bar/line numbers are needed. Knows Geometry(margins) JDisplay Applet : * recalculating the geometry based on the user's font metrics. deciding * whether scroll bars are needed, display, both pretty and plain as a * TextArea that supports copy/paste. Knows Geometry(margins) * Also prepares the iframe html snippets, compacted in snippet/iframe. */ /* * V O C A B U L A R Y : The Payload footprint: widthxheight in pixels. This * the space required to render just the tokens. It does not include the * line numbers, margins, bars etc. However, when scrolling it does include * the parts currently offscreen. So it may sometimes be smaller than the * Applet footprint and sometimes larger. */ private static final int FIRST_COPYRIGHT_YEAR = 2004; // /declarations // methods /** * process one file with the given tokenizer to and leave the result on disk in a *.html file. * * @param tokens array of Tokens forming the snippet. * @param snippetDir directory where the file is. * @param snippetName unqualified snippetName to tokenize e.g. "sample.javafrag" * * @return FNV1a64 checksum of iframe. * @throws java.io.IOException if can't write iframe file. */ private static long buildAndSaveIframe( Token[] tokens, final File snippetDir, String snippetName ) throws IOException { // produce the decorated HTML and put in the iframe dir. final File iframe = new File( snippetDir, "iframe/" + snippetName + ".htm" ); final FastCat sb = new FastCat( tokens.length + 18 ); // unix newline conventions. End user does not directly touch this file. sb.append( DOCTYPE_FOR_IFRAME ); sb.append( "