/*::.
==================================================================================================================================
=================================================¦ Copyright © 2008 Allen Baker ¦=================================================
                                                 +------------------------------+
File:       DelimitedFile.java
Originator: Allen Baker (2008.02.23 12:25)
LayoutRev:  5
================================================================================================================================== */



/*.
==========================================================================================
Package
------------------------------------------------------------------------------------------ */
package cosmicabyss.com.lib;



/*.
==========================================================================================
Imports
------------------------------------------------------------------------------------------ */
import java.io.*;
import java.util.*;
import java.net.*;



/*::
======================================================================================================================== *//**
Instances of this class read delimited files such as tab-separated-values files (.tsv files) and comma-separated-values
files (.csv files such as those used by MS Excel). The file is first cleansed of garbage characters and then parsed into
the tokens that are separated by the delimiter. The default delimiter is the comma and the default quote character is
the double quote. These special characters may be changed by calling the appropriate setter methods.

Salient characteristics of a DelimitedFile include:
   <BLOCKQUOTE>
      => The first line of the file is expected to contain column headers. Instances of this class use the delimited
         Strings on that first line to determine how many columns are in the file as well as to identify each
         column.<BR>
      => The comma character is considered the delimiter by default.<BR>
      => The delimiter can be changed from the comma to anything else through a method call,<BR>
      => The double quote character is considered the quote character by default.<BR>
      => The quote character can be changed from the double quote to anything else through a method call,<BR>
      => Instances of this class can return lines of the file in already parsed token arrays.
   </BLOCKQUOTE>

Subclass<P>
   This class is derived from the TextFile class. The abstraction it layers on top of TextFile includes the following:
      <BLOCKQUOTE>
         => It interprets each line of text in the file as a series of deliited Strings.<BR>
         => The first line of the file is expected to contain column headers.<BR>
         => All lines subsequent to the first line (column headers line) are expected to contain a series of Strings
            Separated by the delimiter, which is a comma by default.<BR>
         => All lines subsequent to the first are expected to have as many delimiter-separated Strings as there are in
            The First line.
      </BLOCKQUOTE>

<P>
   <DL>
      <DT>
         <B>
            Example usage:
         </B>
         <DD>
            <BLOCKQUOTE>
               <PRE id="unindent">
                  ========================================================================================
                  create an object and send its output to the ConsoleStream
                  ----------------------------------------------------------------------------------------
                  DelimitedFile  obj = new DelimitedFile(pArgs[0]);

                  ========================================================================================
                  the file we will read uses the double quote to enclose tokens and
                  uses the tab as a delimiter.
                  ----------------------------------------------------------------------------------------
                  obj.setQuote('\"');
                  obj.setDelimiter('\t');

                  ========================================================================================
                  test code
                  ----------------------------------------------------------------------------------------
                  obj.test();

                  ========================================================================================
                  parse each line of the delimited file into its separate tokens and
                  print each line of tokens out on a line with each tken separated by
                  a comma.
                  ----------------------------------------------------------------------------------------
                  while (obj.hasNext())
                     {
                     ArrayList<XString>  tokens = obj.parsedLine();
                     for (XString tkn : tokens)
                        {
                        cOut.print(tkn.concat(","));
                        }
                     cOut.println();
                     }
                  }
               </PRE>
            </BLOCKQUOTE>
         </DD>
      </DT>
      <DT>
         <B>
            View Source:
         </B>
         <DD>
            <A href="DelimitedFile.java.html">
               DelimitedFile.java
            </A>
         </DD>
      </DT>
      <DT>
         <B>
            Author:
         </B>
         <DD>
            <A href="mailto:sourcecode.v01@cosmicabyss.com">
               Allen Baker
            </A>
         </DD>
      </DT>
   </DL>
*//*
======================================================================================================================== */
public class DelimitedFile extends TextFile
   {



   /*:                                    
   ====================================================================================================
   [][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][]
   ==================================================================================================== *//**
   This method creates an instance of the DelimitedFile class.

   <P><B>Implementation: </B><A HREF="DelimitedFile.java.html#000">View source</A>

   *//*
   ---------------------------------------------------------------------------------------------------- */
   public DelimitedFile(XFile xfile) throws Exception
      {
      super(xfile.getCanonicalPath());
      initialize();
      }



   /*:                                    
   ====================================================================================================
   [][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][]
   ==================================================================================================== *//**
   This method creates an instance of the DelimitedFile class.

   <P><B>Implementation: </B><A HREF="DelimitedFile.java.html#001">View source</A>

   *//*
   ---------------------------------------------------------------------------------------------------- */
   public DelimitedFile(TextFile tfile) throws Exception
      {
      super(tfile.getCanonicalPath());
      initialize();
      }



   /*:                                    
   ====================================================================================================
   [][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][]
   ==================================================================================================== *//**
   This method creates an instance of the DelimitedFile class.

   <P><B>Implementation: </B><A HREF="DelimitedFile.java.html#002">View source</A>

   *//*
   ---------------------------------------------------------------------------------------------------- */
   public DelimitedFile(DelimitedFile tfile) throws Exception
      {
      super(tfile.getCanonicalPath());
      initialize();
      }



   /*:                                    
   ====================================================================================================
   [][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][]
   ==================================================================================================== *//**
   This method creates an instance of the DelimitedFile class.

   <P><B>Implementation: </B><A HREF="DelimitedFile.java.html#003">View source</A>

   *//*
   ---------------------------------------------------------------------------------------------------- */
   public DelimitedFile(URI uri) throws Exception
      {
      super(uri);
      initialize();
      }



   /*:                                    
   ====================================================================================================
   [][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][]
   ==================================================================================================== *//**
   This method creates an instance of the DelimitedFile class.

   <P><B>Implementation: </B><A HREF="DelimitedFile.java.html#004">View source</A>

   *//*
   ---------------------------------------------------------------------------------------------------- */
   public DelimitedFile(File file) throws Exception
      {
      super(file.getCanonicalPath());
      initialize();
      }



   /*:                                    
   ====================================================================================================
   [][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][]
   ==================================================================================================== *//**
   This method creates an instance of the DelimitedFile class.

   <P><B>Implementation: </B><A HREF="DelimitedFile.java.html#005">View source</A>

   *//*
   ---------------------------------------------------------------------------------------------------- */
   public <Type1> DelimitedFile(File parent, Type1 child) throws Exception
      {
      super(parent,child);
      initialize();
      }



   /*:                                    
   ====================================================================================================
   [][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][]
   ==================================================================================================== *//**
   This method creates an instance of the DelimitedFile class.

   <P><B>Implementation: </B><A HREF="DelimitedFile.java.html#006">View source</A>

   *//*
   ---------------------------------------------------------------------------------------------------- */
   public <Type1> DelimitedFile(Type1 pathname) throws Exception
      {
      super(pathname);
      initialize();
      }



   /*:                                    
   ====================================================================================================
   [][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][]
   ==================================================================================================== *//**
   This method creates an instance of the DelimitedFile class.

   <P><B>Implementation: </B><A HREF="DelimitedFile.java.html#007">View source</A>

   *//*
   ---------------------------------------------------------------------------------------------------- */
   public <Type1,Type2> DelimitedFile(Type1 parent, Type2 child) throws Exception
      {
      super(parent,child);
      initialize();
      }



   /*:                                    
   ====================================================================================================
   [][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][]
   ==================================================================================================== *//**
   This method returns true if the iteration has more elements. (In other words, returns true if next
   would return an element rather than throwing an exception.)

   <P><B>Implementation: </B><A HREF="DelimitedFile.java.html#008">View source</A>

   @return
      True if the iterator has more elements.
   *//*
   ---------------------------------------------------------------------------------------------------- */
   public boolean hasNext()
      {
      return (iIterator.hasNext());
      }



   /*:                                    
   ====================================================================================================
   [][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][]
   ==================================================================================================== *//**
   This method returns the number of columns in the delimited file. It uses the number of columns in
   the header line as the number of columns in the file.

   <P><B>Implementation: </B><A HREF="DelimitedFile.java.html#009">View source</A>

   @return
      A count of the number of columns in the delimited file.
   *//*
   ---------------------------------------------------------------------------------------------------- */
   public int numCols()
      {
      return iHeaderLine.size();
      }



   /*:                                    
   ====================================================================================================
   [][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][]
   ==================================================================================================== *//**
   This method returns the column header line of the delimited file in an already tokenized form.

   <P><B>Implementation: </B><A HREF="DelimitedFile.java.html#010">View source</A>

   @return
      An ArrayList in which each element is the next sequential token from the column header line of
      the delimited file.
   *//*
   ---------------------------------------------------------------------------------------------------- */
   public ArrayList<XString> headerLine()
      {
      return iHeaderLine;
      }



   /*:                                    
   ====================================================================================================
   [][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][]
   ==================================================================================================== *//**
   This method returns the most recent line read from the delimited file in an already tokenized form.

   <P><B>Implementation: </B><A HREF="DelimitedFile.java.html#011">View source</A>

   @return
      An ArrayList in which each element is the next sequential token from the current line of the
      delimited file.
   *//*
   ---------------------------------------------------------------------------------------------------- */
   public ArrayList<XString> currLine()
      {
      return iCurrentLineTokenized;
      }



   /*:                                    
   ====================================================================================================
   [][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][]
   ==================================================================================================== *//**
   This method returns the most recent line read from the delimited file in its raw, single string
   form.

   <P><B>Implementation: </B><A HREF="DelimitedFile.java.html#012">View source</A>

   @return
      An XString containing the most recent line read from the delimited file in its raw, single string
      form.
   *//*
   ---------------------------------------------------------------------------------------------------- */
   public XString currLineRaw()
      {
      return iCurrentLineRaw;
      }



   /*:                                    
   ====================================================================================================
   [][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][]
   ==================================================================================================== *//**
   This method returns the most recent fixed-up line read from the delimited file in its raw, single
   string form. A fixed-up line is one that has had spurious delimiters and quotes removed and
   quote-enclosed empty strings placed into all empty fields.

   <P><B>Implementation: </B><A HREF="DelimitedFile.java.html#013">View source</A>

   @return
      An XString containing the most recent fixed-up line read from the delimited file in its raw,
      single string form.
   *//*
   ---------------------------------------------------------------------------------------------------- */
   public XString currLineFixed()
      {
      return iCurrentLineFixed;
      }



   /*:                                    
   ====================================================================================================
   [][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][]
   ==================================================================================================== *//**
   This method reads the next line of the delimited file. It is really just a simplified way of calling
   parsedCsvLine() because it does not return the newly read line. Instead, the calling program would
   use the currLine() method to retrieve the newly read line. Alternatively, the calling program can
   use the field accessor methods (like valCi(), valAssetAlias(), ...) to get the actual column values
   from the newly read line.

   <P><B>Implementation: </B><A HREF="DelimitedFile.java.html#014">View source</A>

   *//*
   ---------------------------------------------------------------------------------------------------- */
   public void readNext() throws Exception
      {
      parsedLine();
      }



   /*:                                    
   ====================================================================================================
   [][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][]
   ==================================================================================================== *//**
   This method returns the next line of the delimited file in an already tokenized form.

   <P><B>Implementation: </B><A HREF="DelimitedFile.java.html#015">View source</A>

   @return
      An ArrayList in which each element is the next sequential token from the current line of the
      delimited file.
   *//*
   ---------------------------------------------------------------------------------------------------- */
   public ArrayList<XString> parsedLine() throws Exception
      {
      /*.
      ==========================================================================================
      We cache the first line of the delinmited file because it is the one that contains the
      column headers. It is pre-feched in the initialization() method, which makes the first
      call to this method. The second time this method is called, the cached line is returned.
      Thereafter, each call to this method returns the next un-cached line from the input file.
      ------------------------------------------------------------------------------------------ */
      if ( ++iParsedLineCallCount == 2)
         {
         if (! iSkipHeaderLine)
            {
            return iHeaderLine;
            }
         }
      /*.
      ==========================================================================================
      Read the next line from the file.
      ------------------------------------------------------------------------------------------ */
      XString  line = iIterator.next();
      iCurrentLineRaw = line;
      /*.
      ==========================================================================================
      Replace all those insidious delimiters that sometimes are found within quotes. Replace
      them with a token that can later be found and restored to the original delimiter after the
      line is parsed.
      ------------------------------------------------------------------------------------------ */
      line = replaceDelimitersInsideQuotes(line);
      /*.
      ==========================================================================================
      If the line begins with a delimeter, replace it with a iNullReplacement followed by a
      delimiter
      ------------------------------------------------------------------------------------------ */
      if (line.charAt(0)==iDelimiterChar)
         {
         line = iNullReplacement.concat(line);
         }
      /*.
      ==========================================================================================
      Put a token into the line in all the positions that are empty and then tokenize the line.

      iNullReplacement is a token that is used to replace "" because "" is ignored and passed
      over by tokenizedString(). Later, after tokenization, all the iNullReplacements will be
      replaced with the original "".
      ------------------------------------------------------------------------------------------ */
      line = line.iteratingReplace
         (
         iDelimiterString.concat(iDelimiterString),                          // replace things like this  ,,
         iDelimiterString.concat(iNullReplacement).concat(iDelimiterString)  // with things like this  ,Digest("NULL_REPLACEMENT"),
         );
      ArrayList<XString>  tempTokens = line.tokenizedString(iDelimiterString);
      if (line.endsWith(iDelimiterString)) tempTokens.add(iNullReplacement);
      /*.
      ==========================================================================================
      Later (after tokenization is complete)... Replace all the iNullReplacements with nulls and
      replace all the iDelimiterRepacements with iDelimiters
      ------------------------------------------------------------------------------------------ */
      ArrayList<XString>  tokens = new ArrayList<XString>();
      XStringsIterator    tIter  = new XStringsIterator(tempTokens);
      while (tIter.hasNext())
         {
         /*.
         ==========================================================================================
         ------------------------------------------------------------------------------------------ */
         XString  tkn = tIter.next().trim();
         tkn = tkn.replace(iNullReplacement,"").replace(iDelimiterReplacement,iDelimiterString);
         /*.
         ==========================================================================================
         This part removes unneeded enclosing quote string from the token
         ------------------------------------------------------------------------------------------ */
         tkn = tkn.trimLeft(iQuoteString).trimRight(iQuoteString);
         if (tkn.contains(iDelimiterString))
            {
            tkn = iQuoteString.concat(tkn).concat(iQuoteString);
            }
         /*.
         ==========================================================================================
         This part "fixes" (actually "works around") unmatched quote strings within the token.

         I've already thought through the need for this so dont delete this code. The explanation
         would take too long here.
         ------------------------------------------------------------------------------------------ */
         int  numQuotes    = 0;
         int  lastQuoteIdx = 0;
         for (int i=0; i<tkn.length(); i++)
            {
            if (tkn.charAt(i) == iQuoteChar)
               {
               lastQuoteIdx = i;
               numQuotes++;
               }
            }
         if (UMath.isOdd(numQuotes))
            {
            StringBuffer  sBuf = new StringBuffer(tkn.string());
            sBuf.insert(lastQuoteIdx+1,iQuoteString.string());
            tkn = new XString(sBuf.toString());
            }
         /*.
         ==========================================================================================
         ------------------------------------------------------------------------------------------ */
         tokens.add(tkn);
         }
      /*.
      ==========================================================================================
      Remember what the line looks like after being cleaned up
      ------------------------------------------------------------------------------------------ */
      int  tokenNum = 0;
      tIter = new XStringsIterator(tokens);
      while (tIter.hasNext())
         {
         if (tokenNum == 0)
            {
            iCurrentLineFixed = tIter.next();
            }
         else
            {
            iCurrentLineFixed = iCurrentLineFixed.concat(iDelimiterString).concat(tIter.next());
            }
         }
      /*.
      ==========================================================================================
      Remember what the line looks like after being tokenized
      ------------------------------------------------------------------------------------------ */
      iCurrentLineTokenized = tokens;
      /*.
      ==========================================================================================
      And return the tokenized line
      ------------------------------------------------------------------------------------------ */
      return tokens;
      }



   /*:                                    
   ====================================================================================================
   [][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][]
   ==================================================================================================== *//**
   This method returns the column number of the column with the given header. If no such column is
   found, it returns -1.

   <P><B>Implementation: </B><A HREF="DelimitedFile.java.html#016">View source</A>

   @return
      The column number with the given header or -1 if it is not found

   @param
      pHeader is the column header to search for
   *//*
   ---------------------------------------------------------------------------------------------------- */
   public <Type1> int colNum(Type1 pColHeader)
      {
      for (int i=0; i<iHeaderLine.size(); i++)
         {
         if (iHeaderLine.get(i).equals(pColHeader))
            {
            return i;
            }
         }
      return (-1);
      }



   /*:                                    
   ====================================================================================================
   [][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][]
   ==================================================================================================== *//**
   This method returns the column number of the column with the given header ignoring case. If no such
   column is found, it returns -1.

   <P><B>Implementation: </B><A HREF="DelimitedFile.java.html#017">View source</A>

   @return
      The column number with the given header or -1 if it is not found

   @param
      pHeader is the column header to search for
   *//*
   ---------------------------------------------------------------------------------------------------- */
   public <Type1> int colNumIgnoreCase(Type1 pColHeader)
      {
      for (int i=0; i<iHeaderLine.size(); i++)
         {
         if (iHeaderLine.get(i).equalsIgnoreCase(pColHeader))
            {
            return i;
            }
         }
      return (-1);
      }



   /*:                                    
   ====================================================================================================
   [][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][]
   ==================================================================================================== *//**
   This method sets the quote character for this object.

   <P><B>Implementation: </B><A HREF="DelimitedFile.java.html#018">View source</A>

   @return
      A reference to this object

   @param
      pQuoteChar is the character to use as a quote for this object
   *//*
   ---------------------------------------------------------------------------------------------------- */
   public DelimitedFile setQuote(char pQuoteChar)
      {
      iQuoteChar   = pQuoteChar;
      iQuoteString = new XString(iQuoteChar);
      return this;
      }



   /*:                                    
   ====================================================================================================
   [][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][]
   ==================================================================================================== *//**
   This method sets the delimiter character for this object.

   <P><B>Implementation: </B><A HREF="DelimitedFile.java.html#019">View source</A>

   @return
      A reference to this object

   @param
      pDelimiterChar is the character to use as a delimiter for this object
   *//*
   ---------------------------------------------------------------------------------------------------- */
   public DelimitedFile setDelimiter(char pDelimiterChar)
      {
      iDelimiterChar   = pDelimiterChar;
      iDelimiterString = new XString(iDelimiterChar);
      return this;
      }



   /*:                                    
   ====================================================================================================
   [][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][]
   ==================================================================================================== *//**
   This method sets the flag that determines whether or not the parseLine() method wil return the
   header line.

   <P><B>Implementation: </B><A HREF="DelimitedFile.java.html#020">View source</A>

   @return
      A reference to this object

   @param
      pSkipHeaderLines is the boolean value to turn on or off the skipping of header lines. If
      pSkipHeaderLines is true, then the parsedLine() will skip the header line. If false, it will not
      skip header lines.
   *//*
   ---------------------------------------------------------------------------------------------------- */
   public DelimitedFile setSkipHeaderLine(boolean pSkipHeaderLine)
      {
      iSkipHeaderLine = pSkipHeaderLine;
      return this;
      }



   /*:                                    
   ====================================================================================================
   [][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][]
   ==================================================================================================== *//**
   This method tells this object whether to skip white space lines or not. A white space line is a line
   of the text file that contains only white space characters.<P>

   This method is a superset of setSkipBlankLines() because white space lines include blank lines.

   <P><B>Implementation: </B><A HREF="DelimitedFile.java.html#021">View source</A>

   @return
      A reference to this object

   @param
      pSkipWhiteSpaceLines is the boolean value to turn on or off the skipping of white space lines. If
      pSkipWhiteSpaceLines is true, then the TextReaderIterator will skip white space lines. If false,
      it will not skip white space lines.
   *//*
   ---------------------------------------------------------------------------------------------------- */
   public DelimitedFile setSkipWhiteSpaceLines(boolean pSkipWhiteSpaceLines)
      {
      iIterator.setSkipWhiteSpaceLines(pSkipWhiteSpaceLines);
      return this;
      }



   /*:                                    
   ====================================================================================================
   [][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][]
   ==================================================================================================== *//**
   This method sets the ConsoleStream for this object.

   <P><B>Implementation: </B><A HREF="DelimitedFile.java.html#022">View source</A>

   @return
      A reference to this object

   @param
      pConsole is a ConsoleStream through which this objects sends its output.
   *//*
   ---------------------------------------------------------------------------------------------------- */
   public DelimitedFile setOut(ConsoleStream pConsole)
      {
      cOut = pConsole;
      return this;
      }



   /*:                                    
   ====================================================================================================
   [][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][]
   ==================================================================================================== *//**
   This boilerplate method is used for testing an instantiated object of this class and may include any
   code the developer chooses.

   <P><B>Implementation: </B><A HREF="DelimitedFile.java.html#023">View source</A>

   *//*
   ---------------------------------------------------------------------------------------------------- */
   public DelimitedFile test() throws Exception
      {
      cOut.titledPrintf
         (
         "\"HELLO WORLD!\"",
         "%s  %s  %s",
         "I'm an object of the", CLASS_NAME, "class, and I approved this message."
         );
      return this;
      }



   /*:.
   ==============================================================================================================
   @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@[  Protected  ]@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
   ============================================================================================================== */



   /*:.
   ==============================================================================================================
   @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@[  Private  ]@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
   ============================================================================================================== */



   /*.
   ==========================================================================================
   Class Constants
      CLASS_NAME:
         The name of this class
      DFLT_LINE_LEN:
         The default line length for word wrapping
   ------------------------------------------------------------------------------------------ */
   private static final XString  CLASS_NAME            = new XString(DelimitedFile.class.getName());
   private static final int      DFLT_LINE_LEN         = ConsoleMessage.defaultLineLength();
   /*.
   ==========================================================================================
   Class variables
      cOut : console output.
   ------------------------------------------------------------------------------------------ */
   private static ConsoleStream  cOut = ConsoleStream.getSingleton();
   /*.
   ==========================================================================================
   Instance variables
      <BLOCKQUOTE>
         <PRE id="unindent">
            iQuoteChar            : the character to use as a quote
            iQuoteString          : an XString representation of the quote character
            iDelimiterChar        : the character to use as a delimiter
            iDelimiterString      : an XString representation of the delimiter character
            iIterator             : an iterator for iterating over the lines of the delimited file
            iDelimiterReplacement : a uniquely recognizable string that is substituted for a delimiter character
            iNullReplacement      : a uniquely recognizable string that is substituted for a null character
            iCurrentLineRaw       : remembers the most recent line of the input file that was fetched by the iterator
            iCurrentLineFixed     : remembers the most recent line of the input file that was cleaned
         </PRE>
      </BLOCKQUOTE>
   ------------------------------------------------------------------------------------------ */
   private char                iQuoteChar            = '\"';
   private XString             iQuoteString          = new XString(iQuoteChar);
   private char                iDelimiterChar        = ',';
   private XString             iDelimiterString      = new XString(iDelimiterChar);
   private TextReaderIterator  iIterator             = null;
   private XString             iDelimiterReplacement = null;
   private XString             iNullReplacement      = null;
   private XString             iCurrentLineRaw       = null;
   private XString             iCurrentLineFixed     = null;
   private ArrayList<XString>  iCurrentLineTokenized = null;
   private ArrayList<XString>  iHeaderLine           = null;
   private long                iParsedLineCallCount  = 0L;
   private boolean             iSkipHeaderLine       = false;



   /*:.
   ==============================================================================================================
   @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@[  Public Static Methods  ]@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
   ============================================================================================================== */



   /*:                                    
   ====================================================================================================
   [][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][]
   ==================================================================================================== *//**
   This method is the common mechanism used by all constructors to initialize an object

   <P><B>Implementation: </B><A HREF="DelimitedFile.java.html#024">View source</A>

   *//*
   ---------------------------------------------------------------------------------------------------- */
   private void initialize() throws Exception
      {
      /*.
      ==========================================================================================
      ------------------------------------------------------------------------------------------ */
      iDelimiterReplacement = new XString(new Digest("DELIMITER_REPLACEMENT").string());
      iNullReplacement      = new XString(new Digest("NULL_REPLACEMENT").string());
      /*.
      ==========================================================================================
      ------------------------------------------------------------------------------------------ */
      fixEmbeddedNewlines();
      trim();   // dont do this trim() until after removing embedded newlines
      replaceNonPrintingCharacters();
      /*.
      ==========================================================================================
      ------------------------------------------------------------------------------------------ */
      iIterator = new TextReaderIterator(this);
      /*.
      ==========================================================================================
      ------------------------------------------------------------------------------------------ */
      iHeaderLine = parsedLine();
      }



   /*:                                    
   ====================================================================================================
   [][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][]
   ==================================================================================================== *//**
   This method replaces all non-printing (except tabs and newlines) and non-standard (chars 127 thru
   255) ascii characters with the uptick character - '^'.

   <P><B>Implementation: </B><A HREF="DelimitedFile.java.html#025">View source</A>

   *//*
   ---------------------------------------------------------------------------------------------------- */
   private void replaceNonPrintingCharacters() throws Exception
      {
      TextFile    thisFile = new TextFile(this);
      String[][]  strPairs = new String[30 + 129][2];
      int  idx = 0;
      for (int i=0; i<32; i++)
         {
         if ((i==10) || (i==9)) continue;
         strPairs[idx][0] = new Character((char)i).toString();
         strPairs[idx][1] = new String("^");
         idx++;
         }
      for (int i=127; i<256; i++)
         {
         strPairs[idx][0] = new Character((char)i).toString();
         strPairs[idx][1] = new String("^");
         idx++;
         }
      thisFile.replace(strPairs);
      thisFile = null;
      }



   /*:                                    
   ====================================================================================================
   [][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][]
   ==================================================================================================== *//**
   This method eliminates the insidious delimiters that are sometimes put into quote-enclosed fields by
   the csv file generators. Those delimiters obviously mess up any parsing based on delimiters.

   <P><B>Implementation: </B><A HREF="DelimitedFile.java.html#026">View source</A>

   @return
      An XString without any delimiters within quote-enclosed fields

   @param
      pIn is the XString to be searched for the offending quote-enclosed delimiters.
   *//*
   ---------------------------------------------------------------------------------------------------- */
   private XString replaceDelimitersInsideQuotes(XString pIn)
      {
      char[]  chars    = pIn.toCharArray();
      boolean inQuote  = false;
      boolean fixedOne = false;

      for (int i=0; i<chars.length; i++)
         {
         char  thisChar = chars[i];
         if (thisChar==iQuoteChar)
            {
            inQuote = !inQuote;
            }
         else if (inQuote)
            {
            if (thisChar==iDelimiterChar)
               {
               chars[i] = '\376';  // 376 octal == 254 decimal
               fixedOne = true;
               }
            }
         }
      XString  result = new XString(chars).replace("\376",iDelimiterReplacement);
      return result;
      }



   /*:                                    
   ====================================================================================================
   [][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][]
   ==================================================================================================== *//**
   This method eliminates newlines that are embedded within quote strings in a file.

   <P><B>Implementation: </B><A HREF="DelimitedFile.java.html#027">View source</A>

   *//*
   ---------------------------------------------------------------------------------------------------- */
   private void fixEmbeddedNewlines() throws Exception
      {
      /*.
      ==========================================================================================
      ------------------------------------------------------------------------------------------ */
      File  iFile = new File(this.getCanonicalPath());
      /*.
      ==========================================================================================
      This method makes no sense on directory files, this must be a normal file
      ------------------------------------------------------------------------------------------ */
      if (iFile.isDirectory()) return;
      /*.
      ==========================================================================================
      We'll output the processed version of the file to a temp file so if the process fails
      midway through, we don't corrupt the original file
      ------------------------------------------------------------------------------------------ */
      File  oFile = File.createTempFile("fixEmbeddedNewlines-",".tmp");
      /*.
      ==========================================================================================
      Process the file
      ------------------------------------------------------------------------------------------ */
      fixEmbeddedNewlines
         (
         iFile.getCanonicalPath(),
         oFile.getCanonicalPath()
         );
      /*.
      ==========================================================================================
      Delete the original file
      ------------------------------------------------------------------------------------------ */
      if ( ! iFile.delete())
         {
         throw new Exception("Cannot delete " + iFile.getCanonicalPath());
         }
      /*.
      ==========================================================================================
      And replace it with the processed file.
      ------------------------------------------------------------------------------------------ */
      if ( ! oFile.renameTo(iFile))
         {
         throw new Exception("Cannot rename " + oFile.getCanonicalPath());
         }
      }



   /*:                                    
   ====================================================================================================
   [][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][]
   ==================================================================================================== *//**
   This method eliminates newlines that are embedded within quote strings in a file. To accomplish this
   task, it reads the file as a stream of chars and executes the following state machine.
      <BLOCKQUOTE>
         <PRE id="unindent">
            +-------------------+-------+----------------------------------------------------+
            | Meaning of state  | State |                 Next character                     |
            |                   |       +- - - - - - - - - - +- - - - - - - -+- - - - - - - -+
            |                   |       |     newline        |  iQuoteChar   |  any other    |
            +-------------------+-------+--------------------+---------------+---------------+
            | Start of line     |   0   | (do nothing)       | go to state 2 | go to state 1 |
            + -  -  -  -  -  - -+ -  - -+ -  -  -  -  -  - - + -  -  -  -  - + -  -  -  -  - +
            | Not inside quotes |   1   | go to state 0      | go to state 2 | (do nothing)  |
            + -  -  -  -  -  - -+ -  - -+ -  -  -  -  -  - - + -  -  -  -  - + -  -  -  -  - +
            | Inside quotes     |   2   | replace with blank | go to state 1 | (do nothing)  |
            + -  -  -  -  -  - -+ -  - -+ -  -  -  -  -  - - + -  -  -  -  - + -  -  -  -  - +
         </PRE>
      </BLOCKQUOTE>

   <P><B>Implementation: </B><A HREF="DelimitedFile.java.html#028">View source</A>

   @param
      pFileIn is the name of the file that will be copied to the output file with embedded newlines
      tossed. PFileOut must not be the same as pFileIn. PFileOut and pFileIn must not be null.
   @param
      pFileOut is the name that will be given to the file that is produced. PFileOut must not be the
      same as pFileIn. PFileOut and pFileIn must not be null.
   *//*
   ---------------------------------------------------------------------------------------------------- */
   private <Type1,Type2> void fixEmbeddedNewlines
      (
      Type1  pFileIn,
      Type2  pFileOut
      )
      throws Exception
      {
      /*.
      ==========================================================================================
      Check the parameters.
      ------------------------------------------------------------------------------------------ */
      if (pFileIn == null)
         {
         throw new Exception("pFileIn is null");
         }
      if (pFileOut == null)
         {
         throw new Exception("pFileOut is null");
         }
      if (pFileIn.equals(pFileOut))
         {
         throw new Exception("pFileIn and pFileOut are the same file");
         }
      /*.
      ==========================================================================================
      ------------------------------------------------------------------------------------------ */
      XString  fileIn  = new XString(pFileIn);
      XString  fileOut = new XString(pFileOut);
      /*.
      ==========================================================================================
      Set up the files
      ------------------------------------------------------------------------------------------ */
      CharIterator  iter   = (new TextFile(fileIn)).charIterator();
      PrintWriter   output = (new TextFile(fileOut)).printWriter();
      /*.
      ==========================================================================================
      Read each line of the source file and write a tossed version of it to the output file.
      ------------------------------------------------------------------------------------------ */
      int state = 0;
      while (iter.hasNext())
         {
         char  c = iter.next();
         switch (state)
            {
            case 0: // BEGIN LINE
               {
               if (c=='\n')
                  {
                  output.print(c);
                  }
               else if (c==iQuoteChar)
                  {
                  state = 2;
                  output.print(c);
                  }
               else
                  {
                  output.print(c);
                  state = 1;
                  }
               break;
               }
            case 1: // OUT OF QUOTE
               {
               if (c=='\n')
                  {
                  state = 0;
                  output.print(c);
                  }
               else if (c==iQuoteChar)
                  {
                  state = 2;
                  output.print(c);
                  }
               else
                  {
                  output.print(c);
                  }
               break;
               }
            case 2: // IN QUOTE
               {
               if (c=='\n')
                  {
                  output.print(' ');
                  }
               else if (c==iQuoteChar)
                  {
                  state = 1;
                  output.print(c);
                  }
               else
                  {
                  output.print(c);
                  }
               break;
               }
            }
         }
      /*.
      ==========================================================================================
      If we were still in state 2 when the end of file was reached, then that means the file
      ended without a required close quote followed by a required newline. In that situation,
      fix the file by finishing it off with a close quote and newline.
      ------------------------------------------------------------------------------------------ */
      if (state == 2)
         {
         output.print("\"\n");
         }
      /*.
      ==========================================================================================
      Flush the output buffers
      ------------------------------------------------------------------------------------------ */
      output.flush();
      /*.
      ==========================================================================================
      Close the files
      ------------------------------------------------------------------------------------------ */
      output.close();
      }



   /*:                                    
   ====================================================================================================
   [][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][]
   ==================================================================================================== *//**
   This method allows this class file to be unit tested as a standalone application. It's the method
   that's called when the class is invoked from the command line by using the java application launcher
   - "java". Main() is not a required method, but the practice of putting one in each class and
   wrapping class test code within it allows easy unit testing of the class; and main does not need to
   be removed when testing is complete.

   <P>
      <DL>
         <DT>
            <B>
               Command line usage:
            </B>
            <DD>
               Java cosmicabyss.com.lib.DelimitedFile
            </DD>
         </DT>
      </DL>

   <P><B>Implementation: </B><A HREF="DelimitedFile.java.html#029">View source</A>

   @param
      pArgs contains the command line arguments with which this class was invoked as an application.
   *//*
   ---------------------------------------------------------------------------------------------------- */
   public static void main(String[] pArgs) throws Exception
      {
      /*.
      ==========================================================================================
      Greetings !
      ------------------------------------------------------------------------------------------ */
      cOut.banner(CLASS_NAME);
      /*.
      ==========================================================================================
      Create an object and send its output to the ConsoleStream
      ------------------------------------------------------------------------------------------ */
      DelimitedFile  obj = new DelimitedFile("c:/temp/problems.csv");
//      DelimitedFile  obj = new DelimitedFile(pArgs[0]);
//      /*
//      ==========================================================================================
//      the file we will read uses the double quote to enclose tokens and
//      uses the tab as a delimiter.
//      ------------------------------------------------------------------------------------------ */
//      obj.setQuote('\"');
//      obj.setDelimiter('\t');
      /*.
      ==========================================================================================
      Test code
      ------------------------------------------------------------------------------------------ */
      obj.test();
//      /*
//      ==========================================================================================
//      parse each line of the delimited file into its separate tokens and
//      print each line of tokens out on a line with each tken separated by
//      a comma.
//      ------------------------------------------------------------------------------------------ */
//      while (obj.hasNext())
//         {
//         ArrayList<XString>  tokens = obj.parsedLine();
//         for (XString tkn : tokens)
//            {
//            cOut.print(tkn.concat(","));
//            }
//         cOut.println();
//         }
      }



   }  // class DelimitedFile



   /*:.
   ==============================================================================================================
   @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@[  Notes  ]@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
   ============================================================================================================== */