/* Copyright (c) 2003-2006 Ricebridge. All Rights Reserved.
 *
 * This file is available under the terms and conditions of the
 * Ricebridge "Open Source API" policy; Ricebridge grants use of this
 * copyrighted work under the terms of a BSD-style license only. See
 * http://www.opensource.org/licenses/bsd-license.php for more
 * information.
 * 
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 
 *  - Redistributions of source code must retain the above copyright
 *  notice, this list of conditions and the following disclaimer.
 *
 *  - Redistributions in binary form must reproduce the above
 *  copyright notice, this list of conditions and the following
 *  disclaimer in the documentation and/or other materials provided
 *  with the distribution.
 *
 *  - Neither the name of the Ricebridge nor the names of its
 *  contributors may be used to endorse or promote products derived
 *  from this software without specific prior written permission.
 * 
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
 * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
 * OF THE POSSIBILITY OF SUCH DAMAGE.  
 */

package com.ricebridge.csvman.test;


import com.ricebridge.csvman.*;

import org.jostraca.util.*;

import junit.framework.*;
import junit.textui.*;

import javax.swing.table.*;
import java.sql.ResultSet;
import java.util.*;
import java.io.*;


/** Test cases for {@link com.ricebridge.csvman.CsvManager}.
 *    <p>The <b><a href="CsvManagerTest.java.html">Source Code</a></b> of this Java class 
 *    is available under a <a href="http://www.opensource.org/licenses/bsd-license.php">BSD-style license</a>.</p>
 */
public class CsvManagerTest extends TestCase {

  // standard test methods
  
  public CsvManagerTest( String pName ) {
    super( pName );
  }

  public static TestSuite suite() {
    return new TestSuite( CsvManagerTest.class );
  }

  public static void main( String[] pArgs ) {
    TestRunner.run( suite() );
  }


  /** An example of a very simple <code>LineProvider</code>.
   *  Note that the {@link LineProvider#hasNextLine} method is called <i>before</i>
   *  the {@link LineProvider#nextLine} method. This means that you should be take care
   *  initialise any array index variables to -1, so that they will be 0 after the first call to
   *  <code>nextLine</code>. This also means that the test in <code>hasNextLine</code> should be
   *  against the next (+1), not the current, value of the row index.
   *    <p>See the <b><a href="CsvManagerTest.java.html">Source Code</a></b> of this method
   *  for the full example.</p>
   */
  public static LineProvider makeTestLineProvider() {
    LineProvider lp = new CustomLineProvider() {
        String[][] data  = new String[][] { {"a","foo","1"}, {"b","bar","2"}, {"c","baz","3"} }; 
        int        row   = -1;

        // reset row to -1
        public void startProcessImpl()    { row = -1; } 
        public void endProcessImpl()      {}

        // test if next (current + 1) row exists 
        protected boolean hasNextLineImpl() { return row + 1 < data.length;}        

        // move to next row, reset field index
        protected String[] nextLineImpl() { row++; return data[row]; };                  

      };
    return lp;
  }



  
  public void testLineProvider() throws Exception {
    System.out.println( "testLineProvider" );

    LineProvider lp     = makeTestLineProvider();
    CsvManager   csvm   = new CsvManager();
    csvm.getCsvSpec().setEndOfLine("\n");
    String       out    = csvm.saveToString( lp );
    String       expect = "a,foo,1\nb,bar,2\nc,baz,3\n";
    assertEquals( expect, out );

    csvm.getCsvSpec().setEncoding("UTF-16");
    out = csvm.saveToString( lp );
    assertEquals( expect, out );
  }



  public void testSeparator() throws Exception {
    System.out.println( "testSeparator" );

    CsvManager cm = new CsvManager();
    //cm.setFieldListener( new DebugFieldListener() );
    List lines = cm.loadAsLists( TestUtil.getTextCsvFile("01") );
    String[] match = new String[] { "[1, foo, a'a]", "[2, f\"o, a]", "[3, \", a]", "[, , ]", "[4, ;a, 1.1]", "[, a, ]",
                                    "[5, , ]", "[6, a, b, c, d]", "[7, 1.1, ]", "[, , ]", "[8, b, ]", "[b,b, a,a, 1.1]",
                                    "[9, , a, ]", "[10, a, , ]", "[11, , , a, ]", "[12, , ]", "[13, , ]", "[14, , , ]",
                                    "[, , ]", "[, , ]", "[, , , ]", "[, , , , ]"  };
    //TestUtil.displayTimes( cm );
    //TestUtil.displayLists( Arrays.asList(match), lines );
    assertEquals( Arrays.asList(match).toString(), lines.toString() );

    CsvSpec cs = cm.getCsvSpec();
    cs.setSeparator( ",;" );
    lines = cm.loadAsLists( TestUtil.getTextCsvFile("01") );
    match = new String[] { "[1, foo, a'a]", "[2, f\"o, a]", "[3, \", a]", "[, , ]", "[4, , a, 1.1]", "[, a, ]",
                           "[5, , ]", "[6, a, b, c, d]", "[7, 1.1, ]", "[, , ]", "[8, b, ]", "[b,b, a,a, 1.1]",
                           "[9, , a, ]", "[10, a, , ]", "[11, , , a, ]", "[12, , ]", "[13, , ]", "[14, , , ]",
                           "[, , ]", "[, , ]", "[, , , ]", "[, , , , ]" };
    //TestUtil.displayTimes( cm );
    assertEquals( Arrays.asList(match).toString(), lines.toString() );
    
    assertTrue( -1 == cm.getStatsSummary().indexOf( "ERROR" ) );

    cs.setMergeSeparators( true );
    lines = cm.loadAsLists( TestUtil.getTextCsvFile("01") );
    match = new String[] { "[1, foo, a'a]", "[2, f\"o, a]", "[3, \", a]", "[, , ]", "[4, a, 1.1]", "[, a, ]",
                           "[5, , ]", "[6, a, b, c, d]", "[7, 1.1, ]", "[, , ]", "[8, b, ]", "[b,b, a,a, 1.1]",
                           "[9, a, ]", "[10, a, ]", "[11, a, ]", "[12, , ]", "[13, , ]", "[14, , ]",
                           "[, , ]", "[, , ]", "[, , ]", "[, , ]" };
    //TestUtil.displayTimes( cm );
    assertEquals( Arrays.asList(match).toString(), lines.toString() );


    cs.setSeparator( ";" );
    cs.setMergeSeparators( false );
    lines = cm.loadAsLists( TestUtil.getTextCsvFile("02") );
    //TestUtil.displayTimes( cm );
    assertEquals( "[[1, foo, a'a], [2, f\"o, a], [3, \", a], [, , ], [4, ,a, 1.1], [, a, ], "
                  +"[5, , ], [6, a, b, c, d], [7, 1.1, ], [, , ], [8, b, ], [b;b, a;a, 1.1]]", 
                  lines.toString() );                                                

    cs.setSeparator( "\t" );
    lines = cm.loadAsLists( TestUtil.getTextCsvFile("03") );
    //TestUtil.displayTimes( cm );
    assertEquals( "[[1, foo, a'a], [2, f\"o, a], [3, \", a], [, , ], [4, ;a, 1.1], [, a, ], "
                  +"[5, , ], [6, a, b, c, d], [7, 1.1, ], [, , ], [8, b, ], [b\tb, a\ta, 1.1]]", 
                  lines.toString() );                                                

  }



  public void testQuotes() throws Exception {
    System.out.println( "testQuotes" );

    CsvManager cm = new CsvManager();
    //System.out.println( "-- quotes 01" );
    List lines = cm.loadAsLists( TestUtil.getTextCsvFile("quotes-01") );
    String[] match = new String[] { "[1, b, c]", "[2, b, c]", "[3, b, c]", "[4, b, c]", "[5, b, c]", "[6, b, ]", "[7, , ]", "[8, , ]", 
                                    "[9, , ]", "[, 10, ]", "[, , 11]", "[12, , ]", "[13, , ]", "[14, , ]", "[15\"a, b, c]", 
                                    "[16, b\"b, c]", "[17, b, c\"c]", "[, 18, c]", "[19, , c]", "[20, b, ]", 
                                    "[\", 21, c]", "[22, \", c]", "[23, b, \"]", "[24, a, b]" };
    //TestUtil.displayLists( Arrays.asList(match), lines );
    //TestUtil.displayTimes( cm );
    assertEquals( Arrays.asList(match).toString(), lines.toString() );

    cm.getCsvSpec().setUseQuote( false );
    lines = cm.loadAsLists( TestUtil.getTextCsvFile("quotes-01") );
    match = new String[] { "[1, \"b\", c]", "[2, \"b\", c]", "[3, \"b\", c]", "[4, \"b\", c]", 
                           "[\"5\", b, c]", "[\"6\", b, ]", "[\"7\", , ]", "[\"8\", , ]", 
                           "[\"9\", , ]", "[, \"10\", ]", "[, , \"11\"]", "[\"12\", , ]", 
                           "[\"13\", , ]", "[\"14\", , ]", "[\"15\"\"a\", b, c]", 
                           "[16, \"b\"\"b\", c]", "[17, b, \"c\"\"c\"]", "[\"\", 18, c]", "[19, \"\", c]", "[20, b, \"\"]", 
                           "[\"\"\"\", 21, c]", "[22, \"\"\"\", c]", "[23, b, \"\"\"\"]", "[24, \"a\", \"b\"]" };
    //TestUtil.displayTimes( cm );
    //TestUtil.displayLists( Arrays.asList(match), lines );
    assertEquals( Arrays.asList(match).toString(), lines.toString() );
  }


  public void testTrim() throws Exception {
    System.out.println( "testTrim" );

    CsvManager cm  = new CsvManager();

    String[]   m01 = new String[] { "[1, foo, a]",  "[2, bar, b]",  "[3, baz, c]",  "[4, b o, d]",  
                                    "[5,  oz, d]",  "[6, bo , e]",  "[7,  o , f]", "[8,  , g]" };
    
    List lines = cm.loadAsLists( TestUtil.getTextCsvFile("07") );
    //TestUtil.displayLists( Arrays.asList(m01),  lines );
    assertEquals( Arrays.asList(m01).toString(),  lines.toString() );


    String[]   m02 = new String[] { "[1, foo, a]",  "[2, bar, b]",  "[3, baz , c ]",  "[4, b o, d]",  
                                    "[5,  oz, d]",  "[6, bo , e]",  "[7 ,  o , f ]", "[8,  , g]" };
    cm.setTrimType( TrimType.Start );
    lines = cm.loadAsLists( TestUtil.getTextCsvFile("07") );
    //TestUtil.displayLists( Arrays.asList(m02),  lines );
    assertEquals( Arrays.asList(m02).toString(),  lines.toString() );


    String[]   m03 = new String[] { "[1, foo, a]",  "[2,  bar,  b]",  "[ 3,  baz,  c]",  "[4, b o, d]",  
                                    "[5,  oz,  d]",  "[6, bo , e]",  "[ 7,  o ,  f]", "[8,  ,  g]" };
    cm.setTrimType( TrimType.End );
    lines = cm.loadAsLists( TestUtil.getTextCsvFile("07") );
    //TestUtil.displayLists( Arrays.asList(m03),  lines );
    assertEquals( Arrays.asList(m03).toString(),  lines.toString() );


    String[]   m04 = new String[] { "[1, foo, a]",  "[2,  bar,  b]",  "[ 3,  baz ,  c ]",  "[4, b o, d]",  
                                    "[5,  oz,  d]",  "[6, bo , e]",  "[ 7 ,  o ,  f ]", "[8,  ,  g]" };
    cm.setTrimType( TrimType.None );
    lines = cm.loadAsLists( TestUtil.getTextCsvFile("07") );
    //TestUtil.displayLists( Arrays.asList(m04),  lines );
    assertEquals( Arrays.asList(m04).toString(),  lines.toString() );
    
  }




  public void testBadLines() throws Exception {
    System.out.println( "testBadLines" );

    CsvManager cm = new CsvManager();
    cm.setEscape('\\');
    cm.setIgnoreBadLines(true);
    List lines = cm.loadAsLists( TestUtil.getTextCsvFile("fail01") );
    String[] match    = new String[] { "[1, foo, a'a]",  "[2, f\"o, b]", "[3, f\"o, c]", "[5, foy, e]", "[7, foz, g]"};
    String[] badlines = new String[] { "4,\"fo\"x\",d", "6,\"fo\"n,f", "8,\",h\n9,bar,i" };

    System.out.println( "lines:"+lines );
    System.out.println( "badlines:"+cm.getBadLines() );

    //TestUtil.displayTimes( cm );
    assertEquals( badlines[0], ((BadLine)cm.getBadLines().get(0)).getOriginalLine() );
    assertEquals( badlines[1], ((BadLine)cm.getBadLines().get(1)).getOriginalLine() );
    assertEquals( badlines[2], TextUtil.replace(((BadLine)cm.getBadLines().get(2)).getOriginalLine(),"\r\n","\n") );
    assertEquals( Arrays.asList(match).toString(), lines.toString() );

    assertEquals( 3, cm.getBadLineCount() );
    assertEquals( 8, cm.getLineCount() );

    cm.setIgnoreBadLines( false );
    try {
      cm.loadAsLists( TestUtil.getTextCsvFile("fail01") );
      fail();
    }
    catch( CsvManagerException e ) {
      assertEquals( CsvManagerException.CODE_bad_line, e.getCode() );
      assertEquals( "An incorrectly formatted line caused processing to halt. The line was: '4,\"fo\"x\",d'. The parse error was: 'unexpected token: x\"'.", e.toString() );
      assertEquals( "4,\"fo\"x\",d", e.getBadLine().getOriginalLine() );
    }
  }


  public void testEmptyLines() throws Exception {
    CsvManager cm = new CsvManager();
    cm.setIgnoreEmptyLines( true );
    List lines = cm.loadAsLists( TestUtil.getTextCsvFile("01") );
    String[] match = new String[] { "[1, foo, a'a]", "[2, f\"o, a]", "[3, \", a]", "[4, ;a, 1.1]", "[, a, ]",
                                    "[5, , ]", "[6, a, b, c, d]", "[7, 1.1, ]", "[8, b, ]", "[b,b, a,a, 1.1]",
                                    "[9, , a, ]", "[10, a, , ]", "[11, , , a, ]", "[12, , ]", "[13, , ]", "[14, , , ]",
                                    "[, , ]", "[, , ]", "[, , , ]", "[, , , , ]"  };
    //TestUtil.displayTimes( cm );
    //TestUtil.displayLists( Arrays.asList(match), lines );
    assertEquals( Arrays.asList(match).toString(), lines.toString() );
  }


  public void testStartEndLineLoad() throws Exception {
    System.out.println( "testStartEndLineLoad" );

    CsvManager cm = new CsvManager();

    cm.setNumLines( 2 );
    List lines = cm.loadAsLists( TestUtil.getTextCsvFile("01") );
    String[] match = new String[] { "[1, foo, a'a]", "[2, f\"o, a]" };
    //TestUtil.displayLists( Arrays.asList(match), lines );
    assertEquals( 2, cm.getLineCount() );
    assertEquals( 0, cm.getBadLineCount() );
    assertEquals( Arrays.asList(match).toString(), lines.toString() );

    cm.setStartLine( 3 );
    lines = cm.loadAsLists( TestUtil.getTextCsvFile("01") );
    match = new String[] { "[3, \", a]", "[, , ]" };
    //TestUtil.displayLists( Arrays.asList(match), lines );
    assertEquals( 2, cm.getLineCount() );
    assertEquals( 0, cm.getBadLineCount() );
    assertEquals( Arrays.asList(match).toString(), lines.toString() );
    
    cm.setIgnoreEmptyLines( true );
    lines = cm.loadAsLists( TestUtil.getTextCsvFile("01") );
    match = new String[] { "[3, \", a]", "[4, ;a, 1.1]" };
    //TestUtil.displayLists( Arrays.asList(match), lines );
    assertEquals( 2, cm.getLineCount() );
    assertEquals( 0, cm.getBadLineCount() );
    assertEquals( Arrays.asList(match).toString(), lines.toString() );

    cm.setStartLine( 4 );
    cm.setEndLine( 8 );
    lines = cm.loadAsLists( TestUtil.getTextCsvFile("01") );
    match = new String[] { "[4, ;a, 1.1]", "[, a, ]", "[5, , ]", "[6, a, b, c, d]", "[7, 1.1, ]" };
    //TestUtil.displayLists( Arrays.asList(match), lines );
    assertEquals( 5, cm.getLineCount() );
    assertEquals( 0, cm.getBadLineCount() );
    assertEquals( Arrays.asList(match).toString(), lines.toString() );

    cm.setStartLine( 4 );
    cm.setEndLine( 4 );
    lines = cm.loadAsLists( TestUtil.getTextCsvFile("01") );
    match = new String[] { "[4, ;a, 1.1]" };
    //TestUtil.displayLists( Arrays.asList(match), lines );
    assertEquals( 1, cm.getLineCount() );
    assertEquals( 0, cm.getBadLineCount() );
    assertEquals( Arrays.asList(match).toString(), lines.toString() );

    cm.setStartLine( 5 );
    try {
      lines = cm.loadAsLists( TestUtil.getTextCsvFile("01") );
      fail();
    }
    catch( CsvManagerException e ) {
      assertEquals( CsvManagerException.CODE_bad_end_line, e.getCode() );
      assertEquals( "5", e.getContextValues().get("start") );
      assertEquals( "4", e.getContextValues().get("end") );
    }

    try {
      cm.setEndLine( 3 );
      fail();
    }
    catch( CsvManagerException e ) {
      assertEquals( CsvManagerException.CODE_bad_end_line, e.getCode() );
      assertEquals( "5", e.getContextValues().get("start") );
      assertEquals( "3", e.getContextValues().get("end") );
    }


    try {
      cm.setStartLine( 0 );
      fail();
    }
    catch( CsvManagerException e ) {
      assertEquals( CsvManagerException.CODE_bad_start_line, e.getCode() );
      assertEquals( "0", (String)e.getContext() );
    }

  }



  public void testStartEndLineSave() throws Exception {
    System.out.println( "testStartEndLineSave" );

    CsvManager cm = new CsvManager();

    List lines = cm.load( TestUtil.getTextCsvFile("01") );
    assertEquals( 22, cm.getLineCount() );

    System.out.println( "a" );

    String s = cm.saveToString( lines );    
    System.out.println( "b" );
    lines = cm.loadFromString( s );
    System.out.println( "c" );
    assertEquals( 22, cm.getLineCount() );

    cm.setStartLine(3);
    s = cm.saveToString( lines );    
    cm.setStartLine(1);
    lines = cm.loadFromString( s );
    assertEquals( 20, lines.size() );
    assertEquals( 20, cm.getLineCount() );

    cm.setStartLine(4);
    cm.setEndLine(10);
    s = cm.saveToString( lines );    
    cm.setStartLine(1);
    lines = cm.loadFromString( s );
    assertEquals( 7, cm.getLineCount() );

  }


  public void testEscapeMap() throws Exception {
    System.out.println( "testEscapeMap" );

    CsvManager cm = new CsvManager();

    cm.getCsvSpec().setUseEscapeMap( true );
    cm.setSeparator( ":" );
    List lines = cm.loadAsLists( TestUtil.getTextCsvFile("04") );
    String[] match = new String[] { "[1, foo, a]", "[2, fo:y, b]", "[3, fo\nx, c]", 
                                    "[4, fo:\nz, d]", "[5, fo\n:a, e]", "[6, fo\r:a, f]", "[7, fo\ra, g]" };

    //TestUtil.displayLists( Arrays.asList(match), lines );
    assertEquals( Arrays.asList(match).toString(), lines.toString() );

    String csvout = cm.saveAsListsToString( lines );

    // notice: "fo:y" = : not escaped as quoted, and fo\ra is quoted as internal newline
    assertEquals( "1:foo:a\n2:\"fo:y\":b\n3:\"fo\\nx\":c\n4:\"fo:\\nz\":d\n5:\"fo\\n:a\":e\n6:\"fo\\r:a\":f\n7:\"fo\\ra\":g\n",
                  TextUtil.replace( csvout, "\r\n", "\n" ) );
  }



  public void testLoadSave() throws Exception {
    System.out.println( "testLoadSave" );

    CsvManager cm = new CsvManager();

    String s01 = "1,foo,a\n2,bar,b\n3,baz,c\n";    

    // LineListener
    BasicLineListener bll = new BasicLineListener();

    cm.load( TestUtil.getTextCsvFile("05"), bll );
    String d01s = TextUtil.replace( cm.saveToString( bll.getData() ), "\r", "" );
    assertEquals( s01, d01s );

    bll = new BasicLineListener();
    cm.load( TestUtil.getTextCsvFile("05").getAbsolutePath(), bll );
    d01s = TextUtil.replace( cm.saveToString( bll.getData() ), "\r", "" );
    assertEquals( s01, d01s );

    bll = new BasicLineListener();
    cm.load( new FileInputStream( TestUtil.getTextCsvFile("05") ), bll );
    d01s = TextUtil.replace( cm.saveToString( bll.getData() ), "\r", "" );
    assertEquals( s01, d01s );

    bll = new BasicLineListener();
    cm.loadFromString( s01, bll );
    d01s = TextUtil.replace( cm.saveToString( bll.getData() ), "\r", "" );
    assertEquals( s01, d01s );


    // List of String[]
    
    List   d01  = cm.load( TestUtil.getTextCsvFile("05") );
    d01s = TextUtil.replace( cm.saveToString( d01 ), "\r", "" );
    assertEquals( s01, d01s );

    d01  = cm.load( TestUtil.getTextCsvFile("05").getAbsolutePath() );
    d01s = TextUtil.replace( cm.saveToString( d01 ), "\r", "" );
    assertEquals( s01, d01s );

    d01  = cm.load( new FileInputStream( TestUtil.getTextCsvFile("05") ) );
    d01s = TextUtil.replace( cm.saveToString( d01 ), "\r", "" );
    assertEquals( s01, d01s );

    d01  = cm.loadFromString( s01 );
    d01s = TextUtil.replace( cm.saveToString( d01 ), "\r", "" );
    assertEquals( s01, d01s );


    // List of List

    d01  = cm.loadAsLists( TestUtil.getTextCsvFile("05") );
    d01s = TextUtil.replace( cm.saveAsListsToString( d01 ), "\r", "" );
    assertEquals( s01, d01s );

    d01  = cm.loadAsLists( TestUtil.getTextCsvFile("05").getAbsolutePath() );
    d01s = TextUtil.replace( cm.saveAsListsToString( d01 ), "\r", "" );
    assertEquals( s01, d01s );

    d01  = cm.loadAsLists( new FileInputStream( TestUtil.getTextCsvFile("05") ) );
    d01s = TextUtil.replace( cm.saveAsListsToString( d01 ), "\r", "" );
    assertEquals( s01, d01s );

    d01  = cm.loadAsListsFromString( s01 );
    d01s = TextUtil.replace( cm.saveAsListsToString( d01 ), "\r", "" );
    assertEquals( s01, d01s );


    // TableModel

    TableModel tm01  = cm.loadTableModel( TestUtil.getTextCsvFile("05"), false );
    String     tm01s = TextUtil.replace( cm.saveToString( tm01, false ), "\r", "" );
    assertEquals( s01, tm01s );

    tm01  = cm.loadTableModel( TestUtil.getTextCsvFile("05").getAbsolutePath(), false );
    tm01s = TextUtil.replace( cm.saveToString( tm01, false ), "\r", "" );
    assertEquals( s01, tm01s );

    tm01  = cm.loadTableModel( new FileInputStream( TestUtil.getTextCsvFile("05") ), false );
    tm01s = TextUtil.replace( cm.saveToString( tm01, false ), "\r", "" );
    assertEquals( s01, tm01s );

    tm01  = cm.loadTableModelFromString( s01, false );
    tm01s = TextUtil.replace( cm.saveToString( tm01, false ), "\r", "" );
    assertEquals( s01, tm01s );


    // ResultSet

    String s02 = "Identifier,Name,Code\n1,foo,a\n2,bar,b\n3,baz,c\n";    

    StringBuffer sb = new StringBuffer();
    //cm.setFieldListener( new DebugFieldListener() );
    ResultSet rs02  = cm.loadResultSet( TestUtil.getTextCsvFile("06"), true );
    while( rs02.next() ) {
      sb.append( rs02.getString("Identifier")+", "+rs02.getString("Name")+", "+rs02.getString("Code")+"|" );
    }
    assertEquals( "1, foo, a|2, bar, b|3, baz, c|", sb.toString() );
    rs02.beforeFirst();



    sb = new StringBuffer();
    rs02  = cm.loadResultSet( TestUtil.getTextCsvFile("06"), false );
    while( rs02.next() ) {
      sb.append( rs02.getString(1)+", "+rs02.getString(2)+", "+rs02.getString(3)+"|" );
    }
    assertEquals( "Identifier, Name, Code|1, foo, a|2, bar, b|3, baz, c|", sb.toString() );


    rs02  = cm.loadResultSet( TestUtil.getTextCsvFile("06"), true );
    String    rs02s = TextUtil.replace( cm.saveToString( rs02, true ), "\r", "" );
    assertEquals( s02, rs02s );
    rs02.beforeFirst();
    String    rs02snh = TextUtil.replace( cm.saveToString( rs02, false ), "\r", "" );
    assertEquals( s01, rs02snh );

    rs02  = cm.loadResultSet( TestUtil.getTextCsvFile("06").getAbsolutePath(), true );
    rs02s = TextUtil.replace( cm.saveToString( rs02, true ), "\r", "" );
    assertEquals( s02, rs02s );

    rs02  = cm.loadResultSet( new FileInputStream( TestUtil.getTextCsvFile("06") ), true );
    rs02s = TextUtil.replace( cm.saveToString( rs02, true ), "\r", "" );
    assertEquals( s02, rs02s );

    rs02  = cm.loadResultSetFromString( s02, true );
    rs02s = TextUtil.replace( cm.saveToString( rs02, true ), "\r", "" );
    assertEquals( s02, rs02s );


    // can handle empty lists
    String s03 = cm.saveToString( new ArrayList() );
    assertEquals( "", s03 );
  }

      


  public void testFieldListener() throws Exception {
    System.out.println( "testFieldListener" );

    // test normal
    CsvManager       cm  = new CsvManager();
    TestFieldListener tcl = new TestFieldListener();
    cm.setFieldListener( tcl );
    cm.load( TestUtil.getTextCsvFile("01") );    
    assertEquals( "1:foo:a'a:|2:f\"o:a:|3:\":a:|:|4:;a:1.1:|:a::|5::|6:a:b:c:d:|7:1.1::|:|8:b::|b,b:a,a:1.1:|9::a::|"
                  +"10:a:::|11:::a::|12:::|13:::|14::::|:::|:::|::::|:::::|", tcl.buffer.toString() );
    assertEquals( "1:1,foo,a'a|2:2,\"f\"\"o\",a|3:3,\"\"\"\",a|4:|5:4,;a,1.1|6:,a|7:5|8:6, a, \"b\", \"c\" , d |9:7,1.1|10:|11:8,b,|12:\"b,b\",\"a,a\",1.1|13:9,,a,|14:10,a,,|15:11,,,a,|16:12,|17:13,,|18:14,,,|19:,|20:,,|21:,,,|22:,,,,|", tcl.origbuf.toString() );
    assertEquals( 22, tcl.linecount );    

    // test semantic badline
    tcl.buffer    = new StringBuffer();
    tcl.origbuf   = new StringBuffer();
    tcl.linecount = 0;
    tcl.failon3   = true;
    cm.setIgnoreBadLines( true );
    cm.load( TestUtil.getTextCsvFile("01") );    
    assertEquals( "1:foo:a'a:|2:f\"o:a:|3:\":a:|:|4:;a:1.1:|:a::|5::|6:a:b:c:d:|7:1.1::|:|8:b::|b,b:a,a:1.1:|9::a::|"
                  +"10:a:::|11:::a::|12:::|13:::|14::::|:::|:::|::::|:::::|", tcl.buffer.toString() );
    assertEquals( "1:1,foo,a'a|2:2,\"f\"\"o\",a|3:3,\"\"\"\",a|4:|5:4,;a,1.1|6:,a|7:5|8:6, a, \"b\", \"c\" , d |9:7,1.1|10:|11:8,b,|12:\"b,b\",\"a,a\",1.1|13:9,,a,|14:10,a,,|15:11,,,a,|16:12,|17:13,,|18:14,,,|19:,|20:,,|21:,,,|22:,,,,|", tcl.origbuf.toString() );
    assertEquals( 22, tcl.linecount );    
    assertEquals( 1, cm.getBadLineCount() );
    assertEquals( 22, cm.getLineCount() );


    tcl.buffer    = new StringBuffer();
    tcl.origbuf   = new StringBuffer();
    tcl.linecount = 0;
    tcl.failon3   = true;
    cm.setIgnoreBadLines( false );
    try {
      cm.load( TestUtil.getTextCsvFile("01") );    
    }
    catch( Exception e ) {
      CsvManagerException ce = (CsvManagerException) e;
      assertEquals( "A line was rejected due to invalid data and caused processing to halt. The line was: '3,\"\"\"\",a'.", ce.toString() );
      assertEquals( "[CSV:BadLine:3:failon3:SEMANTIC:3,\"\"\"\",a]", ce.getBadLine().toString() );
    }
    assertEquals( "1:foo:a'a:|2:f\"o:a:|3:\":a:|", tcl.buffer.toString() );
    assertEquals( "1:1,foo,a'a|2:2,\"f\"\"o\",a|3:3,\"\"\"\",a|", tcl.origbuf.toString() );
    assertEquals( 3, tcl.linecount );    
    assertEquals( 1, cm.getBadLineCount() );
    assertEquals( 3, cm.getLineCount() );


    // test syntactic badlines
    tcl.failon3   = false;
    tcl.buffer    = new StringBuffer();
    tcl.origbuf   = new StringBuffer();
    tcl.linecount = 0;
    cm.setIgnoreBadLines( true );
    cm.load( TestUtil.getTextCsvFile("fail01") );    
    assertEquals( "1:1,foo,a'a|3:3,f\"o,c|5:5,foy,e|7:7,f\\oz,g|", tcl.origbuf.toString() );
    ass