/* Copyright (c) 2005-2007 Ricebridge. All Rights Reserved.
 *
 * This file is available under the terms and conditions of the
 * Ricebridge "Open Source API" policy; Ricebridge grants use of this
 * copyrighted work under the terms of a BSD-style license only. See
 * http://www.opensource.org/licenses/bsd-license.php for more
 * information.
 * 
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 
 *  - Redistributions of source code must retain the above copyright
 *  notice, this list of conditions and the following disclaimer.
 *
 *  - Redistributions in binary form must reproduce the above
 *  copyright notice, this list of conditions and the following
 *  disclaimer in the documentation and/or other materials provided
 *  with the distribution.
 *
 *  - Neither the name of the Ricebridge nor the names of its
 *  contributors may be used to endorse or promote products derived
 *  from this software without specific prior written permission.
 * 
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
 * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
 * OF THE POSSIBILITY OF SUCH DAMAGE.  
 */

package com.ricebridge.csvman.test;


import com.ricebridge.csvman.*;
import com.ricebridge.data.Text;

import junit.framework.*;
import junit.textui.*;

import java.util.*;
import java.io.*;



/** Test cases for {@link com.ricebridge.csvman.BadLine}.
 *    <p>The <b><a href="BadLineTest.java.html">Source Code</a></b> of this Java class 
 *    is available under a <a href="http://www.opensource.org/licenses/bsd-license.php">BSD-style license</a>.</p>
 */
public class BadLineTest extends TestCase {

  // standard test methods
  
  public BadLineTest( String pName ) {
    super( pName );
  }

  public static TestSuite suite() {
    return new TestSuite( BadLineTest.class );
  }

  public static void main( String[] pArgs ) throws Exception {
    TestRunner.run( suite() );
  }


  public void testNormal() throws Exception {
    String csv = "1,a\n2,\"b\"b\n3,\"c\"";
    CsvManager cm = new CsvManager();
    cm.getCsvSpec().setIgnoreBadLines(true);
    List data = cm.loadAsLists( new Text(csv) );
    assertEquals( "[[1, a], [3, c]]", data.toString() );
    assertEquals( "[[CSV:BadLine:2:unexpected token: b:SYNTAX:2,\"b\"b]]", cm.getBadLines().toString() );
    assertEquals( 1, cm.getBadLineCount() );

    cm.getCsvSpec().setCollectBadLines(false);
    data = cm.loadAsLists( new Text(csv) );
    assertEquals( "[[1, a], [3, c]]", data.toString() );
    assertEquals( "[]", cm.getBadLines().toString() );
    assertEquals( 1, cm.getBadLineCount() );
  }


  public void testResume() throws Exception {
    String csv = "1,a,AA\n2,\"b\"b,BB\n3,\"c\",CC";
    CsvManager cm = new CsvManager();
    cm.getCsvSpec().setIgnoreBadLines(true);
    List data = cm.loadAsLists( new Text(csv) );
    assertEquals( "[[1, a, AA], [3, c, CC]]", data.toString() );
    assertEquals( "[[CSV:BadLine:2:unexpected token: b:SYNTAX:2,\"b\"b,BB]]", cm.getBadLines().toString() );
    assertEquals( 1, cm.getBadLineCount() );

    cm.getCsvSpec().setCollectBadLines(false);
    data = cm.loadAsLists( new Text(csv) );
    assertEquals( "[[1, a, AA], [3, c, CC]]", data.toString() );
    assertEquals( "[]", cm.getBadLines().toString() );
    assertEquals( 1, cm.getBadLineCount() );
  }



  public void testFollowing() throws Exception {
    String csv = "1,a,AA\n2,bbbb,BB\n3,c,CC";
    CsvManager cm = new CsvManager();
    cm.getCsvSpec().setDataFieldMaxLength(2);

    try {
      List data = cm.loadAsLists( new Text(csv) );
      fail();
    }
    catch( Exception e ) {
      CsvManagerException csvex = (CsvManagerException) e;
      assertEquals( CsvManagerException.CODE_dfmaxlen, csvex.getCode() );
      assertEquals( "Data field was longer then maximum allowed length (CsvSpec.getDataFieldMaxLength=2): 'bb...'.", csvex.toString() );
      BadLine bl = csvex.getBadLine();

      // contains entire line as CsvReader has 2048 buffer
      assertEquals( "[CSV:BadLine:2:Data field was longer then maximum allowed length (CsvSpec.getDataFieldMaxLength=2): 'bb...'.:SEMANTIC:2,bbbb,BB]", 
                    bl.toString() );
    }



    cm.getCsvSpec().setIgnoreBadLines(true);
    List data = cm.loadAsLists( new Text(csv) );

    // contains entire line as CsvReader has 2048 buffer
    assertEquals( "2,bbbb,BB", ((BadLine)cm.getBadLines().get(0)).getOriginalLine() );
    //System.out.println( cm.getBadLines() );
    assertEquals( 1, cm.getBadLineCount() );
    assertEquals( 3, cm.getLineCount() );
    assertEquals( "[[1, a, AA], [3, c, CC]]", data.toString() );


    cm.getCsvSpec().setDataFieldMaxLength(30);
    csv = "1,a,AA\n2,123456789012345678901234567890a,BB\n3,c,CC";
    data = cm.loadAsLists( new Text(csv) );
    assertEquals( "[[1, a, AA], [3, c, CC]]", data.toString() );
    assertEquals( "2,123456789012345678901234567890a,BB", ((BadLine)cm.getBadLines().get(0)).getOriginalLine() );
    assertEquals( 1, cm.getBadLineCount() );
    assertEquals( 3, cm.getLineCount() );
  }




  public void testIgnoreBadLinesWithFailingListener() {
    Text csv = new Text("1,a,AA\n2,b,BB\n3,c,CC\n4,d,DD\n5,e,EE\n");
    
    CsvManager csvman = new CsvManager();
    csvman.getCsvSpec().setIgnoreBadLines(true);

    FailingListener fl       = null;
    List            data     = null;
    List            badlines = null;
    String[]        blstr    = null;
    int[][]         cut      = null;


    String[] datastr_one = new String[] { "", "[[2, b, BB], [3, c, CC], [4, d, DD], [5, e, EE]]", 
                                            "[[1, a, AA], [3, c, CC], [4, d, DD], [5, e, EE]]",
                                            "[[1, a, AA], [2, b, BB], [4, d, DD], [5, e, EE]]",
                                            "[[1, a, AA], [2, b, BB], [3, c, CC], [5, e, EE]]",
                                            "[[1, a, AA], [2, b, BB], [3, c, CC], [4, d, DD]]" };

    String[] datastr_two = new String[] { "", "[[2, b, BB], [4, d, DD], [5, e, EE]]", 
                                            "[[1, a, AA], [3, c, CC], [5, e, EE]]",
                                            "[[1, a, AA], [2, b, BB], [4, d, DD]]",
                                            "[[2, b, BB], [3, c, CC], [5, e, EE]]",
                                            "[[1, a, AA], [3, c, CC], [4, d, DD]]" };


    // FT_BADLINE 1x

    blstr = new String[] { "", "[[CSV:BadLine:1:failtype=badline:SEMANTIC:1,a,AA]]",
                           "[[CSV:BadLine:2:failtype=badline:SEMANTIC:2,b,BB]]",
                           "[[CSV:BadLine:3:failtype=badline:SEMANTIC:3,c,CC]]",
                           "[[CSV:BadLine:4:failtype=badline:SEMANTIC:4,d,DD]]",
                           "[[CSV:BadLine:5:failtype=badline:SEMANTIC:5,e,EE]]" };
    
    for( int i = 1; i < 6; i++ ) {
      fl = new FailingListener( new int[] {i,-1}, new int[] { FailingListener.FT_BADLINE} );
      csvman.load(csv,fl);
      data     = fl.getData();
      badlines = csvman.getBadLines();
      assertEquals(datastr_one[i], data.toString() );
      assertEquals(blstr[i], badlines.toString() );
    }


    // FT_BADLINE 2x

    blstr = new String[] { "", "[[CSV:BadLine:1:failtype=badline:SEMANTIC:1,a,AA], [CSV:BadLine:3:failtype=badline:SEMANTIC:3,c,CC]]",
                           "[[CSV:BadLine:2:failtype=badline:SEMANTIC:2,b,BB], [CSV:BadLine:4:failtype=badline:SEMANTIC:4,d,DD]]",
                           "[[CSV:BadLine:3:failtype=badline:SEMANTIC:3,c,CC], [CSV:BadLine:5:failtype=badline:SEMANTIC:5,e,EE]]",
                           "[[CSV:BadLine:1:failtype=badline:SEMANTIC:1,a,AA], [CSV:BadLine:4:failtype=badline:SEMANTIC:4,d,DD]]",
                           "[[CSV:BadLine:2:failtype=badline:SEMANTIC:2,b,BB], [CSV:BadLine:5:failtype=badline:SEMANTIC:5,e,EE]]" };

    cut = new int[][] { {}, {1,3,-1}, {2,4,-1}, {3,5,-1},  {1,4,-1}, {2,5,-1} };
    
    for( int i = 1; i < 6; i++ ) {
      fl = new FailingListener( cut[i], new int[] { FailingListener.FT_BADLINE, FailingListener.FT_BADLINE} );
      csvman.load(csv,fl);
      data     = fl.getData();
      badlines = csvman.getBadLines();
      assertEquals(datastr_two[i], data.toString() );
      assertEquals(blstr[i], badlines.toString() );
    }



    // FT_EXCEPTION 1x

    blstr = new String[] { "", "[[CSV:BadLine:1:The LineListener com.ricebridge.csvman.test.BadLineTest$FailingListener threw an Exception from method addLineImpl: java.lang.Exception: 1 1,a,AA failtype=exception:SEMANTIC:2]]",
                           "[[CSV:BadLine:2:The LineListener com.ricebridge.csvman.test.BadLineTest$FailingListener threw an Exception from method addLineImpl: java.lang.Exception: 2 2,b,BB failtype=exception:SEMANTIC:3]]",
                           "[[CSV:BadLine:3:The LineListener com.ricebridge.csvman.test.BadLineTest$FailingListener threw an Exception from method addLineImpl: java.lang.Exception: 3 3,c,CC failtype=exception:SEMANTIC:4]]",
                           "[[CSV:BadLine:4:The LineListener com.ricebridge.csvman.test.BadLineTest$FailingListener threw an Exception from method addLineImpl: java.lang.Exception: 4 4,d,DD failtype=exception:SEMANTIC:5]]",
                           "[[CSV:BadLine:5:The LineListener com.ricebridge.csvman.test.BadLineTest$FailingListener threw an Exception from method addLineImpl: java.lang.Exception: 5 5,e,EE failtype=exception:SEMANTIC:]]" };
    
    for( int i = 1; i < 6; i++ ) {
      fl = new FailingListener( new int[] {i,-1}, new int[] { FailingListener.FT_EXCEPTION} );
      csvman.load(csv,fl);
      data     = fl.getData();
      badlines = csvman.getBadLines();
      assertEquals(datastr_one[i], data.toString() );
      assertEquals(blstr[i], badlines.toString() );
    }


    // FT_EXCEPTION 2x

    blstr = new String[] { "", "[[CSV:BadLine:1:The LineListener com.ricebridge.csvman.test.BadLineTest$FailingListener threw an Exception from method addLineImpl: java.lang.Exception: 1 1,a,AA failtype=exception:SEMANTIC:2], [CSV:BadLine:3:The LineListener com.ricebridge.csvman.test.BadLineTest$FailingListener threw an Exception from method addLineImpl: java.lang.Exception: 3 3,c,CC failtype=exception:SEMANTIC:4]]",
                           "[[CSV:BadLine:2:The LineListener com.ricebridge.csvman.test.BadLineTest$FailingListener threw an Exception from method addLineImpl: java.lang.Exception: 2 2,b,BB failtype=exception:SEMANTIC:3], [CSV:BadLine:4:The LineListener com.ricebridge.csvman.test.BadLineTest$FailingListener threw an Exception from method addLineImpl: java.lang.Exception: 4 4,d,DD failtype=exception:SEMANTIC:5]]",
                           "[[CSV:BadLine:3:The LineListener com.ricebridge.csvman.test.BadLineTest$FailingListener threw an Exception from method addLineImpl: java.lang.Exception: 3 3,c,CC failtype=exception:SEMANTIC:4], [CSV:BadLine:5:The LineListener com.ricebridge.csvman.test.BadLineTest$FailingListener threw an Exception from method addLineImpl: java.lang.Exception: 5 5,e,EE failtype=exception:SEMANTIC:]]",
                           "[[CSV:BadLine:1:The LineListener com.ricebridge.csvman.test.BadLineTest$FailingListener threw an Exception from method addLineImpl: java.lang.Exception: 1 1,a,AA failtype=exception:SEMANTIC:2], [CSV:BadLine:4:The LineListener com.ricebridge.csvman.test.BadLineTest$FailingListener threw an Exception from method addLineImpl: java.lang.Exception: 4 4,d,DD failtype=exception:SEMANTIC:5]]",
                           "[[CSV:BadLine:2:The LineListener com.ricebridge.csvman.test.BadLineTest$FailingListener threw an Exception from method addLineImpl: java.lang.Exception: 2 2,b,BB failtype=exception:SEMANTIC:3], [CSV:BadLine:5:The LineListener com.ricebridge.csvman.test.BadLineTest$FailingListener threw an Exception from method addLineImpl: java.lang.Exception: 5 5,e,EE failtype=exception:SEMANTIC:]]" };

    cut = new int[][] { {}, {1,3,-1}, {2,4,-1}, {3,5,-1},  {1,4,-1}, {2,5,-1} };
    
    for( int i = 1; i < 6; i++ ) {
      fl = new FailingListener( cut[i], new int[] { FailingListener.FT_EXCEPTION, FailingListener.FT_EXCEPTION} );
      csvman.load(csv,fl);
      data     = fl.getData();
      badlines = csvman.getBadLines();
      assertEquals(datastr_two[i], data.toString() );
      assertEquals(blstr[i], badlines.toString() );
    }


    // FT_CSVMAN_EXCEPTION 1x

    blstr = new String[] { "", "[[CSV:BadLine:1:An incorrectly formatted line caused processing to halt. The line was: '1 1,a,AA failtype=csvmanex'. The parse error was: 'none'.:SEMANTIC:2]]",
                           "[[CSV:BadLine:2:An incorrectly formatted line caused processing to halt. The line was: '2 2,b,BB failtype=csvmanex'. The parse error was: 'none'.:SEMANTIC:3]]",
                           "[[CSV:BadLine:3:An incorrectly formatted line caused processing to halt. The line was: '3 3,c,CC failtype=csvmanex'. The parse error was: 'none'.:SEMANTIC:4]]",
                           "[[CSV:BadLine:4:An incorrectly formatted line caused processing to halt. The line was: '4 4,d,DD failtype=csvmanex'. The parse error was: 'none'.:SEMANTIC:5]]",
                           "[[CSV:BadLine:5:An incorrectly formatted line caused processing to halt. The line was: '5 5,e,EE failtype=csvmanex'. The parse error was: 'none'.:SEMANTIC:]]" };
    
    for( int i = 1; i < 6; i++ ) {
      fl = new FailingListener( new int[] {i,-1}, new int[] { FailingListener.FT_CSVMAN_EXCEPTION} );
      csvman.load(csv,fl);
      data     = fl.getData();
      badlines = csvman.getBadLines();
      assertEquals(datastr_one[i], data.toString() );
      assertEquals(blstr[i], badlines.toString() );
    }


    // FT_CSVMAN_EXCEPTION 2x

    blstr = new String[] { "", "[[CSV:BadLine:1:An incorrectly formatted line caused processing to halt. The line was: '1 1,a,AA failtype=csvmanex'. The parse error was: 'none'.:SEMANTIC:2], [CSV:BadLine:3:An incorrectly formatted line caused processing to halt. The line was: '3 3,c,CC failtype=csvmanex'. The parse error was: 'none'.:SEMANTIC:4]]",
                           "[[CSV:BadLine:2:An incorrectly formatted line caused processing to halt. The line was: '2 2,b,BB failtype=csvmanex'. The parse error was: 'none'.:SEMANTIC:3], [CSV:BadLine:4:An incorrectly formatted line caused processing to halt. The line was: '4 4,d,DD failtype=csvmanex'. The parse error was: 'none'.:SEMANTIC:5]]",
                           "[[CSV:BadLine:3:An incorrectly formatted line caused processing to halt. The line was: '3 3,c,CC failtype=csvmanex'. The parse error was: 'none'.:SEMANTIC:4], [CSV:BadLine:5:An incorrectly formatted line caused processing to halt. The line was: '5 5,e,EE failtype=csvmanex'. The parse error was: 'none'.:SEMANTIC:]]",
                           "[[CSV:BadLine:1:An incorrectly formatted line caused processing to halt. The line was: '1 1,a,AA failtype=csvmanex'. The parse error was: 'none'.:SEMANTIC:2], [CSV:BadLine:4:An incorrectly formatted line caused processing to halt. The line was: '4 4,d,DD failtype=csvmanex'. The parse error was: 'none'.:SEMANTIC:5]]",
                           "[[CSV:BadLine:2:An incorrectly formatted line caused processing to halt. The line was: '2 2,b,BB failtype=csvmanex'. The parse error was: 'none'.:SEMANTIC:3], [CSV:BadLine:5:An incorrectly formatted line caused processing to halt. The line was: '5 5,e,EE failtype=csvmanex'. The parse error was: 'none'.:SEMANTIC:]]" };

    cut = new int[][] { {}, {1,3,-1}, {2,4,-1}, {3,5,-1},  {1,4,-1}, {2,5,-1} };
    
    for( int i = 1; i < 6; i++ ) {
      fl = new FailingListener( cut[i], new int[] { FailingListener.FT_CSVMAN_EXCEPTION, FailingListener.FT_CSVMAN_EXCEPTION} );
      csvman.load(csv,fl);
      data     = fl.getData();
      badlines = csvman.getBadLines();
      assertEquals(datastr_two[i], data.toString() );
      assertEquals(blstr[i], badlines.toString() );
    }



    // FT_BADLINE, FT_EXCEPTION

    blstr = new String[] { "", "[[CSV:BadLine:1:failtype=badline:SEMANTIC:1,a,AA], [CSV:BadLine:3:The LineListener com.ricebridge.csvman.test.BadLineTest$FailingListener threw an Exception from method addLineImpl: java.lang.Exception: 3 3,c,CC failtype=exception:SEMANTIC:4]]",
                           "[[CSV:BadLine:2:failtype=badline:SEMANTIC:2,b,BB], [CSV:BadLine:4:The LineListener com.ricebridge.csvman.test.BadLineTest$FailingListener threw an Exception from method addLineImpl: java.lang.Exception: 4 4,d,DD failtype=exception:SEMANTIC:5]]",
                           "[[CSV:BadLine:3:failtype=badline:SEMANTIC:3,c,CC], [CSV:BadLine:5:The LineListener com.ricebridge.csvman.test.BadLineTest$FailingListener threw an Exception from method addLineImpl: java.lang.Exception: 5 5,e,EE failtype=exception:SEMANTIC:]]",
                           "[[CSV:BadLine:1:failtype=badline:SEMANTIC:1,a,AA], [CSV:BadLine:4:The LineListener com.ricebridge.csvman.test.BadLineTest$FailingListener threw an Exception from method addLineImpl: java.lang.Exception: 4 4,d,DD failtype=exception:SEMANTIC:5]]",
                           "[[CSV:BadLine:2:failtype=badline:SEMANTIC:2,b,BB], [CSV:BadLine:5:The LineListener com.ricebridge.csvman.test.BadLineTest$FailingListener threw an Exception from method addLineImpl: java.lang.Exception: 5 5,e,EE failtype=exception:SEMANTIC:]]" };

    cut = new int[][] { {}, {1,3,-1}, {2,4,-1}, {3,5,-1},  {1,4,-1}, {2,5,-1} };
    
    for( int i = 1; i < 6; i++ ) {
      fl = new FailingListener( cut[i], new int[] { FailingListener.FT_BADLINE, FailingListener.FT_EXCEPTION } );
      csvman.load(csv,fl);
      data     = fl.getData();
      badlines = csvman.getBadLines();
      assertEquals(datastr_two[i], data.toString() );
      assertEquals(blstr[i], badlines.toString() );
    }


    // FT_EXCEPTION, FT_BADLINE

    blstr = new String[] { "", "[[CSV:BadLine:1:The LineListener com.ricebridge.csvman.test.BadLineTest$FailingListener threw an Exception from method addLineImpl: java.lang.Exception: 1 1,a,AA failtype=exception:SEMANTIC:2], [CSV:BadLine:3:failtype=badline:SEMANTIC:3,c,CC]]",
                           "[[CSV:BadLine:2:The LineListener com.ricebridge.csvman.test.BadLineTest$FailingListener threw an Exception from method addLineImpl: java.lang.Exception: 2 2,b,BB failtype=exception:SEMANTIC:3], [CSV:BadLine:4:failtype=badline:SEMANTIC:4,d,DD]]",
                           "[[CSV:BadLine:3:The LineListener com.ricebridge.csvman.test.BadLineTest$FailingListener threw an Exception from method addLineImpl: java.lang.Exception: 3 3,c,CC failtype=exception:SEMANTIC:4], [CSV:BadLine:5:failtype=badline:SEMANTIC:5,e,EE]]",
                           "[[CSV:BadLine:1:The LineListener com.ricebridge.csvman.test.BadLineTest$FailingListener threw an Exception from method addLineImpl: java.lang.Exception: 1 1,a,AA failtype=exception:SEMANTIC:2], [CSV:BadLine:4:failtype=badline:SEMANTIC:4,d,DD]]",
                           "[[CSV:BadLine:2:The LineListener com.ricebridge.csvman.test.BadLineTest$FailingListener threw an Exception from method addLineImpl: java.lang.Exception: 2 2,b,BB failtype=exception:SEMANTIC:3], [CSV:BadLine:5:failtype=badline:SEMANTIC:5,e,EE]]" };

    cut = new int[][] { {}, {1,3,-1}, {2,4,-1}, {3,5,-1},  {1,4,-1}, {2,5,-1} };
    
    for( int i = 1; i < 6; i++ ) {
      fl = new FailingListener( cut[i], new int[] { FailingListener.FT_EXCEPTION, FailingListener.FT_BADLINE } );
      csvman.load(csv,fl);
      data     = fl.getData();
      badlines = csvman.getBadLines();
      assertEquals(datastr_two[i], data.toString() );
      assertEquals(blstr[i], badlines.toString() );
    }


    // FT_EXCEPTION, FT_CSVMAN_EXCEPTION

    blstr = new String[] { "", "[[CSV:BadLine:1:The LineListener com.ricebridge.csvman.test.BadLineTest$FailingListener threw an Exception from method addLineImpl: java.lang.Exception: 1 1,a,AA failtype=exception:SEMANTIC:2], [CSV:BadLine:3:An incorrectly formatted line caused processing to halt. The line was: '3 3,c,CC failtype=csvmanex'. The parse error was: 'none'.:SEMANTIC:4]]",
                           "[[CSV:BadLine:2:The LineListener com.ricebridge.csvman.test.BadLineTest$FailingListener threw an Exception from method addLineImpl: java.lang.Exception: 2 2,b,BB failtype=exception:SEMANTIC:3], [CSV:BadLine:4:An incorrectly formatted line caused processing to halt. The line was: '4 4,d,DD failtype=csvmanex'. The parse error was: 'none'.:SEMANTIC:5]]",
                           "[[CSV:BadLine:3:The LineListener com.ricebridge.csvman.test.BadLineTest$FailingListener threw an Exception from method addLineImpl: java.lang.Exception: 3 3,c,CC failtype=exception:SEMANTIC:4], [CSV:BadLine:5:An incorrectly formatted line caused processing to halt. The line was: '5 5,e,EE failtype=csvmanex'. The parse error was: 'none'.:SEMANTIC:]]",
                           "[[CSV:BadLine:1:The LineListener com.ricebridge.csvman.test.BadLineTest$FailingListener threw an Exception from method addLineImpl: java.lang.Exception: 1 1,a,AA failtype=exception:SEMANTIC:2], [CSV:BadLine:4:An incorrectly formatted line caused processing to halt. The line was: '4 4,d,DD failtype=csvmanex'. The parse error was: 'none'.:SEMANTIC:5]]",
                           "[[CSV:BadLine:2:The LineListener com.ricebridge.csvman.test.BadLineTest$FailingListener threw an Exception from method addLineImpl: java.lang.Exception: 2 2,b,BB failtype=exception:SEMANTIC:3], [CSV:BadLine:5:An incorrectly formatted line caused processing to halt. The line was: '5 5,e,EE failtype=csvmanex'. The parse error was: 'none'.:SEMANTIC:]]" };

    cut = new int[][] { {}, {1,3,-1}, {2,4,-1}, {3,5,-1},  {1,4,-1}, {2,5,-1} };
    
    for( int i = 1; i < 6; i++ ) {
      fl = new FailingListener( cut[i], new int[] { FailingListener.FT_EXCEPTION, FailingListener.FT_CSVMAN_EXCEPTION } );
      csvman.load(csv,fl);
      data     = fl.getData();
      badlines = csvman.getBadLines();
      assertEquals(datastr_two[i], data.toString() );
      assertEquals(blstr[i], badlines.toString() );
    }


    // FT_CSVMAN_EXCEPTION, FT_EXCEPTION

    blstr = new String[] { "", "[[CSV:BadLine:1:An incorrectly formatted line caused processing to halt. The line was: '1 1,a,AA failtype=csvmanex'. The parse error was: 'none'.:SEMANTIC:2], [CSV:BadLine:3:The LineListener com.ricebridge.csvman.test.BadLineTest$FailingListener threw an Exception from method addLineImpl: java.lang.Exception: 3 3,c,CC failtype=exception:SEMANTIC:4]]",
                           "[[CSV:BadLine:2:An incorrectly formatted line caused processing to halt. The line was: '2 2,b,BB failtype=csvmanex'. The parse error was: 'none'.:SEMANTIC:3], [CSV:BadLine:4:The LineListener com.ricebridge.csvman.test.BadLineTest$FailingListener threw an Exception from method addLineImpl: java.lang.Exception: 4 4,d,DD failtype=exception:SEMANTIC:5]]",
                           "[[CSV:BadLine:3:An incorrectly formatted line caused processing to halt. The line was: '3 3,c,CC failtype=csvmanex'. The parse error was: 'none'.:SEMANTIC:4], [CSV:BadLine:5:The LineListener com.ricebridge.csvman.test.BadLineTest$FailingListener threw an Exception from method addLineImpl: java.lang.Exception: 5 5,e,EE failtype=exception:SEMANTIC:]]",
                           "[[CSV:BadLine:1:An incorrectly formatted line caused processing to halt. The line was: '1 1,a,AA failtype=csvmanex'. The parse error was: 'none'.:SEMANTIC:2], [CSV:BadLine:4:The LineListener com.ricebridge.csvman.test.BadLineTest$FailingListener threw an Exception from method addLineImpl: java.lang.Exception: 4 4,d,DD failtype=exception:SEMANTIC:5]]",
                           "[[CSV:BadLine:2:An incorrectly formatted line caused processing to halt. The line was: '2 2,b,BB failtype=csvmanex'. The parse error was: 'none'.:SEMANTIC:3], [CSV:BadLine:5:The LineListener com.ricebridge.csvman.test.BadLineTest$FailingListener threw an Exception from method addLineImpl: java.lang.Exception: 5 5,e,EE failtype=exception:SEMANTIC:]]" };

    cut = new int[][] { {}, {1,3,-1}, {2,4,-1}, {3,5,-1},  {1,4,-1}, {2,5,-1} };
    
    for( int i = 1; i < 6; i++ ) {
      fl = new FailingListener( cut[i], new int[] { FailingListener.FT_CSVMAN_EXCEPTION, FailingListener.FT_EXCEPTION } );
      csvman.load(csv,fl);
      data     = fl.getData();
      badlines = csvman.getBadLines();
      assertEquals(datastr_two[i], data.toString() );
      assertEquals(blstr[i], badlines.toString() );
    }


    // FT_BADLINE, FT_CSVMAN_EXCEPTION

    blstr = new String[] { "", "[[CSV:BadLine:1:failtype=badline:SEMANTIC:1,a,AA], [CSV:BadLine:3:An incorrectly formatted line caused processing to halt. The line was: '3 3,c,CC failtype=csvmanex'. The parse error was: 'none'.:SEMANTIC:4]]",
                           "[[CSV:BadLine:2:failtype=badline:SEMANTIC:2,b,BB], [CSV:BadLine:4:An incorrectly formatted line caused processing to halt. The line was: '4 4,d,DD failtype=csvmanex'. The parse error was: 'none'.:SEMANTIC:5]]",
                           "[[CSV:BadLine:3:failtype=badline:SEMANTIC:3,c,CC], [CSV:BadLine:5:An incorrectly formatted line caused processing to halt. The line was: '5 5,e,EE failtype=csvmanex'. The parse error was: 'none'.:SEMANTIC:]]",
                           "[[CSV:BadLine:1:failtype=badline:SEMANTIC:1,a,AA], [CSV:BadLine:4:An incorrectly formatted line caused processing to halt. The line was: '4 4,d,DD failtype=csvmanex'. The parse error was: 'none'.:SEMANTIC:5]]",
                           "[[CSV:BadLine:2:failtype=badline:SEMANTIC:2,b,BB], [CSV:BadLine:5:An incorrectly formatted line caused processing to halt. The line was: '5 5,e,EE failtype=csvmanex'. The parse error was: 'none'.:SEMANTIC:]]" };

    cut = new int[][] { {}, {1,3,-1}, {2,4,-1}, {3,5,-1},  {1,4,-1}, {2,5,-1} };
    
    for( int i = 1; i < 6; i++ ) {
      fl = new FailingListener( cut[i], new int[] { FailingListener.FT_BADLINE, FailingListener.FT_CSVMAN_EXCEPTION } );
      csvman.load(csv,fl);
      data     = fl.getData();
      badlines = csvman.getBadLines();
      assertEquals(datastr_two[i], data.toString() );
      assertEquals(blstr[i], badlines.toString() );
    }


    // FT_CSVMAN_EXCEPTION, FT_BADLINE

    blstr = new String[] { "", "[[CSV:BadLine:1:An incorrectly formatted line caused processing to halt. The line was: '1 1,a,AA failtype=csvmanex'. The parse error was: 'none'.:SEMANTIC:2], [CSV:BadLine:3:failtype=badline:SEMANTIC:3,c,CC]]",
                           "[[CSV:BadLine:2:An incorrectly formatted line caused processing to halt. The line was: '2 2,b,BB failtype=csvmanex'. The parse error was: 'none'.:SEMANTIC:3], [CSV:BadLine:4:failtype=badline:SEMANTIC:4,d,DD]]",
                           "[[CSV:BadLine:3:An incorrectly formatted line caused processing to halt. The line was: '3 3,c,CC failtype=csvmanex'. The parse error was: 'none'.:SEMANTIC:4], [CSV:BadLine:5:failtype=badline:SEMANTIC:5,e,EE]]",
                           "[[CSV:BadLine:1:An incorrectly formatted line caused processing to halt. The line was: '1 1,a,AA failtype=csvmanex'. The parse error was: 'none'.:SEMANTIC:2], [CSV:BadLine:4:failtype=badline:SEMANTIC:4,d,DD]]",
                           "[[CSV:BadLine:2:An incorrectly formatted line caused processing to halt. The line was: '2 2,b,BB failtype=csvmanex'. The parse error was: 'none'.:SEMANTIC:3], [CSV:BadLine:5:failtype=badline:SEMANTIC:5,e,EE]]" };

    cut = new int[][] { {}, {1,3,-1}, {2,4,-1}, {3,5,-1},  {1,4,-1}, {2,5,-1} };
    
    for( int i = 1; i < 6; i++ ) {
      fl = new FailingListener( cut[i], new int[] { FailingListener.FT_CSVMAN_EXCEPTION, FailingListener.FT_BADLINE } );
      csvman.load(csv,fl);
      data     = fl.getData();
      badlines = csvman.getBadLines();
      assertEquals(datastr_two[i], data.toString() );
      assertEquals(blstr[i], badlines.toString() );
    }

    
  }



  public static class FailingListener extends AsListsLineListener {
    public static final int FT_BADLINE          = 1;
    public static final int FT_EXCEPTION        = 2;
    public static final int FT_CSVMAN_EXCEPTION = 3;

    private int[] iFailOnLines;
    private int[] iFailTypes;
    private int   iFailIndex = 0;

    // requires a sentinel matching line -1 at end
    public FailingListener( int[] pFailOnLines, int[] pFailTypes ) {
      iFailOnLines = pFailOnLines;
      iFailTypes   = pFailTypes;
    }

    public BadLine handleLineImpl( String[] pLine, int pNumFields, long pLineNumber, String pOriginalLine ) 
      throws Exception
    {
      if( iFailOnLines[iFailIndex] == pLineNumber ) {
        int failindex = iFailIndex;
        iFailIndex++;
        if( FT_BADLINE == iFailTypes[failindex] ) {
          return new BadLine( pLineNumber, pOriginalLine, "failtype=badline" );
        }
        else if( FT_EXCEPTION == iFailTypes[failindex] ) {
          throw new Exception( pLineNumber+" "+pOriginalLine+" failtype=exception" );
        }
        else {
          throw new CsvManagerException( CsvManagerException.CODE_bad_line, 
                                         new String[] { "line", pLineNumber+" "+pOriginalLine+" failtype=csvmanex", "errormsg", "none" } );
        }
      } 
      else {
        return super.handleLineImpl( pLine, pNumFields, pLineNumber, pOriginalLine );
      }
    }    
  }



  public void manual_testMemory() throws Exception {
    final PipedInputStream  is = new PipedInputStream();
    final PipedOutputStream os = new PipedOutputStream(is);

    final long numlines = 1000000;
    final String[] lines = new String[] {"\"1\"1,a,A","2,\"b\"b,B","3,c,\"C\"C","4,d,D"};

    Thread out = new Thread() {
        public void run() {
          System.out.println( "out t" );
          PrintWriter pw = new PrintWriter(os);
          for( int line = 0; line < numlines; line++ ) {
            pw.println( lines[line%4]);
          }
          pw.close();
        }
      };
    
    Thread in = new Thread() {
        public void run() {
          System.out.println( "in t" );
          CsvManager cm = new CsvManager();
          cm.getCsvSpec().setIgnoreBadLines(true);
          cm.getCsvSpec().setCollectBadLines(false);
          cm.load( is, new NullLineListener() );
          try {
            is.close();
          }
          catch( Exception e ) {
            System.out.println( e );
          }
        }
      };

    out.start();
    in.start();
    System.out.println( "running" );
  }



  public static class NullLineListener extends CustomLineListener {
    public BadLine handleLineImpl( String[] pLine, int pNumFields, long pLineNumber, String pOriginalLine ) {
      if( 0 == pLineNumber % 100 ) {
        Runtime rt = Runtime.getRuntime();
        System.out.println( pLineNumber+": "+(rt.totalMemory()-rt.freeMemory()) );
      }
      return null;
    }
  }
}