Tokenização de palavras em arquivos txt, doc e pdf

/* 
* ConstrutorDeTemplate1.java 
* 
* Created on 9 de Junho de 2007, 14:54 
* 
* @author 
*/ 

package br.unifacs.dis2007.template1; 

// Java core packages 
import java.awt.*; 
import java.awt.event.*; 
import java.io.*; 
import java.text.*; 
import java.util.*; 

// Java extension packages 
import javax.swing.*; 


public class ConstrutorDeTemplate1 extends JFrame 
implements ActionListener { 

private JTextField enterField; 
private JTextArea outputArea; 
private BufferedWriter out; 
private String word; 

// set up GUI 
public ConstrutorDeTemplate1() 
{ 
super( "Testing class File" ); 

enterField = new JTextField( 
"Enter file or directory name here" ); 
enterField.addActionListener( this ); 
outputArea = new JTextArea(); 

ScrollPane scrollPane = new ScrollPane(); 
scrollPane.add( outputArea ); 

Container container = getContentPane(); 
container.add( enterField, BorderLayout.NORTH ); 
container.add( scrollPane, BorderLayout.CENTER ); 

setSize( 400, 400 ); 
show(); 
} 

// display information about file user specifies 
public void actionPerformed( ActionEvent actionEvent ) 
{ 
File name = new File( actionEvent.getActionCommand() ); 

// if name exists, output information about it 
if ( name.exists() ) { 
outputArea.setText( 
name.getName() + " exists\n" + 
( name.isFile() ? 
"is a file\n" : "is not a file\n" ) + 
( name.isDirectory() ? 
"is a directory\n" : "is not a directory\n" ) + 
( name.isAbsolute() ? "is absolute path\n" : 
"is not absolute path\n" ) + 
"Last modified: " + name.lastModified() + 
"\nLength: " + name.length() + 
"\nPath: " + name.getPath() + 
"\nAbsolute path: " + name.getAbsolutePath() + 
"\nParent: " + name.getParent() ); 

// output information if name is a file 
if ( name.isFile() ) { 

// append contents of file to outputArea 

try { 
// Create the tokenizer to read from a file 
FileReader rd = new FileReader(name); 
StreamTokenizer st = new StreamTokenizer(rd); 

// Prepare the tokenizer for Java-style tokenizing rules 
st.parseNumbers(); 
st.wordChars('_', '_'); 
st.eolIsSignificant(true); 

// If whitespace is not to be discarded, make this call 
st.ordinaryChars(0, ' '); 

// These calls caused comments to be discarded 
st.slashSlashComments(true); 
st.slashStarComments(true); 

// Parse the file 
int token = st.nextToken(); 
String word_ant = ""; 
outputArea.append( " \n" ); 
out = new BufferedWriter(new FileWriter(name, true)); 

while (token != StreamTokenizer.TT_EOF) { 
token = st.nextToken(); 
if (token == StreamTokenizer.TT_EOL){ 
out.flush(); 
out = new BufferedWriter(new FileWriter(name, true)); 
} 

switch (token) { 
case StreamTokenizer.TT_NUMBER: 
// A number was found; the value is in nval 
double num = st.nval; 
break; 
case StreamTokenizer.TT_WORD: 
// A word was found; the value is in sval 

word = st.sval; 

outputArea.append( word.toString() + " \n " ); 

break; 

case '"': 
// A double-quoted string was found; sval contains the contents 
String dquoteVal = st.sval; 
break; 
case '\'': 
// A single-quoted string was found; sval contains the contents 
String squoteVal = st.sval; 
break; 
case StreamTokenizer.TT_EOL: 
// End of line character found 
break; 
case StreamTokenizer.TT_EOF: 
// End of file has been reached 
break; 
default: 
// A regular character was found; the value is the token itself 
char ch = (char)st.ttype; 
break; 
} // fim do switch 
} // fim do while 
rd.close(); 
out.close(); 
} // fim do try 


// process file processing problems 
catch( IOException ioException ) { 
JOptionPane.showMessageDialog( this, 
"FILE ERROR", 
"FILE ERROR", JOptionPane.ERROR_MESSAGE ); 
} 
} // fim do if da linha 68 

// output directory listing 
else if ( name.isDirectory() ) { 
String directory[] = name.list(); 

outputArea.append( "\n\nDirectory contents:\n"); 

for ( int i = 0; i < directory.length; i++ ) 
outputArea.append( directory[ i ] + "\n" ); 
} 
} // fim do if da linha 52 

// not file or directory, output error message 
else { 
JOptionPane.showMessageDialog( this, 
actionEvent.getActionCommand() + " Does Not Exist", 
"ERROR", JOptionPane.ERROR_MESSAGE ); 
} 

} // fim do método actionPerformed 

// executa a aplicação 
public static void main( String args[] ) 
{ 
ConstrutorDeTemplate1 application = new ConstrutorDeTemplate1(); 

application.setDefaultCloseOperation( 
JFrame.EXIT_ON_CLOSE ); 
} // fim do método main 

}