Question

Hi I would like to revise the codes below to bring all text files I have text file 0 from 10 I will give you link here

https://drive.google.com/open?id=1LnWqv8ftzARx5Rhf7HtVlVkLVlcA1FtI

when tokenizer read all tokens from the 10 files which attached above, I would like to arrange it in alphabetical order and want to make same word tokens being merged.

For instance,

You are my son?

You are my son and good friend!

you are the only one whom I loved.

The output would be

and

are

friend!

good

I

loved.

my

one

only

son

son?

the

whom

You

you

-> I merged the same words but with the Punctuation, Stemming, and Capitalization, the tokenizers recognize as different word.

Please help me to make this code

import java.io.File;
import java.io.FileNotFoundException;
import java.util.Scanner;

public class TokensGeneration {
   public static void main(String[] args) throws FileNotFoundException {
       Scanner sc = new Scanner(System.in);
       System.out.print("Enter file name: ");
       String file_name = sc.nextLine();
       sc = new Scanner(new File(file_name));
       while(sc.hasNextLine()){
           String[] words = sc.nextLine().split(" ");
           for(int i=0; i<words.length; i++){                  
               System.out.println(words[i]);
           }
       }
   }
}

Answer 1

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;

import java.io.File;
import java.io.FileNotFoundException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.Scanner;
import java.util.Set;
public class Test {

   public static void main(String[] args) throws FileNotFoundException {
       String sclChar[]={"?","!",".",","}; //list all the signs file may have add more if needed
   Set<String> uniqueWord=new HashSet<String>(); //set will contain unique word
      
       Scanner sc1 = new Scanner(System.in);
       System.out.print("Enter no of files: ");
       int n=sc1.nextInt();
       for(int f=0;f<n;f++){
   System.out.print("Enter file name: ");
   sc1 = new Scanner(System.in);
   String file_name = sc1.nextLine();
  
   Scanner sc = new Scanner(new File("d:\\HomeworkLib\\"+file_name));
   while(sc.hasNextLine()){   
       String line=sc.nextLine(); //read here line as string
       for(int j=0;j<sclChar.length;j++){ //for each sign
           line=line.replace(sclChar[j], ""); //replace it with space
       }
   String[] words = line.split(" "); //now split the line into words
   for(int i=0; i<words.length; i++){
       if(uniqueWord.contains(words[i].toLowerCase())){ //make word lower case and put to set to avoid issue of low/upper case
             
           continue; //if set already contain that word skip
       }
       else{
           uniqueWord.add(words[i].toLowerCase()); //else put it into set
       }
         
   }
   }
       }
   ArrayList<String> list = new ArrayList<>(uniqueWord); //convert set to list
   Collections.sort(list); //now sort it
   for(String s:list){ //print list one by one
       System.out.println(s);
   }
      
   }
  
}

output

Enter no of files: 1
Enter file name: f.txt
and
are
friend
good
i
loved
my
one
only
son
the
whom
you