package parser;

import java.io.*;
import java.nio.charset.StandardCharsets;

import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
import org.json.simple.parser.ParseException;

public class DatasetParser{
    static String path = "C:\\Users\\Admin\\Desktop\\Skripts\\Thesis\\Repo\\local\\";
    static long lineCounter = 0;
    static long lineTotal = 22524846;
    static long userCounter = 0;
    static int threadNum = 8;
    static long startDataTime = 1351742400;
    static int roundLength = 3600;

    static BufferedWriter bw;
    public static void main(String[] args) throws IOException, ParseException, InterruptedException {
        File file = new File(path + "round_based_data_" + roundLength + ".txt");
        FileWriter fw = new FileWriter(file, true);
        bw = new BufferedWriter(fw);

        DatasetParser parser = new DatasetParser();
        parser.parse();
        //parser.parseFromTo(0, 11262423);
        //parser.parseFromTo(11262424, 22524846);
        //parser.parseSplitData();
    }

    void parse() throws IOException, ParseException, InterruptedException {
        // parsing dataset from path
        FileInputStream fis = new FileInputStream(path + "tweets-nov-2012.json.gz.out");
        BufferedReader br = new BufferedReader(new InputStreamReader(fis, StandardCharsets.UTF_8));
        JSONObject obj;
        String line;
        int readProgress = -1;
        //ExecutorService executor = Executors.newFixedThreadPool(threadNum);

        while((line = br.readLine()) != null) {
            lineCounter++;
            obj = (JSONObject) new JSONParser().parse(line);
            long timestamp = Long.parseLong(String.valueOf(obj.get("timestamp")));
            long id = Long.parseLong((String) obj.get("user_id"));
            JSONArray arr = (JSONArray) obj.get("hashtags");
            String[] hashtags = new String[arr.size()];
            for(int i=0; i<arr.size(); i++) {
                hashtags[i]= String.valueOf(arr.get(i));
            }

            writeRoundBasedData(id, hashtags, timestamp);

            //write tweet data of user to file using thread pool
            //executor.submit(new WorkerThread(id, line));
            //writeUserData(id, line);

            //counting progress

            int progress =  (int) (((float) lineCounter /(float) lineTotal) * 100F);
            System.out.print("\r");
            System.out.print(progress + " %");
        }
        // Always close files.
        br.close();
        //executor.shutdown();
        //if(executor.awaitTermination(60, TimeUnit.MINUTES)) {
        //    System.out.println("Parsing completed");
        //}
        //else System.out.println("Parsing errors");
        bw.close();
        System.out.println(lineCounter + " lines");
        System.out.println(userCounter + " users");
    }

    private void writeRoundBasedData(long user_id, String[] hashtags, long timestamp) throws IOException {
        String hashtagsString = "";
        for(String s : hashtags)
            hashtagsString = hashtagsString + (new String(s.getBytes(StandardCharsets.UTF_8)) + " ");

        bw.write( calculateTimeDiff(timestamp) + "\t" + user_id + "\t" + hashtagsString.toString().trim());
        if(lineCounter != lineTotal)
            bw.newLine();
    }

    private long calculateTimeDiff(long stamp) {
        long diffStartTime = stamp - startDataTime;
        int inRound = (int)(diffStartTime / roundLength) + 1;
        return startDataTime + (long) roundLength * inRound;
    }


    class WorkerThread implements Runnable {
        private final long param1;
        private final String param2;
        public static long taskDone = 0;
        public static int writeProgress = -1;

        public WorkerThread(long param1, String param2){
            this.param1 = param1;
            this.param2 = param2;
        }
        @Override
        public void run() {
            writeUserData(param1, param2);
        }

        void writeUserData(long id, String data) {
            String wPath = "C:\\Users\\Admin\\OneDrive\\Desktop\\Skripts\\Thesis\\Repo\\local\\data";
            try {
                File file = new File(wPath + "\\"+id+".txt");
                boolean exists = file.exists();
                if (!exists) {
                    file.createNewFile();
                    userCounter++;
                }
                FileWriter fw = new FileWriter(file, true);
                BufferedWriter bw = new BufferedWriter(fw);
                synchronized (file.getCanonicalFile()) {
                    if (exists)
                        bw.newLine();
                    bw.write(data);
                    bw.close();
                }
                incrementCount();
                countProgress();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }

        private static synchronized void incrementCount() {
            taskDone++;
        }

        public static synchronized void countProgress() {
            if(writeProgress != (int)(((double)taskDone/(double)lineTotal) * 100)) {
                writeProgress = (int)(((double)taskDone/(double)lineTotal) * 100);
                System.out.println(writeProgress + "% written");
            }
        }
    }
}
