P2P_Based_Crawling_Framework/Sality/Collector/ConsoleApplication6/DataAggregator/SecondPhase/SecondPhase.cs

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace ConsoleApplication6.SecondPhase
{
    class SecondPhase
    {
        public static void gethoursDiagram(String directory, String outputFile, String OnlineFile)
        {
            string line;

            List<String> fileEntries = DirSearch(directory);
            List<Counter> list1 = new List<Counter>();
            List<Counter> listOnline = new List<Counter>();
            foreach (String url in fileEntries)
            {
                String[] sp = { "/", "_" };
                Console.WriteLine(url);
                String[] spRes = url.Split(sp, StringSplitOptions.None);

                FileInfo fileToDecompress = new FileInfo(url);
                Zipper.Decompress(fileToDecompress);
                // Read the file and display it line by line.
                System.IO.StreamReader file =
                    new System.IO.StreamReader(url.Replace(".gz", ""));
                while ((line = file.ReadLine()) != null)
                {

                    String[] spliters = { "#" };
                    String[] parts = line.Split(spliters, StringSplitOptions.None);
                    //DateTime dateTime = DateTime.Parse(parts[0]);
                    int y = Convert.ToInt32(spRes[spRes.Length - 4]);
                    int m = Convert.ToInt32(spRes[spRes.Length - 3]);
                    int d = Convert.ToInt32(spRes[spRes.Length - 2]);
                    int h = Convert.ToInt32(spRes[spRes.Length-1].Replace(".txt.gz", ""));

                    DateTime dateTime = new DateTime(y, m, d, h, 0, 0);

                    parts[1] = parts[1].Replace("\t", "");
                    parts[1] = parts[1].Replace(" ", "");
                    parts[2] = parts[2].Replace("\t", "");
                    parts[2] = parts[2].Replace(" ", "");

                    Boolean found1 = false;
                    foreach (Counter counter in listOnline)
                    {
                        if (counter.isTheHour(dateTime))
                        {
                            counter.increase(parts[1]);
                            found1 = true;
                        }
                    }
                    if (!found1)
                    {
                        Counter counter1 = new Counter(dateTime);
                        counter1.increase(parts[1]);
                        listOnline.Add(counter1);
                    }

                    Boolean found = false;
                    foreach (Counter counter in list1)
                    {
                        if (counter.isTheHour(dateTime))
                        {
                            counter.increase(parts[1]);
                            counter.increase(parts[2]);
                            found = true;
                        }
                    }
                    if (!found)
                    {
                        Counter counter1 = new Counter(dateTime);
                        counter1.increase(parts[1]);
                        counter1.increase(parts[2]);
                        list1.Add(counter1);
                    }

                }
                file.Close();
                File.Delete(url.Replace(".gz", ""));


            }
            System.IO.StreamWriter finalfile = new System.IO.StreamWriter(outputFile);
            System.IO.StreamWriter Onlinefile = new System.IO.StreamWriter(OnlineFile);
            List<Counter> SortedList = list1.OrderBy(o => o.datetime).ToList();
            List<Counter> SortedListOnline = listOnline.OrderBy(o => o.datetime).ToList();
            foreach (Counter counter in SortedList)
            {
                finalfile.WriteLine(counter.datetime + ".." + counter.count);
            }
            finalfile.Close();

            foreach (Counter counter in SortedListOnline)
            {
                Onlinefile.WriteLine(counter.datetime + ".." + counter.count);
            }

            Onlinefile.Close();

            Console.WriteLine("Reporting DONE !!!!!!");
        }

        private static List<String> DirSearch(string sDir)
        {
            List<String> files = new List<String>();
            try
            {
                foreach (string f in Directory.GetFiles(sDir))
                {
                    files.Add(f);
                }
                foreach (string d in Directory.GetDirectories(sDir))
                {
                    files.AddRange(DirSearch(d));
                }
            }
            catch (System.Exception excpt)
            {

            }

            return files;
        }
    }
}