using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Collections;
using System.IO;
using System.Diagnostics;

namespace ConsoleApplication1
{
    class Program
    {
        static void Main(string[] args)
        {
            Stopwatch stopwatch1 = new Stopwatch();
            stopwatch1.Start();
            Console.WriteLine("Stopwatch started.");
            Console.WriteLine("Loading File...");
            List<string> al1 = new List<string>();
            StreamReader sr1 = new StreamReader(@"C:\dics\All.dic");
            string theLine = sr1.ReadToEnd();
            al1.AddRange(theLine.Split(Environment.NewLine.ToCharArray()));
            sr1.Close(); 
            Console.WriteLine("Entries: {0}", al1.Count);
            Console.WriteLine("Elapsed: {0}s", stopwatch1.ElapsedMilliseconds / 1000);
            Console.WriteLine("Removing duplicates...");
            al1 = removeDuplicates(al1);
            Console.WriteLine("Entries: {0}", al1.Count);
            Console.WriteLine("Elapsed: {0}s", stopwatch1.ElapsedMilliseconds / 1000);
            Console.WriteLine("Sorting the entries...");
            al1.Sort();
            Console.WriteLine("Elapsed: {0}s", stopwatch1.ElapsedMilliseconds / 1000);
            Console.WriteLine("Editing the entries...");
            string theVal = "";
            for (int j = 0; j < al1.Count; j++)
            {
                theVal = al1[j].ToString();
                al1[j] = theVal + ":" + md5It(theVal);
                if (j % 500000 == 0)
                {
                    Console.WriteLine("Hashed: " + j.ToString() + " entries.");
                }

            }
            Console.WriteLine("Elapsed: {0}s", stopwatch1.ElapsedMilliseconds / 1000);
            Console.WriteLine("Rewriting new data...");
            StreamWriter sw1 = new StreamWriter(@"C:\dics\Done.dic");
            for (int j = 0; j < al1.Count; j++)
            {
                sw1.WriteLine(al1[j].ToString());
                if (j % 500000 == 0)
                {
                    Console.WriteLine("Written: " + j.ToString() + " entries.");
                }
            }
            sw1.Flush();
            sw1.Close();
            stopwatch1.Stop();
            Console.WriteLine("Done in {0} seconds!", stopwatch1.ElapsedMilliseconds / 1000);
            Console.ReadLine();
        }

        public static List<string> removeDuplicates(List<string> inputList)
        {
            Dictionary<string, int> uniqueStore = new Dictionary<string, int>();
            List<string> finalList = new List<string>();

            foreach (string currValue in inputList)
            {
                if (!uniqueStore.ContainsKey(currValue))
                {
                    uniqueStore.Add(currValue, 0);
                    finalList.Add(currValue);
                }
            }
            return finalList;
        }

        public static string md5It(string txt)
        {
            System.Security.Cryptography.MD5CryptoServiceProvider x = new System.Security.Cryptography.MD5CryptoServiceProvider();
            byte[] data = System.Text.Encoding.ASCII.GetBytes(txt);
            data = x.ComputeHash(data);
            string ret = "";
            for (int i = 0; i < data.Length; i++)
                ret += data[i].ToString("x2").ToLower();
            return ret;
        }
    }
}