c++ 統計英文文本中每個單詞的詞頻並且按照詞頻對每行排序

前幾天寫了Apriori算法之後效率很差，在網上查了很多資料，另外的最大模式挖掘算法一般有FP-Growth 和 Ecalt 算法，這兩種效率比較高而FP-Growth算法構建FP-Tree的過程非常複雜，後來在網上找到了另外一種效率更優化且結構更簡單的算法，那就是北大鄧志鴻老師的PPV算法，看了論文以後覺得這確實是一個好的算法，因此打算加以實現。在實現的第一步就卡住了。在構造PPV-TREE/ FP-TREE 的第一步都是要統計每個事務的頻度並且對項集中按照頻度進行排序，雖然只是簡單的兩句話，可是當實現的時候就沒有說的這麼簡單了。我也用了一天的時間來實現這第一步。。。。。確實有些事不動手不知道難。在如何實現的問題上也經過了很多的思考，最初打算用map 來實現，後來爲了鍛鍊下就改用了Trie樹的方式，如果數據大最好的方式應該是map/reduce的成名函數 wordCount。好了下面直接上代碼.

#ifndef TRIE_H
#define TRIE_H

#include <fstream>

class trie{

    public:
        trie(std::string fname,char spl):filename(fname), sp(spl),root(NULL){}
        ~trie();
        void insert(std::string);
        int search(std::string);
        void openfile();
        std::string getFname();
        char getsp();
    private:
        struct node{
            char key;
            int count;
            node();
            node *child[26];
        };
        node *root;
        std::ifstream in;
        std::string filename;
        char sp;
};
#endif //end tire define

#include <iostream>
#include <cassert>
#include "trie.h"

using namespace std;

trie::node::node()
{
    count = 0;
    for(int i = 0; i<26; ++i)
        child[i] = NULL;
}


trie::~trie()
{
    if(in)
        in.close();
    delete root;
}

void trie::openfile()
{
    in.open(filename.c_str());
    string line;
    assert(in != NULL);

    while(getline(in, line)){
        string tmp;

        while(line.find(sp) != -1)
        {
            tmp = line.substr(0,line.find(sp));
            insert(tmp);
            line = line.substr(line.find(sp)+1, line.size()-1);
        }
    }

    insert(line);
}

//trie 樹的插入過程，排除含有非英文字母的單詞,並且統計單詞出現的頻度
void trie::insert(string word){
    if(root == NULL)
    {
        root =new node;
    }

    int index;
    node *loc = root;
    int pos = 0;
    while(loc != NULL && word[pos] != '\0')
    {
        if(word[pos] >='A' && word[pos] <= 'Z')
            index = word[pos] - 'A';
        else if(word[pos]>= 'a' && word[pos]<='z')
            index = word[pos] - 'a';
        else
            return;

        if(loc->child[index] == NULL)
        {
            loc->child[index] = new node;
        }
        ++pos;
        loc =loc->child[index];
    }

    ++(loc->count);
}

string trie::getFname()
{
    return filename;
}

char trie::getsp()
{
    return sp;
}

//查找某個單詞出現的頻度，如果不存在返回-1
int trie::search(string word)
{
    node *loc = root;
    assert(root != NULL);
    int index = 0;

    while(loc && word[index] != '\0')
    {
        int pos;
        if(word[index]>='A' && word[index]<='Z')
            pos = word[index] - 'A';
        else if(word[index]>='a' && word[index]<='z')
            pos = word[index] - 'a';
        else
            return  -1;
        loc = loc->child[pos];
        ++index;
    }

    if(loc ==NULL)
        return -1;

    return loc->count;
}

#ifndef SORT_DATA
#define SORT_DATA
#include "trie.h"
#include <vector>
#include <fstream>

class sort_data{
    public:
        sort_data(trie *itree, int sup):tree(itree),support(sup){filename = itree->getFname(); split = itree->getsp();}
        ~sort_data();
        void openfile();
        void count_data();
    private:
        void sorts();
        int partion(int *a, int *b,int begin, int end);
        void quicksort(int *a, int *b, int begin,int end);
        std::vector<std::string> spl;
        trie *tree;
        std::string filename;
        std::ifstream in;
        std::ofstream out;
        int support;
        char split;
};
#endif //end define

// 整體功能是對一個文本中單詞進行詞頻統計，並且對該文本的每行進行排序，如果有相同單詞則合併爲一個
// 如果有兩個單詞詞頻相同那麼按照字典序進行排序。輸出文件名稱爲輸入文件名字+你定義的support
#include "sort_data.h"
#include "trie.h"
#include <iostream>
#include <cassert>
#include <sstream>
#include <map>

using namespace std;

sort_data::~sort_data()
{
    in.close();
    out.close();
//  delete tree;
}

void sort_data::openfile()
{
    in.open(filename.c_str());
    assert(in != NULL);

    stringstream ss;
    ss<<support;

    string tmp = filename + ss.str();

    out.open(tmp.c_str());
}

//將文本的每一行進行拆分，將每個單詞放到vector<string> spl 中
void sort_data::count_data()
{
    string line;

    while(getline(in,line))
    {
        spl.clear();
        string tmp;

        while(line.find(split) != -1)
        {
            tmp = line.substr(0,line.find(split));
            spl.push_back(tmp);
            line = line.substr(line.find(split)+1,line.size()-1);
        }

        spl.push_back(line);
        sorts();
    }
}

//對排序後的單詞進行去重和按照字典序排列的過程，使用了map<string, int>
void sort_data::sorts()
{
    int n = spl.size();
    int a[n];
    int tmp = 0;

    for(int i = 0;i<n; ++i)
    {
        if(tree->search(spl[i])>=support)
        {
            a[tmp] = i;
            ++tmp;
        }
    }

    int b[tmp];
    int c[tmp];

    for(int k = 0; k<tmp;++k)
    {
        c[k] = k;
    }

    for(int j = 0; j<tmp; ++j)
    {
        int in = a[j];
        b[j] = tree->search(spl[in]);
    }
    quicksort(b,c,0,tmp-1);

    for(int i = 0; i<tmp; ++i)
    {
       int j = c[i];
         j = a[j];
    }

    map<string, int> qop;
    string line;
    int cp = 0;
    if(tmp == 1)
    {
        int i = a[0];
        line = spl[i]+'\n';
    }

    else{
    while(cp<tmp){
        int i;
        if(cp+1 == tmp)
        {
            i = c[cp];
            i = a[i];
            line =line +spl[i]+'\n';
            break;
        }

        if(b[cp]>b[cp+1])
        {
            i = c[cp];
            i = a[i];
            line = line+spl[i]+' ';
            ++cp;
        }

        else
        {
            while(cp+1 != tmp && (b[cp] == b[cp+1]))
            {
                i = c[cp];
                i = a[i];
                string s = spl[i];
                ++qop[s];
                ++cp;
            }
             i = c[cp];
             i = a[i];
             string s =spl[i];
             ++qop[s];
             ++cp;
            for(map<string,int>::iterator it = qop.begin(); it != qop.end(); ++it)
            {
                line = line+it->first+' ';
            }
            qop.clear();
        }
    }
    line = line + '\n';
    }

    out<<line;
    out<<endl;
}

//對每一行出現次數大於support的單詞進行排序過程，a爲詞頻數組，b爲伴隨數組
int sort_data::partion(int *a,int *b,int begin, int end)
{
    int index = a[end];
    int start = begin;
    for(int i = begin; i<end; ++i)
    {
        if(a[i]>= index)
        {
            swap(a[start],a[i]);
            swap(b[start],b[i]);
            ++start;
        }
    }
    swap(a[start],a[end]);
    swap(b[start],b[end]);
    return start;
}

void sort_data::quicksort(int *a,int *b,int begin,int end)
{
    if(begin<end)
    {
        int i = partion(a,b,begin,end);

        quicksort(a,b,begin,i-1);
        quicksort(a,b,i+1,end);
    }
}

#include "trie.h"
#include <iostream>
#include "sort_data.h"

using namespace std;

int main()
{
    trie tr("in.txt", ' ');

    tr.openfile();
    cout<<tr.search("balancing")<<endl;

    sort_data sort(&tr, 5);

    sort.openfile();
    sort.count_data();
    return 0;
}