word2vec源码解析

//下面是我对 word2vec.c 的注释

//详细算法可以参考论文，或者看这篇博客 http://www.cnblogs.com/downtjs/p/3784440.html

// Copyright 2013 Google Inc. All Rights Reserved.

//

// Licensed under the Apache License, Version 2.0 (the "License");

// you may not use this file except in compliance with the License.

// You may obtain a copy of the License at

//

// limitations under the License.

#include <stdio.h>

#include <stdlib.h>

#include <string.h>

#include <math.h>

#include <pthread.h>

#define MAX_STRING 100

#define EXP_TABLE_SIZE 1000

#define MAX_EXP 6

#define MAX_SENTENCE_LENGTH 1000

int *point;//huffman 编码对应内节点的路径

char *word, *code, codelen;//huffman 编码

};

char train_file[MAX_STRING], output_file[MAX_STRING];

char save_vocab_file[MAX_STRING], read_vocab_file[MAX_STRING];

struct vocab_word *vocab;

int binary = 0, cbow = 0, debug_mode = 2, window = 5, min_count = 5, num_threads = 1,

min_reduce = 1;

int *vocab_hash;

long long vocab_max_size = 1000, vocab_size = 0, layer1_size = 100;

long long train_words = 0, word_count_actual = 0, file_size = 0, classes = 0;

real alpha = 0.025, starting_alpha, sample = 0;

real *syn0, *syn1, *syn1neg, *expTable;

clock_t start;

int hs = 1, negative = 0;

{

long long train_words_pow = 0;

real d1, power = 0.75;

table = (int *)malloc(table_size * sizeof(int));

i = 0;

的比例

for (a = 0; a < table_size; a++)//遍历 table。a 表示 table 的位置，i 表示词汇表的位置

table[a] = i;//单词 i 占用 table 的 a 位置

//table 反映的是一个单词能量的分布，一个单词能量越大，所占用的 table 的位置越多

if (a / (real)table_size > d1)

i++;//移动到下一个词

if (i >= vocab_siInitNetze) i = vocab_size - 1;

}

}

// Reads a single word from a file, assuming space + tab + EOL to be word boundaries

void ReadWord(char *word, FILE *fin) {

int a = 0, ch;

while (!feof(fin)) {

ch = fgetc(fin);

if (ch == 13) continue;

if ((ch == ' ') || (ch == '\t') || (ch == '\n')) {

if (a > 0) {

if (ch == '\n') ungetc(ch, fin);

break;

}

if (ch == '\n') {

strcpy(word, (char *)"</s>");

}

}

int GetWordHash(char *word)

unsigned long long a, hash = 0;

for (a = 0; a < strlen(word); a++)

hash = hash * 257 + word[a];//采取 257 进制

hash = hash % vocab_hash_size;

// Returns position of a word in the vocabulary; if the word is not found, returns -1

while (1)

{

if (vocab_hash[hash] == -1) return -1;

if (!strcmp(word, vocab[vocab_hash[hash]].word))

return vocab_hash[hash];

hash = (hash + 1) % vocab_hash_size;

}

return -1;

}

// Reads a word and returns its index in the vocabulary

int ReadWordIndex(FILE *fin)

{

char word[MAX_STRING];

ReadWord(word, fin);

if (feof(fin)) return -1;

return SearchVocab(word);

// Adds a word to the vocabulary 将一个词添加到一个词汇中

if (length > MAX_STRING)

vocab[vocab_size].word = (char *)calloc(length, sizeof(char));

strcpy(vocab[vocab_size].word, word);

vocab[vocab_size].cn = 0;

vocab_size++;

// Reallocate memory if needed

{

vocab_max_size += 1000;

vocab = (struct vocab_word *)realloc(vocab, vocab_max_size * sizeof(struct vocab_word));

}

while (vocab_hash[hash] != -1)//如果 hash 值冲突了

hash = (hash + 1) % vocab_hash_size;//使用开放地址法解决冲突

vocab_hash[hash] = vocab_size - 1;//由词的 hash 值找到她所在词汇表的排序位置

int VocabCompare(const void *a, const void *b)

{

return ((struct vocab_word *)b)->cn - ((struct vocab_word *)a)->cn;

}

// Sorts the vocabulary by frequency using word counts

// 根据词频排序

void SortVocab()

{

int a, size;

unsigned int hash;

// Sort the vocabulary and keep </s> at the first position

qsort(&vocab[1], vocab_size - 1, sizeof(struct vocab_word), VocabCompare);

for (a = 0; a < vocab_hash_size; a++)

vocab_hash[a] = -1;

size = vocab_size;

train_words = 0;

for (a = 0; a < size; a++)

//出现太少的词直接丢弃

free(vocab[vocab_size].word);

else

{

// Hash will be re-computed, as after the sorting it is not actual

hash=GetWordHash(vocab[a].word);

vocab_hash[hash] = a;

train_words += vocab[a].cn;

}

}

{

// Reduces the vocabulary by removing infrequent tokens

// 再次移除词频过小的词，缩减词汇表

void ReduceVocab()

{

int a, b = 0;

unsigned int hash;

for (a = 0; a < vocab_size; a++)//我草，这很容易看错啊