基于Lucene 7.1.0 实现搜索引擎

package org.welkin.lucene; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.StringField; import org.apache.lucene.index.*; import org.apache.lucene.search.*; import org.apache.lucene.search.spell.*; import org.apache.lucene.search.suggest.InputIterator; import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import java.io.Closeable; import java.io.IOException; import java.util.ArrayList; import java.util.Comparator; import java.util.Iterator; import java.util.List; /** * 搜索纠错 * * @author Hyoutei * @version 1.0 * @date 2019/1/11 15:54 */ public class CnSpellChecker implements Closeable { private Directory spellIndex; private float bStart; private float bEnd; private IndexSearcher searcher; private final Object searcherLock; private final Object modifyCurrentIndexLock; private volatile boolean closed; private float accuracy; private StringDistance sd; private Comparator<SuggestWord> comparator; private CnSpellChecker(Directory spellIndex, StringDistance sd) throws IOException { this(spellIndex, sd, SuggestWordQueue.DEFAULT_COMPARATOR); } CnSpellChecker(Directory spellIndex) throws IOException { this(spellIndex, new LevensteinDistance()); } private CnSpellChecker(Directory spellIndex, StringDistance sd, Comparator<SuggestWord> comparator) throws IOException { this.bStart = 2.0F; this.bEnd = 1.0F; this.searcherLock = new Object(); this.modifyCurrentIndexLock = new Object(); this.closed = false; this.accuracy = 0.5F; this.setSpellIndex(spellIndex); this.setStringDistance(sd); this.comparator = comparator; } private void setSpellIndex(Directory spellIndexDir) throws IOException { synchronized (this.modifyCurrentIndexLock) { this.ensureOpen(); if (!DirectoryReader.indexExists(spellIndexDir)) { IndexWriter writer = new IndexWriter(spellIndexDir, new IndexWriterConfig(null)); writer.close(); } this.swapSearcher(spellIndexDir); } } public void setComparator(Comparator<SuggestWord> comparator) { this.comparator = comparator; } public Comparator<SuggestWord> getComparator() { return this.comparator; } public void setStringDistance(StringDistance sd) { this.sd = sd; } public StringDistance getStringDistance() { return this.sd; } public void setAccuracy(float acc) { this.accuracy = acc; } public float getAccuracy() { return this.accuracy; } public String[] suggestSimilar(String word, int numSug) throws IOException { return this.suggestSimilar(word, numSug, (IndexReader) null, (String) null, SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX); } public String[] suggestSimilar(String word, int numSug, float accuracy) throws IOException { return this.suggestSimilar(word, numSug, (IndexReader) null, (String) null, SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX, accuracy); } public String[] suggestSimilar(String word, int numSug, IndexReader ir, String field, SuggestMode suggestMode) throws IOException { return this.suggestSimilar(word, numSug, ir, field, suggestMode, this.accuracy); } public String[] suggestSimilar(String word, int numSug, IndexReader ir, String field, SuggestMode suggestMode, float accuracy) throws IOException { IndexSearcher indexSearcher = this.obtainSearcher(); try { if (ir == null || field == null) { suggestMode = SuggestMode.SUGGEST_ALWAYS; } if (suggestMode == SuggestMode.SUGGEST_ALWAYS) { ir = null; field = null; } int lengthWord = word.length(); int freq = ir != null ? ir.docFreq(new Term(field, word)) : 0; int goalFreq = suggestMode == SuggestMode.SUGGEST_MORE_POPULAR ? freq : 0; if (suggestMode == SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX && freq > 0) { return new String[]{word}; } else { BooleanQuery.Builder query = new BooleanQuery.Builder(); int ng; for (ng = getMin(lengthWord); ng <= getMax(lengthWord); ++ng) { String key = "gram" + ng; String[] grams = formGrams(word, ng); if (grams.length != 0) { if (this.bStart > 0.0F) { add(query, "start" + ng, grams[0], this.bStart); } if (this.bEnd > 0.0F) { add(query, "end" + ng, grams[grams.length - 1], this.bEnd); } for (String gram : grams) { add(query, key, gram); } } } ng = 10 * numSug; ScoreDoc[] hits = indexSearcher.search(query.build(), ng).scoreDocs; SuggestWordQueue sugQueue = new SuggestWordQueue(numSug, this.comparator); int stop = Math.min(hits.length, ng); SuggestWord sugWord = new SuggestWord(); for (int i = 0; i < stop; ++i) { sugWord.string = indexSearcher.doc(hits[i].doc).get("word"); sugWord.score = this.sd.getDistance(word, sugWord.string); if (sugWord.score >= accuracy) { if (ir != null) { sugWord.freq = ir.docFreq(new Term(field, sugWord.string)); boolean exits = suggestMode == SuggestMode.SUGGEST_MORE_POPULAR && goalFreq > sugWord.freq || sugWord.freq < 1; if (exits) { continue; } } sugQueue.insertWithOverflow(sugWord); if (sugQueue.size() == numSug) { accuracy = sugQueue.top().score; } sugWord = new SuggestWord(); } } String[] list = new String[sugQueue.size()]; for (int i = sugQueue.size() - 1; i >= 0; --i) { list[i] = sugQueue.pop().string; } return list; } } finally { this.releaseSearcher(indexSearcher); } } private static void add(BooleanQuery.Builder q, String name, String value, float boost) { Query tq = new TermQuery(new Term(name, value)); q.add(new BooleanClause(new BoostQuery(tq, boost), BooleanClause.Occur.SHOULD)); } private static void add(BooleanQuery.Builder q, String name, String value) { q.add(new BooleanClause(new TermQuery(new Term(name, value)), BooleanClause.Occur.SHOULD)); } private static String[] formGrams(String text, int ng) { int len = text.length(); String[] res = new String[len - ng + 1]; for (int i = 0; i < len - ng + 1; ++i) { res[i] = text.substring(i, i + ng); } return res; } public void clearIndex() throws IOException { synchronized (this.modifyCurrentIndexLock) { this.ensureOpen(); Directory dir = this.spellIndex; IndexWriter writer = new IndexWriter(d