Lucene.net站内搜索—5、搜索引擎第一版实现

 ParameterizedThreadStart threadStart = (obj) =>                    {                        txtLog.AppendText(obj + "\n");                    };                    txtLog.Invoke(threadStart, item);

using System;using System.Collections.Generic;using System.Linq;using System.Web;using System.Threading;using log4net;using System.Configuration;using System.Web.Hosting;using RuPeng.Utils;using RuPengSite.DataTier.DataSetThreadTableAdapters;using System.Text;using Lucene.Net.Store;using Lucene.Net.Index;using System.IO;using Lucene.Net.Analysis.PanGu;using Lucene.Net.Documents;namespace RuPengSite.Search{    public class IndexManager    {        public readonly static IndexManager Instance = new IndexManager();        private HashSet<IndexJobItem> jobs = new HashSet<IndexJobItem>();//任务的集合        private bool isStopped;//任务是否停止                private static ILog log = LogManager.GetLogger(typeof(IndexManager));        private IndexManager()        {                    }        //启动任务        public void Start()        {            isStopped = false;            Thread thread = new Thread(ScanThread);            thread.Start();                    }        //停止任务        public void Stop()        {            isStopped = true;        }        /// <summary>        /// 扫描线程        /// </summary>        private void ScanThread()        {            //如果停止，则不再无限循环            while (!isStopped)            {                Thread.Sleep(5000);//休息5秒钟，尽可能多的累积任务                if (jobs.Count <= 0)                {                    continue;//如果没任务继续睡                }                log.Debug("开始索引预处理");                string indexPath = SearchHelper.GetSearchIndexFullPath();                log.Debug("索引路径是：" + indexPath);                FSDirectory directory = FSDirectory.Open(new DirectoryInfo(indexPath), new NativeFSLockFactory());                //判断索引目录是否已经存在                bool isUpdate = IndexReader.IndexExists(directory);                log.Debug("索引路径存在状态是" + isUpdate);                if (isUpdate)                {                    //如果索引目录被锁定（比如索引过程中程序异常退出），则首先解锁                    if (IndexWriter.IsLocked(directory))                    {                        log.Debug("开始解锁索引路径");                        IndexWriter.Unlock(directory);                    }                }                IndexWriter writer = new IndexWriter(directory, new PanGuAnalyzer(), !isUpdate,                     Lucene.Net.Index.IndexWriter.MaxFieldLength.UNLIMITED);                try                {                    ProcessJobItems(directory, writer);                }                finally                {                    log.Debug("开始关闭reader、writer");                    writer.Close();                    directory.Close();                    log.Debug("完成关闭reader、writer");                }                                   }        }        /// <summary>        /// 处理队列中的任务        /// </summary>        /// <param name="directory"></param>        /// <param name="writer"></param>        private void ProcessJobItems(FSDirectory directory, IndexWriter writer)        {            log.Debug("开始处理队列中的"+jobs.Count+"个任务");            foreach (var jobItem in jobs.ToArray())//转换为数组，避免读的时候不能修改的问题            {                try                {                    ProcessJobItem(writer, jobItem);                    jobs.Remove(jobItem);//将处理完成的任务移除                }                catch (Exception ex)                {                    log.Error("对任务进行处理失败" + jobItem, ex);                }                            }            log.Debug("队列中的任务处理完毕");        }        private static void ProcessJobItem(IndexWriter writer, IndexJobItem jobItem)        {            long threadId = jobItem.ThreadId;            JobType jobType = jobItem.ItemType;            string url = SearchHelper.GetThreadUrl(threadId);            if (jobType == JobType.Delete)//判断任务的类型            {                log.Debug("将帖子从索引中移除,threadId=" + threadId);                writer.DeleteDocuments(new Term(SearchHelper.URL, url));//删除旧的收录            }            else if (jobType == JobType.Add)            {                writer.DeleteDocuments(new Term(SearchHelper.URL, url));//删除旧的收录                var threads = new rp_threadsTableAdapter().GetDataById(threadId);                if (threads.Count <= 0)                {                    log.Debug("id为"+threadId+"的帖子不存在！");                    return;                }                string body = SearchHelper.GetThreadContent(threadId);//帖子内容                string title = threads.Single().Subject;//主题                Document document = new Document();                document.Add(new Field(SearchHelper.URL, url, Field.Store.YES, Field.Index.NOT_ANALYZED));                document.Add(new Field(SearchHelper.TITLE, title, Field.Store.YES, Field.Index.NOT_ANALYZED));                document.Add(new Field(SearchHelper.BODY, body, Field.Store.YES, Field.Index.ANALYZED,
 Lucene.Net.Documents.Field.TermVector.WITH_POSITIONS_OFFSETS));                writer.AddDocument(document);                log.Debug("索引帖子" + threadId+"完成");            }            else            {                throw new Exception("错误的jobType：" + jobType);            }        }        /// <summary>        /// 添加帖子的索引任务        /// </summary>        /// <param name="threadId"></param>        public void AddThread(long threadId)        {            IndexJobItem jobItem = new IndexJobItem() { ItemType=JobType.Add,ThreadId=threadId};            jobs.Add(jobItem);        }        /// <summary>        /// 移除帖子的索引任务        /// </summary>        /// <param name="threadId"></param>        public void DeleteThread(long threadId)        {            IndexJobItem jobItem = new IndexJobItem() { ItemType = JobType.Delete, ThreadId = threadId };            jobs.Add(jobItem);        }        class IndexJobItem        {            public JobType ItemType { get; set; }            public long ThreadId { get; set; }            public override bool Equals(object obj)            {                IndexJobItem item = obj as IndexJobItem;                if (item == null)                {                    return false;                }                return this.ItemType==item.ItemType&&this.ThreadId==item.ThreadId;            }            public override int GetHashCode()            {                return ToString().GetHashCode();            }            public override string ToString()            {                return ItemType+":"+ThreadId;            }        }        enum JobType {Delete,Add }//任务类型    }}

PhraseQuery queryMsg = new PhraseQuery();foreach (string word in CommonHelper.SplitWords(txtKW.Text))            {                queryMsg.Add(new Term("msg", word));            }queryMsg.SetSlop(100);PhraseQuery queryTitle = new PhraseQuery();foreach (string word in CommonHelper.SplitWords(txtKW.Text))            {                queryTitle.Add(new Term("title", word));            }queryTitle.SetSlop(100);BooleanQuery query = new BooleanQuery();
query.Add(queryMsg, BooleanClause.Occur.SHOULD);query.Add(queryTitle, BooleanClause.Occur.SHOULD);

private static String highLight(string keyword,String content)        {            PanGu.HighLight.SimpleHTMLFormatter formatter = new PanGu.HighLight.SimpleHTMLFormatter("<font color='red'>", "</font>");            PanGu.HighLight.Highlighter highlighter = new PanGu.HighLight.Highlighter(formatter, new Segment());            highlighter.FragmentSize = 500;            string msg = highlighter.GetBestFragment(keyword,content);
if (string.IsNullOrEmpty(msg))            {                return content;            }            else            {                return msg;            }        } String hightlightTitle = highLight(keyword, title); String hightlightBody = HttpUtility.HtmlEncode(body);//防止XSS攻击 hightlightBody = highLight(keyword, hightlightBody);