add Full-Text Search

Author: Rigorich

DatabaseService/ArchiveDatabase.backup

@@ -7,6 +7,7 @@
 using ServiceStack.Text;
 using static ServiceStack.Text.JsonSerializer;
 using ServiceStack.OrmLite;
+using ServiceStack.OrmLite.PostgreSQL;
 
 namespace DatabaseService
 {
@@ -19,33 +20,48 @@ public static class Database
         static Database()
         {
             dbFactory = new OrmLiteConnectionFactory(ConnectionString, PostgreSqlDialect.Provider);
+            bool empty;
             using (var db = dbFactory.Open())
             {
-                if (db.CreateTableIfNotExists<NewsArticle>())
+                empty = db.CreateTableIfNotExists<NewsArticle>();
+                if (empty)
                 {
-                    List<NewsArticle> News = LoadFromFile();
-                    News = News.OrderBy(art => art.Date).ToList();
-                    foreach (NewsArticle article in News)
-                    {
-                        db.Insert(article);
-                    }
-                }
-                SaveToFile();
-                db.ExecuteSql(@"
+                    db.ExecuteSql(@"
 DROP TABLE IF EXISTS text_analysis
 ;");
-                db.ExecuteSql(@"
+                    db.ExecuteSql(@"
 CREATE TABLE text_analysis(
 id SERIAL PRIMARY KEY,
-article_id SERIAL REFERENCES news_article(id),
+article_id SERIAL REFERENCES news_article(id) ON DELETE CASCADE,
 tsvec tsvector
 );");
+                }
                 db.ExecuteSql(@"
-INSERT INTO text_analysis (article_id, tsvec)
-SELECT id, to_tsvector(text)
-FROM news_article
+CREATE INDEX IF NOT EXISTS url_index ON news_article (url)
 ;");
+                //db.CreateIndex<NewsArticle>(art => art.URL);
+                NewsArticle egg = new NewsArticle(
+                    "tg: @Rigorich",
+                    "-",
+                    "Author",
+                    "Ãðèøàåâ Íèêèòà Ãðèãîðüåâè÷",
+                    System.Data.SqlTypes.SqlDateTime.MinValue.Value
+                );
+                if (!db.Exists<NewsArticle>(art => art.URL == egg.URL))
+                {
+                    db.Insert(egg);
+                }
+            }
+            if (empty)
+            {
+                List<NewsArticle> News = LoadFromFile();
+                News = News.OrderBy(art => art.Date).ToList();
+                foreach (NewsArticle article in News)
+                {
+                    Add(article);
+                }
             }
+            SaveToFile();
         }
         static List<NewsArticle> LoadFromFile()
         {
@@ -90,12 +106,11 @@ public static void Add(NewsArticle article)
                 if (!db.Exists<NewsArticle>(m => m.URL == article.URL && m.Text == article.Text))
                 {
                     db.Insert(article);
-                    long articleId = db.LastInsertId();
                     db.ExecuteSql(@"
 INSERT INTO text_analysis (article_id, tsvec)
-SELECT id, to_tsvector(text)
+SELECT id, setweight(to_tsvector(name), 'A') || setweight(to_tsvector(text), 'D')
 FROM news_article
-WHERE id=" + articleId + @"
+WHERE url='" + article.URL + @"'
 ;");
                 }
             }
@@ -121,13 +136,61 @@ public static List<NewsArticle> GetFilteredList(ListRequest request)
             {
                 var q = db.From<NewsArticle>();
                 q = q.Where(art => request.LeftBoundDate <= art.Date && art.Date <= request.RightBoundDate);
-                q = q.Where(art => art.URL.Contains(request.Url));
-                //q = q.Where(request.Keywords);
-                //q = q.Where(request.Entitities);
-                q = request.OldestFirst ? q.OrderBy(art => art.Date) : q.OrderByDescending(art => art.Date);
+                q = q.Where(art => art.URL.StartsWith(request.Url));
+
+                string keywords = request.Keywords;
+                if (!keywords.IsNullOrEmpty())
+                {
+                    keywords = keywords.Replace("\'", "\'\'");
+                    var results = db.Select<(int ArtId, float Rank)>(@"
+WITH query_rank AS (
+SELECT article_id AS art_id, ts_rank(tsvec, websearch_to_tsquery('" + keywords + @"')) AS ts_rank_value
+FROM text_analysis
+)
+
+SELECT id, ts_rank_value
+FROM query_rank
+
+INNER JOIN news_article
+    ON query_rank.ts_rank_value > 0
+    AND query_rank.art_id = news_article.id
+
+ORDER BY ts_rank_value DESC
+;")
+                        .Map(tup =>
+                        new SearchRankResult()
+                        {
+                            ArtId = tup.ArtId,
+                            Rank = tup.Rank,
+                            Query = keywords
+                        });
+                    db.CreateTableIfNotExists<SearchRankResult>();
+                    foreach (SearchRankResult result in results)
+                    {
+                        if (!db.Exists<SearchRankResult>(r => r.ArtId == result.ArtId && r.Query == result.Query))
+                        {
+                            db.Insert(result);
+                        }
+                    }
+                    q = q.Join<NewsArticle, SearchRankResult>((art, rnk) => art.Id == rnk.ArtId && rnk.Query == keywords);
+                    q = q.OrderByDescending<NewsArticle, SearchRankResult>((art, rnk) => rnk.Rank);
+                }
+                else
+                {
+                    q = request.OldestFirst ? q.OrderBy(art => art.Date) : q.OrderByDescending(art => art.Date);
+                }
+
+                if (!(request.Entitities is null || request.Entitities.IsEmpty()))
+                {
+                    q = q.UnsafeWhere(PgSql.Array(request.Entitities) + " <@ entities");
+                }
                 q = q.Limit(request.Skip, request.Count);
-                q = q.Select(art => new { art.URL, art.Name, art.Date });
+                q = q.Select(art => new { art.Id, art.URL, art.Name, art.Date });
                 FilteredNews = db.Select(q).ToList();
+                if (!keywords.IsNullOrEmpty())
+                {
+                    db.Delete<SearchRankResult>(rnk => rnk.Query == keywords);
+                }
             }
             return FilteredNews;
         }

DatabaseService/Database.cs

@@ -21,7 +21,9 @@ public class ListRequest : IReturn<ListResponse>
         public string Url { get; set; } = null;
         public DateTime LeftBoundDate { get; set; } = DateTime.MinValue;
         public DateTime RightBoundDate { get; set; } = DateTime.MaxValue;
-        public string[] Keywords { get; set; } = null;
+
+        public string Keywords { get; set; } = null;
+
         public string[] Entitities { get; set; } = null;
 
         public int Skip { get; set; } = 0;

DatabaseService/DatabaseRequest.cs

@@ -0,0 +1,14 @@
+using ServiceStack.DataAnnotations;
+
+namespace DatabaseService
+{
+    class SearchRankResult
+    {
+        [AutoIncrement]
+        public long Id { get; set; }
+
+        public int ArtId { get; set; }
+        public float Rank { get; set; }
+        public string Query { get; set; }
+    }
+}

DatabaseService/SearchRankResult.cs

@@ -111,21 +111,22 @@ static void Main(string[] args)
                 Console.WriteLine("GET FILTER LIST");
                 string url;
                 DateTime? ldate, rdate;
-                string[] keywords, entities;
+                string keywords;
+                string[] entities;
                 Console.WriteLine("Enter left bound date:"); ldate = Console.ReadLine().ToDateTime();
                 Console.WriteLine("Enter right bound date:"); rdate = Console.ReadLine().ToDateTime();
                 Console.WriteLine("Enter url:"); url = Console.ReadLine();
 
                 static string[] ParseLine(string s)
                 {
-                    return s.Split(",").Select(w => w.ToLowerInvariant().Trim()).Where(w => !w.IsNullOrEmpty()).ToArray();
+                    return s.Split(",").Select(w => w.Trim()).Where(w => !w.IsNullOrEmpty()).ToArray();
                 }
-                Console.WriteLine("Enter keywords separated by comma:"); keywords = ParseLine(Console.ReadLine());
+                Console.WriteLine("Enter keywords query:"); keywords = Console.ReadLine();
                 Console.WriteLine("Enter entities separated by comma:"); entities = ParseLine(Console.ReadLine());
 
                 Console.WriteLine($"URL = {url}");
                 Console.WriteLine($"Dates = {ldate} - {rdate}");
-                Console.WriteLine($"Keywords = {keywords.Join(", ")}");
+                Console.WriteLine($"Keywords = {keywords}");
                 Console.WriteLine($"Entities = {entities.Join(", ")}");
 
                 Console.WriteLine("OK? (Press Enter)");