Introduce cache database and reindex command

Author: barnabasdomozi

analyzer/codechecker_analyzer/cachedb.py

@@ -0,0 +1,98 @@
+# -------------------------------------------------------------------------
+#
+#  Part of the CodeChecker project, under the Apache License v2.0 with
+#  LLVM Exceptions. See LICENSE for license information.
+#  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# -------------------------------------------------------------------------
+
+import sqlite3
+import itertools
+import os
+from typing import List
+
+
+class CacheDB:
+    """
+    SQLite database located in the report directory,
+    designed to speed up the parsing process.
+    """
+
+    __sqlitedb_path: str
+    __con: sqlite3.Connection
+    __cur: sqlite3.Cursor
+
+    def __init__(self, report_dir: str, clean: bool = False):
+        """
+        Initiates the cache database and creates the necessary tables.
+
+        Args:
+            report_dir (str): path to the report directory
+            clean (bool): If set to True, the previous database
+                will be dropped and a new one is created.
+        """
+        self.__sqlitedb_path = os.path.join(report_dir, "cache.db")
+
+        if clean and os.path.exists(self.__sqlitedb_path):
+            os.remove(self.__sqlitedb_path)
+
+        self.__create_connection()
+
+    def __create_connection(self):
+        self.__con = sqlite3.connect(self.__sqlitedb_path)
+        self.__cur = self.__con.cursor()
+        self.__create_tables()
+
+    def close_connection(self):
+        """
+        Closes the connection to the cache database and writes
+        changes to the disk.
+        """
+        self.__con.close()
+
+    def __table_exists(self, name: str) -> bool:
+        res = self.__cur.execute("SELECT name FROM sqlite_master WHERE name=?",
+                                 [name])
+        return res.fetchone() is not None
+
+    def __create_tables(self):
+        if not self.__table_exists("plist_lookup"):
+            self.__cur.execute("CREATE TABLE plist_lookup(plist, source)")
+
+    def insert_plist_sources(self, plist_file: str, source_files: List[str]):
+        """
+        Inserts the plist file and its associated source files into the
+        cache database. These source files are located in the 'files' section
+        of an individual plist file.
+
+        Args:
+            plist_file (str): path to the plist file
+            source_files (List[str]): list of source files mapped to
+                the plist file
+        """
+
+        data = list(zip(itertools.repeat(plist_file), source_files))
+        self.__cur.executemany("INSERT INTO plist_lookup VALUES(?, ?)", data)
+        self.__con.commit()
+
+    def plist_query(self, source_files: List[str]) -> List[str]:
+        """
+        Returns all plist files associated with any of the given source files
+        by querying the cache database.
+
+        Args:
+            source_files (List[str]): list of source files to be looked up
+                from the cache database.
+        """
+
+        placeholders = ','.join('?' for _ in source_files)
+        res = self.__cur.execute("SELECT plist FROM plist_lookup WHERE source"
+                                 f" IN ({placeholders})", source_files)
+        return list(map(lambda e: e[0], res))
+
+    def get_indexed_plist_files(self) -> List[str]:
+        """
+        Returns already indexed plist files from the cache database.
+        """
+        res = self.__cur.execute("SELECT DISTINCT plist FROM plist_lookup")
+        return list(map(lambda e: e[0], res))

analyzer/codechecker_analyzer/cli/reindex.py

@@ -0,0 +1,108 @@
+# -------------------------------------------------------------------------
+#
+#  Part of the CodeChecker project, under the Apache License v2.0 with
+#  LLVM Exceptions. See LICENSE for license information.
+#  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# -------------------------------------------------------------------------
+
+import os
+import sys
+from codechecker_analyzer.cachedb import CacheDB
+from codechecker_common import arg, logger
+from codechecker_report_converter.report.parser import plist as plistparser
+from multiprocessing import Pool, cpu_count
+from typing import List, Tuple
+
+LOG = logger.get_logger('system')
+
+
+def get_argparser_ctor_args():
+    """
+    This method returns a dict containing the kwargs for constructing an
+    argparse.ArgumentParser (either directly or as a subparser).
+    """
+
+    return {
+        'prog': 'CodeChecker reindex',
+        'formatter_class': arg.RawDescriptionDefaultHelpFormatter,
+        'description': """
+The analysis cache database is a SQLite database located in the
+report directory, designed to speed up the parsing process.
+In case it is missing or outdated, one can use the 'reindex' command to
+recreate/update this database.""",
+        'help': "Recreate/update the cache database given a report directory."
+    }
+
+
+def add_arguments_to_parser(parser):
+    """
+    Add the subcommand's arguments to the given argparse.ArgumentParser.
+    """
+
+    parser.add_argument('input',
+                        type=str,
+                        nargs='+',
+                        metavar='folder',
+                        help="The analysis result folder(s) containing "
+                             "analysis results which should be "
+                             "reindexed.")
+
+    parser.add_argument('-j', '--jobs',
+                        type=int,
+                        dest="jobs",
+                        required=False,
+                        default=cpu_count(),
+                        help="Number of threads to use for reindex. More "
+                             "threads mean faster reindex at the cost of "
+                             "using more memory.")
+
+    parser.add_argument('-f', '--force',
+                        action="store_true",
+                        dest="force",
+                        required=False,
+                        default=False,
+                        help="Drop the previous cache database and do a "
+                             "clean reindex.")
+
+    logger.add_verbose_arguments(parser)
+    parser.set_defaults(func=main)
+
+
+def main(args):
+    logger.setup_logger(args.verbose if 'verbose' in args else None)
+    for i in args.input:
+        update_cache_db(i, args.force, args.jobs)
+
+
+def __process_file(file_path: str) -> Tuple[str, List[str]]:
+    with open(file_path, 'rb') as fp:
+        plist = plistparser.parse(fp)
+
+        file_list = [] if plist is None else \
+            plistparser.get_file_list(plist, os.path.dirname(file_path))
+        return (file_path, file_list)
+
+
+def update_cache_db(report_dir: str, force: bool, jobs: int):
+    if not os.path.isdir(report_dir):
+        LOG.error(f"Directory {report_dir} does not exist!")
+        sys.exit(1)
+
+    report_dir = os.path.abspath(report_dir)
+    cachedb = CacheDB(report_dir, force)
+    indexed_files = cachedb.get_indexed_plist_files()
+
+    plist_files = list(filter(lambda f: f.endswith(
+                    plistparser.EXTENSION), os.listdir(report_dir)))
+    plist_files = list(map(lambda f: os.path.abspath(
+                    os.path.join(report_dir, f)), plist_files))
+    plist_files = list(filter(lambda f: f not in indexed_files, plist_files))
+
+    with Pool(jobs) as p:
+        res = p.map(__process_file, plist_files)
+        for (plist_file, sources) in res:
+            if sources != []:
+                cachedb.insert_plist_sources(plist_file, sources)
+
+    cachedb.close_connection()

tools/report-converter/codechecker_report_converter/report/parser/plist.py

@@ -178,15 +178,25 @@ def get_file_index_map(
 ) -> Dict[int, File]:
     """ Get file index map from the given plist object. """
     file_index_map: Dict[int, File] = {}
+    file_list = get_file_list(plist, source_dir_path)
 
-    for i, orig_file_path in enumerate(plist.get('files', [])):
-        file_path = os.path.normpath(os.path.join(
-            source_dir_path, orig_file_path))
+    for i, file_path in enumerate(file_list):
         file_index_map[i] = get_or_create_file(file_path, file_cache)
 
     return file_index_map
 
 
+def get_file_list(
+    plist: Any,
+    source_dir_path: str
+) -> List[str]:
+    """ Get file list section from the given plist object. """
+    return list(map(
+        lambda f: os.path.normpath(os.path.join(
+            source_dir_path, f)),
+        plist.get('files', [])))
+
+
 class Parser(BaseParser):
     def get_reports(
         self,