Merge pull request #139 from rowingdude/sql_support

Sql support - TESTING!!!!!

Author: rowingdude

requirements.txt

@@ -1 +1,2 @@
-openpyxl==3.0.10
+openpyxl==3.0.10
+sqlite3 

setup.py

@@ -11,6 +11,9 @@
     author_email='[email protected]',
     package_dir={'': 'src'},
     packages=find_packages(where='src'),
+    package_data={
+        'analyzeMFT': ['sql/*.sql'],
+    },
     url='http://github.com/rowingdude/analyzeMFT',
     license='LICENSE.txt',
     description='Analyze the $MFT from a NTFS filesystem.',
@@ -31,6 +34,7 @@
     python_requires=">=3.7",
     install_requires=[
         "pywin32;platform_system=='Windows'",
+        "openpyxl==3.0.10",
     ],
     entry_points={
         'console_scripts': [

src/analyzeMFT/cli.py

@@ -27,6 +27,11 @@ async def main():
                             help="Export as TSK timeline")
     export_group.add_option("--l2t", action="store_const", const="l2t", dest="export_format",
                             help="Export as log2timeline CSV")
+    export_group.add_option("--sqlite", action="store_const", const="sqlite", dest="export_format",
+                            help="Export as SQLite database")
+    export_group.add_option("--tsk", action="store_const", const="tsk", dest="export_format",
+                            help="Export as TSK bodyfile format")
+    
     parser.add_option_group(export_group)
 
     verbosity_group = OptionGroup(parser, "Verbosity Options")

src/analyzeMFT/file_writers.py

@@ -1,7 +1,9 @@
+import asyncio
 import csv
+import os
 import json
+import sqlite3
 import xml.etree.ElementTree as ET
-import asyncio
 from typing import List, Dict, Any
 from .mft_record import MftRecord
 from .constants import *
@@ -86,4 +88,69 @@ async def write_l2t(records: List[MftRecord], output_file: str) -> None:
                         date_str, time_str, 'UTC', macb, 'MFT', 'FILESYSTEM', time_type, '', '', '',
                         f"{record.filename} {time_type}", '', record.filename, record.recordnum, '', '', ''
                     ])
+            await asyncio.sleep(0)
+
+
+    @staticmethod
+    async def write_sqlite(records: List[MftRecord], output_file: str) -> None:
+        conn = sqlite3.connect(output_file)
+        cursor = conn.cursor()
+
+        # Create and populate static tables
+        sql_dir = os.path.join(os.path.dirname(__file__), 'sql')
+        for sql_file in os.listdir(sql_dir):
+            with open(os.path.join(sql_dir, sql_file), 'r') as f:
+                cursor.executescript(f.read())
+
+        # Create MFT records table
+        cursor.execute('''
+            CREATE TABLE mft_records (
+                record_number INTEGER PRIMARY KEY,
+                filename TEXT,
+                parent_record_number INTEGER,
+                file_size INTEGER,
+                is_directory INTEGER,
+                creation_time TEXT,
+                modification_time TEXT,
+                access_time TEXT,
+                entry_time TEXT,
+                attribute_types TEXT
+            )
+        ''')
+
+        # Insert MFT records
+        for record in records:
+            cursor.execute('''
+                INSERT INTO mft_records (
+                    record_number, filename, parent_record_number, file_size,
+                    is_directory, creation_time, modification_time, access_time,
+                    entry_time, attribute_types
+                ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+            ''', (
+                record.recordnum,
+                record.filename,
+                record.get_parent_record_num(),
+                record.filesize,
+                1 if record.flags & FILE_RECORD_IS_DIRECTORY else 0,
+                record.fn_times['crtime'].dtstr,
+                record.fn_times['mtime'].dtstr,
+                record.fn_times['atime'].dtstr,
+                record.fn_times['ctime'].dtstr,
+                ','.join(map(str, record.attribute_types))
+            ))
+
+        conn.commit()
+        conn.close()
+        await asyncio.sleep(0)
+
+    @staticmethod
+    async def write_tsk(records: List[MftRecord], output_file: str) -> None:
+        with open(output_file, 'w', newline='', encoding='utf-8') as tskfile:
+            for record in records:
+                # TSK body file format:
+                # MD5|name|inode|mode_as_string|UID|GID|size|atime|mtime|ctime|crtime
+                tskfile.write(f"0|{record.filename}|{record.recordnum}|{record.flags:04o}|0|0|"
+                              f"{record.filesize}|{record.fn_times['atime'].unixtime}|"
+                              f"{record.fn_times['mtime'].unixtime}|{record.fn_times['ctime'].unixtime}|"
+                              f"{record.fn_times['crtime'].unixtime}\n")
             await asyncio.sleep(0)

src/analyzeMFT/mft_analyzer.py

@@ -1,7 +1,9 @@
 import asyncio
 import csv
 import io
+import os
 import signal
+import sqlite3
 import sys
 import traceback
 from typing import Dict, Set, List, Optional, Any
@@ -245,11 +247,52 @@ async def write_output(self) -> None:
             await FileWriters.write_xml(list(self.mft_records.values()), self.output_file)
         elif self.export_format == "excel":
             await FileWriters.write_excel(list(self.mft_records.values()), self.output_file)
+        elif self.export_format == "sqlite":
+            await FileWriters.write_sqlite(list(self.mft_records.values()), self.output_file)
+        elif self.export_format == "tsk":
+            await FileWriters.write_tsk(list(self.mft_records.values()), self.output_file)
         else:
             print(f"Unsupported export format: {self.export_format}")
 
     async def cleanup(self):
         self.log("Performing cleanup...", 1)
          # to-do add more cleanup after database stuff is integrated.
         await self.write_remaining_records()
-        self.log("Cleanup complete.", 1)
+        self.log("Cleanup complete.", 1)
+
+    async def create_sqlite_database(self):
+        conn = sqlite3.connect(self.output_file)
+        cursor = conn.cursor()
+
+        # Create and populate static tables
+        sql_dir = os.path.join(os.path.dirname(__file__), 'sql')
+        for sql_file in os.listdir(sql_dir):
+            with open(os.path.join(sql_dir, sql_file), 'r') as f:
+                cursor.executescript(f.read())
+
+        # Create MFT records table
+        cursor.execute('''
+            CREATE TABLE mft_records (
+                record_number INTEGER PRIMARY KEY,
+                filename TEXT,
+                parent_record_number INTEGER,
+                -- Add other fields as needed
+                FOREIGN KEY (attribute_type) REFERENCES attribute_types(id)
+            )
+        ''')
+
+        conn.commit()
+        return conn
+
+    async def write_sqlite(self):
+        conn = await self.create_sqlite_database()
+        cursor = conn.cursor()
+
+        for record in self.mft_records.values():
+            cursor.execute('''
+                INSERT INTO mft_records (record_number, filename, parent_record_number)
+                VALUES (?, ?, ?)
+            ''', (record.recordnum, record.filename, record.get_parent_record_num()))
+
+        conn.commit()
+        conn.close()

src/analyzeMFT/sql/attribute_types.sql

@@ -0,0 +1,21 @@
+CREATE TABLE attribute_types (
+    id INTEGER PRIMARY KEY,
+    name TEXT NOT NULL
+);
+
+INSERT INTO attribute_types (id, name) VALUES
+(16, '$STANDARD_INFORMATION'),
+(32, '$ATTRIBUTE_LIST'),
+(48, '$FILE_NAME'),
+(64, '$OBJECT_ID'),
+(80, '$SECURITY_DESCRIPTOR'),
+(96, '$VOLUME_NAME'),
+(112, '$VOLUME_INFORMATION'),
+(128, '$DATA'),
+(144, '$INDEX_ROOT'),
+(160, '$INDEX_ALLOCATION'),
+(176, '$BITMAP'),
+(192, '$REPARSE_POINT'),
+(208, '$EA_INFORMATION'),
+(224, '$EA'),
+(256, '$LOGGED_UTILITY_STREAM');

src/analyzeMFT/sql/file_record_flags.sql

@@ -0,0 +1,10 @@
+CREATE TABLE file_record_flags (
+    id INTEGER PRIMARY KEY,
+    name TEXT NOT NULL
+);
+
+INSERT INTO file_record_flags (id, name) VALUES
+(1, 'FILE_RECORD_IN_USE'),
+(2, 'FILE_RECORD_IS_DIRECTORY'),
+(4, 'FILE_RECORD_IS_EXTENSION'),
+(8, 'FILE_RECORD_HAS_SPECIAL_INDEX');