feat(reader): refactor classes for generalization

Author: Kyle Hernandez

gdc_fastq_splitter/fastq/base.py

@@ -20,8 +20,9 @@ def __str__(self):
 
 class FastqRecord:
     """The base class for a Fastq record"""
-    def __init__(self, seqid, sequence, qid, qual, seqid_cls=SequenceIdentifier):
-        self.seqid = seqid_cls.from_string(seqid)
+    seqid_cls=SequenceIdentifier
+    def __init__(self, seqid, sequence, qid, qual):
+        self.seqid = self.seqid_cls.from_string(seqid)
         self.sequence = sequence
         self.qid = qid
         self.qual = qual

gdc_fastq_splitter/fastq/illumina.py

@@ -72,8 +72,10 @@ def is_valid(seqid):
         return True
 
 class IlluminaFastqRecord(base.FastqRecord):
-    def __init__(self, seqid, sequence, qid, qual, seqid_cls=IlluminaSequenceIdentifier):
-        super().__init__(seqid, sequence, qid, qual, seqid_cls=seqid_cls)
+    seqid_cls = IlluminaSequenceIdentifier
+
+    def __init__(self, seqid, sequence, qid, qual):
+        super().__init__(seqid, sequence, qid, qual)
 
     @property
     def read_key(self):
@@ -95,9 +97,15 @@ def flowcell(self):
     def lane(self):
         return self.seqid.lane
 
+    @classmethod
+    def is_valid_seqid(cls, seqid):
+        return cls.seqid_cls.is_valid(seqid)
+
 class IlluminaNoBarcodeFastqRecord(base.FastqRecord):
-    def __init__(self, seqid, sequence, qid, qual, seqid_cls=IlluminaSequenceIdentifierNoBarcode):
-        super().__init__(seqid, sequence, qid, qual, seqid_cls=seqid_cls)
+    seqid_cls = IlluminaSequenceIdentifierNoBarcode
+
+    def __init__(self, seqid, sequence, qid, qual):
+        super().__init__(seqid, sequence, qid, qual)
 
     @property
     def read_key(self):
@@ -115,15 +123,19 @@ def flowcell(self):
     def lane(self):
         return self.seqid.lane
 
+    @classmethod
+    def is_valid_seqid(cls, seqid):
+        return cls.seqid_cls.is_valid(seqid)
+
 def infer_fastq_type(fil):
     """
     Infer the type of fastq based on the first line.
     """
     def predicate(obj):
         """A predicate to get all classes that are subclasses of
-        base.SequenceIdentifier"""
-        return inspect.isclass(obj) and issubclass(obj, base.SequenceIdentifier) \
-            and hasattr(obj, 'is_valid')
+        base.FastqRecord"""
+        return inspect.isclass(obj) and issubclass(obj, base.FastqRecord) \
+            and hasattr(obj, 'is_valid_seqid')
 
     fh = gzip.open(fil, 'rt') if fil.endswith('.gz') else \
         open(fil, 'rt')
@@ -132,10 +144,11 @@ def predicate(obj):
 
     try:
         line = fh.readline().rstrip('\r\n')
+        #mod = sys.modules["gdc_fastq_splitter.fastq.illumina"]
         mod = sys.modules["gdc_fastq_splitter.fastq.illumina"]
         # Get all available seqidentifier types 
         for m in inspect.getmembers(mod, predicate):
-            if m[1].is_valid(line):
+            if m[1].is_valid_seqid(line):
                 cls_mod = m
                 break 
     finally:

gdc_fastq_splitter/fastq/report.py

@@ -6,9 +6,10 @@
 
 class BaseReport:
     """Base report for a fastq file"""
-    def __init__(self, filename, flowcell_barcode=None, lane_number=None):
-        self._filename = filename
-        self.filename = os.path.basename(filename)
+    def __init__(self, report_filename, fastq_filename, flowcell_barcode=None, lane_number=None):
+        self._report_filename = report_filename
+        self.report_filename = os.path.basename(report_filename)
+        self.fastq_filename = os.path.basename(fastq_filename)
         self.flowcell_barcode = flowcell_barcode
         self.lane_number = lane_number
         self.record_counts = 0
@@ -18,27 +19,27 @@ def __iadd__(self, record):
         return self
 
     def __str__(self):
-        return '<flowcell>({0.flowcell_barcode}),<lane>({0.lane_number})'.format(self)
+        return json.dumps(self.to_dict(), indent=2, sort_keys=True)
 
     def to_dict(self):
         return {
             'metadata': {
-                'filename': self.filename,
+                'fastq_filename': self.fastq_filename,
                 'flowcell_barcode': self.flowcell_barcode,
                 'lane_number': self.lane_number,
                 'record_count': self.record_counts
             }
         }
 
     def write_to_json(self):
-        with open(self._filename, 'wt') as o:
+        with open(self._report_filename, 'wt') as o:
             json.dump(self.to_dict(), o, indent=2, sort_keys=True)
 
 
-class BarcodeFastqReport(BaseReport):
+class ReportWithBarcodes(BaseReport):
     """Report that contains barcode frequencies."""
-    def __init__(self, filename, flowcell_barcode=None, lane_number=None):
-        super().__init__(filename, flowcell_barcode=flowcell_barcode, lane_number=lane_number)
+    def __init__(self, report_filename, fastq_filename, flowcell_barcode=None, lane_number=None):
+        super().__init__(report_filename, fastq_filename, flowcell_barcode=flowcell_barcode, lane_number=lane_number)
         self.barcode_frequency = Counter()
 
     @property
@@ -57,13 +58,10 @@ def __iadd__(self, record):
         self._add_barcode(record.index)
         return self
 
-    def __str__(self):
-        return '<flowcell>({0.flowcell_barcode}),<barcode>({0.multiplex_barcode}),<lane>({0.lane_number})'.format(self)
-
     def to_dict(self):
         return {
             'metadata': {
-                'filename': self.filename,
+                'filename': self.fastq_filename,
                 'flowcell_barcode': self.flowcell_barcode,
                 'multiplex_barcode': self.most_common_barcode,
                 'lane_number': self.lane_number,

gdc_fastq_splitter/fastq/writer.py

@@ -1,7 +1,7 @@
 """Module containing writer classes for writing Fastq files"""
 import gzip
 import io
-from gdc_fastq_splitter.fastq.report import BaseReport
+from gdc_fastq_splitter.fastq.report import BaseReport, ReportWithBarcodes
 
 class FastqWriter:
     """Base Fastq writer class"""
@@ -28,11 +28,12 @@ def __init__(self, fname, reporter, **kwargs):
         self.reporter = reporter
 
     @classmethod
-    def from_record_and_prefix(cls, prefix, record, report_cls=BaseReport):
-        fbase = '{0}_{1}_R{2}'.format(prefix, record.read_key, record.read_pair)
+    def from_record_and_prefix(cls, record, prefix):
+        report_cls = ReportWithBarcodes if hasattr(record, 'index') else BaseReport
+        fbase = '{0}{1}_R{2}'.format(prefix, record.read_key, record.read_pair)
         fname = '{0}.fq.gz'.format(fbase)
         rname = '{0}.report.json'.format(fbase)
-        return cls(fname, report_cls(rname, flowcell_barcode=record.flowcell, lane_number=record.lane))
+        return cls(fname, report_cls(rname, fname, flowcell_barcode=record.flowcell, lane_number=record.lane))
 
     def __iadd__(self, record):
         self.reporter += record
@@ -42,4 +43,3 @@ def __iadd__(self, record):
     def close(self):
         super().close()
         self.reporter.write_to_json()
-

setup.py

@@ -15,6 +15,7 @@
     license = "Apache 2.0",
     packages = [
         "gdc_fastq_splitter",
+        "gdc_fastq_splitter.fastq",
     ],
     classifiers = [
         "Development Status :: 3 - Alpha",

tests/test_illumina.py

@@ -2,7 +2,10 @@
 import os
 import logging
 
-from gdc_fastq_splitter.fastq.illumina import IlluminaSequenceIdentifierNoBarcode, IlluminaSequenceIdentifier, IlluminaFastqRecord, infer_fastq_type
+from gdc_fastq_splitter.fastq.illumina import (
+    IlluminaFastqRecord, IlluminaSequenceIdentifierNoBarcode, 
+    IlluminaSequenceIdentifier, IlluminaNoBarcodeFastqRecord,
+    infer_fastq_type) 
 
 class TestIlluminaSequenceIdentifier(unittest.TestCase):
     """Test modern illumina sequence identifiers"""
@@ -58,18 +61,21 @@ def test_valid_record(self):
 class TestInferFastqType(unittest.TestCase):
     """Test the infer_fastq_type functionality"""
     def test_modern(self):
+        """Testing IlluminaFastqRecord"""
         fil = os.path.join(os.path.dirname(__file__), 'etc/fake_IlluminaSequenceIdentifier.fastq')
         m = infer_fastq_type(fil)
-        expected = ('IlluminaSequenceIdentifier', IlluminaSequenceIdentifier)
+        expected = ('IlluminaFastqRecord', IlluminaFastqRecord)
         self.assertEqual(expected, m)
 
     def test_nobarcode(self):
+        """Testing IlluminaNoBarcodeFastqRecord"""
         fil = os.path.join(os.path.dirname(__file__), 'etc/fake_IlluminaSequenceIdentifierNoBarcode.fastq')
         m = infer_fastq_type(fil)
-        expected = ('IlluminaSequenceIdentifierNoBarcode', IlluminaSequenceIdentifierNoBarcode)
+        expected = ('IlluminaNoBarcodeFastqRecord', IlluminaNoBarcodeFastqRecord)
         self.assertEqual(expected, m)
 
     def test_unknown(self):
+        """Testing raise exception"""
         fil = os.path.join(os.path.dirname(__file__), 'etc/fake_Unknown.fastq')
 
         with self.assertRaises(Exception):