diff --git a/src/decode.rs b/src/decode.rs
index 191cab357..b5bd58884 100644
--- a/src/decode.rs
+++ b/src/decode.rs
@@ -4262,22 +4262,16 @@ pub(crate) unsafe fn rav1d_decode_frame_init(
     c: &Rav1dContext,
     f: &mut Rav1dFrameData,
 ) -> Rav1dResult {
-    if f.sbh > f.lf.start_of_tile_row_sz {
-        free(f.lf.start_of_tile_row as *mut c_void);
-        f.lf.start_of_tile_row = malloc(f.sbh as usize * ::core::mem::size_of::<u8>()) as *mut u8;
-        if f.lf.start_of_tile_row.is_null() {
-            f.lf.start_of_tile_row_sz = 0;
-            return Err(ENOMEM);
-        }
-        f.lf.start_of_tile_row_sz = f.sbh;
-    }
+    // TODO: Fallible allocation
+    f.lf.start_of_tile_row.resize(f.sbh as usize, 0);
+
     let frame_hdr = &***f.frame_hdr.as_ref().unwrap();
     let mut sby = 0;
     for tile_row in 0..frame_hdr.tiling.rows {
-        *f.lf.start_of_tile_row.offset(sby as isize) = tile_row as u8;
+        f.lf.start_of_tile_row[sby as usize] = tile_row as u8;
         sby += 1;
         while sby < frame_hdr.tiling.row_start_sb[(tile_row + 1) as usize] as c_int {
-            *f.lf.start_of_tile_row.offset(sby as isize) = 0;
+            f.lf.start_of_tile_row[sby as usize] = 0;
             sby += 1;
         }
     }
diff --git a/src/internal.rs b/src/internal.rs
index 02139885d..8676b60cb 100644
--- a/src/internal.rs
+++ b/src/internal.rs
@@ -467,8 +467,7 @@ pub struct Rav1dFrameContext_lf {
     pub lr_lpf_line: [*mut DynPixel; 3], /* plane */
 
     // in-loop filter per-frame state keeping
-    pub start_of_tile_row: *mut u8,
-    pub start_of_tile_row_sz: c_int,
+    pub start_of_tile_row: Vec<u8>,
     pub p: [*mut DynPixel; 3],
     pub sr_p: [*mut DynPixel; 3],
     pub restore_planes: c_int, // enum LrRestorePlanes
diff --git a/src/lib.rs b/src/lib.rs
index 76b5f5165..cd2cd4924 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -350,6 +350,7 @@ pub(crate) unsafe fn rav1d_open(c_out: &mut *mut Rav1dContext, s: &Rav1dSettings
         addr_of_mut!(f.lf.tx_lpf_right_edge).write(Default::default());
         addr_of_mut!(f.lf.cdef_line_buf).write(Default::default());
         addr_of_mut!(f.lf.lr_line_buf).write(Default::default());
+        addr_of_mut!(f.lf.start_of_tile_row).write(Default::default());
         f.lf.last_sharpness = -(1 as c_int);
         rav1d_refmvs_init(&mut f.rf);
         n = n.wrapping_add(1);
@@ -926,7 +927,7 @@ impl Drop for Rav1dContext {
                 let _ = mem::take(&mut f.lf.lr_mask); // TODO: remove when context is owned
                 let _ = mem::take(&mut f.lf.level);
                 let _ = mem::take(&mut f.lf.tx_lpf_right_edge); // TODO: remove when context is owned
-                free(f.lf.start_of_tile_row as *mut c_void);
+                let _ = mem::take(&mut f.lf.start_of_tile_row); // TODO: remove when context is owned
                 rav1d_refmvs_clear(&mut f.rf);
                 let _ = mem::take(&mut f.lf.cdef_line_buf); // TODO: remove when context is owned
                 let _ = mem::take(&mut f.lf.lr_line_buf); // TODO: remove when context is owned
diff --git a/src/recon.rs b/src/recon.rs
index 7c46594f8..01c1f67b5 100644
--- a/src/recon.rs
+++ b/src/recon.rs
@@ -4536,7 +4536,7 @@ pub(crate) unsafe fn rav1d_filter_sbrow_deblock_cols<BD: BitDepth>(
         &p_offset,
         mask_offset as usize,
         sby,
-        *(f.lf.start_of_tile_row).offset(sby as isize) as c_int,
+        f.lf.start_of_tile_row[sby as usize] as c_int,
     );
 }