This is a major, structural change which represents a new release. Th…

…e entire structure of the data storage has been changed. Instead of a single raw_logs table, it's now broken into months. Additionally, session information has been broken out of the raw_logs and is solely stored in the sessions table. In order to accomplish this, there is a MAJOR database restructuring, requiring the execution of 'db_upgrade.php' which can take HOURS to run depending on how much data you have. PLEASE BACK UP YOUR DATABASE BEFORE RUNNING THAT. PLEASE LOOK AT THE ACTUAL CODE I AM RUNNING and consider running it manually. The most important user-facing changes...1) session filtering is now Year_Month instead of year AND month. 2) The GPS coordinates have been changed from float to Double, making the lines smoother. 3) You can now select favorite PIDs in the PID Edit page, and those are now auto-selected to be populated for plotting when the app loads.
surfrock66 · Feb 3, 2019 · e555902 · e555902
1 parent 5a7d7d6
commit e555902
Show file tree

Hide file tree

Showing 17 changed files with 477 additions and 352 deletions.
diff --git a/README.md b/README.md
@@ -136,3 +136,10 @@ LimitRequestLine 15000
   * Email-receiver for this?  LONG SHOT, but have the server read an email address so you can email tracks from the app
 * Ian Hawkins has a google map pin show up when hovering over his graphs on his reference viewer...different system, but it'd be cool to implement.
 * Idea: speed heatmap for the map track. (Google Maps iOS API has gradient polylines, javascript API does not...may not be possible for now).
+
+### Credits and Thanks ###
+
+* [Ian Hawkins](http://ian-hawkins.com/) - Creator of the Torque app, none of this happens without that
+* [Matt Nicklay/econpy](https://github.com/econpy) - This is the project I forked from, so all credit where credit is due
+* [Takashi Saito/takashisite](https://github.com/takashisite) - Spawn for the patch to change the GPS from float to double; I didn't take his pull request because it came with a greater structural update, but he deserves credit for pointing it out
+* [marvinwankersteen](https://github.com/marvinwankersteen) - Gave me the idea to implement the "favorite variables" system.
diff --git a/scripts/create_torque_keys_table.sql b/scripts/create_torque_keys_table.sql
diff --git a/scripts/create_torque_log_table.sql b/scripts/create_torque_log_table.sql
@@ -16,18 +16,8 @@ PREPARE stmt FROM @query;
 EXECUTE stmt;
 #DROP TABLE IF EXISTS `raw_logs`;
 CREATE TABLE `raw_logs` (
-  `v` varchar(1) NOT NULL,
   `session` varchar(15) NOT NULL,
-  `id` varchar(32) NOT NULL,
   `time` varchar(15) NOT NULL,
-  `eml` varchar(255),
-  `profileName` varchar(255) NOT NULL DEFAULT '0',
-  `profileFuelType` varchar(255) NOT NULL DEFAULT 'Not Specified',
-  `profileWeight` float NOT NULL DEFAULT '0',
-  `profileVe` float NOT NULL DEFAULT '0',
-  `profileFuelCost` float NOT NULL DEFAULT '0',
-  `notice` varchar(255),
-  `noticeClass` varchar(255),
   `k10` float NOT NULL DEFAULT '0',
   `k11` float NOT NULL DEFAULT '0',
   `k12` float NOT NULL DEFAULT '0',
@@ -92,8 +82,8 @@ CREATE TABLE `raw_logs` (
   `kf` float NOT NULL DEFAULT '0',
   `kfe1805` float NOT NULL DEFAULT '0',
   `kff1001` float NOT NULL DEFAULT '0',
-  `kff1005` float NOT NULL DEFAULT '0',
-  `kff1006` float NOT NULL DEFAULT '0',
+  `kff1005` double NOT NULL DEFAULT '0',
+  `kff1006` double NOT NULL DEFAULT '0',
   `kff1007` float NOT NULL DEFAULT '0',
   `kff1010` float NOT NULL DEFAULT '0',
   `kff1201` float NOT NULL DEFAULT '0',
@@ -169,6 +159,12 @@ CREATE TABLE `raw_logs` (
   `kff1275` float NOT NULL DEFAULT '0',
   `kff1276` float NOT NULL DEFAULT '0',
   `kff1277` float NOT NULL DEFAULT '0',
+  `kff1278` float NOT NULL DEFAULT '0',
+  `kff1280` float NOT NULL DEFAULT '0',
+  `kff1296` float NOT NULL DEFAULT '0',
+  `kff1297` float NOT NULL DEFAULT '0',
+  `kff1298` float NOT NULL DEFAULT '0',
+  `kff129a` float NOT NULL DEFAULT '0',
   `kff5201` float NOT NULL DEFAULT '0',
   `kff5202` float NOT NULL DEFAULT '0',
   `kff5203` float NOT NULL DEFAULT '0',

diff --git a/web/db_upgrade.php b/web/db_upgrade.php
@@ -0,0 +1,49 @@
+<?php
+  require_once ('db.php');
+  require_once ('auth_app.php');
+
+  mysqli_query($con, "ALTER TABLE $db_keys_table ADD COLUMN favorite TINYINT(1) NOT NULL DEFAULT 0") or die(mysqli_error($con));
+  // Update existing tables to handle new data structures
+  $table_list = mysqli_query($con, "SELECT table_name FROM INFORMATION_SCHEMA.tables WHERE table_schema = '$db_name' and table_name like '$db_table%' ORDER BY table_name DESC;");
+  while( $row = mysqli_fetch_assoc($table_list) ) {
+    $db_table_name = $row["table_name"];
+    // Change the GPS Latitude and Longitude datapoints from Float to Double to improve accuracy
+    $sqlLatQuery = "ALTER TABLE $db_table_name MODIFY kff1006 DOUBLE NOT NULL DEFAULT '0'";
+    mysqli_query($con, $sqlLatQuery) or die(mysqli_error($con));
+    $sqlLongQuery = "ALTER TABLE $db_table_name MODIFY kff1005 DOUBLE NOT NULL DEFAULT '0'";
+    mysqli_query($con, $sqlLongQuery) or die(mysqli_error($con));
+    // Delete columns which are now redundant and just stored with the session
+    $sqlVQuery = "ALTER TABLE $db_table_name DROP COLUMN v";
+    mysqli_query($con, $sqlVQuery) or die(mysqli_error($con));
+    $sqlIdQuery = "ALTER TABLE $db_table_name DROP COLUMN id";
+    mysqli_query($con, $sqlIdQuery) or die(mysqli_error($con));
+    $sqlEmlQuery = "ALTER TABLE $db_table_name DROP COLUMN eml";
+    mysqli_query($con, $sqlEmlQuery) or die(mysqli_error($con));
+    $sqlProfileNameQuery = "ALTER TABLE $db_table_name DROP COLUMN profileName";
+    mysqli_query($con, $sqlProfileNameQuery) or die(mysqli_error($con));
+    $sqlProfileFuelTypeQuery = "ALTER TABLE $db_table_name DROP COLUMN profileFuelType";
+    mysqli_query($con, $sqlProfileFuelTypeQuery) or die(mysqli_error($con));
+    $sqlProfileWeightQuery = "ALTER TABLE $db_table_name DROP COLUMN profileWeight";
+    mysqli_query($con, $sqlProfileWeightQuery) or die(mysqli_error($con));
+    $sqlProfileVeQuery = "ALTER TABLE $db_table_name DROP COLUMN profileVe";
+    mysqli_query($con, $sqlProfileVeQuery) or die(mysqli_error($con));
+    $sqlProfileFuelCostQuery = "ALTER TABLE $db_table_name DROP COLUMN profileFuelCost";
+    mysqli_query($con, $sqlProfileFuelCostQuery) or die(mysqli_error($con));
+
+  }
+
+  // Split the raw logs table into per-month tables 
+  $sessionYears = mysqli_query($con, "SELECT DISTINCT CONCAT(YEAR(FROM_UNIXTIME(session/1000)), '_', DATE_FORMAT(FROM_UNIXTIME(session/1000),'%m')) as Suffix, YEAR(FROM_UNIXTIME(session/1000)) as Year, MONTH(FROM_UNIXTIME(session/1000)) as Month FROM $db_table");
+  while( $row = mysqli_fetch_assoc( $sessionYears ) ) {
+    $suffix = $row['Suffix'];
+    $year = $row['Year'];
+    $month = $row['Month'];
+    $new_table_name = "{$db_table}_test_{$suffix}";
+    $table_create_query = "CREATE TABLE $new_table_name SELECT * FROM $db_table WHERE YEAR(FROM_UNIXTIME(session/1000)) LIKE '$year' and MONTH(FROM_UNIXTIME(session/1000)) LIKE '$month'";
+    mysqli_query($con, $table_create_query) or die(mysqli_error($con));
+  }
+
+  // Clear the raw_logs table; we still want it as a shell, just empty
+  mysqli_query($con, "DELETE FROM $db_table") or die(mysqli_error($con));
+?>
+
diff --git a/web/del_session.php b/web/del_session.php
@@ -12,13 +12,14 @@
 }
 
 if (isset($deletesession) && !empty($deletesession)) {
-    $delresult = mysqli_query($con, "DELETE FROM $db_table
-                          WHERE session=".quote_value($deletesession)) or die(mysqli_error($con));
+    $tableYear = date( "Y", $deletesession/1000 );
+    $tableMonth = date( "m", $deletesession/1000 );
+    $db_table_full = "{$db_table}_{$tableYear}_{$tableMonth}";
+    $delresult = mysqli_query($con, "DELETE FROM $db_table_full WHERE session=".quote_value($deletesession)) or die(mysqli_error($con));
 
     mysqli_free_result($delresult);
 
-    $delresult = mysqli_query($con, "DELETE FROM $db_sessions_table
-                          WHERE session=".quote_value($deletesession)) or die(mysqli_error($con));
+    $delresult = mysqli_query($con, "DELETE FROM $db_sessions_table WHERE session=".quote_value($deletesession)) or die(mysqli_error($con));
 
     mysqli_free_result($delresult);
 }

diff --git a/web/export.php b/web/export.php
@@ -5,7 +5,10 @@
     $session_id = $_GET['sid'];
     // Get data for session
     $output = "";
-    $sql = mysqli_query($con, "SELECT * FROM $db_table join $db_sessions_table on $db_table.session = $db_sessions_table.session WHERE $db_table.session=".quote_value($session_id)." ORDER BY $db_table.time DESC;") or die(mysqli_error($con));
+    $tableYear = date( "Y", $session_id/1000 );
+    $tableMonth = date( "m", $session_id/1000 );
+    $db_table_full = "{$db_table}_{$tableYear}_{$tableMonth}";
+    $sql = mysqli_query($con, "SELECT * FROM $db_table_full join $db_sessions_table on $db_table_full.session = $db_sessions_table.session WHERE $db_table_full.session=".quote_value($session_id)." ORDER BY $db_table_full.time DESC;") or die(mysqli_error($con));
 
     if ($_GET["filetype"] == "csv") {
         $columns_total = mysqli_num_fields($sql);

diff --git a/web/get_columns.php b/web/get_columns.php
@@ -6,10 +6,10 @@
 // 2015.08.21 - edit by surfrock66 - Rather than pull from the column comments,
 //   oull from a new database created which manages variables. Include
 //   a column flagging whether a variable is populated or not.
-$colqry = mysqli_query($con, "SELECT id,description,type FROM $db_keys_table WHERE populated = 1 ORDER BY description") or die(mysqli_error($con));
+$colqry = mysqli_query($con, "SELECT id,description,type,favorite FROM $db_keys_table WHERE populated = 1 ORDER BY description") or die(mysqli_error($con));
 while ($x = mysqli_fetch_array($colqry)) {
   if ((substr($x[0], 0, 1) == "k") && ($x[2] == "float")) {
-    $coldata[] = array("colname"=>$x[0], "colcomment"=>$x[1]);
+    $coldata[] = array("colname"=>$x[0], "colcomment"=>$x[1], "colfavorite"=>$x[3]);
   }
 }
 

diff --git a/web/get_sessions.php b/web/get_sessions.php
@@ -1,36 +1,17 @@
 <?php
-//echo "<!-- Begin get_session.php at ".date("H:i:s", microtime(true))." -->\r\n";
+//echo "<!-- Begin get_sessions.php at ".date("H:i:s", microtime(true))." -->\r\n";
 // this page relies on being included from another page that has already connected to db
 
 session_set_cookie_params(0,dirname($_SERVER['SCRIPT_NAME']));
 if (!isset($_SESSION)) { session_start(); }
 
-// Process the 4 possibilities for the year filter: Set in POST, Set in GET, select all possible years, or the default: select the current year
-if ( isset($_POST["selyear"]) ) {
-	$filteryear = $_POST["selyear"];
-} elseif ( isset($_GET["year"])) {
-	$filteryear = $_GET["year"];
+// Process the possibilities for the year and month filter: Set in POST, Set in GET, select all possible year/months, or the default: select the current year/month
+if ( isset($_POST["selyearmonth"]) ) {
+	$filteryearmonth = $_POST["selyearmonth"];
+} elseif ( isset($_GET["yearmonth"])) {
+	$filteryearmonth = $_GET["yearmonth"];
 } else {
-	$filteryear = date('Y');
-}
-if ( $filteryear == "ALL" ) {
-	$filteryear = "%";
-}
-
-// Process the 4 possibilities for the month filter: Set in POST, Set in GET, select all possible months, or the default: select the current month
-if ( isset($_POST["selmonth"]) ) {
-	$filtermonth = $_POST["selmonth"];
-} elseif ( isset($_GET["month"])) {
-	$filtermonth = $_GET["month"];
-} else {
-	if ( isset($_POST["selyear"]) || isset($_GET["year"]) ) {
-		$filtermonth = "%";
-	} else {
-		$filtermonth = date('F');
-	}
-}
-if ( $filtermonth == "ALL" ) {
-	$filtermonth = "%";
+	$filteryearmonth = date('Y_m');
 }
 
 // Process the 4 possibilities for the profile filter: Set in POST, Set in GET, select all possible profiles, or no filter as default
@@ -45,22 +26,18 @@
 	$filterprofile = "%";
 }
 
-// Build the MySQL select string based on the inputs (year, month, or session id)
+
+// Build the MySQL select string based on the inputs (year_month or session id)
 $sessionqrystring = "SELECT timestart, timeend, session, profileName, sessionsize FROM $db_sessions_table ";
-$sqlqryyear = "YEAR(FROM_UNIXTIME(session/1000)) LIKE " . quote_value($filteryear) . " ";
-$sqlqrymonth = "MONTHNAME(FROM_UNIXTIME(session/1000)) LIKE " . quote_value($filtermonth) . " ";
+$sqlqryyearmonth = "CONCAT(YEAR(FROM_UNIXTIME(session/1000)), '_', DATE_FORMAT(FROM_UNIXTIME(session/1000),'%m')) LIKE " . quote_value($filteryearmonth) . " ";
 $sqlqryprofile = "profileName LIKE " . quote_value($filterprofile) . " " ;
 $orselector = "WHERE ";
 $andselector = "";
-if ( $filteryear <> "%" || $filtermonth <> "%" || $filterprofile <> "%") {
+if ( $filteryearmonth <> "%" || $filterprofile <> "%") {
 	$orselector = " OR ";
 	$sessionqrystring = $sessionqrystring . "WHERE ( ";
-	if ( $filteryear <> "%" ) {
-		$sessionqrystring = $sessionqrystring . $sqlqryyear;
-		$andselector = " AND ";
-	}
-	if ( $filtermonth <> "%" ) {
-		$sessionqrystring = $sessionqrystring . $andselector . $sqlqrymonth;
+	if ( $filteryearmonth <> "%" ) {
+		$sessionqrystring = $sessionqrystring . $sqlqryyearmonth;
 		$andselector = " AND ";
 	}
 	if ( $filterprofile <> "%" ) {
@@ -101,6 +78,6 @@
 }
 
 mysqli_free_result($sessionqry);
-//echo "<!-- End get_session.php at ".date("H:i:s", microtime(true))." -->\r\n";
+//echo "<!-- End get_sessions.php at ".date("H:i:s", microtime(true))." -->\r\n";
 
 ?>
diff --git a/web/merge_sessions.php b/web/merge_sessions.php
@@ -44,14 +44,18 @@
     $newsessionsize = $mergerow['sessionsize'];
     mysqli_free_result($mergeqry);
 
+    $tableYear = date( "Y", $mergesession/1000 );
+    $tableMonth = date( "m", $mergesession/1000 );
+    $db_table_full = "{$db_table}_{$tableYear}_{$tableMonth}";
+
     foreach ($sessionids as $value) {
         if ($value == $newsession) {
             $updatequery = "UPDATE $db_sessions_table SET timestart=$newtimestart, timeend=$newtimeend, sessionsize=$newsessionsize where session=$newsession";
             mysqli_query($con, $updatequery) or die(mysqli_error($con));
         } else {
             $delquery = "DELETE FROM $db_sessions_table WHERE session = '$value'";
             mysqli_query($con, $delquery) or die(mysqli_error($con));
-            $updatequery = "UPDATE $db_table SET session=$newsession WHERE session=".quote_value($value);
+            $updatequery = "UPDATE $db_table_full SET session=$newsession WHERE session=".quote_value($value);
             mysqli_query($con, $updatequery) or die(mysqli_error($con));
         }
     }

diff --git a/web/pid_commit.php b/web/pid_commit.php
@@ -14,23 +14,33 @@
     $split_data = explode(':', $field_id);
     $id = $split_data[1];
     $field_name = $split_data[0];
-echo "\nField Name: '$field_name'\nField ID: '$id'\nValue: '$val'\n";
-    if(!empty($id) && !empty($field_name) && !empty($val)) {
-      if($field_name == 'populated') {
-        if($val == 'true'){
+//echo "ID: '$id' Field Name: '$field_name' Field ID: '$id' Value: '$val'";
+    if (!empty($id) && !empty($field_name)) {
+      if ($field_name == 'populated') {
+        if ($val == 'true') {
+          $val=1;
+        } else {
+          $val=0;
+       }
+      } elseif ($field_name == 'favorite') {
+        if ($val == 'true') {
           $val=1;
         } else {
           $val=0;
        }
       }
       //update the values
       $query = "UPDATE $db_name.$db_keys_table SET ".quote_name($field_name)." = ".quote_value($val)." WHERE id = ".quote_value($id);
-echo "\n$query\n";
-      mysqli_query($query) || die(mysqli_error($con));
+//echo "<br />$query<br />";
+      mysqli_query($con, $query) || die(mysqli_error($con));
       if($field_name == 'type') {
-      $query = "ALTER TABLE $db_name.$db_table MODIFY ".quote_name($id)." ".mysqli_real_escape_string($con, $val)." NOT NULL DEFAULT '0'";
-echo $query;
-        mysqli_query($query) || die(mysqli_error($con));
+        $table_list = mysqli_query($con, "SELECT table_name FROM INFORMATION_SCHEMA.tables WHERE table_schema = '$db_name' and table_name like '$db_table%' ORDER BY table_name DESC;");
+        while( $row = mysqli_fetch_assoc($table_list) ) {
+          $db_table_name = $row["table_name"];
+          $query = "ALTER TABLE $db_name.$db_table_name MODIFY ".quote_name($id)." ".mysqli_real_escape_string($con, $val)." NOT NULL DEFAULT '0'";
+//echo "<br />$query<br />";
+          mysqli_query($con, $query) || die(mysqli_error($con));
+        }
       }
       echo "Updated";
     } else {

diff --git a/web/pid_edit.php b/web/pid_edit.php
@@ -7,11 +7,11 @@
 // 2015.08.21 - edit by surfrock66 - Rather than pull from the column comments,
 //   oull from a new database created which manages variables. Include
 //   a column flagging whether a variable is populated or not.
-$keyqry = mysqli_query($con, "SELECT id,description,units,type,min,max,populated FROM ".$db_name.".".$db_keys_table." ORDER BY description") or die(mysqli_error($con));
+$keyqry = mysqli_query($con, "SELECT id,description,units,type,min,max,populated,favorite FROM ".$db_name.".".$db_keys_table." ORDER BY description") or die(mysqli_error($con));
 $i = 0;
 while ($x = mysqli_fetch_array($keyqry)) {
 	if ((substr($x[0], 0, 1) == "k") ) {
-		$keydata[$i] = array("id"=>$x[0], "description"=>$x[1], "units"=>$x[2], "type"=>$x[3], "min"=>$x[4], "max"=>$x[5], "populated"=>$x[6]);
+		$keydata[$i] = array("id"=>$x[0], "description"=>$x[1], "units"=>$x[2], "type"=>$x[3], "min"=>$x[4], "max"=>$x[5], "populated"=>$x[6], "favorite"=>$x[7]);
 		$i = $i + 1;
 	}
 }
@@ -38,7 +38,7 @@
     <script language="javascript" type="text/javascript" src="static/js/jquery.peity.min.js"></script>
     <script language="javascript" type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/chosen/1.1.0/chosen.jquery.min.js"></script>
     <script language="javascript" type="text/javascript" src="static/js/torquehelpers.js"></script>
-    <script type="text/javascript" src="//ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
+    <script type="text/javascript" src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
     <script>
       $(function(){
         var message_status = $("#status");
@@ -97,7 +97,8 @@
         <th>Variable Type</th>
         <th>Min Value</th>
         <th>Max Value</th>
-        <th>In Use?</th>
+        <th>Visible?</th>
+        <th>Favorite?</th>
       </thead>
       <tbody>
 <?php $i = 1; ?>
@@ -109,13 +110,15 @@
           <td>
             <select  id="type:<?php echo $keycol['id']; ?>" contenteditable="true">
               <!--<option value="boolean"<?php //if ($keycol['type'] == "boolean") echo ' selected'; ?>>boolean</option>-->
+              <option value="double"<?php if ($keycol['type'] == "double") echo ' selected'; ?>>double</option>
               <option value="float"<?php if ($keycol['type'] == "float") echo ' selected'; ?>>float</option>
               <option value="varchar(255)"<?php if ($keycol['type'] == "varchar(255)") echo ' selected'; ?>>varchar(255)</option>
             </selecT>
           </td>
           <td id="min:<?php echo $keycol['id']; ?>" contenteditable="true"><?php echo $keycol['min']; ?></td>
           <td id="max:<?php echo $keycol['id']; ?>" contenteditable="true"><?php echo $keycol['max']; ?></td>
           <td><input type="checkbox" id="populated:<?php echo $keycol['id']; ?>" contenteditable="true"<?php if ( $keycol['populated'] ) echo " CHECKED"; ?>/></td>
+          <td><input type="checkbox" id="favorite:<?php echo $keycol['id']; ?>" contenteditable="true"<?php if ( $keycol['favorite'] ) echo " CHECKED"; ?>/></td>
         </tr>
 <?php   $i = $i + 1; ?>
 <?php } ?>

diff --git a/web/plot.php b/web/plot.php
@@ -55,7 +55,10 @@
 		$i = $i + 1;
 	}
 	// Get data for session
-	$sessionqry = mysqli_query($con, "SELECT $selectstring FROM $db_table WHERE session=".quote_value($session_id)." ORDER BY time DESC;") or die(mysqli_error($con));
+	$tableYear = date( "Y", $session_id/1000 );
+	$tableMonth = date( "m", $session_id/1000 );
+	$db_table_full = "{$db_table}_{$tableYear}_{$tableMonth}";
+	$sessionqry = mysqli_query($con, "SELECT $selectstring FROM $db_table_full WHERE session=".quote_value($session_id)." ORDER BY time DESC;") or die(mysqli_error($con));
 	while($row = mysqli_fetch_assoc($sessionqry)) {
 	    $i = 1;
 		while (isset(${'v' . $i})) {