Merge branch 'csv-release'

app/controllers/exports_controller.rb

@@ -13,7 +13,9 @@ def download
     else
       data = e.data
     end
-    send_data(data, :type => 'text/csv; charset=utf-8; header=present', :filename => "#{e.name}.csv")
+    # strip off 4 character random string at end of name
+    filename = e.name.gsub(/_[A-Za-z]{4}$/, "") + ".csv"
+    send_data(data, :type => 'text/csv; charset=utf-8; header=present', :filename => filename)
   end
 
 end

app/controllers/questions_controller.rb

@@ -1193,12 +1193,13 @@ def export
     else
       question = @earl.question
 
-      redis_key  = "export_#{@earl.question_id}_#{type}_#{Time.now.to_i}"
-      redis_key += "_#{Digest::SHA1.hexdigest(redis_key + rand(10000000).to_s)}"
+      o = [('a'..'z'), ('A'..'Z')].map { |i| i.to_a }.flatten
+      randstring = (0...4).map { o[rand(o.length)] }.join
+      export_key  = "wikisurvey_#{@earl.question_id}_#{type.gsub("_", "")}_#{Time.now.utc.iso8601}_#{randstring}"
 
-      question.post(:export, :type => type, :response_type => 'redis', :redis_key => redis_key)
+      question.post(:export, :type => type, :key => export_key)
 
-      Delayed::Job.enqueue MungeAndNotifyJob.new(@earl.id, type, current_user.email, @photocracy, redis_key), 15
+      Delayed::Job.enqueue MungeAndNotifyJob.new(@earl.id, type, current_user.email, @photocracy, export_key), 15
     end
 
 

app/models/idea_mailer.rb

@@ -68,11 +68,13 @@ def export_data_ready(email, link, photocracy=false)
      @body[:photocracy] = photocracy
   end
 
-  def export_failed(email, photocracy=false)
+  def export_failed(email, type, date_requested, photocracy=false)
     setup_email(nil, photocracy)
     @recipients = email
     @subject += "Data export failed"
     @body[:photocracy] = photocracy
+    @body[:date_requested] = date_requested
+    @body[:type] = type
   end
 
   protected

app/views/idea_mailer/export_failed.text.html.erb

@@ -1,6 +1,6 @@
 <p><%= t('items.email.hello') %></p>
 
-<p>Your requested export of data from the <%= @photocracy ? 'Photocracy' : 'All Our Ideas'%> site has failed.  We've been notified of the problem and will be working on a fix.</p>
+<p>The <%= @type %> CSV export you requested on <%= @date_requested.to_formatted_s(:long) %> (UTC) from the <%= @photocracy ? 'Photocracy' : 'All Our Ideas'%> site has failed.  We've been notified of the problem and will be working on a fix.</p>
 
 <p><%= t('items.email.thanks') %></p>
 <%= render :partial => 'idea_mailer/signature' %>

app/views/idea_mailer/export_failed.text.plain.erb

@@ -1,6 +1,6 @@
 <%= t('items.email.hello') %>
 
-Your requested export of data from the <%= @photocracy ? 'Photocracy' : 'All Our Ideas'%> site has failed.  We've been notified of the problem and will be working on a fix.
+The <%= @type %> CSV export you requested on <%= @date_requested.to_formatted_s(:long) %> (UTC) from the <%= @photocracy ? 'Photocracy' : 'All Our Ideas'%> site has failed.  We've been notified of the problem and will be working on a fix.
 
 <%= t('items.email.thanks') %>
 <%= render :partial => 'idea_mailer/signature' %>

app/views/questions/admin.html.haml

@@ -101,22 +101,25 @@
               %a.btn{:href => ""}
                 Cancel
 
-        %table.table.csv-links
-          %thead
-            %tr
-              %th
-                = t('admin.download_idea_marketplace_data')
-                = link_to('', 'http://blog.allourideas.org/post/2739358388/download-your-data', :target=> '_blank', :class => 'glyphicon glyphicon-question-sign')
-          %tbody
-            %tr
-              %td.title
-                = link_to(t('admin.list_of_ideas_csv'), {:action => :export, :params => {:type => :ideas}}, :"data-toggle" => 'modal-ajax', :id => 'ideas_request_link')
-            %tr
-              %td.title
-                = link_to(t('admin.list_of_votes_csv'), {:action => :export, :params => {:type => :votes}}, :"data-toggle" => 'modal-ajax',:id => 'votes_request_link')
-            %tr
-              %td.title
-                = link_to(t('admin.list_of_non_votes_csv'), {:action => :export, :params => {:type => :non_votes}}, :"data-toggle" => 'modal-ajax', :id => 'non_votes_request_link')
+        %fieldset.download-data
+          %legend
+            = t('admin.download_idea_marketplace_data')
+            %small= link_to('', 'http://blog.allourideas.org/post/2739358388/download-your-data', :target=> '_blank', :class => 'glyphicon glyphicon-question-sign')
+          .row-fluid
+            .span4.title
+              = t('admin.list_of_ideas_csv')
+            .span2
+              = link_to(t('admin.request'), {:action => :export, :params => {:type => :ideas}}, :"data-toggle" => 'modal-ajax', :class => "btn btn-primary")
+          .row-fluid
+            .span4.title
+              = t('admin.list_of_votes_csv')
+            .span2
+              = link_to(t('admin.request'), {:action => :export, :params => {:type => :votes}}, :"data-toggle" => 'modal-ajax', :class => "btn btn-primary", :id => "votes_request_link")
+          .row-fluid
+            .span4.title
+              = t('admin.list_of_non_votes_csv')
+            .span2
+              = link_to(t('admin.request'), {:action => :export, :params => {:type => :non_votes}}, :"data-toggle" => 'modal-ajax', :class => "btn btn-primary")
 
   #cant-edit-question.modal.hide.fade
     .modal-body

config/locales/allourideas/ar.yml

@@ -295,7 +295,7 @@ ar:
     allow_flag_as_inappropriate: Allow users to flag individual ideas as inappropriate. 
     allow_flag_as_inappropriate_exp: This setting will allow users to flag ideas as inappropriate or offtopic. After a user submits a flag request, the idea will be automatically deactivated and an email will be sent to the owner of the idea marketplace. The email will contain whatever explanation the user provided along with the option to quickly re-activate an idea.
     default_lang_exp: Change the default language of the voting page interface
-    download_idea_marketplace_data: Download Idea Marketplace Data
+    download_idea_marketplace_data: Download Wiki Survey Data
     list_of_ideas_csv: List of Ideas (CSV File)
     list_of_votes_csv: List of Votes (CSV File)
     list_of_non_votes_csv: List of Non Vote Actions (CSV File)

config/locales/allourideas/en.yml

@@ -300,7 +300,7 @@ en:
     allow_flag_as_inappropriate: Allow users to flag individual ideas as inappropriate. 
     allow_flag_as_inappropriate_exp: This setting will allow users to flag ideas as inappropriate or offtopic. After a user submits a flag request, the idea will be automatically deactivated and an email will be sent to the owner of the wiki survey. The email will contain whatever explanation the user provided along with the option to quickly re-activate an idea.
     default_lang_exp: Change the default language of the voting page interface
-    download_idea_marketplace_data: Download Idea Marketplace Data
+    download_idea_marketplace_data: Download Wiki Survey Data
     list_of_ideas_csv: List of Ideas (CSV File)
     list_of_votes_csv: List of Votes (CSV File)
     list_of_non_votes_csv: List of Non Vote Actions (CSV File)

lib/munge_and_notify_job.rb

@@ -1,8 +1,9 @@
-MungeAndNotifyJob = Struct.new(:earl_id, :type, :email, :photocracy, :redis_key)
+MungeAndNotifyJob = Struct.new(:earl_id, :type, :email, :photocracy, :export_key)
 class MungeAndNotifyJob
 
   def on_permanent_failure
-    IdeaMailer.deliver_export_failed([APP_CONFIG[:ERRORS_EMAIL], email], photocracy)
+    date_requested = DateTime.iso8601(export_key.split('_')[-2])
+    IdeaMailer.deliver_export_failed([APP_CONFIG[:ERRORS_EMAIL], email], type, date_requested, photocracy)
   end
 
   # munges CSV file generated by pairwise to augment or remove the CSV
@@ -15,16 +16,17 @@ def perform
     # so we do this manually
     current_user = User.find_by_email(email)
 
-    r = Redis.new(:host => REDIS_CONFIG['hostname'])
-
-    thekey, zlibcsv = r.blpop(redis_key, (60*10).to_s) # Timeout - 10 minutes
-
-    r.del(redis_key) # client is responsible for deleting key
-
-    zstream = Zlib::Inflate.new
-    csvdata = zstream.inflate(zlibcsv)
-    zstream.finish
-    zstream.close
+    # make HTTP request to pairwise to get export data
+    url = URI.parse("#{APP_CONFIG[:API_HOST]}/exports/#{export_key}")
+    req = Net::HTTP::Get.new(url.path)
+    # important to trigger basic HTTP Auth on pairwise
+    req["Accept"] = "text/csv"
+    req.basic_auth APP_CONFIG[:PAIRWISE_USERNAME], APP_CONFIG[:PAIRWISE_PASSWORD]
+    res = Net::HTTP.start(url.host, url.port) { |http| http.request(req) }
+    if res.code != "200"
+      raise "Export URL returned response code of #{res.code} for #{url.to_s}"
+    end
+    csvdata = res.body.force_encoding('UTF-8')
 
     # for creating zlibed CSV at the end
     zoutput = Zlib::Deflate.new
@@ -36,7 +38,7 @@ def perform
 
     num_slugs = earl.slugs.size
 
-    modified_csv = CSVBridge.generate do |csv|
+    CSVBridge.generate do |csv|
       CSVBridge.parse(csvdata, {:headers => :first_row, :return_headers => true}) do |row|
 
         if row.header_row?
@@ -142,9 +144,8 @@ def perform
 
 
                 row << ['Hashed IP Address', Digest::MD5.hexdigest([user_session.ip_addr, APP_CONFIG[:IP_ADDR_HASH_SALT]].join(""))]
-                # we've had some referrers be UTF-8, rest of CSV is ASCII-8BIT
-                row << ['URL Alias', url_alias.force_encoding('ASCII-8BIT')]
-                row << ['User Agent', user_session.user_agent.try(:force_encoding, 'ASCII-8BIT')]
+                row << ['URL Alias', url_alias]
+                row << ['User Agent', user_session.user_agent]
 
                 # grab most recent referrer from clicks
                 # that is older than this current vote
@@ -158,21 +159,21 @@ def perform
                 referrer = (session_start) ? session_start.referrer : 'REFERRER_NOT_FOUND'
                 referrer = 'DIRECT_VISIT' if referrer == '/'
                 # we've had some referrers be UTF-8, rest of CSV is ASCII-8BIT
-                row << ['Referrer', referrer.force_encoding('ASCII-8BIT')]
+                row << ['Referrer', referrer]
 
                 vote_click = user_session.find_click_for_vote(row)
                 widget = (vote_click.widget?) ? 'TRUE' : 'FALSE'
                 row << ['Widget', widget]
 
                 info = user_session.find_info_value(row)
                 info = 'NA' unless info 
-                row << ['Info', info.force_encoding('ASCII-8BIT')]
+                row << ['Info', info]
 
                 if current_user.admin?
                   #row << ['Geolocation Info', user_session.loc_info.to_s]
                 end
               end
-            end
+          end
             # Zlib the CSV as we create it
             znewcsv << zoutput.deflate(row.to_csv, Zlib::SYNC_FLUSH)
         end
@@ -181,9 +182,9 @@ def perform
     znewcsv << zoutput.finish
     zoutput.close
 
-    export_id = Export.connection.insert("INSERT INTO `exports` (`name`, `data`, `compressed`) VALUES (#{Export.connection.quote(redis_key)}, #{Export.connection.quote(znewcsv)}, 1)")
+    export_id = Export.connection.insert("INSERT INTO `exports` (`name`, `data`, `compressed`) VALUES (#{Export.connection.quote(export_key)}, #{Export.connection.quote(znewcsv)}, 1)")
     Delayed::Job.enqueue DestroyOldExportJob.new(export_id), 20, 3.days.from_now
-    url = "/export/#{redis_key}"
+    url = "/export/#{export_key}"
     IdeaMailer.deliver_export_data_ready(email, url, photocracy)
 
     return true

public/javascripts/application.js

@@ -15,7 +15,6 @@ AOI.app = (function($) {
         modalAjax();
         preventDefaultLinks();
         makeRowsClickable('.ideas-table tbody tr');
-        makeRowsClickable('.csv-links tbody tr');
         dataVisualizationOpen();
     };
 

public/stylesheets/sass/screen.sass

@@ -433,10 +433,6 @@ p.add-idea
       width: 80%
     .score
       font-size: 120%
-.csv-links
-  tbody
-    tr
-      cursor: pointer
 
 .vote-count-span
   text-align: center

public/stylesheets/sass/styles.scss

@@ -425,3 +425,11 @@ a i.glyphicon {
         text-decoration: none;
     }
 }
+.download-data {
+    .row-fluid {
+        margin-bottom: 0.5em;
+    }
+    .title {
+        text-align: right;
+    }
+}