Merge pull request #189 from tosdr/mj-old-db-import

WiP: db import

Author: michielbdejong

app/models/case.rb

@@ -0,0 +1,3 @@
+class Case < ApplicationRecord
+  belongs_to :topic
+end

app/models/comment.rb

@@ -0,0 +1,3 @@
+class Comment < ApplicationRecord
+  belongs_to :point
+end

db/import_cases_from_old_db.rb

@@ -0,0 +1,31 @@
+# in repo root, run:
+# rails runner db/import_cases_from_old_db.rb
+
+filename_cases = "old_db/cases.json"
+
+def importCase(data)
+  puts 'old data:'
+  puts data 
+  puts 'new data:'
+  topic = Topic.find_by_oldId(data['topic'])
+  imported_case = Case.new(
+    title: data['name'],
+    classification: data['point'],
+    score: data['score'],
+    topic: topic
+  )
+  puts imported_case
+  unless imported_case.valid?
+    puts "### #{imported_case.title} not imported ! ###" #+ panic
+    puts topic
+  end
+  imported_case.save
+  puts 'saved.'
+end
+
+puts "Importing cases..."
+file = File.read(filename_cases)
+data = JSON.parse(file)
+data.each { |line| importCase(line) }
+puts "Finishing importing cases"
+puts "Done!"

db/import_points_from_old_db.rb

@@ -1,31 +1,70 @@
 require 'json'
 require 'pry'
 
-# in repo root, run
+# in repo root, run:
 # rails runner db/import_points_from_old_db.rb
 
 filepath_points = "old_db/points/"
 
-puts "Importing points..."
-Dir.foreach(filepath_points) do |filename|
-  next if filename == '.' or filename == '..'
-  file = File.read(filepath_points + filename)
-  data = JSON.parse(file)
+def importPoint(data, service)
+  puts 'old data:'
   puts data
+  puts service
+  puts 'new data:'
+  userObj = User.find_by_email('[email protected]')
+  topicObj = Topic.find_by_title('Personal Data')
+  serviceObjDefault = Service.find_by_name('amazon')
+  serviceObj = Service.find_by_name(service) || serviceObjDefault
+  caseObjDefault = Case.find_by_title('info given about security practices')
+  caseObj = Case.find_by_title(data['tosdr']['case']) || caseObjDefault
   imported_point = Point.new(
-#   user_id: # go make the script look for the user, if no user, make it default
+    # old_id: data['id'] + '-' + service,
     title: data['title'],
-    source: data['discussion'],
-    status: (data['needModeration'] == "false" ? "approved" : "pending"),
-    analysis: data['tldr'],
-    rating: data['tosdr']['score']
-#    service = Service.find_by_name(line['services'])
+    user: userObj,
+    source: "http://perdu.com",
+    status: "pending",
+    analysis: "Bla bla bla",
+    rating: 3,
+    topic: topicObj,
+    service: serviceObj,
+    case_id: caseObj.id
+  )
+
+#  validates :title, presence: true
+#  validates :title, length: { in: 5..140 }
+#  validates :source, presence: true
+#  validates :status, inclusion: { in: ["approved", "pending", "declined", "disputed", "draft"], allow_nil: false }
+#  validates :analysis, presence: true
+#  validates :rating, presence: true
+#  validates :rating, numericality: true
+
+  unless imported_point.valid?
+    puts "### #{imported_point.title} not imported ! ###"
+    # panic
+  end
 #    binding.pry
 #    service_id: service.id,
 #    topic = Topic.find_by_title(line['topics']) #need to import the services first and match it by string
 #    topic_id: topic.id, #need to import topics first and match it by string
-    )
   imported_point.save
+  puts imported_point.id
+  puts data['id']
+  creationComment = Comment.new(
+    point: imported_point,
+    summary: 'imported from '+data['id']
+  )
+  creationComment.save
+  puts 'saved.'
+end
+
+puts "Importing points..."
+Dir.foreach(filepath_points) do |filename|
+  next if filename == '.' or filename == '..' or filename == 'README.md'
+  file = File.read(filepath_points + filename)
+  data = JSON.parse(file)
+  for i in 0 ... data['services'].size
+    importPoint(data, data['services'][i])
+  end
 end
 puts "Finishing importing points"
 puts "Done!"

db/import_services_from_old_db.rb

@@ -1,20 +1,32 @@
-require 'json'
-require 'pry'
+# in repo root, run:
+# rails runner db/import_services_from_old_db.rb
 
-puts "Starting the loop and the import..."
-Dir[File.join(Rails.root,"old_db/services/*.json")].each do |json_file|
-  hash = JSON.parse(File.read(json_file))
+filepath_services = "old_db/services/"
+
+def importService(data)
+  puts 'old data:'
+  puts data 
+  puts 'new data:'
   imported_service = Service.new(
-    name: hash['name'],
-    url: hash['fulltos']['terms']['url']
-    )
-  imported_service.save
-  byebug
+    # old_id: data['id'],
+    name: data['name'].downcase,
+    url: data['urls'][0] || 'url',
+    grade: 'grade'
+  )
+  puts imported_service
   unless imported_service.valid?
-    puts "### #{imported_service.name} not imported ! ###"
-  else
-    puts "Imported and saved: #{imported_service.name}"
+    puts "### #{imported_service.name} not imported ! ###" #+ panic
   end
-  puts "Exiting loop"
+  imported_service.save
+  puts 'saved.'
+end
+
+puts "Importing services..."
+Dir.foreach(filepath_services) do |filename|
+  next if filename == '.' or filename == '..' or filename == 'README.md'
+  file = File.read(filepath_services + filename)
+  data = JSON.parse(file)
+  importService(data)
 end
-puts "Finishing importing topics!"
+puts "Finishing importing services"
+puts "Done!"

db/import_topics_from_old_db.rb

@@ -1,19 +1,33 @@
-require 'json'
+# in repo root, run:
+# rails runner db/import_topics_from_old_db.rb
 
-puts "Starting the loop and the import..."
-Dir[File.join(Rails.root,"old_db/topics/*.json")].each do |json_file|
-  hash = JSON.parse(File.read(json_file))
+filepath_topics = "old_db/topics/"
+
+def importTopic(data)
+  puts 'old data:'
+  puts data 
+  puts 'new data:'
   imported_topic = Topic.new(
-    title: hash['name'],
-    subtitle: hash['subtitle'] || 'no text here',
-    description: hash['description'] || 'no text here'
-    )
-  imported_topic.save #add topic error handling
+    title: data['title'] || 'title',
+    subtitle: data['subtitle'] || 'subtitle',
+    description: data['description'] || 'description',
+    oldId: data['id']
+  )
+  puts imported_topic
   unless imported_topic.valid?
     puts "### #{imported_topic.title} not imported ! ###"
-  else
-    puts "Imported and saved: #{imported_topic.title}"
+    panic
   end
-  puts "Exiting loop"
+  imported_topic.save
+  puts 'saved.'
+end
+
+puts "Importing topics..."
+Dir.foreach(filepath_topics) do |filename|
+  next if filename == '.' or filename == '..' or filename == 'README.md'
+  file = File.read(filepath_topics + filename)
+  data = JSON.parse(file)
+  importTopic(data)
 end
-puts "Finishing importing topics!"
+puts "Finishing importing topics"
+puts "Done!"

db/migrate/20171214143220_create_cases.rb

@@ -0,0 +1,13 @@
+class CreateCases < ActiveRecord::Migration[5.1]
+  def change
+    create_table :cases do |t|
+      t.string :classification
+      t.integer :score
+      t.string :title
+      t.text :description
+      t.references :topic, foreign_key: true
+
+      t.timestamps
+    end
+  end
+end

db/migrate/20171214143221_create_comments.rb

@@ -0,0 +1,10 @@
+class CreateComments < ActiveRecord::Migration[5.1]
+  def change
+    create_table :comments do |t|
+      t.references :point, foreign_key: true
+      t.string :summary
+
+      t.timestamps
+    end
+  end
+end

db/migrate/20171214143222_add_quote_to_points.rb

@@ -0,0 +1,5 @@
+class AddQuoteToPoints < ActiveRecord::Migration[5.1]
+  def change
+    add_column :points, :quote, :string
+  end
+end

db/migrate/20171214143225_add_refs_to_points.rb

@@ -0,0 +1,5 @@
+class AddRefsToPoints < ActiveRecord::Migration[5.1]
+  def change
+    add_reference :points, :case, foreign_key: true
+  end
+end

db/migrate/20171215063327_add_old_id_to_topics.rb

@@ -0,0 +1,5 @@
+class AddOldIdToTopics < ActiveRecord::Migration[5.1]
+  def change
+    add_column :topics, :oldId, :string
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20171120110106) do
+ActiveRecord::Schema.define(version: 20171215063327) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -29,6 +29,25 @@
     t.index ["resource_type", "resource_id"], name: "index_active_admin_comments_on_resource_type_and_resource_id"
   end
 
+  create_table "cases", force: :cascade do |t|
+    t.string "classification"
+    t.integer "score"
+    t.string "title"
+    t.text "description"
+    t.bigint "topic_id"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["topic_id"], name: "index_cases_on_topic_id"
+  end
+
+  create_table "comments", force: :cascade do |t|
+    t.bigint "point_id"
+    t.string "summary"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["point_id"], name: "index_comments_on_point_id"
+  end
+
   create_table "points", force: :cascade do |t|
     t.bigint "user_id"
     t.integer "rank", default: 0
@@ -42,6 +61,9 @@
     t.datetime "updated_at", null: false
     t.bigint "topic_id"
     t.bigint "service_id"
+    t.string "quote"
+    t.bigint "case_id"
+    t.index ["case_id"], name: "index_points_on_case_id"
     t.index ["service_id"], name: "index_points_on_service_id"
     t.index ["topic_id"], name: "index_points_on_topic_id"
     t.index ["user_id"], name: "index_points_on_user_id"
@@ -72,6 +94,7 @@
     t.string "description"
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
+    t.string "oldId"
   end
 
   create_table "users", force: :cascade do |t|
@@ -94,6 +117,9 @@
     t.index ["reset_password_token"], name: "index_users_on_reset_password_token", unique: true
   end
 
+  add_foreign_key "cases", "topics"
+  add_foreign_key "comments", "points"
+  add_foreign_key "points", "cases"
   add_foreign_key "points", "services"
   add_foreign_key "points", "topics"
   add_foreign_key "points", "users"

db/seeds.rb

@@ -13,27 +13,20 @@
 Service.destroy_all
 Topic.destroy_all
 
-# puts "Importing topics"
-# load File.join(Rails.root,"db","import_topics_from_old_db.rb")
-# puts "Importing services"
-# load File.join(Rails.root,"db","import_services_from_old_db.rb")
-# load File.join(Rails.root,"db","import_points_from_old_db.rb")
-
 puts "Starts new seeding"
 test_user = User.new(email: "[email protected]", username: "test user", password: "testnonadminuser", password_confirmation: "testnonadminuser")
 test_user.save
 curator_test_user = User.new(email: "[email protected]", password: "testcuratoruser", password_confirmation: "testcuratoruser", curator: true)
 curator_test_user.save
 admin_test_user = User.new(email: "[email protected]", username: "admin test user", password: "testadminuser", password_confirmation: "testadminuser", admin: true)
 admin_test_user.save
-test_service_1 = Service.new(name: "Test service 1", url: "http://perdu.com", grade: "A")
-test_service_1.save
-test_service_2 = Service.new(name: "Test service 2", url: "http://perdu.com", grade: "B")
-test_service_2.save
-test_topic = Topic.new(title: "Test topic", subtitle: "Test subtitle", description: "Test topic description")
-test_topic.save
-test_point_1 = Point.new(user_id: 1, title: "Test point 1", source: "http://perdu.com", status: "pending", analysis: "Bla bla bla", rating: 3, topic_id: 1, service_id: 1)
-test_point_1.save
-test_point_2 = Point.new(user_id: 1, title: "Test point 2", source: "http://perdu.com", status: "pending", analysis: "Bla bla bla", rating: 5, topic_id: 1, service_id: 1)
-test_point_2.save
+
+puts "Importing topics"
+load File.join(Rails.root,"db","import_topics_from_old_db.rb")
+puts "Importing services"
+load File.join(Rails.root,"db","import_services_from_old_db.rb")
+puts "Importing cases"
+load File.join(Rails.root,"db","import_cases_from_old_db.rb")
+puts "Importing points"
+load File.join(Rails.root,"db","import_points_from_old_db.rb")
 puts "Done!"