wip: rewrite to use DB

.bundle/config

@@ -0,0 +1,2 @@
+---
+BUNDLE_JOBS: 5

.gitignore

@@ -1,2 +1,3 @@
 /config/travis.yml
 /results
+tags

Gemfile

@@ -8,30 +8,34 @@ gem 'travis-config',      '~> 0.1.0'
 # .lib/travis/test-results/services/process_test_results.rb
 #gem 'travis-core',        github: 'finalci/travis-core' #, branch: 'feature/test_results_models'
 
-gem 'metriks'
 
-platform :jruby do
+gem 'sequel',             '~> 4.28.0'
+gem 'pusher'
+gem 'metriks'
+gem 'sentry-raven',       github: 'getsentry/raven-ruby'
 
-  gem 'march_hare',         '~> 2.3.0'
+gem 'connection_pool'
 
-  gem 'jdbc-postgres',      '9.3.1101'
-  gem 'activerecord-jdbcpostgresql-adapter'
-  gem 'activerecord-jdbc-adapter'
-end
+gem 'sidekiq'
 
-platform :mri do
-  gem 'bunny',            '~> 1.7.0'
-  gem 'pg'
-end
+gem 'bunny',              '~> 1.7.0'
+gem 'pg'
 
 #gem 'json',               '~> 1.8.0'
 gem 'multi_json'
 
+
+gem 'sinatra', '~> 1.4'
+gem 'connection_pool'
+gem 'rack-ssl'
+gem 'unicorn'
+
 group :test do
   gem 'rspec',            '~> 2.14.1'
 end
 
 
 group :test, :development do
   gem 'rake'
+  gem 'rack-test'
 end

Gemfile.lock

@@ -1,51 +1,67 @@
 GIT
   remote: git://github.com/finalci/travis-support.git
-  revision: e496e14fb6bba39a0c1a8dcee9b935ce9e7de481
+  revision: 61d556e38746241e2ad22f3521de1454b557d4b9
   specs:
     travis-support (0.0.1)
 
+GIT
+  remote: git://github.com/getsentry/raven-ruby.git
+  revision: 8e338102938275e96484583c3cce818cc6f05f37
+  specs:
+    sentry-raven (0.16.0.dev0)
+      faraday (>= 0.7.6)
+
 GEM
   remote: https://rubygems.org/
   specs:
-    activemodel (3.2.21)
-      activesupport (= 3.2.21)
-      builder (~> 3.0.0)
-    activerecord (3.2.21)
-      activemodel (= 3.2.21)
-      activesupport (= 3.2.21)
-      arel (~> 3.0.2)
-      tzinfo (~> 0.3.29)
-    activerecord-jdbc-adapter (1.3.15)
-      activerecord (>= 2.2)
-    activerecord-jdbcpostgresql-adapter (1.3.15)
-      activerecord-jdbc-adapter (~> 1.3.15)
-      jdbc-postgres (>= 9.1)
     activesupport (3.2.21)
       i18n (~> 0.6, >= 0.6.4)
       multi_json (~> 1.0)
     amq-protocol (1.9.2)
-    arel (3.0.3)
     atomic (1.1.99)
     atomic (1.1.99-java)
     avl_tree (1.2.1)
       atomic (~> 1.1)
-    builder (3.0.4)
     bunny (1.7.0)
       amq-protocol (>= 1.9.2)
+    celluloid (0.16.0)
+      timers (~> 4.0.0)
+    connection_pool (2.2.0)
     diff-lcs (1.2.5)
+    faraday (0.9.2)
+      multipart-post (>= 1.2, < 3)
     hashr (0.0.22)
     hitimes (1.2.3)
     hitimes (1.2.3-java)
+    httpclient (2.7.0)
     i18n (0.7.0)
-    jdbc-postgres (9.3.1101)
-    march_hare (2.3.0-java)
+    json (1.8.3)
+    json (1.8.3-java)
+    kgio (2.9.3)
     metriks (0.9.9.7)
       atomic (~> 1.0)
       avl_tree (~> 1.2.0)
       hitimes (~> 1.1)
     multi_json (1.11.0)
+    multipart-post (2.0.0)
     pg (0.18.1)
+    pusher (0.14.6)
+      httpclient (~> 2.5)
+      multi_json (~> 1.0)
+      pusher-signature (~> 0.1.8)
+    pusher-signature (0.1.8)
+    rack (1.6.4)
+    rack-protection (1.5.3)
+      rack
+    rack-ssl (1.3.4)
+      rack
+    rack-test (0.6.3)
+      rack (>= 1.0)
+    raindrops (0.14.0)
     rake (10.4.2)
+    redis (3.2.1)
+    redis-namespace (1.5.2)
+      redis (~> 3.0, >= 3.0.4)
     rspec (2.14.1)
       rspec-core (~> 2.14.0)
       rspec-expectations (~> 2.14.0)
@@ -54,28 +70,50 @@ GEM
     rspec-expectations (2.14.5)
       diff-lcs (>= 1.1.3, < 2.0)
     rspec-mocks (2.14.6)
+    sequel (4.28.0)
+    sidekiq (3.4.2)
+      celluloid (~> 0.16.0)
+      connection_pool (~> 2.2, >= 2.2.0)
+      json (~> 1.0)
+      redis (~> 3.2, >= 3.2.1)
+      redis-namespace (~> 1.5, >= 1.5.2)
+    sinatra (1.4.6)
+      rack (~> 1.4)
+      rack-protection (~> 1.4)
+      tilt (>= 1.3, < 3)
+    tilt (2.0.1)
+    timers (4.0.1)
+      hitimes
     travis-config (0.1.0)
       hashr (~> 0.0)
-    tzinfo (0.3.44)
+    unicorn (4.9.0)
+      kgio (~> 2.6)
+      rack
+      raindrops (~> 0.7)
 
 PLATFORMS
   java
   ruby
 
 DEPENDENCIES
-  activerecord-jdbc-adapter
-  activerecord-jdbcpostgresql-adapter
   activesupport (~> 3.2)
   bunny (~> 1.7.0)
-  jdbc-postgres (= 9.3.1101)
-  march_hare (~> 2.3.0)
+  connection_pool
   metriks
   multi_json
   pg
+  pusher
+  rack-ssl
+  rack-test
   rake
   rspec (~> 2.14.1)
+  sentry-raven!
+  sequel (~> 4.28.0)
+  sidekiq
+  sinatra (~> 1.4)
   travis-config (~> 0.1.0)
   travis-support!
+  unicorn
 
 BUNDLED WITH
    1.10.6

Procfile

@@ -1 +1,2 @@
 test_results: ./bin/receive_test_results
+web: script/server

README.md

@@ -5,8 +5,14 @@ Process test results from Travis Worker (particularly travis-guest-api)
 and store it in DB.
 
 
-*Work in progress*
-
-
 Implementation is based (copy & paste) from travis-logs (therefore 
 travis-logs license is attached as well).
+
+# Setup
+
+     # if needed:
+     dropdb travis_test_results_${ENV} --user ... --host .. --port ...
+     createdb travis_test_results_${ENV} --user ... --host ...
+
+     rake db:migrate
+

Rakefile

@@ -0,0 +1,38 @@
+require "bundler/setup"
+require "rake"
+
+$LOAD_PATH << File.expand_path("../lib", __FILE__)
+require "travis/test-results.rb"
+require "travis/support"
+require "travis/test-results/helpers/database"
+
+namespace :db do
+  desc "Apply migrations"
+  task :migrate, [:version] do |t, args|
+    Sequel.extension(:migration)
+    db = Travis::TestResults::Helpers::Database.create_sequel
+
+    if args[:version]
+      puts "Migrating to version #{args[:version]}"
+      Sequel::Migrator.run(db, "db/migrations", target: args[:version].to_i)
+    else
+      puts "Migrating to latest"
+      Sequel::Migrator.run(db, "db/migrations")
+    end
+  end
+
+  desc "List status for migrations"
+  task :status do
+    Sequel.extension(:migration)
+    db = Travis::TestResults::Helpers::Database.create_sequel
+    applied = Sequel::TimestampMigrator.new(db, "db/migrations").applied_migrations
+    all_migrations = Dir["db/migrations/*.rb"].map { |file| File.basename(file) }.sort
+    all_migrations.each do |migration_file|
+      if applied.include?(migration_file)
+        puts "   up   #{migration_file}"
+      else
+        puts "  down  #{migration_file}"
+      end
+    end
+  end
+end

bin/receive_test_results

@@ -6,9 +6,6 @@ require 'bundler/setup'
 require 'travis/test-results'
 require 'travis/test-results/receive'
 
-#initialize cache
-Travis::TestResults.cache
-
 app = Travis::TestResults::Receive.new
 app.setup
 Travis.logger.info "Starting travis-test-results"

config.ru

@@ -0,0 +1,4 @@
+$: << 'lib'
+require 'bundler/setup'
+require 'travis/test-results/app'
+run Travis::TestResults::App.new

db/migrations/20151111151946_create_step_results.rb

@@ -0,0 +1,20 @@
+Sequel.migration do
+  up do
+    create_table(:step_results) do
+      primary_key :id
+      integer :job_id
+      text    :data
+
+      timestamp :created_at
+      timestamp :updated_at
+
+      index :job_id, name: "index_step_results_on_job_id"
+    end
+
+    run "ALTER TABLE step_results ALTER COLUMN data TYPE JSON USING data::JSON;"
+  end
+
+  down do
+    drop_table(:step_results)
+  end
+end

lib/travis/redis_pool.rb

@@ -0,0 +1,32 @@
+require 'connection_pool'
+require 'redis'
+require 'metriks'
+
+module Travis
+  class RedisPool
+    attr_reader :pool
+
+    def initialize(options = {})
+      pool_options = options.delete(:pool) || {}
+      pool_options[:size] ||= 10
+      pool_options[:timeout] ||= 10
+      @pool = ConnectionPool.new(pool_options) do
+        ::Redis.new(options)
+      end
+    end
+
+    def method_missing(name, *args)
+      timer = Metriks.timer('redis.pool.wait').time
+      @pool.with do |redis|
+        timer.stop
+        if redis.respond_to?(name)
+          Metriks.timer("redis.operations").time do
+            redis.send(name, *args)
+          end
+        else
+          super
+        end
+      end
+    end
+  end
+end

lib/travis/test-results.rb

@@ -1,5 +1,4 @@
 require 'travis/test-results/config'
-require 'travis/test-results/cache'
 
 module Travis
   def self.config
@@ -12,13 +11,6 @@ def config
         @config ||= Config.load
       end
 
-      def cache
-        @cache ||= Travis::TestResults::Cache.new(
-          config.test_results.gc_remove_after,
-          config.test_results.gc_pooling_interval
-        )
-      end
-
       def database_connection=(connection)
         @database_connection = connection
       end