diff --git a/.gitignore b/.gitignore index fc500a41fe..fbe55f493e 100644 --- a/.gitignore +++ b/.gitignore @@ -39,3 +39,4 @@ api.html brakeman-output.* .idea +TEMP_NOTES.txt diff --git a/Jenkinsfile b/Jenkinsfile index 5e07add90a..eb1d6a2614 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -58,6 +58,9 @@ pipeline { stage('API') { steps { sh 'cd ci && ./test --cucumber-api' } } + stage('Rotators') { + steps { sh 'cd ci && ./test --cucumber-rotators' } + } } post { always { diff --git a/app/domain/rotation/conjur_facade.rb b/app/domain/rotation/conjur_facade.rb index 3608825c7c..bdd2b5ff23 100644 --- a/app/domain/rotation/conjur_facade.rb +++ b/app/domain/rotation/conjur_facade.rb @@ -14,14 +14,20 @@ class ConjurFacade def initialize(rotated_variable:, secret_model: Secret, + resource_model: Resource, db: Sequel::Model.db) @rotated_variable = rotated_variable @secret_model = secret_model + @resource_model = resource_model @db = db end - def current_values(variables) - @secret_model.current_values(variables) + def current_values(variable_ids) + @secret_model.current_values(variable_ids) + end + + def annotations + @resource_model.annotations(@rotated_variable.resource_id) end # new_values is a Hash of {resource_id: new_value} pairs that represent a @@ -38,6 +44,7 @@ def current_values(variables) # def update_variables(new_values, &rotator_code) @db.transaction do + new_values.each do |resource_id, value| update_secret(resource_id, value) end diff --git a/app/domain/rotation/base58_password.rb b/app/domain/rotation/password.rb similarity index 94% rename from app/domain/rotation/base58_password.rb rename to app/domain/rotation/password.rb index bf825fa743..3e73f01427 100644 --- a/app/domain/rotation/base58_password.rb +++ b/app/domain/rotation/password.rb @@ -1,4 +1,5 @@ require 'base58' +require 'securerandom' module Rotation @@ -7,13 +8,13 @@ module Rotation # # This means that your password will have a space of 58 ^ length. Passwords # are generated using the ruby `SecureRandom` library. - class Base58Password + class Password # Use the first char of the default alphabet to pad zeros # ZERO_PAD_CHAR = Base58::ALPHABETS[:flickr][0] - def self.new(length:) + def self.base58(length:) valid = length > 0 && length.is_a?(Integer) raise ArgumentError, "length must be a positive integer" unless valid diff --git a/app/domain/rotation/rotated_variable.rb b/app/domain/rotation/rotated_variable.rb index 27a8ccced5..19375b5f05 100644 --- a/app/domain/rotation/rotated_variable.rb +++ b/app/domain/rotation/rotated_variable.rb @@ -39,7 +39,6 @@ def name # and returns everything up to th first slash # def prefix - puts '@resource_id', @resource_id @resource_id.match(%r{(.*)/.*})[1] end diff --git a/app/domain/rotation/rotators/postgresql/password.rb b/app/domain/rotation/rotators/postgresql/password.rb index e6364475e8..440d83ebe0 100644 --- a/app/domain/rotation/rotators/postgresql/password.rb +++ b/app/domain/rotation/rotators/postgresql/password.rb @@ -6,22 +6,31 @@ module Postgresql class Password - def initialize(password_factory: ::Rotation::Base58Password, pg: ::PG) + PASSWORD_LENGTH_ANNOTATION = 'rotation/postgresql/password/length' + + def initialize(password_factory: ::Rotation::Password, pg: ::PG) @password_factory = password_factory @pg = pg end # 1. Generate new pw - # 2. Update of variable in Conjur. + # 2. Update variable in Conjur. # 3. Update variable in the DB itself. # # NOTE: Both 2 and 3 are executed inside a single transaction, so either # both updates happen, or neither do. + # + # NOTE: The order matters: + # 1. Capture the *current* credentials + # 2. The Database object `db` then uses those to perform the + # update to the new credentials. # def rotate(facade) + resource_id = facade.rotated_variable.resource_id - db = rotated_db(facade) - new_pw = db.new_password + credentials = DbCredentials.new(facade) + new_pw = new_password(facade) + db = Database.new(credentials, @pg) pw_update = Hash[resource_id, new_pw] facade.update_variables(pw_update) do @@ -31,63 +40,47 @@ def rotate(facade) private - def rotated_db(facade) - RotatedDb.new(facade, @pg, @password_factory) + def new_password(facade) + @password_factory.base58(length: pw_length(facade)) end - class RotatedDb - - def initialize(facade, pg, password_factory) - @facade = facade - @pg = pg - @password_factory = password_factory - end + def pw_length(facade) + @pw_length ||= facade.annotations[PASSWORD_LENGTH_ANNOTATION].to_i || 20 + end - def new_password - len = policy_vals[pw_length_id] || 20 - @password_factory.new(length: len) - end + class DbCredentials - def update_password(new_pw) - conn = connection - conn.prepare('update_pw', "ALTER ROLE $1 WITH PASSWORD '$2'") - conn.exec_prepared('update_pw', [current.username, new_pw]) - conn.close + # NOTE: It's important that @credentials is initialized on + # intialization, because we need to capture the *current* + # credentials so that we can access the db to *update* to + # the new password. + # + def initialize(facade) + @facade = facade + @credentials = current_credentials end - def connection - connection = @pg.connect(db_uri) - connection.exec('SELECT 1') - connection + def db_uri + url, uname, password = credential_resource_ids.map(&@credentials) + "postgresql://#{uname}:#{password}@#{url}" end - def db_uri - url, username, password = credential_ids.map { |x| policy_vals[x] } - "postgresql://#{username}:#{password}@#{url}" + def username + @credentials[username_id] end private - # Values of the postgres rotator related variables in policy.yml - # - def policy_vals - @policy_vals ||= @facade.current_values( - credential_ids << pw_length_id - ) - end - - # Variables containing database connection info are expected to exist - # - def credential_ids - @credential_ids ||= [url_id, username_id, password_id] + def current_credentials + @facade.current_values(credential_resource_ids) end def rotated_variable @facade.rotated_variable end - def pw_length_id - rotated_variable.sibling_id('password/length') + def credential_resource_ids + [url_id, username_id, password_id] end def password_id @@ -103,6 +96,29 @@ def username_id end end + class Database + + def initialize(credentials, pg) + @credentials = credentials + @pg = pg + end + + def update_password(new_pw) + username = @credentials.username + conn = connection + conn.exec("ALTER ROLE #{username} WITH PASSWORD '#{new_pw}'") + conn.close + end + + private + + def connection + connection = @pg.connect(@credentials.db_uri) + connection.exec('SELECT 1') + connection + end + end + end end diff --git a/app/models/resource.rb b/app/models/resource.rb index 2a3f4e46bd..cbf6e161ce 100644 --- a/app/models/resource.rb +++ b/app/models/resource.rb @@ -58,6 +58,15 @@ def find_if_visible role, *a def [] *args args.length == 3 ? super(args.join ':') : super end + + def annotations(resource_id) + natural_join(:annotations) + .where(resource_id: resource_id) + .all + .map(&:values) + .reduce([]) { |m,x| m << [x[:name], x[:value]] } + .to_h + end end def extended_associations diff --git a/ci/docker-compose.yml b/ci/docker-compose.yml index 3cae1e4fa2..cf3c83d6f1 100644 --- a/ci/docker-compose.yml +++ b/ci/docker-compose.yml @@ -3,6 +3,9 @@ services: pg: image: postgres:9.3 + testdb: + image: postgres:9.3 + conjur: image: "conjur:${TAG}" environment: @@ -42,6 +45,7 @@ services: links: - conjur - pg + - testdb ldap-server: image: osixia/openldap diff --git a/ci/test b/ci/test index 3d939f5cce..8d0d30f4b0 100755 --- a/ci/test +++ b/ci/test @@ -2,6 +2,7 @@ # shellcheck disable=SC1091 + # Display CLI usage information function print_help { cat << EOF @@ -17,6 +18,8 @@ GLOBAL OPTIONS --cucumber-authenticators - Runs Cucumber Authenticator features + --cucumber-rotators - Runs Cucumber Rotator features + --cucumber-policy - Runs Cucumber Policy features -h | --help - Show this message @@ -67,11 +70,13 @@ export COMPOSE_PROJECT_NAME="$(openssl rand -hex 3)" services="pg conjur" RUN_ALL=true RUN_AUTHENTICATORS=false +RUN_ROTATORS=false RUN_API=false RUN_POLICY=false RUN_RSPEC=false while true ; do case "$1" in + --cucumber-rotators ) RUN_ALL=false ; RUN_ROTATORS=true ; shift ;; --cucumber-authenticators ) RUN_ALL=false ; RUN_AUTHENTICATORS=true ; shift ;; --cucumber-api ) RUN_ALL=false ; RUN_API=true ; shift ;; --cucumber-policy ) RUN_ALL=false ; RUN_POLICY=true ; shift ;; @@ -91,6 +96,11 @@ export TAG="$(version_tag)" cd ci # Run tests based on what flags were passed +if [[ $RUN_ROTATORS = true || $RUN_ALL = true ]]; then + services="$services testdb" + run_cucumber_tests 'rotators' +fi + if [[ $RUN_AUTHENTICATORS = true || $RUN_ALL = true ]]; then services="$services ldap-server" run_cucumber_tests 'authenticators' diff --git a/cucumber.yml b/cucumber.yml index 8f79af2f25..31eeb883db 100644 --- a/cucumber.yml +++ b/cucumber.yml @@ -12,3 +12,8 @@ authenticators: > --format pretty -r cucumber/authenticators/features/support -r cucumber/authenticators/features/step_definitions cucumber/authenticators + +rotators: > + --format pretty + -r cucumber/rotators/features/support + -r cucumber/rotators/features/step_definitions cucumber/rotators diff --git a/cucumber/policy/features/support/env.rb b/cucumber/policy/features/support/env.rb index a57d211157..3d5c856158 100644 --- a/cucumber/policy/features/support/env.rb +++ b/cucumber/policy/features/support/env.rb @@ -1,6 +1,5 @@ require 'aruba' require 'aruba/cucumber' -require 'conjur-api' +require_relative './world.rb' -Conjur.configuration.appliance_url = ENV['CONJUR_APPLIANCE_URL'] || 'http://conjur' -Conjur.configuration.account = ENV['CONJUR_ACCOUNT'] || 'cucumber' +World(PossumWorld) diff --git a/cucumber/policy/features/support/world.rb b/cucumber/policy/features/support/world.rb index feba7a1b8d..9488fe76b8 100644 --- a/cucumber/policy/features/support/world.rb +++ b/cucumber/policy/features/support/world.rb @@ -1,3 +1,8 @@ +require 'conjur-api' + +Conjur.configuration.appliance_url = ENV['CONJUR_APPLIANCE_URL'] || 'http://conjur' +Conjur.configuration.account = ENV['CONJUR_ACCOUNT'] || 'cucumber' + module FullId def make_full_id id, account: Conjur.configuration.account tokens = id.split(":", 3) @@ -91,5 +96,3 @@ def login_as_role login, api_key = nil @conjur_api = Conjur::API.new_from_key login, api_key end end - -World(PossumWorld) diff --git a/cucumber/rotators/features/password.feature b/cucumber/rotators/features/password.feature new file mode 100644 index 0000000000..131134600a --- /dev/null +++ b/cucumber/rotators/features/password.feature @@ -0,0 +1,34 @@ +Feature: Postgres password rotation + + Background: Configure a postgres rotator + Given I reset my root policy + And I have the root policy: + """ + - !policy + id: db-reports + body: + - &variables + - !variable url + - !variable username + - !variable + id: password + annotations: + rotation/rotator: postgresql/password + rotation/ttl: PT1S + rotation/postgresql/password/length: 32 + """ + And I create a db user "test" with password "secret" + And I add the value "testdb" to variable "db-reports/url" + And I add the value "test" to variable "db-reports/username" + And I add the value "secret" to variable "db-reports/password" + + # NOTE: To make these tests robust on unreliable Jenkins servers that may + # not be able to respect `sleep` times exactly, we give ourselves + # some leeway, waiting for 3 rotations but allowing for more than + # 3 seconds of time. + # + Scenario: Values are rotated according to the policy + Given I poll "db-reports/password" and db user "test" for 3 rotations in 20 seconds + Then the first 3 db and conjur passwords match + Then the first 3 conjur passwords are distinct + And the generated passwords have length 32 diff --git a/cucumber/rotators/features/step_definitions/password_steps.rb b/cucumber/rotators/features/step_definitions/password_steps.rb new file mode 100644 index 0000000000..e7faff7338 --- /dev/null +++ b/cucumber/rotators/features/step_definitions/password_steps.rb @@ -0,0 +1,64 @@ +Then(/^I create a db user "(.*?)" with password "(.*?)"$/) do |user, pw| + # drop them first, in case we're re-running during dev + run_sql_in_testdb("DROP DATABASE #{user};") + run_sql_in_testdb("DROP USER #{user};") + run_sql_in_testdb("CREATE USER #{user} WITH PASSWORD '#{pw}';") + run_sql_in_testdb("CREATE DATABASE #{user};") +end + +re = /^I poll "(.+)" and db user "(.+)" for (\d+) rotations in (\d+) seconds$/ +Then(re) do |var, user, num_rots_str, timeout_str| + poll_for_N_rotations( + var_id: var, + db_user: user, + num_rots: num_rots_str.to_i, + timeout: timeout_str.to_i + ) +end + +Then(/^the first (\d+) db and conjur passwords match$/) do |num_str| + num = num_str.to_i + db_pws = db_passwords.first(num) + conjur_pws = conjur_passwords.first(num) + expect(db_pws).to match_array(conjur_pws) +end + +Then(/^the first (\d+) conjur passwords are distinct$/) do |num_str| + num = num_str.to_i + conjur_pws = conjur_passwords.first(num) + expect(conjur_pws.size).to eq(num) + expect(conjur_pws.uniq.size).to eq(num) +end + +Then(/^the generated passwords have length (\d+)$/) do |len_str| + length = len_str.to_i + conjur_pw = conjur_passwords.last + expect(conjur_pw.length).to eq(length) +end + +Given(/^I have the root policy:$/) do |policy| + invoke do + load_root_policy policy + end +end + +Given(/^I reset my root policy$/) do + invoke do + load_root_policy <<~EOS + - !policy + id: db-reports + body: + EOS + end +end + +Given(/^I add the value "(.*)" to variable "(.+)"$/) do |val, var| + variable = variable_resource(var) + variable.add_value(val) +end + + +Then(/^I wait for (\d+) seconds?$/) do |num_seconds| + puts "Sleeping #{num_seconds}...." + sleep(num_seconds.to_i) +end diff --git a/cucumber/rotators/features/support/env.rb b/cucumber/rotators/features/support/env.rb new file mode 100644 index 0000000000..41e11548e0 --- /dev/null +++ b/cucumber/rotators/features/support/env.rb @@ -0,0 +1,10 @@ +# Bring in the policy's World +# +require_relative './world.rb' +require_relative '../../../policy/features/support/world.rb' + +Conjur.configuration.appliance_url = ENV['CONJUR_APPLIANCE_URL'] || 'http://possum' +Conjur.configuration.account = ENV['CONJUR_ACCOUNT'] || 'cucumber' + +World(PossumWorld) +World(RotatorWorld) diff --git a/cucumber/rotators/features/support/world.rb b/cucumber/rotators/features/support/world.rb new file mode 100644 index 0000000000..39b94ecf11 --- /dev/null +++ b/cucumber/rotators/features/support/world.rb @@ -0,0 +1,91 @@ +module RotatorWorld + + # stores history of all rotated passwords + # + attr_reader :db_passwords, :conjur_passwords + + def pg_host + 'testdb' + end + + def pg_login_result(user, pw) + system("PGPASSWORD=#{pw} psql -c \"\\q\" -h #{pg_host} -U #{user}") + end + + def run_sql_in_testdb(sql, user='postgres', pw='postgres_secret') + system("PGPASSWORD=#{pw} psql -h #{pg_host} -U #{user} -c \"#{sql}\"") + end + + def variable_resource(var) + conjur_api.resource("cucumber:variable:#{var}") + end + + # + # Polling / Watching for changes + # + + def poll_for_N_rotations(var_id:, db_user:, num_rots:, timeout:) + @conjur_passwords = [] + @db_passwords = [] + timer = Timer.new + error_msg = "Failed to detect #{num_rots} rotations in #{timeout} seconds" + + loop do + + # NOTE: We rescue here because we don't want errors in these lines + # to kill the entire threads. It's perfectly valid to attempt + # to read the variables or access the db when we cannot. + # + pw = variable_resource(var_id)&.value rescue nil + pw_works_in_db = pg_login_result(db_user, pw) if pw rescue nil + + # we only record it if they're synced -- avoids race conditions + if pw_works_in_db + add_conjur_password(pw) if new_conjur_pw?(pw) + add_db_password(pw) if new_db_pw?(pw) + end + + return if total_rots >= num_rots + raise error_msg if timer.has_exceeded?(timeout) + sleep(0.3) + end + end + + class Timer + def initialize + @started_at = Time.new + end + + def time_elapsed + Time.new - @started_at + end + + def has_exceeded?(seconds) + time_elapsed > seconds + end + end + + private + + def total_rots + [@db_passwords, @conjur_passwords].map(&:size).min + end + + def add_db_password(pw) + @db_passwords = (@db_passwords || []) << pw + end + + def add_conjur_password(pw) + @conjur_passwords = (@conjur_passwords || []) << pw + end + + def new_conjur_pw?(pw) + pw != @conjur_passwords.last + end + + def new_db_pw?(pw) + pw != @db_passwords.last + end + + +end diff --git a/dev/docker-compose.yml b/dev/docker-compose.yml index 73f0a41918..c72d4d6cc1 100644 --- a/dev/docker-compose.yml +++ b/dev/docker-compose.yml @@ -3,6 +3,11 @@ services: pg: image: postgres:9.4 + testdb: + image: postgres:9.3 + environment: + POSTGRES_PASSWORD: postgres_secret + conjur: build: context: ../ @@ -30,7 +35,8 @@ services: cucumber: image: conjur-dev - entrypoint: bash + entrypoint: sleep + command: infinity environment: LDAP_URI: ldap://ldap-server:389 LDAP_BASE: dc=conjur,dc=net @@ -43,8 +49,9 @@ services: - ..:/src/conjur-server - authn-local:/run/authn-local links: - - conjur:conjur - - pg:pg + - conjur + - pg + - testdb client: image: conjurinc/cli5 diff --git a/dev/start b/dev/start index 6a4c415715..1b6d5c7774 100755 --- a/dev/start +++ b/dev/start @@ -8,6 +8,10 @@ To start the application server, run: Usage: start [options] --authn-ldap Starts OpenLDAP server and loads a demo policy to enable authentication via: 'curl -X POST -d "alice" http://localhost:3000/authn-ldap/test/cucumber/alice/authenticate' + --rotators Starts a cucumber and test postgres container. + Drops you into the cucumber container. + You then manually start `conjurctl server` in another tab. + -h, --help Shows this help message. EOF exit @@ -17,9 +21,11 @@ unset COMPOSE_PROJECT_NAME # Determine which extra services should be loaded when working with authenticators ENABLE_AUTHN_LDAP=false +ENABLE_ROTATORS=false while true ; do case "$1" in --authn-ldap ) ENABLE_AUTHN_LDAP=true ; shift ;; + --rotators ) ENABLE_ROTATORS=true ; shift ;; -h | --help ) print_help ; shift ;; * ) if [ -z "$1" ]; then break; else echo "$1 is not a valid option"; exit 1; fi;; esac @@ -52,8 +58,22 @@ if [[ $ENABLE_AUTHN_LDAP = true ]]; then docker-compose exec conjur conjurctl policy load cucumber /src/conjur-server/dev/files/authn-ldap/policy.yml fi +if [[ $ENABLE_ROTATORS = true ]]; then + services="$services testdb cucumber" +fi + docker-compose up -d --no-deps $services api_key=$(docker-compose exec -T conjur conjurctl \ role retrieve-key cucumber:user:admin | tr -d '\r') -docker exec -e CONJUR_AUTHN_API_KEY=$api_key $env_args -it --detach-keys 'ctrl-\' $(docker-compose ps -q conjur) bash + +if [[ $ENABLE_ROTATORS = true ]]; then + container_name=cucumber +else + container_name=conjur +fi + +echo container_name $container_name + +docker exec -e CONJUR_AUTHN_API_KEY=$api_key $env_args \ + -it --detach-keys 'ctrl-\' "$(docker-compose ps -q "$container_name")" bash diff --git a/test/postgres_rotator/docker-compose.yml b/test/postgres_rotator/docker-compose.yml deleted file mode 100644 index b112bef63b..0000000000 --- a/test/postgres_rotator/docker-compose.yml +++ /dev/null @@ -1,55 +0,0 @@ -version: "3" -services: - pg: - image: postgres:9.3 - - testdb: - image: postgres:9.3 - - conjur: - build: - context: ../ - dockerfile: dev/Dockerfile.dev - image: conjur-dev - environment: - CONJUR_APPLIANCE_URL: http://localhost:3000 - DATABASE_URL: postgres://postgres@pg/postgres - CONJUR_ADMIN_PASSWORD: admin - CONJUR_ACCOUNT: cucumber - CONJUR_PASSWORD_ALICE: secret - CONJUR_DATA_KEY: - RAILS_ENV: - ports: - - "3000:3000" - expose: - - "3000" - volumes: - - ..:/src/conjur-server - - ../../conjur-policy-parser:/src/conjur-policy-parser - - authn-local:/run/authn-local - links: - - pg - - testdb - - ldap-server - - cucumber: - image: conjur-dev - entrypoint: bash - environment: - LDAP_URI: ldap://ldap-server:389 - LDAP_BASE: dc=conjur,dc=net - CONJUR_APPLIANCE_URL: http://conjur:3000 - DATABASE_URL: postgres://postgres@pg/postgres - CONJUR_ADMIN_PASSWORD: admin - CONJUR_DATA_KEY: - RAILS_ENV: - volumes: - - ..:/src/conjur-server - - authn-local:/run/authn-local - links: - - conjur - - pg - - test-db - -volumes: - authn-local: