Merge branch '7353-decomposed-db-ha-poc' into 'master'

Patroni and PgBouncer support for decomposed Rails database

Closes #7353

See merge request https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/7379

Merged-by: Balasankar 'Balu' C <balasankar@gitlab.com>
Approved-by: Andrew Patterson <apatterson@gitlab.com>
Approved-by: Balasankar 'Balu' C <balasankar@gitlab.com>
Co-authored-by: Hossein Pursultani <hpursultani@gitlab.com>

Author: balasankarc

files/gitlab-cookbooks/gitlab/libraries/helpers/gitlab_rails.rb

@@ -8,7 +8,11 @@ def public_attributes
       'gitlab' => {
         'gitlab_rails' => node['gitlab']['gitlab_rails'].select do |key, value|
           %w(db_database).include?(key)
-        end
+        end.merge(
+          'databases' => node['gitlab']['gitlab_rails']['databases'].transform_values do |value|
+            value['db_database']
+          end
+        )
       }
     }
   end

files/gitlab-cookbooks/pgbouncer/recipes/user.rb

@@ -33,12 +33,18 @@
 end
 
 if pgb_helper.create_pgbouncer_user?('postgresql') || pgb_helper.create_pgbouncer_user?('patroni')
-  pgbouncer_user 'rails' do
-    helper lazy { PgHelper.new(node) }
-    user node['postgresql']['pgbouncer_user']
-    password node['postgresql']['pgbouncer_user_password']
-    database node['gitlab']['gitlab_rails']['db_database']
-    add_auth_function default_auth_query.eql?(auth_query)
-    action :create
+  ignored_databases = %w[geo]
+  database_host = node['gitlab']['gitlab_rails']['db_host']
+  databases = node['gitlab']['gitlab_rails']['databases'].select { |db, details| details['enable'] && details['db_host'] == database_host && !ignored_databases.include?(db) }
+
+  databases.each do |db, settings|
+    pgbouncer_user "rails:#{db}" do
+      helper lazy { PgHelper.new(node) }
+      user node['postgresql']['pgbouncer_user']
+      password node['postgresql']['pgbouncer_user_password']
+      database settings['db_database']
+      add_auth_function default_auth_query.eql?(auth_query)
+      action :create
+    end
   end
 end

files/gitlab-cookbooks/pgbouncer/resources/pgbouncer_user.rb

@@ -16,16 +16,15 @@
     helper new_resource.helper
     password "md5#{new_resource.password}"
     action :create
-    notifies :run, "execute[Add pgbouncer auth function]", :immediately
   end
 
   pgbouncer_auth_function = new_resource.helper.pg_shadow_lookup
 
   execute 'Add pgbouncer auth function' do
     command %(/opt/gitlab/bin/#{new_resource.helper.service_cmd} -d #{new_resource.database} -c '#{pgbouncer_auth_function}')
     user new_resource.account_helper.postgresql_user
-    not_if { new_resource.helper.has_function?(new_resource.database, "pg_shadow_lookup") }
-    only_if { new_resource.add_auth_function }
-    action :nothing
+    only_if { new_resource.add_auth_function && new_resource.helper.is_running? && new_resource.helper.is_ready? }
+    not_if { new_resource.helper.is_offline_or_readonly? || new_resource.helper.has_function?(new_resource.database, "pg_shadow_lookup") }
+    action :run
   end
 end

files/gitlab-ctl-commands-ee/lib/pgbouncer.rb

@@ -40,13 +40,12 @@ def initialize(options, install_path, base_data_path)
     end
 
     def update_databases(original = {})
+      rails_databases = attributes.dig('gitlab', 'gitlab_rails', 'databases')&.values
+      rails_databases = rails_databases ? rails_databases.uniq : [DEFAULT_RAILS_DATABASE]
+
       updated = {}
 
-      if original.empty?
-        original = {
-          database => {}
-        }
-      end
+      original = rails_databases.to_h { |db| [db, {}] } if original.empty?
 
       original.each do |db, settings|
         settings.delete('password')
@@ -55,11 +54,18 @@ def update_databases(original = {})
 
         settings['auth_user'] = settings.delete('user') if settings.key?('user')
 
-        if db == @database
+        is_rails_db = rails_databases.include?(db)
+
+        if db == @database || is_rails_db
           settings['host'] = options['newhost'] if options['newhost']
           settings['port'] = options['port'] if options['port']
           settings['auth_user'] = options['user'] if options['user']
-          settings['dbname'] = options['pg_database'] if options['pg_database']
+
+          if is_rails_db
+            settings['dbname'] = db
+          elsif options['pg_database']
+            settings['dbname'] = options['pg_database']
+          end
         end
 
         settings.each do |setting, value|

spec/chef/cookbooks/gitlab-ee/resources/pgbouncer_user_spec.rb

@@ -14,14 +14,10 @@
 
     it 'should create the pg_shadow_lookup function' do
       postgresql_user = chef_run.postgresql_user('pgbouncer-geo')
-      expect(postgresql_user).to notify('execute[Add pgbouncer auth function]')
+      expect(postgresql_user.username).to eq('pgbouncer-geo')
       resource = chef_run.execute('Add pgbouncer auth function')
       expect(resource.command).to match(%r{^/opt/gitlab/bin/gitlab-geo-psql -d fakedb-geo})
     end
-
-    it 'should not try and recreate the function' do
-      expect(chef_run).not_to run_execute('Add pgbouncer auth function')
-    end
   end
 
   context 'create rails' do
@@ -36,13 +32,9 @@
 
     it 'should create the pg_shadow_lookup function' do
       postgresql_user = chef_run.postgresql_user('pgbouncer-rails')
-      expect(postgresql_user).to notify('execute[Add pgbouncer auth function]')
+      expect(postgresql_user.username).to eq('pgbouncer-rails')
       resource = chef_run.execute('Add pgbouncer auth function')
       expect(resource.command).to match(%r{^/opt/gitlab/bin/gitlab-psql -d fakedb-rails})
     end
-
-    it 'should not try and recreate the function' do
-      expect(chef_run).not_to run_execute('Add pgbouncer auth function')
-    end
   end
 end

spec/chef/cookbooks/patroni/recipes/patroni_spec.rb

@@ -236,7 +236,8 @@
       expect(chef_run).not_to run_execute('/opt/gitlab/embedded/bin/initdb -D /var/opt/gitlab/postgresql/data -E UTF8')
       expect(chef_run).to create_postgresql_user('gitlab')
       expect(chef_run).to create_postgresql_user('gitlab_replicator')
-      expect(chef_run).to create_pgbouncer_user('rails')
+      expect(chef_run).to create_pgbouncer_user('rails:ci')
+      expect(chef_run).to create_pgbouncer_user('rails:main')
       expect(chef_run).to create_postgresql_database('gitlabhq_production')
       expect(chef_run).to enable_postgresql_extension('pg_trgm')
       expect(chef_run).to enable_postgresql_extension('btree_gist')

spec/chef/cookbooks/pgbouncer/recipes/pgbouncer_spec.rb

@@ -433,7 +433,7 @@
 
   it 'should create the pgbouncer user on the database' do
     expect(chef_run).to include_recipe('pgbouncer::user')
-    expect(chef_run).to create_pgbouncer_user('rails').with(
+    expect(chef_run).to create_pgbouncer_user('rails:main').with(
       password: 'fakeuserpassword'
     )
   end

spec/chef/cookbooks/pgbouncer/recipes/pgbouncer_user_spec.rb

@@ -37,7 +37,7 @@
     end
 
     it 'should call pgbouncer_user with the correct values for rails' do
-      expect(chef_run).to create_pgbouncer_user('rails').with(
+      expect(chef_run).to create_pgbouncer_user('rails:main').with(
         database: 'gitlabhq_production',
         password: 'fakepassword',
         user: 'pgbouncer-rails',
@@ -64,7 +64,10 @@
     end
 
     it 'should not create the pg_shadow_lookup function' do
-      expect(chef_run).to create_pgbouncer_user('rails').with(
+      expect(chef_run).to create_pgbouncer_user('rails:main').with(
+        add_auth_function: false
+      )
+      expect(chef_run).to create_pgbouncer_user('rails:ci').with(
         add_auth_function: false
       )
     end
@@ -85,7 +88,8 @@
     end
 
     it 'should not create the pgbouncer user' do
-      expect(chef_run).not_to create_pgbouncer_user('rails')
+      expect(chef_run).not_to create_pgbouncer_user('rails:main')
+      expect(chef_run).not_to create_pgbouncer_user('rails:ci')
       expect(chef_run).not_to create_pgbouncer_user('geo')
     end
   end
@@ -112,7 +116,8 @@
         user: 'pgbouncer-geo',
         add_auth_function: true
       )
-      expect(chef_run).not_to create_pgbouncer_user('rails')
+      expect(chef_run).not_to create_pgbouncer_user('rails:main')
+      expect(chef_run).not_to create_pgbouncer_user('rails:ci')
     end
   end
 end

spec/chef/gitlab-ctl-commands-ee/lib/pgbouncer_spec.rb

@@ -8,8 +8,12 @@
   let(:fake_ohai) do
     {
       'gitlab' => {
-        'gitlab-rails' => {
-          'db_database' => 'fake_database'
+        'gitlab_rails' => {
+          'db_database' => 'fake_database',
+          'databases' => {
+            'main' => 'fake_database',
+            'ci' => 'fake_database_ci'
+          },
         },
       },
       'pgbouncer' => {
@@ -94,7 +98,7 @@
 
     it 'should generate an databases.ini with the default rails database' do
       expect(@obj.render).to eq(
-        "[databases]\n\nfake_database = \n\n"
+        "[databases]\n\nfake_database = dbname=fake_database\n\nfake_database_ci = dbname=fake_database_ci\n\n"
       )
     end
   end