Commit a9ff9489 authored by Nels Nelson's avatar Nels Nelson

Add repo size comparison check

parent 6a054b2a
......@@ -10,10 +10,12 @@
#
# Staging example:
#
# gitlab-rails runner /var/opt/gitlab/scripts/storage_rebalance.rb --verbose --dry-run=yes --current-file-server=nfs-file01 --target-file-server=nfs-file09 --staging --count
# gitlab-rails runner /var/opt/gitlab/scripts/storage_rebalance.rb --verbose --dry-run=yes --wait=10800 --current-file-server=nfs-file01 --target-file-server=nfs-file09 --staging --max-failures=200 --refresh-stats
#
# Production examples:
#
# gitlab-rails runner /var/opt/gitlab/scripts/storage_rebalance.rb --verbose --dry-run=yes --current-file-server=nfs-file34 --target-file-server=nfs-file40 --count
# gitlab-rails runner /var/opt/gitlab/scripts/storage_rebalance.rb --verbose --dry-run=yes --wait=10800 --current-file-server=nfs-file25 --target-file-server=nfs-file36
# gitlab-rails runner /var/opt/gitlab/scripts/storage_rebalance.rb --verbose --dry-run=no --wait=10800 --move-amount=10 --current-file-server=nfs-file27 --target-file-server=nfs-file38 --skip=9271929
#
......@@ -66,6 +68,7 @@ class NoCommits < StandardError; end
class MigrationTimeout < StandardError; end
class CommitsMismatch < StandardError; end
class ChecksumsMismatch < StandardError; end
class RepositorySizesMismatch < StandardError; end
Options = {
dry_run: true,
......@@ -77,9 +80,16 @@ Options = {
move_amount: 0,
timeout: 10,
max_failures: 3,
clauses: {
delete_error: nil,
pending_delete: false,
project_statistics: {commit_count: 1..Float::INFINITY},
mirror: false,
},
verify_only: false,
checksum: false,
list: false,
repository_size: false,
list_nodes: false,
black_list: [],
refresh_statistics: false,
stats: [:commit_count, :storage_size, :repository_size],
......@@ -150,6 +160,10 @@ def parse_args
Options[:checksum] = true
end
opt.on('-S', '--verify-size', 'Verify project repository size is constant post-migration') do |checksum|
Options[:repository_size] = true
end
opt.on('-f', '--max-failures=<N>', Integer, "Maximum failed migrations; default: #{Options[:max_failures]}") do |failures|
Options[:max_failures] = failures
end
......@@ -158,6 +172,10 @@ def parse_args
Options[:group] = group
end
opt.on('-M', '--include-mirrors', 'Include mirrors in the selection clauses') do |include_mirrors|
Options[:clauses].delete(:mirror)
end
opt.on('--staging', 'Use the staging environment') do |env|
Options[:env] = :staging
end
......@@ -328,6 +346,7 @@ class Rebalancer
project.repository.expire_exists_cache
original_checksum = project.repository.checksum if Options[:checksum]
original_repository_size = project.repository.statistics[:repository_size] if Options[:repository_size]
log_artifact = {
id: project.id,
......@@ -345,18 +364,19 @@ class Rebalancer
project.save
wait_for_repository_storage_update(project)
post_migration_project = Project.find_by(id: project.id)
if project.repository_storage != Options[:target_file_server]
if post_migration_project.repository_storage != Options[:target_file_server]
raise MigrationTimeout.new("Timed out waiting for migration of " +
"project id: #{project.id}")
"project id: #{post_migration_project.id}")
end
log.debug "Refreshing all statistics for project id: #{project.id}"
project.statistics.refresh!
log.debug "Refreshing all statistics for project id: #{post_migration_project.id}"
post_migration_project.statistics.refresh!
log.info "Validating project integrity by comparing latest commit " +
"identifers before and after"
current_commit_id = get_commit_id(project.id)
current_commit_id = get_commit_id(post_migration_project.id)
if original_commit_id != current_commit_id
raise CommitsMismatch.new("Current commit id #{current_commit_id} " +
"does not match original commit id #{original_commit_id}")
......@@ -365,18 +385,28 @@ class Rebalancer
if Options[:checksum]
log.info "Validating project integrity by comparing checksums " +
"before and after"
project.repository.expire_exists_cache
current_checksum = project.repository.checksum
post_migration_project.repository.expire_exists_cache
current_checksum = post_migration_project.repository.checksum
if original_checksum != current_checksum
raise ChecksumsMismatch.new("Current checksum #{current_checksum} " +
"does not match original checksum #{original_checksum}")
end
end
log.info "Migrated project id: #{project.id}"
log.debug " Name: #{project.name}"
log.debug " Storage: #{project.repository_storage}"
log.debug " Path: #{project.disk_path}"
if Options[:repository_size]
log.info "Validating project integrity by comparing repository size " +
"before and after"
current_repository_size = post_migration_project.repository.statistics[:repository_size]
if original_repository_size != current_repository_size
raise RepositorySizesMismatch.new("Current repository size #{current_checksum} " +
"does not match original checksum #{original_checksum}")
end
end
log.info "Migrated project id: #{post_migration_project.id}"
log.debug " Name: #{post_migration_project.name}"
log.debug " Storage: #{post_migration_project.repository_storage}"
log.debug " Path: #{post_migration_project.disk_path}"
log_artifact[:date] = DateTime.now.iso8601(ISO8601_FRACTIONAL_SECONDS_LENGTH)
@migration_log.info log_artifact.to_json
end
......@@ -393,20 +423,13 @@ class Rebalancer
end
Project.transaction do
ActiveRecord::Base.connection.execute 'SET statement_timeout = 600000'
clauses = {
repository_storage: current_file_server,
delete_error: nil,
pending_delete: false,
project_statistics: {commit_count: 1..Float::INFINITY},
}
clauses = Options[:clauses].dup
clauses.merge!(repository_storage: current_file_server)
if namespace_id
log.info "Filtering projects by group: #{group}"
clauses.merge!(namespace_id: namespace_id)
end
Project
.joins(:statistics)
.where(**clauses)
.size
Project.joins(:statistics).where(**clauses).size
end
end
......@@ -429,21 +452,16 @@ class Rebalancer
project_identifiers = []
Project.transaction do
ActiveRecord::Base.connection.execute 'SET statement_timeout = 600000'
clauses = {
repository_storage: current_file_server,
delete_error: nil,
pending_delete: false,
project_statistics: {commit_count: 1..Float::INFINITY},
}
clauses = Options[:clauses].dup
clauses.merge!(repository_storage: current_file_server)
if namespace_id
log.info "Filtering projects by group: #{group}"
clauses.merge!(namespace_id: namespace_id)
end
query = Project
.joins(:statistics)
.where(**clauses)
.order('project_statistics.repository_size DESC')
.order('last_activity_at ASC')
query = Project.joins(:statistics).where(**clauses).
order('project_statistics.repository_size DESC').
order('last_activity_at ASC')
query = query.limit(limit) if limit > 0
project_identifiers = query.pluck(:id)
end
......@@ -476,6 +494,11 @@ class Rebalancer
log.error "Failed to validate integrity of project id: #{project.id}"
log.error "Error: #{e}"
log.warn "Skipping migration"
rescue RepositorySizesMismatch => e
migration_errors << { project_id: project_id, message: e.message }
log.error "Failed to validate integrity of project id: #{project.id}"
log.error "Error: #{e}"
log.warn "Skipping migration"
rescue MigrationTimeout => e
migration_errors << { project_id: project_id, message: e.message }
log.error "Timed out migrating project id: #{project.id}"
......@@ -520,6 +543,21 @@ class Rebalancer
log.error "Failed to validate integrity of project id: #{project.id}"
log.error "Error: #{e}"
log.warn "Skipping migration"
rescue ChecksumsMismatch => e
migration_errors << { project_id: project_id, message: e.message }
log.error "Failed to validate integrity of project id: #{project.id}"
log.error "Error: #{e}"
log.warn "Skipping migration"
rescue RepositorySizesMismatch => e
migration_errors << { project_id: project_id, message: e.message }
log.error "Failed to validate integrity of project id: #{project.id}"
log.error "Error: #{e}"
log.warn "Skipping migration"
rescue MigrationTimeout => e
migration_errors << { project_id: project_id, message: e.message }
log.error "Timed out migrating project id: #{project.id}"
log.error "Error: #{e}"
log.warn "Skipping migration"
rescue StandardError => e
migration_errors << { project_id: project_id, message: e.message }
log.error "Unexpected error migrating project id: #{project.id}: #{e}"
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment