gitlab高可用部署方案
- 两台 Gitlab Rails Server 作为 Praefect nodes 同时在线
- NFS 共享储存
- Redis 共享
- PortgreSQL 共享DB
- OpenLDAP 提供统一认证
- 官方推荐多节点部署参考地址
推荐配置
- 预计支持 2000 左右用户
- centos 7.8.2003
- gitlab 13.1.4
- postgresql 11
- nfs v4
- redis 5.0.9
- ldap 2.4.44
| Server Name | CPU核心 | 内存 | SSD 磁盘大小 | 节点数 |
|---|---|---|---|---|
| Gitlab Rails Server | 16 vCPU | 32G | >= 200G | 2 |
| NFS Server | 4 vCPU | 8G | >= 1T | 1 |
| PostGreSQL Server | 4 vCPU | 8G | >= 300G | 1 |
| Redis Server | 2 vCPU | 4G | >= 200G | 1 |
| OpenLDAP Server | 2 vCPU | 4G | >= 100G | 1 |
Gitlab服务构成
|—|—| |gitlab-shell|用于处理Git命令和修改authorized keys列表| |gitlab-workhorse|轻量级的反向代理服务器| |logrotate|日志文件管理工具| |sidekiq|用于在后台执行队列任务（异步执行）| |postgresql|数据库| |redis|缓存数据库| |Nginx|前端web服务器| |NFS|文件共享存储|
初始化服务器
- IP地址规划
- 172.16.11.30 gitlab-1
- 172.16.11.31 gitlab-2
- 172.16.11.32 gitlab-db (nfs, postgresql, redis)
- 172.16.11.33 gitlab-nginx (OpenLDAP)
- disable firewalld selinux
- 配置yum源，安装初始软件
- /etc/yum.repos.d/gitlab-ce.repo
- /etc/yum.repos.d/epel.repo
- /etc/yum.repos.d/CentOS-Base.repo
配置ldap的yum源
# yum install https://mirrors.tuna.tsinghua.edu.cn/remi/enterprise/remi-release-7.rpm
部署 Rails Server 1
安装初始化软件
# yum install curl policycoreutils-python openssh-server openssh-clients postfix
# sudo systemctl enable sshd
# sudo systemctl start sshd
# sudo systemctl enable postfix
# sudo systemctl start postfix
安装gitlab-ce
下载软件包 https://mirrors.tuna.tsinghua.edu.cn/gitlab-ce/yum/el7/
# yum localinstall -y gitlab-ce-13.1.4-ce.0.el7.x86_64.rpm
添加 /etc/gitlab/gitlab.rb 配置文件
- cp /etc/gitlab/gitlab.rb{,.bak}
- reset gitlab.rb 文件内容如下
[root@RJYF-GRS-01 ~]# egrep -v ^"$|#" /etc/gitlab/gitlab.rb
external_url 'https://git.cloud.com'
roles ['application_role']
gitlab_rails['time_zone'] = 'Asia/Shanghai'
gitlab_rails['gitlab_default_can_create_group'] = false
gitlab_rails['gitlab_username_changing_enabled'] = false
gitlab_rails['ldap_enabled'] = true
gitlab_rails['prevent_ldap_sign_in'] = false
gitlab_rails['ldap_servers'] = YAML.load <<-'EOS'
main:
label: 'CLOUD-AD'
host: 'CLOUD-AD01.CLOUD.local'
port: 389
uid: 'sAMAccountName'
bind_dn: 'CN=rjyf_svc,ou=CLOUD-User,OU=VMware-pool,DC=cloud,DC=local'
password: 'CoDev$!2020'
encryption: 'plain'
verify_certificates: true
smartcard_auth: false
active_directory: true
allow_username_or_email_login: true
lowercase_usernames: false
block_auto_created_users: false
base: 'OU=集团,DC=cloud,DC=local'
user_filter: ''
attributes:
username: ['cn']
email: ['mail']
name: 'displayName'
first_name: 'name'
last_name: 'sn'
EOS
gitlab_rails['manage_backup_path'] = true
gitlab_rails['backup_path'] = "/var/opt/gitlab/gitlab_backup_data/backups"
gitlab_rails['backup_archive_permissions'] = 0644
gitlab_rails['backup_keep_time'] = 172800
gitlab_rails['shared_path'] = '/var/opt/gitlab/gitlab-rails/shared'
high_availability['mountpoint'] = ["/var/opt/gitlab/git-data", "/var/opt/gitlab/gitlab-rails/shared"]
gitlab_rails['uploads_directory'] = "/var/opt/gitlab/gitlab-rails/uploads"
gitlab_rails['db_adapter'] = "postgresql"
gitlab_rails['db_encoding'] = "unicode"
gitlab_rails['db_database'] = "gitlabhq_production"
gitlab_rails['db_username'] = "gitlab"
gitlab_rails['db_password'] = "git_lab_p2d"
gitlab_rails['db_host'] = "172.20.60.10"
gitlab_rails['db_port'] = 5432
gitlab_rails['redis_host'] = "172.20.60.11"
gitlab_rails['redis_port'] = 6379
puma['enable'] = true
puma['worker_timeout'] = 600
puma['worker_processes'] = 4
puma['min_threads'] = 8
puma['max_threads'] = 8
puma['per_worker_max_memory_mb'] = 850
gitlab_rails['env'] = {
'GITLAB_RAILS_RACK_TIMEOUT' => 1200
}
gitlab_shell['auth_file'] = "/var/opt/gitlab/.ssh/authorized_keys"
postgresql['enable'] = false
redis['enable'] = false
nginx['enable'] = true
nginx['redirect_http_to_https'] = true
nginx['ssl_certificate'] = "/etc/gitlab/ssl/_.cloud.com.ssl/STAR_cloud_com_integrated.crt"
nginx['ssl_certificate_key'] = "/etc/gitlab/ssl/_.cloud.com.ssl/STAR_cloud_com.key"
nginx['listen_addresses'] = ['*', '[::]']
gitlab_ci['builds_directory'] = '/var/opt/gitlab/gitlab-ci/builds'
[root@RJYF-GRS-01 ~]#
- 微软 AD 认证。不能首次登陆，需要修改密码。
- 前期测试使用OpenLDAP，等微软AD部署好了，切换到AD。
- 部署 OpenLDAP Server 参考地址：https://www.openldap.org/doc/
- 证书的⽂件的位置 /etc/gitlab/ssl/server.crt /etc/gitlab/ssl/server-key.key
启动gitlab服务
注意：
在启动gitlab服务之前，需要完成NFS、PostgresqlDB、Redis的服务器部署。
# touch/etc/gitlab/skip-auto-reconfigure
# SKIP_POST_DEPLOYMENT_MIGRATIONS=truegitlab-ctlreconfigure
# gitlab-rakegitlab:env:info
# gitlab-rakegitlab:env:check
# gitlab-rakegitlab:ldap:check
# gitlab-ctlstart
部署 PostgreSQL Server
添加 pgsql yum 源
/etc/yum.repos.d/pgdg-redhat-all.repo
# wget https://download.postgresql.org/pub/repos/yum/reporpms/EL-7-x86_64/pgdgredhat-repo-latest.noarch.rpm
# yum install -y pgdg-redhat-repo-latest.noarch.rpm
# ls -l /etc/yum.repos.d/pgdg-redhat-all.repo
安装PostgreSQL 11
# yum install -y postgresql11-contrib postgresql11-server
# systemctl enable postgresql-11
# systemctl start postgresql-11
初始化数据库
/usr/pgsql-11/bin/postgresql-11-setup initdb
创建用户
# sudo -u postgres createuser -P -s -e gitlab
连续两次输入 nil 作为密码
配置文件目录
# ls -l /var/lib/pgsql/11/data
添加授权
# vim /var/lib/pgsql/11/data/pg_hba.conf
在最后一行添加授权策略
host all all 0.0.0.0/0 md5
修改监听地址
# vim /var/lib/pgsql/11/data/postgresql.conf
listen_addresses = '*'
重启pgsql 服务
# systemctl restart postgresql-11
部署 NFS Server
安装软件
# yum install -y nfs-utils
创建目录
# mkdir -pv /data/nfs; chmod 777 /data/nfs
添加配置文件
# vim /etc/exports
/data/nfs 172.16.11.0/24(rw,sync,no_root_squash)
生效配置文件
# exportfs -arv
# showmount -e
# systemctl restart nfs
# systemctl enable nfs
在nfs上创建目录
# mkdir -pv /data/nfs/gitlab/.ssh
# mkdir -pv /data/nfs/gitlab/gitlab-rails/uploads
# mkdir -pv /data/nfs/gitlab/gitlab-rails/shared
# mkdir -pv /data/nfs/gitlab/gitlab-ci/builds
# mkdir -pv /data/nfs/gitlab/git-data
在 gitlab-1 和 gitlab-2 上创建挂载目录
# mkdir -pv /var/opt/gitlab/.ssh
# mkdir -pv /var/opt/gitlab/gitlab-rails/uploads
# mkdir -pv /var/opt/gitlab/gitlab-rails/shared
# mkdir -pv /var/opt/gitlab/gitlab-ci/builds
# mkdir -pv /var/opt/gitlab/git-data
nfs 挂载
在 gitlab-1 和 gitlab-2 上的 /etc/fstab 中添加 nfs 配置如下行
10.1.0.1:/var/opt/gitlab/.ssh /var/opt/gitlab/.ssh nfs4 defaults,vers=4.1,hard,rsize=1048576,wsize=1048576,noatime,nofail,lookupcache=positive 0 2
10.1.0.1:/var/opt/gitlab/gitlab-rails/uploads /var/opt/gitlab/gitlab-rails/uploads nfs4 defaults,vers=4.1,hard,rsize=1048576,wsize=1048576,noatime,nofail,lookupcache=positive 0 2
10.1.0.1:/var/opt/gitlab/gitlab-rails/shared /var/opt/gitlab/gitlab-rails/shared nfs4 defaults,vers=4.1,hard,rsize=1048576,wsize=1048576,noatime,nofail,lookupcache=positive 0 2
10.1.0.1:/var/opt/gitlab/gitlab-ci/builds /var/opt/gitlab/gitlab-ci/builds nfs4 defaults,vers=4.1,hard,rsize=1048576,wsize=1048576,noatime,nofail,lookupcache=positive 0 2
10.1.0.1:/var/opt/gitlab/git-data /var/opt/gitlab/git-data nfs4 defaults,vers=4.1,hard,rsize=1048576,wsize=1048576,noatime,nofail,lookupcache=positive 0 2
mount -a
部署 Redis Server
下载 redis 源码包
wget http://download.redis.io/releases/redis-6.0.6.tar.gz
安装redis
解决环境依赖
# yum -y install gcc gcc-c++ kernel-devel
升级gcc
# yum -y install centos-release-scl
# yum -y install devtoolset-9-gcc devtoolset-9-gcc-c++ devtoolset-9-binutils
永久⽣效
# echo "source /opt/rh/devtoolset-9/enable" >> /etc/profile
临时⽣效
# scl enable devtoolset-9 bash
验证版本
# gcc --version
解压源码⽂件
# tar -xf redis-6.0.6.tar.gz -C /usr/local
制作软连并部署⼆进制⽂件
# ln -sv redis-6.0.6/ redis
# cp /usr/local/redis/src/redis-server /usr/local/bin/
# cp /usr/local/redis/src/redis-cli /usr/local/bin/
创建 /etc/redis/redis.conf 配置⽂件
# mkdir /etc/redis
# cp /usr/local/redis/redis.conf /etc/redis/
创建 rdb ⽂件夹
# mkdir /var/lib/redis/
设置 /etc/redis/redis.conf ⽂件
bind 0.0.0.0
daemonize yes
dbfilename gitlab-dump.rdb
dir /var/lib/redis/
创建开机启动脚本
# vim /usr/lib/systemd/system/redis.service
[Unit]
Description=Redis Server
After=network.target
After=network-online.target
Wants=network-online.target
[Service]
Type=forking
ExecStart=/usr/local/bin/redis-server /etc/redis/redis.conf
ExecStop=/usr/local/bin/redis-cli -h 127.0.0.1 -p 6379 shutdown
[Install]
WantedBy=multi-user.target
设置开机启动
# systemctl enable redis
# systemc start redis
测试redis
[root@RJYF-RS-01 ~]# redis-cli
127.0.0.1:6379> info
...
查看redis info信息
部署 Rails server 2
安装软件
# yum install gitlab-ce-13.1.4-ce.0.el7.x86_64.rpm
从gitlab-1 server上copy配置文件
# scp /etc/gitlab/gitlab.rb gitlab-2:/etc/gitlab/
# scp /etc/gitlab/gitlab-secrets.json gitlab-2:/etc/gitlab/
第二台文件示例
[root@RJYF-GRS-02 ~]# egrep -v ^"$|#" /etc/gitlab/gitlab.rb
external_url 'https://git.cloud.com'
roles ['application_role']
gitlab_rails['time_zone'] = 'Asia/Shanghai'
gitlab_rails['gitlab_default_can_create_group'] = false
gitlab_rails['gitlab_username_changing_enabled'] = false
gitlab_rails['ldap_enabled'] = true
gitlab_rails['prevent_ldap_sign_in'] = false
gitlab_rails['ldap_servers'] = YAML.load <<-'EOS'
main:
label: 'CLOUD-AD'
host: 'CLOUD-AD01.CLOUD.local'
port: 389
uid: 'sAMAccountName'
bind_dn: 'CN=rjyf_svc,ou=CLOUD-User,OU=VMware-pool,DC=cloud,DC=local'
password: 'CoDev$!2020'
encryption: 'plain'
verify_certificates: true
smartcard_auth: false
active_directory: true
allow_username_or_email_login: true
lowercase_usernames: false
block_auto_created_users: false
base: 'OU=集团,DC=cloud,DC=local'
user_filter: ''
attributes:
username: ['cn']
email: ['mail']
name: 'displayName'
first_name: 'name'
last_name: 'sn'
EOS
gitlab_rails['manage_backup_path'] = true
gitlab_rails['backup_path'] = "/var/opt/gitlab/gitlab_backup_data/backups"
gitlab_rails['backup_archive_permissions'] = 0644
gitlab_rails['backup_keep_time'] = 172800
gitlab_rails['shared_path'] = '/var/opt/gitlab/gitlab-rails/shared'
high_availability['mountpoint'] = ["/var/opt/gitlab/git-data", "/var/opt/gitlab/gitlab-rails/shared"]
gitlab_rails['uploads_directory'] = "/var/opt/gitlab/gitlab-rails/uploads"
gitlab_rails['db_adapter'] = "postgresql"
gitlab_rails['db_encoding'] = "unicode"
gitlab_rails['db_database'] = "gitlabhq_production"
gitlab_rails['db_username'] = "gitlab"
gitlab_rails['db_password'] = "git_lab_p2d"
gitlab_rails['db_host'] = "172.20.60.10"
gitlab_rails['db_port'] = 5432
gitlab_rails['redis_host'] = "172.20.60.11"
gitlab_rails['redis_port'] = 6379
puma['enable'] = true
puma['worker_timeout'] = 600
puma['worker_processes'] = 4
puma['min_threads'] = 8
puma['max_threads'] = 8
puma['per_worker_max_memory_mb'] = 850
gitlab_rails['env'] = {
'GITLAB_RAILS_RACK_TIMEOUT' => 1200
}
gitlab_shell['auth_file'] = "/var/opt/gitlab/.ssh/authorized_keys"
postgresql['enable'] = false
redis['enable'] = false
nginx['enable'] = true
nginx['redirect_http_to_https'] = true
nginx['ssl_certificate'] = "/etc/gitlab/ssl/_.cloud.com.ssl/STAR_cloud_com_integrated.crt"
nginx['ssl_certificate_key'] = "/etc/gitlab/ssl/_.cloud.com.ssl/STAR_cloud_com.key"
nginx['listen_addresses'] = ['*', '[::]']
gitlab_ci['builds_directory'] = '/var/opt/gitlab/gitlab-ci/builds'
[root@RJYF-GRS-02 ~]#
启动 gitlab 服务
# touch /etc/gitlab/skip-auto-reconfigure
# SKIP_POST_DEPLOYMENT_MIGRATIONS=true gitlab-ctl reconfigure
# gitlab-ctl start
至此两台master、redis、nfs、postgresql已经部署完成。
配置前端代理使用haproxy
https://docs.gitlab.com/ee/administration/high_availability/load_balancer.html
# yum install haproxy
配置相同的sshd 秘钥
# scp /etc/ssh/ssh_host_* gitlab-2:/etc/ssh/
haproxy 配置文件
[root@RJYF-HP-01 ~]# cat /etc/haproxy/haproxy.cfg
global
log 127.0.0.1 local2
chroot /var/lib/haproxy
pidfile /var/run/haproxy.pid
maxconn 4000
user haproxy
group haproxy
daemon
stats socket /var/lib/haproxy/stats
defaults
mode tcp
log global
option httplog
option dontlognull
option redispatch
retries 3
timeout http-request 10s
timeout queue 1m
timeout connect 10s
timeout client 1m
timeout server 1m
timeout http-keep-alive 10s
timeout check 10s
maxconn 4000
# enable gitlab 80
frontend gitlab_http *:80
option tcplog
default_backend backend_gitlab_http
backend backend_gitlab_http
balance source
option tcp-check
option httpchk GET /-/readiness?all=1
http-check expect status 200
default-server inter 5s fall 3 rise 2
server backend_gitlab_http_server1 172.20.60.7:80 check port 80
server backend_gitlab_http_server2 172.20.60.8:80 check port 80
# enable gitlab 443
frontend gitlab_https *:443
option tcplog
default_backend backend_gitlab_https
backend backend_gitlab_https
balance source
option httpchk GET /-/readiness?all=1
http-check expect status 200
default-server inter 5s fall 3 rise 2
server backend_gitlab_https_server1 172.20.60.7:443 check port 80
server backend_gitlab_https_server2 172.20.60.8:443 check port 80
# enable gitlab 22
frontend gitlab_ssh *:22
option tcplog
default_backend backend_gitlab_ssh
backend backend_gitlab_ssh
balance source
option tcp-check
option httpchk GET /-/readiness?all=1
http-check expect status 200
default-server inter 5s fall 3 rise 2
server backend_gitlab_ssh_server1 172.20.60.7:22 check port 80
server backend_gitlab_ssh_server2 172.20.60.8:22 check port 80
[root@RJYF-HP-01 ~]#
信任自签名证书
# git config --global http.sslVerify false