Harbor ist ja eine sehr beliebte und deswegen auch oft eingesetzte Docker-Registry zum selber hosten.
Im Grunde scheint es zwei Wege für das Deployment von Harbor zu geben:
- Harbor per Docker-Compose in einer VM ausrollen
- Harbor per Helm-Chart in Kubernetes deployen
Egal welchen Weg man wählt: Wenn man das Setup produktionsreif machen möchte, muß man sich in beiden Fällen um
- ein ausfallsicheres Postgres Setup
- ein ausfallsicheres Redis Setup
- und ein hochverfügbares Storage für die Registry kümmern.
Denn auch beim Deployment in Kubernetes übernimmt das Harbor Helm-Chart diese Aufgaben nicht.
Um dem Fernziel, Harbor mal in Nomad ausrollen zu können, näher zu kommen, habe ich erstmal den Weg mit Docker-Compose gewählt, da ich ja schon über ein ausfallsicheres Redis Setup (auf Basis des Redis Cluster Creators) und über ein schönes Postgres Patroni Cluster (auf Basis der Terraform Konfiguration von John Terrell) verfüge. Als Storage habe ich S3 von Plusserver gewählt.
Da ich - schon allein für das Patroni Cluster - auch schon ein Consul Cluster habe, hat auch die designierte Harbor-VM eine Consul-Client Konfiguration erhalten:
root@regha:/data/harbor# cat /etc/consul/consul.hcl
datacenter = "de-gt-2"
data_dir = "/var/lib/consul"
node_name = "regha"
ports {
grpc = 8502
}
connect {
enabled = true
}
recursors = [ "8.8.8.8" ]
server = false
leave_on_terminate = true
bind_addr = "{{ GetInterfaceIP \"net1\" }}"
Der Start von consul läuft per systemd:
root@regha:/data/harbor# cat /lib/systemd/system/consul.service
[Unit]
Description=Consul agent
Documentation=https://www.consul.io/docs/
After=network-online.target
Wants=network-online.target
[Service]
EnvironmentFile=-/etc/default/%p
ExecStart=/usr/bin/consul agent -data-dir=/var/lib/consul -config-dir=/etc/consul -retry-join consul.svc.a5ba23f5-8237-6879-e7e1-ea6f574fbde9.de-gt-2.snc.tgos.xyz -retry-interval 10s -rejoin
ExecReload=/bin/kill -HUP $MAINPID
Restart=on-failure
User=consul
KillSignal=SIGINT
#AmbientCapabilities=CAP_NET_BIND_SERVICE
NoNewPrivileges=true
PrivateDevices=true
PrivateTmp=true
ProtectSystem=full
ProtectKernelTunables=true
[Install]
WantedBy=multi-user.target
Durch den vom Triton-CNS bereitgestellten DNS-Record, kann der Consul-Client den Cluster sofort "joinen".
Für die eigentliche Harbor-Installation müssen die Zugänge zu Storage und Redis sowie die Postgres-Datenbanken schon angelegt sein. Meine harbor.yml sah damit so aus:
# Configuration file of Harbor
# The IP address or hostname to access admin UI and registry service.
# DO NOT use localhost or 127.0.0.1, because Harbor needs to be accessed by external clients.
hostname: registry.code667.net
# http related config
http:
# port for http, default is 80. If https enabled, this port will redirect to https port
port: 80
# https related config
https:
# https port for harbor, default is 443
port: 443
# The path of cert and key files for nginx
certificate: /mnt/letsencrypt/certs/registry.code667.net/fullchain.pem
private_key: /mnt/letsencrypt/certs/registry.code667.net/privkey.pem
# Uncomment external_url if you want to enable external proxy
# And when it enabled the hostname will no longer used
# external_url: https://reg.mydomain.com:8433
# The initial password of Harbor admin
# It only works in first time to install harbor
# Remember Change the admin password from UI after launching Harbor.
harbor_admin_password: geheim
# Harbor DB configuration
#database:
# The password for the root user of Harbor DB. Change this before any production use.
# password: root123
# The maximum number of connections in the idle connection pool. If it <=0, no idle connections are retained.
# max_idle_conns: 50
# The maximum number of open connections to the database. If it <= 0, then there is no limit on the number of open connections.
# Note: the default number of connections is 100 for postgres.
# max_open_conns: 100
# The default data volume
data_volume: /data/harbor
# Harbor Storage settings by default is using /data dir on local filesystem
# Uncomment storage_service setting If you want to using external storage
storage_service:
# # ca_bundle is the path to the custom root ca certificate, which will be injected into the truststore
# # of registry's and chart repository's containers. This is usually needed when the user hosts a internal storage with self signed certificate.
#ca_bundle:
# # storage backend, default is filesystem, options include filesystem, azure, gcs, s3, swift and oss
# # for more info about this configuration please refer https://docs.docker.com/registry/configuration/
s3:
accesskey: geheimer-access-key
secretkey: geheimer-secret-key
region: us-west-1
regionendpoint: de-2.s3.psmanaged.com
bucket: harbor
secure: true
# # set disable to true when you want to disable registry redirect
# redirect:
# disabled: false
# Clair configuration
clair:
# The interval of clair updaters, the unit is hour, set to 0 to disable the updaters.
updaters_interval: 12
jobservice:
# Maximum number of job workers in job service
max_job_workers: 10
notification:
# Maximum retry count for webhook job
webhook_job_max_retry: 10
chart:
# Change the value of absolute_url to enabled can enable absolute url in chart
absolute_url: disabled
# Log configurations
log:
# options are debug, info, warning, error, fatal
level: info
# configs for logs in local storage
local:
# Log files are rotated log_rotate_count times before being removed. If count is 0, old versions are removed rather than rotated.
rotate_count: 50
# Log files are rotated only if they grow bigger than log_rotate_size bytes. If size is followed by k, the size is assumed to be in kilobytes.
# If the M is used, the size is in megabytes, and if G is used, the size is in gigabytes. So size 100, size 100k, size 100M and size 100G
# are all valid.
rotate_size: 200M
# The directory on your host that store log
location: /var/log/harbor
# Uncomment following lines to enable external syslog endpoint.
# external_endpoint:
# # protocol used to transmit log to external endpoint, options is tcp or udp
# protocol: tcp
# # The host of external endpoint
# host: localhost
# # Port of external endpoint
# port: 5140
#This attribute is for migrator to detect the version of the .cfg file, DO NOT MODIFY!
_version: 1.10.0
# Uncomment external_database if using external database.
external_database:
harbor:
host: master.inortap.service.consul
port: 5432
db_name: harbor
username: harbor
password: geheim
ssl_mode: disable
max_idle_conns: 2
max_open_conns: 0
clair:
host: master.inortap.service.consul
port: 5432
db_name: clair
username: clair
password: geheimer
ssl_mode: disable
notary_signer:
host: master.inortap.service.consul
port: 5432
db_name: notary_signer
username: notary_signer
password: vielgeheimer
ssl_mode: disable
notary_server:
host: master.inortap.service.consul
port: 5432
db_name: notary_server
username: notary_server
password: geheimst
ssl_mode: disable
# Uncomment external_redis if using external Redis server
external_redis:
host: rtest-redis.svc.a5ba23f5-8237-6879-e7e1-ea6f574fbde9.de-gt-2.snc.tgos.xyz
port: 6379
password: auchgeheim
# db_index 0 is for core, it's unchangeable
registry_db_index: 5
jobservice_db_index: 6
chartmuseum_db_index: 7
clair_db_index: 8
# Uncomment uaa for trusting the certificate of uaa instance that is hosted via self-signed cert.
# uaa:
# ca_file: /path/to/ca
# Global proxy
# Config http proxy for components, e.g. http://my.proxy.com:3128
# Components doesn't need to connect to each others via http proxy.
# Remove component from `components` array if want disable proxy
# for it. If you want use proxy for replication, MUST enable proxy
# for core and jobservice, and set `http_proxy` and `https_proxy`.
# Add domain to the `no_proxy` field, when you want disable proxy
# for some special registry.
proxy:
http_proxy:
https_proxy:
# no_proxy endpoints will appended to 127.0.0.1,localhost,.local,.internal,log,db,redis,nginx,core,portal,postgresql,jobservice,registry,registryctl,clair,chartmuseum,notary-server
no_proxy:
components:
- core
- jobservice
- clair
Wichtig dabei: Auch wenn man alle "stateful" Services ausgelagert hat, benötigt Harbor noch sein data_volume. Da lagert dann zwar nicht mehr allzuviel - aber es wird noch gebraucht:
root@regha:/data/harbor# ls -la
total 36
drwxr-xr-x 9 root root 4096 Feb 23 13:53 .
drwxr-xr-x 4 root root 4096 Feb 23 13:52 ..
drwxr-xr-x 2 10000 10000 4096 Feb 23 13:53 ca_download
drwx------ 2 systemd-coredump systemd-coredump 4096 Feb 23 13:53 database
drwxr-xr-x 2 10000 10000 4096 Feb 23 13:53 job_logs
drwxr-xr-x 2 10000 10000 4096 Feb 23 13:53 psc
drwxr-xr-x 2 systemd-coredump systemd-coredump 4096 Feb 23 13:53 redis
drwxr-xr-x 2 10000 10000 4096 Feb 23 13:53 registry
drwxr-xr-x 6 root root 4096 Feb 23 15:50 secret
root@regha:/data/harbor# du -hs .
68K .
Leider stellte sich beim Start von Harbor mit docker-compose heraus, dass innerhalb der relevanten Docker-Container das Service-Discovery per DNS und Consul nicht funktionierte.
Durch den Blog-Post von Felix Ehrenpfort (zusätzlicher stub-resolver auf der IP der Docker-Bridge), lies sich dieses Problem aber beheben:
root@regha:~# cat /etc/systemd/resolved.conf.d/docker.conf
[Resolve]
DNSStubListener=yes
DNSStubListenerExtra=172.17.0.1
Zusätzlich zu dem schon bestehenden für Consul:
root@regha:~# cat /etc/systemd/resolved.conf.d/consul.conf
[Resolve]
DNS=127.0.0.1:8600
DNSSEC=false
Domains=~consul
Damit finden sich dann brav alle Komponenten. Ob Harbor damit jetzt allerdings läuft, muß ich noch ermitteln. Ich habe den Verdacht, dass die Datenbankinitialisierung der externen Datenbank nicht so wie bei einer lokalen Datenbank durchgeführt wird.
Die docker-compose.yml Datei, die bei der Installation mit ./install.sh --with-clair erzeugt wird, sieht so aus:
version: '2.3'
services:
log:
image: goharbor/harbor-log:v1.10.16
container_name: harbor-log
restart: always
dns_search: .
cap_drop:
- ALL
cap_add:
- CHOWN
- DAC_OVERRIDE
- SETGID
- SETUID
volumes:
- /var/log/harbor/:/var/log/docker/:z
- type: bind
source: ./common/config/log/logrotate.conf
target: /etc/logrotate.d/logrotate.conf
- type: bind
source: ./common/config/log/rsyslog_docker.conf
target: /etc/rsyslog.d/rsyslog_docker.conf
ports:
- 127.0.0.1:1514:10514
networks:
- harbor
registry:
image: goharbor/registry-photon:v1.10.16
container_name: registry
restart: always
cap_drop:
- ALL
cap_add:
- CHOWN
- SETGID
- SETUID
volumes:
- /data/harbor/registry:/storage:z
- ./common/config/registry/:/etc/registry/:z
- type: bind
source: /data/harbor/secret/registry/root.crt
target: /etc/registry/root.crt
networks:
- harbor
- harbor-clair
dns_search: .
depends_on:
- log
logging:
driver: "syslog"
options:
syslog-address: "tcp://127.0.0.1:1514"
tag: "registry"
registryctl:
image: goharbor/harbor-registryctl:v1.10.16
container_name: registryctl
env_file:
- ./common/config/registryctl/env
restart: always
cap_drop:
- ALL
cap_add:
- CHOWN
- SETGID
- SETUID
volumes:
- /data/harbor/registry:/storage:z
- ./common/config/registry/:/etc/registry/:z
- type: bind
source: ./common/config/registryctl/config.yml
target: /etc/registryctl/config.yml
networks:
- harbor
dns_search: .
depends_on:
- log
logging:
driver: "syslog"
options:
syslog-address: "tcp://127.0.0.1:1514"
tag: "registryctl"
core:
image: goharbor/harbor-core:v1.10.16
container_name: harbor-core
env_file:
- ./common/config/core/env
restart: always
cap_drop:
- ALL
cap_add:
- SETGID
- SETUID
volumes:
- /data/harbor/ca_download/:/etc/core/ca/:z
- /data/harbor/psc/:/etc/core/token/:z
- /data/harbor/:/data/:z
- ./common/config/core/certificates/:/etc/core/certificates/:z
- type: bind
source: ./common/config/core/app.conf
target: /etc/core/app.conf
- type: bind
source: /data/harbor/secret/core/private_key.pem
target: /etc/core/private_key.pem
- type: bind
source: /data/harbor/secret/keys/secretkey
target: /etc/core/key
networks:
harbor:
harbor-clair:
aliases:
- harbor-core
dns_search: .
depends_on:
- log
- registry
logging:
driver: "syslog"
options:
syslog-address: "tcp://127.0.0.1:1514"
tag: "core"
portal:
image: goharbor/harbor-portal:v1.10.16
container_name: harbor-portal
restart: always
cap_drop:
- ALL
cap_add:
- CHOWN
- SETGID
- SETUID
- NET_BIND_SERVICE
networks:
- harbor
dns_search: .
depends_on:
- log
logging:
driver: "syslog"
options:
syslog-address: "tcp://127.0.0.1:1514"
tag: "portal"
jobservice:
image: goharbor/harbor-jobservice:v1.10.16
container_name: harbor-jobservice
env_file:
- ./common/config/jobservice/env
restart: always
cap_drop:
- ALL
cap_add:
- CHOWN
- SETGID
- SETUID
volumes:
- /data/harbor/job_logs:/var/log/jobs:z
- type: bind
source: ./common/config/jobservice/config.yml
target: /etc/jobservice/config.yml
networks:
- harbor
- harbor-clair
dns_search: .
depends_on:
- core
logging:
driver: "syslog"
options:
syslog-address: "tcp://127.0.0.1:1514"
tag: "jobservice"
proxy:
image: goharbor/nginx-photon:v1.10.16
container_name: nginx
restart: always
cap_drop:
- ALL
cap_add:
- CHOWN
- SETGID
- SETUID
- NET_BIND_SERVICE
volumes:
- ./common/config/nginx:/etc/nginx:z
- /data/harbor/secret/cert:/etc/cert:z
networks:
- harbor
dns_search: .
ports:
- 80:8080
- 443:8443
depends_on:
- registry
- core
- portal
- log
logging:
driver: "syslog"
options:
syslog-address: "tcp://127.0.0.1:1514"
tag: "proxy"
clair:
networks:
- harbor-clair
container_name: clair
image: goharbor/clair-photon:v1.10.16
restart: always
cap_drop:
- ALL
cap_add:
- DAC_OVERRIDE
- SETGID
- SETUID
cpu_quota: 50000
dns_search: .
depends_on:
- log
volumes:
- type: bind
source: ./common/config/clair/config.yaml
target: /etc/clair/config.yaml
logging:
driver: "syslog"
options:
syslog-address: "tcp://127.0.0.1:1514"
tag: "clair"
env_file:
./common/config/clair/clair_env
clair-adapter:
networks:
- harbor-clair
container_name: clair-adapter
image: goharbor/clair-adapter-photon:v1.10.16
restart: always
cap_drop:
- ALL
cap_add:
- DAC_OVERRIDE
- SETGID
- SETUID
cpu_quota: 50000
dns_search: .
depends_on:
- clair
logging:
driver: "syslog"
options:
syslog-address: "tcp://127.0.0.1:1514"
tag: "clair-adapter"
env_file:
./common/config/clair-adapter/env
networks:
harbor:
external: false
harbor-clair:
external: false
Den Versuch, Harbor mit Nomad auszurollen, hat übrigens schonmal jemand gestartet. In dem Fall aber vermutlich ohne externes Storage, externe Datenbank und externen Redis.