diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index dc77b85e0..09719d583 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -1,7 +1,7 @@ image: registry: docker.io repository: cybertecpostgresql/cybertec-pg-operator - tag: v0.8.0-1 + tag: v0.9.2-1 pullPolicy: "IfNotPresent" # Optionally specify an array of imagePullSecrets. @@ -38,7 +38,7 @@ configGeneral: # etcd connection string for Patroni. Empty uses K8s-native DCS. etcd_host: "" # Database pod docker image - docker_image: docker.io/cybertecpostgresql/cybertec-pg-container:postgres-17.0-2 + docker_image: containers.cybertec.at/cybertec-pg-container/postgres:rocky9-18.3-2 # key name for annotation to ignore globally configured instance limits # ignore_instance_limits_annotation_key: "" @@ -355,8 +355,6 @@ configLogicalBackup: # logical_backup_memory_limit: "" # logical_backup_memory_request: "" - # image for pods of the logical backup job (example runs pg_dumpall) - logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup:v1.10.1" # path of google cloud service account json file # logical_backup_google_application_credentials: "" diff --git a/docs/hugo/content/en/crd/crd-postgresql.md b/docs/hugo/content/en/crd/crd-postgresql.md index 895d36000..781ea4168 100644 --- a/docs/hugo/content/en/crd/crd-postgresql.md +++ b/docs/hugo/content/en/crd/crd-postgresql.md @@ -168,6 +168,7 @@ key, operator, value, effect and tolerationSeconds | | Name | Type | required | Description | | ------------------------------ |:-------:| ---------:| ------------------:| +| customQueries | string | false | Name of the ConfigMap containing custom queries | | [env](#env) | array | false | Allows you to add custom environment variables to all expoerter-sidecar containers | | image | string | true | Docker-Image for the metric exporter | @@ -251,6 +252,8 @@ key, operator, value, effect and tolerationSeconds | | ------------------------------ |:-------:| ---------:| ------------------:| | standby_host | string | true | Endpoint of the primary cluster | | standby_port | string | true | PostgreSQL port of the primary cluster | +| standby_primary_slot_name | string | true | replication slot of the primary cluster | + {{< back >}} diff --git a/docs/hugo/content/en/monitoring/_index.md b/docs/hugo/content/en/monitoring/_index.md index 48d1366df..8d4cfa21f 100644 --- a/docs/hugo/content/en/monitoring/_index.md +++ b/docs/hugo/content/en/monitoring/_index.md @@ -13,6 +13,14 @@ These Stack is based on: CPO has prepared an own Exporter for the PostgreSQl-Pod which can used as a sidecar. +#### Configure Cluster for monitoring +To deploy a monitoring exporter for Prometheus and create the necessary users and permissions in the cluster, the cluster manifest must be extended to include the `monitor` object. +- `customQueries`: Name of the ConfigMap containing custom queries for PostgreSQL and Prometheus. +- `env`: Additional environment variables for the sidecar container +- `image`: Image for the monitoring sidecar. For example: `containers.cybertec.at/cybertec-pg-container/exporter:rocky9-18.3-2` + +{{< hint type=Info >}}The ConfigMap is mounted in the sidecar container. Any additional permissions required for the queries must be manually granted to the monitoring user if necessary.{{< /hint >}} + #### Setting up the Monitoring Stack To setup the Monitoring-Stack we suggest that you create an own namespace and use the prepared kustomization file inside the Operator-Tutorials. ``` @@ -24,7 +32,7 @@ No resources found in cpo-monitoring namespace. git clone https://github.com/cybertec-postgresql/CYBERTEC-operator-tutorial cd CYBERTEC-operator-tutorial/setup/monitoring -# Hint: Please check if youn want to use a specific storage-class the file pvcs.yaml and add your storageclass on the commented part. Please ensure that you removed the comment-char. +{{< hint type=Info >}}Hint: Please check if you want to use a specific storage-class the file pvcs.yaml and add your storageclass on the commented part. Please ensure that you removed the comment-char.{{< /hint >}} $ kubectl apply -n cpo-monitoring -k . serviceaccount/cpo-monitoring created @@ -47,9 +55,9 @@ deployment.apps/cpo-monitoring-alertmanager created deployment.apps/cpo-monitoring-grafana created deployment.apps/cpo-monitoring-prometheus created -Hint: If you're not running Openshift you will get a error like this: +{{< hint type=Info >}}Hint: If you're not running Openshift you will get a error like this: error: resource mapping not found for name: "grafana" namespace: "" from ".": -no matches for kind "Route" in version "route.openshift.io/v1" ensure CRDs are installed first +no matches for kind "Route" in version "route.openshift.io/v1" ensure CRDs are installed first{{< /hint >}} You can ignore this, because it depends on an object with the type route which is part of Openshift. It is not needed replaced by ingress-rules or an loadbalancer-service. diff --git a/docs/hugo/content/en/release_notes/_index.md b/docs/hugo/content/en/release_notes/_index.md index 9f0b369f9..bbae3c1bf 100644 --- a/docs/hugo/content/en/release_notes/_index.md +++ b/docs/hugo/content/en/release_notes/_index.md @@ -4,6 +4,30 @@ date: 2024-03-11T14:26:51+01:00 draft: false weight: 2500 --- + +### 0.9.2 ### + +#### Features +- Flexible environment variables: Introduction of custom environments, which can be defined both at container level and cluster-wide. +- Advanced label configuration: Custom labels can now be flexibly configured for individual pods or the entire cluster. +- Custom Metrics: Support for custom-queries in the exporter. These can now be easily added via ConfigMap without modifying the image. +- logging: Conversion of Patroni logs from plain text to JSON format. + +#### Fixes +- Optimized upgrade-readiness-check: The trigger for major upgrades has been optimized. +- Standby-cluster: An error when promoting standby clusters has been resolved. + +#### Notification of upcoming deprecation +- PG13 has reached its EoL - Support will be removed in the next release + +#### Supported Versions + +- PG: 13 - 18 +- Patroni: 4.1.0 +- pgBackRest: 2.57.0 - 2.58.0 +- Kubernetes: 1.21 - 1.34 +- Openshift: 4.8 - 4.20 + ### 0.9.1 ### #### Features diff --git a/go.mod b/go.mod index 452da64bd..a0be81c9e 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/sirupsen/logrus v1.9.1 github.com/stretchr/testify v1.8.4 go.etcd.io/etcd/client/v3 v3.5.4 - golang.org/x/crypto v0.45.0 + golang.org/x/crypto v0.46.0 golang.org/x/exp v0.0.0-20230108222341-4b8118a2686a gopkg.in/yaml.v2 v2.4.0 k8s.io/api v0.30.4 @@ -30,7 +30,7 @@ require ( go.etcd.io/etcd/client/pkg/v3 v3.5.4 // indirect golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated // indirect google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect - google.golang.org/grpc v1.75.0 // indirect + google.golang.org/grpc v1.79.3 // indirect ) require ( @@ -65,16 +65,16 @@ require ( go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect go.uber.org/zap v1.19.0 // indirect - golang.org/x/mod v0.29.0 // indirect - golang.org/x/net v0.47.0 // indirect - golang.org/x/oauth2 v0.30.0 // indirect - golang.org/x/sync v0.18.0 // indirect - golang.org/x/sys v0.38.0 // indirect - golang.org/x/term v0.37.0 // indirect - golang.org/x/text v0.31.0 // indirect + golang.org/x/mod v0.30.0 // indirect + golang.org/x/net v0.48.0 // indirect + golang.org/x/oauth2 v0.34.0 // indirect + golang.org/x/sync v0.19.0 // indirect + golang.org/x/sys v0.39.0 // indirect + golang.org/x/term v0.38.0 // indirect + golang.org/x/text v0.32.0 // indirect golang.org/x/time v0.3.0 // indirect - golang.org/x/tools v0.38.0 // indirect - google.golang.org/protobuf v1.36.6 // indirect + golang.org/x/tools v0.39.0 // indirect + google.golang.org/protobuf v1.36.10 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/gengo v0.0.0-20220902162205-c0856e24416d // indirect diff --git a/go.sum b/go.sum index 7a6d0757d..e33237c62 100644 --- a/go.sum +++ b/go.sum @@ -20,6 +20,8 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= @@ -233,18 +235,18 @@ go.etcd.io/etcd/client/pkg/v3 v3.5.4 h1:lrneYvz923dvC14R54XcA7FXoZ3mlGZAgmwhfm7H go.etcd.io/etcd/client/pkg/v3 v3.5.4/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= go.etcd.io/etcd/client/v3 v3.5.4 h1:p83BUL3tAYS0OT/r0qglgc3M1JjhM0diV8DSWAhVXv4= go.etcd.io/etcd/client/v3 v3.5.4/go.mod h1:ZaRkVgBZC+L+dLCjTcF1hRXpgZXQPOvnA/Ak/gq3kiY= -go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= -go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= -go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= -go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= -go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= -go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI= -go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg= -go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc= -go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps= -go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= -go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48= +go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8= +go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0= +go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs= +go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18= +go.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwmt1ENZszdJE= +go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8= +go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew= +go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI= +go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= @@ -259,8 +261,8 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= -golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= +golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= +golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20230108222341-4b8118a2686a h1:tlXy25amD5A7gOfbXdqCGN5k8ESEed/Ee1E5RcrYnqU= golang.org/x/exp v0.0.0-20230108222341-4b8118a2686a/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= @@ -274,8 +276,8 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA= -golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w= +golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk= +golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -290,13 +292,13 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= -golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= +golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= +golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= -golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= +golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= +golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -305,8 +307,8 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= -golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -326,17 +328,17 @@ golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= -golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= +golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU= -golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254= +golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q= +golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= -golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= +golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= +golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -352,8 +354,8 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= -golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= +golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ= +golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ= golang.org/x/tools/go/expect v0.1.0-deprecated h1:jY2C5HGYR5lqex3gEniOQL0r7Dq5+VGVgY1nudX5lXY= golang.org/x/tools/go/expect v0.1.0-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY= golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated h1:1h2MnaIAIXISqTFKdENegdpAgUXz6NrPEsbIeWaBRvM= @@ -381,8 +383,8 @@ google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTp google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.75.0 h1:+TW+dqTd2Biwe6KKfhE5JpiYIBWq865PhKGSXiivqt4= -google.golang.org/grpc v1.75.0/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ= +google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE= +google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -396,8 +398,8 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= -google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= +google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= +google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/manifests/postgres-operator.yaml b/manifests/postgres-operator.yaml index 0ea7e3203..a28741149 100644 --- a/manifests/postgres-operator.yaml +++ b/manifests/postgres-operator.yaml @@ -19,7 +19,7 @@ spec: serviceAccountName: postgres-operator containers: - name: postgres-operator - image: registry.opensource.zalan.do/acid/postgres-operator:v1.10.1 + image: cybertecpostgresql/cybertec-pg-operator:v0.9.2 imagePullPolicy: IfNotPresent resources: requests: diff --git a/pkg/apis/cpo.opensource.cybertec.at/v1/crds.go b/pkg/apis/cpo.opensource.cybertec.at/v1/crds.go index 9ffcf2977..0d90f443a 100644 --- a/pkg/apis/cpo.opensource.cybertec.at/v1/crds.go +++ b/pkg/apis/cpo.opensource.cybertec.at/v1/crds.go @@ -737,6 +737,49 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ "ttl": { Type: "integer", }, + "log": { + Type: "object", + Properties: map[string]apiextv1.JSONSchemaProps{ + "type": { + Type: "string", + Enum: []apiextv1.JSON{ + {Raw: []byte(`"plain"`)}, + {Raw: []byte(`"json"`)}, + }, + }, + "level": { + Type: "string", + Enum: []apiextv1.JSON{ + {Raw: []byte(`"DEBUG"`)}, + {Raw: []byte(`"INFO"`)}, + {Raw: []byte(`"WARNING"`)}, + {Raw: []byte(`"ERROR"`)}, + {Raw: []byte(`"CRITICAL"`)}, + }, + }, + "traceback_level": { + Type: "string", + Enum: []apiextv1.JSON{ + {Raw: []byte(`"DEBUG"`)}, + {Raw: []byte(`"INFO"`)}, + {Raw: []byte(`"WARNING"`)}, + {Raw: []byte(`"ERROR"`)}, + {Raw: []byte(`"CRITICAL"`)}, + }, + }, + "static_fields": { + Type: "object", + AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ + Schema: &apiextv1.JSONSchemaProps{ + Type: "string", + }, + }, + }, + "deduplicate_heartbeat_logs": { + Type: "boolean", + }, + }, + }, }, }, "podAnnotations": { @@ -939,6 +982,9 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ "standby_port": { Type: "string", }, + "standby_primary_slot_name": { + Type: "string", + }, }, OneOf: []apiextv1.JSONSchemaProps{ apiextv1.JSONSchemaProps{Required: []string{"s3_wal_path"}}, @@ -1535,6 +1581,9 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ "image": { Type: "string", }, + "customQueries": { + Type: "string", + }, "env": { Type: "array", Nullable: true, diff --git a/pkg/apis/cpo.opensource.cybertec.at/v1/postgresql_type.go b/pkg/apis/cpo.opensource.cybertec.at/v1/postgresql_type.go index 33c302030..542d4fe2e 100644 --- a/pkg/apis/cpo.opensource.cybertec.at/v1/postgresql_type.go +++ b/pkg/apis/cpo.opensource.cybertec.at/v1/postgresql_type.go @@ -185,14 +185,16 @@ type Patroni struct { SynchronousNodeCount uint32 `json:"synchronous_node_count,omitempty" defaults:"1"` Multisite *Multisite `json:"multisite,omitempty"` FailsafeMode *bool `json:"failsafe_mode,omitempty"` + Log *PatroniLog `json:"log,omitempty"` } // StandbyDescription contains remote primary config or s3/gs wal path type StandbyDescription struct { - S3WalPath string `json:"s3_wal_path,omitempty"` - GSWalPath string `json:"gs_wal_path,omitempty"` - StandbyHost string `json:"standby_host,omitempty"` - StandbyPort string `json:"standby_port,omitempty"` + S3WalPath string `json:"s3_wal_path,omitempty"` + GSWalPath string `json:"gs_wal_path,omitempty"` + StandbyHost string `json:"standby_host,omitempty"` + StandbyPort string `json:"standby_port,omitempty"` + StandbyPrimarySlotName string `json:"standby_primary_slot_name,omitempty"` } // TLSDescription specs TLS properties @@ -330,8 +332,9 @@ type TDE struct { // Monitoring Sidecar defines a container to be run in the same pod as the Postgres container. type Monitoring struct { - Image string `json:"image,omitempty"` - Env []v1.EnvVar `json:"env,omitempty"` + Image string `json:"image,omitempty"` + CustomQueries string `json:"customQueries,omitempty"` + Env []v1.EnvVar `json:"env,omitempty"` } // Multisite enables cross Kubernetes replication coordinated via etcd @@ -350,3 +353,11 @@ type EtcdConfig struct { Protocol *string `json:"protocol,omitempty"` CertSecretName *string `json:"certSecretName,omitempty"` } + +type PatroniLog struct { + Type string `json:"type,omitempty"` + Level string `json:"level,omitempty"` + TracebackLevel string `json:"traceback_level,omitempty"` + StaticFields map[string]string `json:"static_fields,omitempty"` + DeduplicateHeartbeatLogs *bool `json:"deduplicate_heartbeat_logs,omitempty"` +} diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index d3db6146f..11697e461 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -1052,6 +1052,12 @@ func (c *Cluster) Update(oldSpec, newSpec *cpov1.Postgresql) error { syncStatefulSet = true } + //sync sts if there is a change in the monitor-section + if !reflect.DeepEqual(oldSpec.Spec.Monitoring, newSpec.Spec.Monitoring) { + c.logger.Infof("monitoring configuration changed, triggering statefulset sync") + syncStatefulSet = true + } + // Pgbackrest backup job func() { @@ -1152,6 +1158,13 @@ func (c *Cluster) Update(oldSpec, newSpec *cpov1.Postgresql) error { } }() + // add or remove standby_cluster section from Patroni config depending on changes in standby section + if !reflect.DeepEqual(oldSpec.Spec.StandbyCluster, newSpec.Spec.StandbyCluster) { + if err := c.syncStandbyClusterConfiguration(); err != nil { + return fmt.Errorf("could not set StandbyCluster configuration options: %v", err) + } + } + // pod disruption budget if oldSpec.Spec.NumberOfInstances != newSpec.Spec.NumberOfInstances { c.logger.Debug("syncing pod disruption budgets") @@ -1274,13 +1287,18 @@ func (c *Cluster) Update(oldSpec, newSpec *cpov1.Postgresql) error { } if !updateFailed { - // Major version upgrade must only fire after success of earlier operations and should stay last - if err := c.majorVersionUpgrade(); err != nil { - c.logger.Errorf("major version upgrade failed: %v", err) + if upgradeErr := c.executeMajorVersionUpgrade(); upgradeErr != nil { + c.logger.Errorf("major version upgrade failed: %v", upgradeErr) updateFailed = true } } + if updateFailed { + c.logger.Errorf("Update for cluster %s/%s finished with errors..", c.Namespace, c.Name) + } else { + c.logger.Infof("Update for cluster %s/%s completed successfully.", c.Namespace, c.Name) + } + return nil } diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 7f7d7df6d..19c6c4802 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -81,6 +81,7 @@ type pgBootstrap struct { type spiloConfiguration struct { PgLocalConfiguration map[string]interface{} `json:"postgresql"` Bootstrap pgBootstrap `json:"bootstrap"` + Log *spiloLogConfiguration `json:"log,omitempty"` } type TDEConfig struct { @@ -88,6 +89,14 @@ type TDEConfig struct { KeyBits string } +type spiloLogConfiguration struct { + Type string `json:"type,omitempty"` + Level string `json:"level,omitempty"` + TracebackLevel string `json:"traceback_level,omitempty"` + StaticFields map[string]string `json:"static_fields,omitempty"` + DeduplicateHeartbeatLogs *bool `json:"deduplicate_heartbeat_logs,omitempty"` +} + func (c *Cluster) statefulSetName() string { return c.Name } @@ -468,6 +477,26 @@ PatroniInitDBParams: config.PgLocalConfiguration[patroniPGHBAConfParameterName] = patroni.PgHba } + if patroni.Log != nil { + logConfig := &spiloLogConfiguration{} + if patroni.Log.Type != "" { + logConfig.Type = patroni.Log.Type + } + if patroni.Log.Level != "" { + logConfig.Level = patroni.Log.Level + } + if patroni.Log.TracebackLevel != "" { + logConfig.TracebackLevel = patroni.Log.TracebackLevel + } + if patroni.Log.StaticFields != nil { + logConfig.StaticFields = patroni.Log.StaticFields + } + if patroni.Log.DeduplicateHeartbeatLogs != nil { + logConfig.DeduplicateHeartbeatLogs = patroni.Log.DeduplicateHeartbeatLogs + } + config.Log = logConfig + } + res, err := json.Marshal(config) return string(res), err } @@ -1639,6 +1668,12 @@ func (c *Cluster) generateStatefulSet(spec *cpov1.PostgresSpec) (*appsv1.Statefu additionalVolumes = append(additionalVolumes, c.generatePgbackrestCloneConfigVolumes(spec.Clone)...) } + if c.Spec.Monitoring != nil && c.Spec.Monitoring.CustomQueries != "" { + if queryVol := c.generateCustomQueriesVolume(c.Spec.Monitoring.CustomQueries); queryVol != nil { + additionalVolumes = append(additionalVolumes, *queryVol) + } + } + // generate pod template for the statefulset, based on the spilo container and sidecars podTemplate, err = c.generatePodTemplate( c.Namespace, @@ -2174,6 +2209,24 @@ func (c *Cluster) generateCertSecretVolume() cpov1.AdditionalVolume { } } +func (c *Cluster) generateCustomQueriesVolume(customQueryConfigMap string) *cpov1.AdditionalVolume { + defaultMode := int32(0640) + return &cpov1.AdditionalVolume{ + Name: "custom-queries-vol", + MountPath: "/postgres_exporter/custom_queries", + SubPath: "", + TargetContainers: []string{"postgres-exporter"}, + VolumeSource: v1.VolumeSource{ + ConfigMap: &v1.ConfigMapVolumeSource{ + LocalObjectReference: v1.LocalObjectReference{ + Name: customQueryConfigMap, + }, + DefaultMode: &defaultMode, + }, + }, + } +} + func (c *Cluster) generatePodAnnotations(spec *cpov1.PostgresSpec) map[string]string { annotations := make(map[string]string) for k, v := range c.OpConfig.CustomPodAnnotations { diff --git a/pkg/cluster/majorversionupgrade.go b/pkg/cluster/majorversionupgrade.go index 61edf1908..5f204af03 100644 --- a/pkg/cluster/majorversionupgrade.go +++ b/pkg/cluster/majorversionupgrade.go @@ -3,8 +3,10 @@ package cluster import ( "context" "encoding/json" + "errors" "fmt" "strings" + "time" "github.com/Masterminds/semver" "github.com/cybertec-postgresql/cybertec-pg-operator/pkg/spec" @@ -29,6 +31,8 @@ const ( majorVersionUpgradeFailureAnnotation = "last-major-upgrade-failure" ) +var errUpgradePrepNotReady = errors.New("cluster not ready for upgrade") + // IsBiggerPostgresVersion Compare two Postgres version numbers func IsBiggerPostgresVersion(old string, new string) bool { oldN := VersionMap[old] @@ -232,12 +236,14 @@ func (c *Cluster) majorVersionUpgrade() error { continue } if checkStreaming && member.State != "streaming" { - c.logger.Infof("skipping major version upgrade, replica %s is not streaming from primary", member.Name) - return nil + // c.logger.Infof("skipping major version upgrade, replica %s is not streaming from primary", member.Name) + // return nil + return fmt.Errorf("%w: replica %s is not streaming (state: %s)", errUpgradePrepNotReady, member.Name, member.State) } if member.Lag > 16*1024*1024 { - c.logger.Infof("skipping major version upgrade, replication lag on member %s is too high", member.Name) - return nil + // c.logger.Infof("skipping major version upgrade, replication lag on member %s is too high", member.Name) + // return nil + return fmt.Errorf("%w: replication lag on member %s is too high (%d bytes)", errUpgradePrepNotReady, member.Name, member.Lag) } } @@ -246,11 +252,10 @@ func (c *Cluster) majorVersionUpgrade() error { if allRunning { c.logger.Infof("healthy cluster ready to upgrade, current: %d desired: %d", c.currentMajorVersion, desiredVersion) if c.currentMajorVersion < desiredVersion { - defer func() error { - if err = c.criticalOperationLabel(pods, nil); err != nil { - return fmt.Errorf("failed to remove critical-operation label: %s", err) + defer func() { + if err := c.criticalOperationLabel(pods, nil); err != nil { + c.logger.Errorf("failed to remove critical-operation label: %v", err) } - return nil }() val := "true" if err = c.criticalOperationLabel(pods, &val); err != nil { @@ -260,37 +265,72 @@ func (c *Cluster) majorVersionUpgrade() error { podName := &spec.NamespacedName{Namespace: masterPod.Namespace, Name: masterPod.Name} c.logger.Infof("triggering major version upgrade on pod %s of %d pods", masterPod.Name, numberOfPods) c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeNormal, "Major Version Upgrade", "starting major version upgrade on pod %s of %d pods", masterPod.Name, numberOfPods) - upgradeCommand := fmt.Sprintf("set -o pipefail && /usr/bin/python3 /scripts/inplace_upgrade.py %d 2>&1 | tee last_upgrade.log", numberOfPods) - - c.logger.Debug("checking if the spilo image runs with root or non-root (check for user id=0)") + upgradeCommand := fmt.Sprintf("/usr/local/bin/python3 /scripts/inplace_upgrade.py %d 2>&1", numberOfPods) + c.logger.Debug("checking if the container runs with root or non-root (check for user id=0)") resultIdCheck, errIdCheck := c.ExecCommand(podName, "/bin/bash", "-c", "/usr/bin/id -u") if errIdCheck != nil { c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeWarning, "Major Version Upgrade", "checking user id to run upgrade from %d to %d FAILED: %v", c.currentMajorVersion, desiredVersion, errIdCheck) } resultIdCheck = strings.TrimSuffix(resultIdCheck, "\n") - var result, scriptErrMsg string + var result string + if resultIdCheck != "0" { c.logger.Infof("user id was identified as: %s, hence default user is non-root already", resultIdCheck) result, err = c.ExecCommand(podName, "/bin/bash", "-c", upgradeCommand) - scriptErrMsg, _ = c.ExecCommand(podName, "/bin/bash", "-c", "tail -n 1 last_upgrade.log") } else { c.logger.Infof("user id was identified as: %s, using su to reach the postgres user", resultIdCheck) result, err = c.ExecCommand(podName, "/bin/su", "postgres", "-c", upgradeCommand) - scriptErrMsg, _ = c.ExecCommand(podName, "/bin/bash", "-c", "tail -n 1 last_upgrade.log") } + if err != nil { isUpgradeSuccess = false c.annotatePostgresResource(isUpgradeSuccess) - c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeWarning, "Major Version Upgrade", "upgrade from %d to %d FAILED: %v", c.currentMajorVersion, desiredVersion, scriptErrMsg) - return fmt.Errorf("%s", scriptErrMsg) + + finalErrorMsg := strings.TrimSpace(result) + if finalErrorMsg == "" { + finalErrorMsg = err.Error() + } + + lines := strings.Split(finalErrorMsg, "\n") + if len(lines) > 5 { + finalErrorMsg = strings.Join(lines[len(lines)-5:], " | ") + } + + c.logger.Errorf("Major upgrade failed: %v", err) + c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeWarning, "Major Version Upgrade", "upgrade from %d to %d FAILED: %s", c.currentMajorVersion, desiredVersion, finalErrorMsg) + + return fmt.Errorf("upgrade script failed: %s", finalErrorMsg) } c.annotatePostgresResource(isUpgradeSuccess) c.logger.Infof("upgrade action triggered and command completed: %s", result[:100]) - c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeNormal, "Major Version Upgrade", "upgrade from %d to %d finished", c.currentMajorVersion, desiredVersion) + c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeNormal, "Major Version Upgrade", "major version upgrade from version %d to version %d was successfully completed.", c.currentMajorVersion, desiredVersion) } } return nil } + +func (c *Cluster) executeMajorVersionUpgrade() error { + maxRetries := 6 + var lastErr error + + for i := 0; i < maxRetries; i++ { + lastErr = c.majorVersionUpgrade() + if lastErr == nil { + return nil + } + + if errors.Is(lastErr, errUpgradePrepNotReady) { + c.logger.Warnf("Major version upgrade deferred (attempt %d/%d): %v. Retrying in 15s...", i+1, maxRetries, lastErr) + + if i < maxRetries-1 { + time.Sleep(15 * time.Second) + continue + } + } + return lastErr + } + return lastErr +} diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index 351a2def0..26ebb4a20 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -230,7 +230,7 @@ func (c *Cluster) Sync(newSpec *cpov1.Postgresql) error { return err } - if err = c.syncPgbackrestRepoHostConfig(&c.Spec); err != nil { + if err = c.syncPgbackrestConfig(); err != nil { err = fmt.Errorf("could not sync pgbackrest config: %v", err) return err } @@ -261,6 +261,13 @@ func (c *Cluster) Sync(newSpec *cpov1.Postgresql) error { } } + // add or remove standby_cluster section from Patroni config depending on changes in standby section + if !reflect.DeepEqual(oldSpec.Spec.StandbyCluster, newSpec.Spec.StandbyCluster) { + if err := c.syncStandbyClusterConfiguration(); err != nil { + return fmt.Errorf("could not sync StandbyCluster configuration: %v", err) + } + } + c.logger.Debug("syncing pod disruption budgets") if err = c.syncPodDisruptionBudget(false); err != nil { err = fmt.Errorf("could not sync pod disruption budget: %v", err) @@ -1002,6 +1009,63 @@ func (c *Cluster) checkAndSetGlobalPostgreSQLConfiguration(pod *v1.Pod, effectiv return configPatched, requiresMasterRestart, nil } +// syncStandbyClusterConfiguration checks whether standby cluster +// parameters have changed and if necessary sets it via the Patroni API +func (c *Cluster) syncStandbyClusterConfiguration() error { + var ( + err error + pods []v1.Pod + ) + + standbyOptionsToSet := make(map[string]interface{}) + if c.Spec.StandbyCluster != nil { + c.logger.Infof("turning %q into a standby cluster", c.Name) + standbyOptionsToSet["create_replica_methods"] = []string{"bootstrap_standby_with_wale", "basebackup_fast_xlog"} + standbyOptionsToSet["restore_command"] = "envdir \"/run/etc/wal-e.d/env-standby\" /scripts/restore_command.sh \"%f\" \"%p\"" + + if c.Spec.StandbyCluster.StandbyHost != "" { + standbyOptionsToSet["host"] = c.Spec.StandbyCluster.StandbyHost + } else { + standbyOptionsToSet["host"] = nil + } + + if c.Spec.StandbyCluster.StandbyPort != "" { + standbyOptionsToSet["port"] = c.Spec.StandbyCluster.StandbyPort + } else { + standbyOptionsToSet["port"] = nil + } + + if c.Spec.StandbyCluster.StandbyPrimarySlotName != "" { + standbyOptionsToSet["primary_slot_name"] = c.Spec.StandbyCluster.StandbyPrimarySlotName + } else { + standbyOptionsToSet["primary_slot_name"] = nil + } + } else { + c.logger.Infof("promoting standby cluster and detach from source") + standbyOptionsToSet = nil + } + + if pods, err = c.listPods(); err != nil { + return err + } + if len(pods) == 0 { + return fmt.Errorf("could not call Patroni API: cluster has no pods") + } + // try all pods until the first one that is successful, as it doesn't matter which pod + // carries the request to change configuration through + for _, pod := range pods { + podName := util.NameFromMeta(pod.ObjectMeta) + c.logger.Infof("patching Postgres config via Patroni API on pod %s with following options: %s", + podName, standbyOptionsToSet) + if err = c.patroni.SetStandbyClusterParameters(&pod, standbyOptionsToSet); err == nil { + return nil + } + c.logger.Warningf("could not patch postgres parameters within pod %s: %v", podName, err) + } + return fmt.Errorf("could not reach Patroni API to set Postgres options: failed on every pod (%d total)", + len(pods)) +} + func (c *Cluster) syncSecrets() error { c.logger.Info("syncing secrets") c.setProcessName("syncing secrets") diff --git a/pkg/util/patroni/patroni.go b/pkg/util/patroni/patroni.go index a0ffa23f0..db7e0667e 100644 --- a/pkg/util/patroni/patroni.go +++ b/pkg/util/patroni/patroni.go @@ -35,6 +35,7 @@ type Interface interface { GetClusterMembers(master *v1.Pod) ([]ClusterMember, error) Switchover(master *v1.Pod, candidate string) error SetPostgresParameters(server *v1.Pod, options map[string]string) error + SetStandbyClusterParameters(server *v1.Pod, options map[string]interface{}) error GetMemberData(server *v1.Pod) (MemberData, error) Restart(server *v1.Pod) error GetConfig(server *v1.Pod) (cpov1.Patroni, map[string]string, error) @@ -166,6 +167,11 @@ func (p *Patroni) SetPostgresParameters(server *v1.Pod, parameters map[string]st return p.httpPostOrPatch(http.MethodPatch, apiURLString+configPath, buf) } +// SetStandbyClusterParameters sets StandbyCluster options via Patroni patch API call. +func (p *Patroni) SetStandbyClusterParameters(server *v1.Pod, parameters map[string]interface{}) error { + return p.SetConfig(server, map[string]interface{}{"standby_cluster": parameters}) +} + // SetConfig sets Patroni options via Patroni patch API call. func (p *Patroni) SetConfig(server *v1.Pod, config map[string]interface{}) error { buf := &bytes.Buffer{} diff --git a/ui/Dockerfile b/ui/Dockerfile index 63e8817e6..37abc7c87 100644 --- a/ui/Dockerfile +++ b/ui/Dockerfile @@ -38,4 +38,4 @@ ARG VERSION=dev RUN sed -i "s/__version__ = .*/__version__ = '${VERSION}'/" /operator_ui/__init__.py WORKDIR / -CMD ["/usr/bin/python3", "-m", "operator_ui"] +CMD ["/usr/local/bin/python3", "-m", "operator_ui"] diff --git a/ui/requirements.txt b/ui/requirements.txt index f71f80735..65ef21539 100644 --- a/ui/requirements.txt +++ b/ui/requirements.txt @@ -3,13 +3,13 @@ boto3==1.26.51 boto==2.49.0 click==8.1.3 Flask-OAuthlib==0.9.6 -Flask==2.3.2 +Flask==3.1.3 furl==2.1.3 gevent==23.9.0 jq==1.4.0 json_delta>=2.0.2 kubernetes==11.0.0 -requests==2.32.4 +requests==2.33.0 stups-tokens>=1.1.19 wal_e==1.1.1 -werkzeug==3.1.5 +werkzeug==3.1.6 diff --git a/ui/start_server.sh b/ui/start_server.sh index e2c3980cc..7ed2546cc 100644 --- a/ui/start_server.sh +++ b/ui/start_server.sh @@ -1,2 +1,2 @@ #!/bin/bash -/usr/bin/python3 -m operator_ui +/usr/local/bin/python3 -m operator_ui