Skip to content

Commit d4573a7

Browse files
authored
Merge pull request #19 from cybertec-postgresql/fixMajorUpgrade
Fix major upgrade
2 parents 86cce42 + 8c26788 commit d4573a7

10 files changed

Lines changed: 90 additions & 120 deletions

File tree

Makefile

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ PGBACKREST_VERSION ?= 2.50
1313
POSTGIS_VERSION ?= 34
1414
PACKAGER ?= dnf
1515
BUILD ?= 1
16+
ARCH ?= amd64
1617
IMAGE_TAG ?= $(BASEOS)-$(PGVERSION_FULL)-$(BUILD)
1718
POSTGIS_IMAGE_TAG ?= $(BASEOS)-$(PGVERSION_FULL)-$(POSTGIS_VERSION)-$(BUILD)
1819

@@ -72,10 +73,11 @@ postgres-build:
7273
--build-arg PACKAGER=$(PACKAGER) \
7374
--build-arg CONTAINERSUITE=$(CONTAINERSUITE) \
7475
--build-arg BUILD=$(BUILD) \
76+
--build-arg ARCH=$(ARCH) \
7577
--build-arg PGBACKREST_VERSION=$(PGBACKREST_VERSION) \
7678
--build-arg PATRONI_VERSION=$(PATRONI_VERSION) \
7779
--build-arg OLD_PG_VERSIONS="$(OLD_PG_VERSIONS)" \
78-
--build-arg PGVERSION=$(PGVERSION)
80+
--build-arg PGVERSION=$(PGVERSION)
7981

8082
postgres: postgres-build
8183

@@ -92,7 +94,7 @@ postgres-stage-build:
9294
--build-arg PGBACKREST_VERSION=$(PGBACKREST_VERSION) \
9395
--build-arg PATRONI_VERSION=$(PATRONI_VERSION) \
9496
--build-arg OLD_PG_VERSIONS="$(OLD_PG_VERSIONS)" \
95-
--build-arg PGVERSION=$(PGVERSION)
97+
--build-arg PGVERSION=$(PGVERSION)
9698

9799
postgres-stage: postgres-stage-build
98100

@@ -110,7 +112,7 @@ postgres-gis-build:
110112
--build-arg PATRONI_VERSION=$(PATRONI_VERSION) \
111113
--build-arg OLD_PG_VERSIONS="$(OLD_PG_VERSIONS)" \
112114
--build-arg PGVERSION=$(PGVERSION) \
113-
--build-arg POSTGIS_VERSION=$(POSTGIS_VERSION)
115+
--build-arg POSTGIS_VERSION=$(POSTGIS_VERSION)
114116

115117
postgres-gis: postgres-gis-build
116118

docker/pgbackrest/Dockerfile

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,8 @@ RUN if [ "$BASEOS" = "ubi8" ] ; then \
2020
tar \
2121
bzip2 \
2222
lz4 \
23-
#crunchy-backrest-${BACKREST_VER} \
23+
dumb-init \
24+
pgbackrest-${PGBACKREST_VERSION} \
2425
&& ${PACKAGER} -y clean all ; \
2526
else \
2627
${PACKAGER} -y install --nodocs \
@@ -30,8 +31,8 @@ else \
3031
bzip2 \
3132
lz4 \
3233
#postgresql${PGVERSION}-server \
34+
dumb-init \
3335
pgbackrest-${PGBACKREST_VERSION} \
34-
nano \
3536
&& ${PACKAGER} -y clean all ; \
3637
fi
3738

@@ -86,4 +87,4 @@ ENTRYPOINT ["/scripts/nss_wrapper/nss_wrapper.sh"]
8687

8788
USER 26
8889

89-
CMD ["/bin/sh", "/launch.sh", "init"]
90+
CMD ["dumb-init", "/launch.sh", "init"]

docker/postgres/Dockerfile

Lines changed: 16 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -13,11 +13,9 @@ ARG OLD_PG_VERSIONS
1313
ARG PG_SUPPORTED_VERSIONS="$PGVERSION"
1414
ARG PG_SUPPORTED_VERSIONS="$OLD_PG_VERSIONS $PGVERSION"
1515
ARG TIMESCALEDB="1.7.5 2.3.1 2.13.0"
16+
ARG ARCH
1617
# Spilo-specific
1718
ENV SET_USER=REL3_0_0 \
18-
#BG_MON_COMMIT=e22182e129ce357b5aa80090fba2aacfc42b1996 \
19-
#PG_AUTH_MON_COMMIT=52d90bbaa6d3c61195cd9532ebe036b1183808c5 \
20-
#PG_MON_COMMIT=54fbdcc3cfe7e2a626bd96dda644d9a0c6866b58 \
2119
PLPROFILER=REL4_1 \
2220
PG_PROFILE=0.3.6 \
2321
PAM_OAUTH2=v1.0.1 \
@@ -28,6 +26,7 @@ ENV SET_USER=REL3_0_0 \
2826
# Get some Standard-Stuff
2927
RUN if [ "$BASEOS" = "ubi8" ] ; then \
3028
${PACKAGER} -y update && ${PACKAGER} -y install --nodocs --noplugins --setopt=install_weak_deps=0 \
29+
--setopt=skip_missing_names_on_install=False \
3130
openssh-clients \
3231
openssh-server \
3332
shadow-utils \
@@ -40,15 +39,14 @@ RUN if [ "$BASEOS" = "ubi8" ] ; then \
4039
python3-psutil \
4140
python3-requests \
4241
python3-etcd \
43-
#python3-pyyaml \
42+
python3-pyyaml \
43+
python3-setuptools \
4444
git \
4545
patchutils \
4646
binutils \
4747
make \
4848
cmake \
4949
gcc \
50-
#curl \
51-
libcurl-devel \
5250
pam-devel \
5351
wget \
5452
mlocate \
@@ -58,11 +56,11 @@ RUN if [ "$BASEOS" = "ubi8" ] ; then \
5856
redhat-rpm-config \
5957
krb5-devel \
6058
busybox \
61-
dpkg \
6259
jq \
6360
rsync \
61+
dumb-init \
6462
libicu \
65-
pgbackrest-${PGBACKREST_VERSION} \
63+
pgbackrest-${PGBACKREST_VERSION} \
6664
&& ${PACKAGER} -y clean all ; \
6765
else \
6866
${PACKAGER} -y update && ${PACKAGER} -y install --nodocs --noplugins --setopt=install_weak_deps=0 \
@@ -78,17 +76,15 @@ else \
7876
python3-psycopg2 \
7977
python3-psutil \
8078
python3-requests \
81-
python3-etcd \
79+
#python3-etcd \
8280
python3-pyyaml \
83-
python3-etcd \
81+
python3-setuptools \
8482
git \
8583
patchutils \
8684
binutils \
8785
make \
8886
cmake \
8987
gcc \
90-
#curl \
91-
libcurl-devel \
9288
pam-devel \
9389
wget \
9490
mlocate \
@@ -98,7 +94,6 @@ else \
9894
redhat-rpm-config \
9995
krb5-devel \
10096
busybox \
101-
dpkg \
10297
jq \
10398
rsync \
10499
dumb-init \
@@ -107,71 +102,22 @@ else \
107102
&& ${PACKAGER} -y clean all ; \
108103
fi
109104

110-
# RUN ${PACKAGER} -y update && ${PACKAGER} -y install --nodocs --noplugins --setopt=install_weak_deps=0 \
111-
# openssh-clients \
112-
# openssh-server \
113-
# shadow-utils \
114-
# tar \
115-
# bzip2 \
116-
# lz4 \
117-
# python3 \
118-
# python3-pip \
119-
# python3-psycopg2 \
120-
# python3-psutil \
121-
# python3-requests \
122-
# python3-etcd \
123-
# #python3-pyyaml \
124-
# git \
125-
# patchutils \
126-
# binutils \
127-
# make \
128-
# cmake \
129-
# gcc \
130-
# #curl \
131-
# libcurl-devel \
132-
# pam-devel \
133-
# wget \
134-
# mlocate \
135-
# git-clang-format \
136-
# openssl-devel \
137-
# ccache \
138-
# redhat-rpm-config \
139-
# krb5-devel \
140-
# busybox \
141-
# dpkg \
142-
# jq \
143-
# rsync \
144-
# && ${PACKAGER} -y clean all;
145-
146105
# install etcdctl
147-
RUN ETCDVERSION=3.3.27 \
148-
&& curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-$(dpkg --print-architecture).tar.gz \
106+
RUN ETCDVERSION=3.5.13 \
107+
&& curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-${ARCH}.tar.gz \
149108
| tar xz -C /bin --strip=1 --wildcards --no-anchored --no-same-owner etcdctl etcd;
150109

151-
# Install Patroni
152-
# RUN pip3 install 'PyYAML<6.0' setuptools pystache loader dumb-init kazoo meld3 boto
153-
# #swiftclient
154-
# RUN ${PACKAGER} -y install --nodocs --noplugins --setopt=install_weak_deps=0 \
155-
# python3-etcd python3-consul \
156-
# python3-gevent python3-greenlet python3-cachetools \
157-
# python3-rsa python3-pyasn1-modules python3-cffi \
158-
# && ${PACKAGER} -y clean all;
159-
160-
# python3-kazoo python3-meld3 python3-boto python3-swiftclient
161-
162-
#RUN pip3 install patroni[kubernetes$EXTRAS]==$PATRONI_VERSION
163-
164-
#RUN mkdir /usr/lib/postgresql
165110
ENV PATHBACKUP = $PATH
166-
#ENV PATH=$PATH:/usr/pgsql-$PGVERSION/bin
167-
#RUN echo $PATH
111+
168112
# Install pam_oauth2.so
169113
#RUN #git clone -b $PAM_OAUTH2 --recurse-submodules https://github.com/zalando-pg/pam-oauth2.git \
170114
#&& make -C pam-oauth2 install \
171115
#&& curl -sL https://github.com/zalando-pg/bg_mon/archive/$BG_MON_COMMIT.tar.gz | tar xz \
172116
#&& curl -sL https://github.com/zalando-pg/pg_auth_mon/archive/$PG_AUTH_MON_COMMIT.tar.gz | tar xz \
173-
RUN pip3 install 'PyYAML<6.0' setuptools pystache loader dumb-init kazoo meld3 boto \
174-
&& pip3 install patroni[kubernetes$EXTRAS]==$PATRONI_VERSION \
117+
118+
#pip3.11 install 'PyYAML<6.0' setuptools pystache loader dumb-init kazoo meld3 boto \
119+
RUN pip3 install 'PyYAML<6.0' setuptools pystache loader dumb-init kazoo meld3 boto python-etcd --upgrade \
120+
&& pip3 install patroni[kubernetes$EXTRAS]==$PATRONI_VERSION --upgrade \
175121
&& mkdir /usr/lib/postgresql \
176122
&& curl -sL https://github.com/cybertec-postgresql/pg_permissions/archive/$PG_PERMISSIONS_COMMIT.tar.gz | tar xz \
177123
#&& curl -sL https://github.com/x4m/pg_tm_aux/archive/$PG_TM_AUX_COMMIT.tar.gz | tar xz \
@@ -349,7 +295,7 @@ RUN for version in $PG_SUPPORTED_VERSIONS; do \
349295
done
350296

351297
# Clean Up Packages binutils glibc-devel glibc-static
352-
RUN ${PACKAGER} -y remove python3-pip python3-wheel python3-dev git patchutils flatpak glibc-static gcc glibc-devel \
298+
RUN ${PACKAGER} -y remove python3.11-pip python3.11-wheel python3.11-dev git patchutils flatpak glibc-static gcc glibc-devel \
353299
#&& ${PACKAGER} -y autoremove \
354300
&& ${PACKAGER} -y clean dbcache \
355301
&& ${PACKAGER} -y clean all;

launcher/pgbackrest/launch.sh

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,11 @@ PGBACKREST_PATH=${PGBACKREST_PATH:-'/opt/pgbackrest'}
66
source "${PGBACKREST_PATH}/bin/shell_lib.sh"
77
output_info "Start pgBackRest-PreCondition-Check"
88

9-
if [ "$MODE" == "pgbackrest" ] && [ "$COMMAND" == "backup" ]; then
9+
if [ "$MODE" == "pgbackrest" ] && [ "$COMMAND" == "repo-host" ]; then
10+
output_info "pgBackRest: Start Repo-Host"
11+
pgbackrest server
12+
13+
elif [ "$MODE" == "pgbackrest" ] && [ "$COMMAND" == "backup" ]; then
1014
output_info "pgBackRest: Backup-Job found"
1115
source "${PGBACKREST_PATH}/bin/backup/start.sh"
1216
output_success "pgBackRest: Backup-Job completed"

major_upgrade/inplace_upgrade.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -490,7 +490,7 @@ def do_upgrade(self):
490490
self.cluster_version, self.desired_version)
491491
return True
492492

493-
if not (self.postgresql.is_running() and self.postgresql.is_leader()):
493+
if not (self.postgresql.is_running() and self.postgresql.is_primary()):
494494
return logger.error('PostgreSQL is not running or in recovery')
495495

496496
cluster = self.dcs.get_cluster()

major_upgrade/pg_upgrade.py

Lines changed: 34 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,9 @@ def restore_shared_preload_libraries(self):
3535
return True
3636

3737
def start_old_cluster(self, config, version):
38-
self.set_bin_dir(version)
38+
self._new_bin_dir = self._bin_dir
39+
self.set_bin_dir_for_version(version)
40+
self._old_bin_dir = self._bin_dir
3941

4042
# make sure we don't archive wals from the old version
4143
self._old_config_values = {'archive_mode': self.config.get('parameters').get('archive_mode')}
@@ -50,15 +52,17 @@ def get_cluster_version(self):
5052
with open(self._version_file) as f:
5153
return f.read().strip()
5254

53-
def set_bin_dir(self, version):
55+
def set_bin_dir_for_version(self, version):
5456
from spilo_commons import get_bin_dir
57+
self.set_bin_dir(get_bin_dir(version))
5558

56-
self._old_bin_dir = self._bin_dir
57-
self._bin_dir = get_bin_dir(version)
59+
def set_bin_dir(self, bin_dir):
60+
self._bin_dir = bin_dir
61+
self._available_gucs = None
5862

5963
@property
6064
def local_conn_kwargs(self):
61-
conn_kwargs = self.config.local_connect_kwargs
65+
conn_kwargs = self.connection_pool.conn_kwargs
6266
conn_kwargs['options'] = '-c synchronous_commit=local -c statement_timeout=0 -c search_path='
6367
conn_kwargs.pop('connect_timeout', None)
6468
return conn_kwargs
@@ -89,9 +93,10 @@ def drop_possibly_incompatible_objects(self):
8993
conn_kwargs['dbname'] = d
9094
with get_connection_cursor(**conn_kwargs) as cur:
9195

92-
cmd = "REVOKE EXECUTE ON FUNCTION pg_catalog.pg_switch_{0}() FROM admin".format(self.wal_name)
93-
logger.info('Executing "%s" in the database="%s"', cmd, d)
94-
cur.execute(cmd)
96+
# Role admin not existing in the containers
97+
# cmd = "REVOKE EXECUTE ON FUNCTION pg_catalog.pg_switch_{0}() FROM admin".format(self.wal_name)
98+
# logger.info('Executing "%s" in the database="%s"', cmd, d)
99+
# cur.execute(cmd)
95100

96101
logger.info('Executing "DROP FUNCTION metric_helpers.pg_stat_statements" in the database="%s"', d)
97102
cur.execute("DROP FUNCTION IF EXISTS metric_helpers.pg_stat_statements(boolean) CASCADE")
@@ -117,9 +122,15 @@ def update_extensions(self):
117122
for d in self._get_all_databases():
118123
conn_kwargs['dbname'] = d
119124
with get_connection_cursor(**conn_kwargs) as cur:
120-
cur.execute('SELECT quote_ident(extname) FROM pg_catalog.pg_extension')
121-
for extname in cur.fetchall():
122-
query = 'ALTER EXTENSION {0} UPDATE'.format(extname[0])
125+
cur.execute('SELECT quote_ident(extname), extversion FROM pg_catalog.pg_extension')
126+
for extname, version in cur.fetchall():
127+
# require manual update to 5.X+
128+
if extname == 'pg_partman' and int(version[0]) < 5:
129+
logger.warning("Skipping update of '%s' in database=%s. "
130+
"Extension version: %s. Consider manual update",
131+
extname, d, version)
132+
continue
133+
query = 'ALTER EXTENSION {0} UPDATE'.format(extname)
123134
logger.info("Executing '%s' in the database=%s", query, d)
124135
try:
125136
cur.execute(query)
@@ -168,7 +179,7 @@ def pg_upgrade(self, check=False):
168179
os.chdir(upgrade_dir)
169180

170181
pg_upgrade_args = ['-k', '-j', str(psutil.cpu_count()),
171-
'-b', self._old_bin_dir, '-B', self._bin_dir,
182+
'-b', self._old_bin_dir, '-B', self._new_bin_dir,
172183
'-d', self._data_dir, '-D', self._new_data_dir,
173184
'-O', "-c timescaledb.restoring='on'",
174185
'-O', "-c archive_mode='off'"]
@@ -180,19 +191,23 @@ def pg_upgrade(self, check=False):
180191
else:
181192
self.config.write_postgresql_conf()
182193

194+
self.set_bin_dir(self._new_bin_dir)
195+
183196
logger.info('Executing pg_upgrade%s', (' --check' if check else ''))
184197
if subprocess.call([self.pgcommand('pg_upgrade')] + pg_upgrade_args) == 0:
198+
if check:
199+
self.set_bin_dir(self._old_bin_dir)
185200
os.chdir(old_cwd)
186201
shutil.rmtree(upgrade_dir)
187202
return True
188203

189204
def prepare_new_pgdata(self, version):
190205
from spilo_commons import append_extensions
191206

192-
locale = self.query('SHOW lc_collate').fetchone()[0]
193-
encoding = self.query('SHOW server_encoding').fetchone()[0]
207+
locale = self.query('SHOW lc_collate')[0][0]
208+
encoding = self.query('SHOW server_encoding')[0][0]
194209
initdb_config = [{'locale': locale}, {'encoding': encoding}]
195-
if self.query("SELECT current_setting('data_checksums')::bool").fetchone()[0]:
210+
if self.query("SELECT current_setting('data_checksums')::bool")[0][0]:
196211
initdb_config.append('data-checksums')
197212

198213
logger.info('initdb config: %s', initdb_config)
@@ -206,7 +221,9 @@ def prepare_new_pgdata(self, version):
206221
old_version_file = self._version_file
207222
self._version_file = os.path.join(self._data_dir, 'PG_VERSION')
208223

209-
self.set_bin_dir(version)
224+
self._old_bin_dir = self._bin_dir
225+
self.set_bin_dir_for_version(version)
226+
self._new_bin_dir = self._bin_dir
210227

211228
# shared_preload_libraries for the old cluster, cleaned from incompatible/missing libs
212229
old_shared_preload_libraries = self.config.get('parameters').get('shared_preload_libraries')
@@ -239,6 +256,7 @@ def prepare_new_pgdata(self, version):
239256
self._new_data_dir, self._data_dir = self._data_dir, self._new_data_dir
240257
self.config._postgresql_conf = old_postgresql_conf
241258
self._version_file = old_version_file
259+
self.set_bin_dir(self._old_bin_dir)
242260

243261
if old_shared_preload_libraries:
244262
self.config.get('parameters')['shared_preload_libraries'] = old_shared_preload_libraries

scripts/configure_spilo.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@
3232
PROVIDER_LOCAL = "local"
3333
PROVIDER_UNSUPPORTED = "unsupported"
3434
USE_KUBERNETES = os.environ.get('KUBERNETES_SERVICE_HOST') is not None
35-
KUBERNETES_DEFAULT_LABELS = '{"application": "cpo"}'
35+
KUBERNETES_DEFAULT_LABELS = '{"application": "cpo", "member.cpo.opensource.cybertec.at/type": "postgres"}'
3636
PATRONI_DCS = ('kubernetes', 'zookeeper', 'exhibitor', 'consul', 'etcd3', 'etcd')
3737
AUTO_ENABLE_WALG_RESTORE = ('WAL_S3_BUCKET', 'WALE_S3_PREFIX', 'WALG_S3_PREFIX', 'WALG_AZ_PREFIX', 'WALG_SSH_PREFIX')
3838
WALG_SSH_NAMES = ['WALG_SSH_PREFIX', 'SSH_PRIVATE_KEY_PATH', 'SSH_USERNAME', 'SSH_PORT']
@@ -1082,7 +1082,7 @@ def main():
10821082

10831083
if provider == PROVIDER_LOCAL and not any(1 for key in config.keys() if key in PATRONI_DCS):
10841084
link_runit_service(placeholders, 'etcd')
1085-
config['etcd'] = {'host': '127.0.0.1:2379'}
1085+
config['etcd3'] = {'host': '127.0.0.1:2379'}
10861086

10871087
pgdata = config['postgresql']['data_dir']
10881088
version_file = os.path.join(pgdata, 'PG_VERSION')

scripts/pgbackrest/repo-host/start.sh

Whitespace-only changes.

0 commit comments

Comments
 (0)