Skip to content

Commit f1cb621

Browse files
SchmaetzSchmaetz
authored
Merge pull request #20 from cybertec-postgresql/pgbackrest-server
Pgbackrest server
2 parents d4573a7 + b0b418f commit f1cb621

5 files changed

Lines changed: 117 additions & 125 deletions

File tree

‎bootstrap/scripts/configure_spilo.py‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@
3939

4040

4141
def parse_args():
42-
sections = ['all', 'patroni', 'certificate', 'wal-e', 'crontab',
42+
sections = ['all', 'patroni', 'certificate', 'pgbackrest', 'wal-e', 'crontab',
4343
'pam-oauth2', 'pgbouncer', 'bootstrap', 'standby-cluster', 'log']
4444
argp = argparse.ArgumentParser(description='Configures Spilo',
4545
epilog="Choose from the following sections:\n\t{}".format('\n\t'.join(sections)),
@@ -555,6 +555,9 @@ def get_placeholders(provider):
555555
# in Kubernetes with Etcd in a non-default namespace
556556
placeholders.setdefault('NAMESPACE', placeholders.get('POD_NAMESPACE', 'default')
557557
if USE_KUBERNETES and placeholders.get('DCS_ENABLE_KUBERNETES_API') else '')
558+
#pgBackRest
559+
placeholders.setdefault('USE_PGBACKREST', False)
560+
placeholders.setdefault('PGBACKREST_SERVER', False)
558561
# use namespaces to set WAL bucket prefix scope naming the folder namespace-clustername for non-default namespace.
559562
placeholders.setdefault('WAL_BUCKET_SCOPE_PREFIX', '{0}-'.format(placeholders['NAMESPACE'])
560563
if placeholders['NAMESPACE'] not in ('default', '') else '')
@@ -1115,6 +1118,9 @@ def main():
11151118
elif section == 'log':
11161119
if bool(placeholders.get('LOG_S3_BUCKET')):
11171120
write_log_environment(placeholders)
1121+
elif section == 'pgbackrest':
1122+
if placeholders['PGBACKREST_SERVER']:
1123+
link_runit_service(placeholders, 'pgbackrest')
11181124
elif section == 'wal-e':
11191125
if placeholders['USE_WALE']:
11201126
write_wale_environment(placeholders, '', args['force'])

‎docker/postgres/Dockerfile‎

Lines changed: 64 additions & 96 deletions
Original file line numberDiff line numberDiff line change
@@ -36,11 +36,11 @@ RUN if [ "$BASEOS" = "ubi8" ] ; then \
3636
python3 \
3737
python3-pip \
3838
python3-psycopg2 \
39-
python3-psutil \
40-
python3-requests \
41-
python3-etcd \
42-
python3-pyyaml \
43-
python3-setuptools \
39+
# python3-psutil \
40+
# python3-requests \
41+
# python3-etcd \
42+
# python3-pyyaml \
43+
# python3-setuptools \
4444
git \
4545
patchutils \
4646
binutils \
@@ -74,11 +74,11 @@ else \
7474
python3 \
7575
python3-pip \
7676
python3-psycopg2 \
77-
python3-psutil \
78-
python3-requests \
79-
#python3-etcd \
80-
python3-pyyaml \
81-
python3-setuptools \
77+
# python3-psutil \
78+
# python3-requests \
79+
# python3-etcd \
80+
# python3-pyyaml \
81+
# python3-setuptools \
8282
git \
8383
patchutils \
8484
binutils \
@@ -115,8 +115,7 @@ ENV PATHBACKUP = $PATH
115115
#&& curl -sL https://github.com/zalando-pg/bg_mon/archive/$BG_MON_COMMIT.tar.gz | tar xz \
116116
#&& curl -sL https://github.com/zalando-pg/pg_auth_mon/archive/$PG_AUTH_MON_COMMIT.tar.gz | tar xz \
117117

118-
#pip3.11 install 'PyYAML<6.0' setuptools pystache loader dumb-init kazoo meld3 boto \
119-
RUN pip3 install 'PyYAML<6.0' setuptools pystache loader dumb-init kazoo meld3 boto python-etcd --upgrade \
118+
RUN pip3 install 'PyYAML<6.0' setuptools pystache loader dumb-init kazoo meld3 boto python-etcd psutil requests --upgrade \
120119
&& pip3 install patroni[kubernetes$EXTRAS]==$PATRONI_VERSION --upgrade \
121120
&& mkdir /usr/lib/postgresql \
122121
&& curl -sL https://github.com/cybertec-postgresql/pg_permissions/archive/$PG_PERMISSIONS_COMMIT.tar.gz | tar xz \
@@ -139,31 +138,12 @@ RUN pip3 install 'PyYAML<6.0' setuptools pystache loader dumb-init kazoo meld3 b
139138
&& for version in $PG_SUPPORTED_VERSIONS; do \
140139
${PACKAGER} -y update && \
141140
${PACKAGER} -y install postgresql${version}-pltcl \
142-
# postgresql${version}-dirtyread \
143-
# postgresql${version}-extra-window-functions \
144-
# postgresql${version}-first-last-agg \
145-
# postgresql${version}-hll \
146-
# postgresql${version}-hypopg \
147141
pgaudit*${version} \
148142
credcheck_*${version} \
149-
# postgresql${version}-pg-checksums \
150-
# postgresql${version}-pgl-ddl-deploy \
151-
# postgresql${version}-pglogical \
152-
# postgresql${version}-pglogical-ticker \
153-
# postgresql${version}-pgq-node \
154-
# postgresql${version}-pldebugger \
155-
# postgresql${version}-pllua \
156-
# postgresql${version}-plpgsql-check \
157-
# postgresql${version}-plproxy \
158-
# postgresql${version}-repack \
159-
# postgresql${version}-wal2json \
160-
#&& ${PACKAGER} -y clean all; \
161143
# Install PostgreSQL binaries, contrib, plproxy and multiple pl's
162144
&& ${PACKAGER} -y install -y postgresql${version}-contrib \
163145
postgresql${version}-plpython3 postgresql${version}-devel \
164146
pg_cron_${version} \
165-
#postgresql-${version}-pgq3 \
166-
#postgresql-${version}-pg-stat-kcache $EXTRAS \
167147
# Modify for using origial-spilo scripts
168148
&& ln -s /usr/pgsql-${version} /usr/lib/postgresql/${version} \
169149
&& export PATH=$PATHBACKUP:/usr/pgsql-${version}/bin \
@@ -189,15 +169,23 @@ RUN pip3 install 'PyYAML<6.0' setuptools pystache loader dumb-init kazoo meld3 b
189169
done \
190170
#&& cd /pgextwlist && make clean && make && make install \
191171
#&& cd /pg_stat_kcache && make clean && make && make install \
192-
&& cd /pgnodemx && make USE_PGXS=1 clean && make USE_PGXS=1 && make USE_PGXS=1 install \
172+
&& cd /pgnodemx && make USE_PGXS=1 clean && make USE_PGXS=1 && make USE_PGXS=1 install; \
193173
#&& cd /pgqd && ./configure --prefix=/opt && make && make install \
194-
&& ${PACKAGER} -y clean all; \
195-
done
174+
done \
175+
&& ${PACKAGER} -y clean all \
176+
&& rm -rf /pgextwlist /pg_stat_kcache /pgnodemx;
177+
178+
COPY runit-2.1.2.tar.gz /package/runit-2.1.2.tar.gz
179+
180+
RUN ${PACKAGER} -y install --nodocs --noplugins --setopt=install_weak_deps=0 glibc-devel glibc-static \
181+
&& ${PACKAGER} -y clean all \
182+
&& mkdir -p /package \
183+
&& cd /package && tar -xvzf runit-2.1.2.tar.gz && rm runit-2.1.2.tar.gz \
184+
&& cd admin/runit-2.1.2 && package/install \
185+
&& ln -s /usr/local/bin/runsvdir /usr/bin/runsvdir \
186+
&& ${PACKAGER} -y remove $(rpm -qa "*-devel") \
187+
&& ${PACKAGER} -y clean all;
196188

197-
# Clean Up git-repos
198-
RUN rm -rf /pgextwlist \
199-
&& rm -rf /pg_stat_kcache \
200-
&& rm -rf /pgnodemx ;
201189

202190
EXPOSE 5432 8008 8080
203191
ENV LC_ALL=en_US.utf-8 \
@@ -220,70 +208,50 @@ COPY motd /etc/
220208

221209
COPY runit /etc/runit/runsvdir/default/
222210
COPY pgq_ticker.ini $PGHOME/
223-
RUN rm -rf /etc/service
224-
225-
RUN mkdir /home/postgres/pgdata && chown -R postgres:postgres /home/postgres/pgdata && chmod -R g=u /home/postgres/pgdata
226-
227-
RUN sed -i "s|/var/lib/pgsql.*|$PGHOME:/bin/bash|" /etc/passwd \
228-
&& chown -R postgres:postgres $PGHOME $PGHOME/pgdata $RW_DIR \
229-
&& rm -fr /var/spool/cron /var/tmp \
230-
&& mkdir -p /var/spool \
231-
&& ln -s $RW_DIR/cron /var/spool/cron \
232-
&& ln -s $RW_DIR/tmp /var/tmp \
233-
&& for d in /etc/runit/runsvdir/default/*; do \
234-
chmod 755 $d/* \
235-
&& ln -s /run/supervise/$(basename $d) $d/supervise; \
236-
done \
237-
&& ln -snf $RW_DIR/service /etc/service \
238-
#&& ln -s $RW_DIR/pam.d-postgresql /etc/pam.d/postgresql \
239-
&& ln -s $RW_DIR/postgres.yml $PGHOME/postgres.yml \
240-
&& ln -s $RW_DIR/.bash_history /root/.bash_history \
241-
&& ln -s $RW_DIR/postgresql/.bash_history $PGHOME/.bash_history \
242-
&& ln -s $RW_DIR/postgresql/.psql_history $PGHOME/.psql_history \
243-
&& ln -s $RW_DIR/etc $PGHOME/etc \
244-
&& for d in $PGHOME /root; do \
245-
d=$d/.config/patroni \
246-
&& mkdir -p $d \
247-
&& ln -s $PGHOME/postgres.yml $d/patronictl.yaml; \
248-
done \
249-
&& sed -i 's/set compatible/set nocompatible/' /etc/vimrc \
250-
&& echo "PATH=\"$PATH\"" > /etc/environment \
251-
&& for e in TERM=linux LC_ALL=C.UTF-8 LANG=C.UTF-8 EDITOR=editor; \
252-
do echo "export $e" >> /etc/bash.bashrc; \
253-
done \
254-
&& ln -s /etc/skel/.bashrc $PGHOME/.bashrc \
255-
&& echo "source /etc/motd" >> /root/.bashrc \
256-
# Allow users in the root group to access the following files and dirs
257-
&& if [ "$COMPRESS" != "true" ]; then \
258-
chmod 664 /etc/passwd \
259-
&& chmod o+r /etc/shadow \
260-
&& chgrp -R 0 $PGHOME $RW_DIR \
261-
&& chmod -R g=u $PGHOME $RW_DIR \
262-
&& usermod -a -G root postgres; \
263-
fi
211+
RUN rm -rf /etc/service && mkdir /home/postgres/pgdata && chown -R postgres:postgres /home/postgres/pgdata && chmod -R g=u /home/postgres/pgdata \
212+
&& sed -i "s|/var/lib/pgsql.*|$PGHOME:/bin/bash|" /etc/passwd \
213+
&& chown -R postgres:postgres $PGHOME $PGHOME/pgdata $RW_DIR \
214+
&& rm -fr /var/spool/cron /var/tmp \
215+
&& mkdir -p /var/spool \
216+
&& ln -s $RW_DIR/cron /var/spool/cron \
217+
&& ln -s $RW_DIR/tmp /var/tmp \
218+
&& for d in /etc/runit/runsvdir/default/*; do \
219+
chmod 755 $d/* \
220+
&& ln -s /run/supervise/$(basename $d) $d/supervise; \
221+
done \
222+
&& ln -snf $RW_DIR/service /etc/service \
223+
#&& ln -s $RW_DIR/pam.d-postgresql /etc/pam.d/postgresql \
224+
&& ln -s $RW_DIR/postgres.yml $PGHOME/postgres.yml \
225+
&& ln -s $RW_DIR/.bash_history /root/.bash_history \
226+
&& ln -s $RW_DIR/postgresql/.bash_history $PGHOME/.bash_history \
227+
&& ln -s $RW_DIR/postgresql/.psql_history $PGHOME/.psql_history \
228+
&& ln -s $RW_DIR/etc $PGHOME/etc \
229+
&& for d in $PGHOME /root; do \
230+
d=$d/.config/patroni \
231+
&& mkdir -p $d \
232+
&& ln -s $PGHOME/postgres.yml $d/patronictl.yaml; \
233+
done \
234+
&& sed -i 's/set compatible/set nocompatible/' /etc/vimrc \
235+
&& echo "PATH=\"$PATH\"" > /etc/environment \
236+
&& for e in TERM=linux LC_ALL=C.UTF-8 LANG=C.UTF-8 EDITOR=editor; \
237+
do echo "export $e" >> /etc/bash.bashrc; \
238+
done \
239+
&& ln -s /etc/skel/.bashrc $PGHOME/.bashrc \
240+
&& echo "source /etc/motd" >> /root/.bashrc \
241+
# Allow users in the root group to access the following files and dirs
242+
&& if [ "$COMPRESS" != "true" ]; then \
243+
chmod 664 /etc/passwd \
244+
&& chmod o+r /etc/shadow \
245+
&& chgrp -R 0 $PGHOME $RW_DIR \
246+
&& chmod -R g=u $PGHOME $RW_DIR \
247+
&& usermod -a -G root postgres; \
248+
fi
264249

265250
# Remove default pgbackrest-config and spool-path
266251
RUN rm /etc/pgbackrest.conf && rm -rf /var/spool/pgbackrest;
267252

268253
COPY scripts bootstrap major_upgrade /scripts/
269254
COPY launcher/postgres/launch.sh /
270-
271-
RUN ${PACKAGER} -y install --nodocs --noplugins --setopt=install_weak_deps=0 \
272-
glibc-devel glibc-static \
273-
&& mkdir -p /package;
274-
275-
COPY runit-2.1.2.tar.gz /package/runit-2.1.2.tar.gz
276-
#wget -nv http://smarden.org/runit/runit-2.1.2.tar.gz \
277-
278-
RUN cd /package && tar -xvzf runit-2.1.2.tar.gz && rm runit-2.1.2.tar.gz \
279-
&& cd admin/runit-2.1.2 && package/install \
280-
&& ${PACKAGER} -y remove $(rpm -qa "*-devel") \
281-
&& ${PACKAGER} -y clean all ;
282-
283-
# Modify for using origial-spilo scripts
284-
# RUN ln -s /usr/local/bin/dumb-init /usr/bin/dumb-init &&
285-
RUN ln -s /usr/local/bin/runsvdir /usr/bin/runsvdir
286-
287255
#RUN curl -LO https://dl.k8s.io/release/v1.26.0/bin/linux/amd64/kubectl && chmod +x kubectl && mv ./kubectl /usr/local/bin/
288256

289257
# Cleanup

‎launcher/pgbackrest/launch.sh‎

Lines changed: 35 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -4,37 +4,46 @@
44
#Define Path
55
PGBACKREST_PATH=${PGBACKREST_PATH:-'/opt/pgbackrest'}
66
source "${PGBACKREST_PATH}/bin/shell_lib.sh"
7+
78
output_info "Start pgBackRest-PreCondition-Check"
89

9-
if [ "$MODE" == "pgbackrest" ] && [ "$COMMAND" == "repo-host" ]; then
10-
output_info "pgBackRest: Start Repo-Host"
11-
pgbackrest server
10+
if [ "$USE_PGBACKREST" == true ]; then
11+
output_info "Check if RepoHost-Server needs to start"
12+
if [ "$USE_PGBACKREST" == true ] && [ "$PGBACKREST_SERVER" == true ]; then
13+
output_info "pgBackRest: Start Repo-Host"
14+
pgbackrest server &
15+
else
16+
output_info "RepoHost-Server not needed. Skip Step"
17+
fi
1218

13-
elif [ "$MODE" == "pgbackrest" ] && [ "$COMMAND" == "backup" ]; then
14-
output_info "pgBackRest: Backup-Job found"
15-
source "${PGBACKREST_PATH}/bin/backup/start.sh"
16-
output_success "pgBackRest: Backup-Job completed"
17-
else
18-
#For Restore with pgBackrest
19-
if [ "$RESTORE_ENABLE" == "true" ]; then
20-
output_info "pgBackRest: Restore-Job found"
21-
source "${PGBACKREST_PATH}/bin/restore/start.sh"
22-
output_success "Restore-Job completed"
19+
if [ "$USE_PGBACKREST" == true ] && [ "$PGBACKREST_MODE" == "backup" ]; then
20+
output_info "pgBackRest: Backup-Job found"
21+
source "${PGBACKREST_PATH}/bin/backup/start.sh"
22+
output_success "pgBackRest: Backup-Job completed"
2323
else
24-
output_info "Restore not defined - Skip Restore-Step"
25-
if [ "$RESTORE_BASEBACKUP" == "false" ]; then
26-
output_info "pgBackRest: Backup-Job found"
27-
export SELECTOR="cluster-name=${SCOPE},spilo-role=master"
28-
export COMMAND_OPTS="--type=full --stanza=db --repo=1"
29-
source "${PGBACKREST_PATH}/bin/backup/start.sh"
30-
output_info "pgBackRest: Update Restore-Configmap"
31-
configmap="${SCOPE}-pgbackrest-restore"
32-
kubectl get cm $configmap -o yaml | \
33-
sed -e 's|restore_basebackup: "false"|restore_basebackup: "true"|' | \
34-
kubectl apply -f -
35-
output_success "pgBackRest: Backup-Job completed"
24+
#For Restore with pgBackrest
25+
if [ "$RESTORE_ENABLE" == "true" ]; then
26+
output_info "pgBackRest: Restore-Job found"
27+
source "${PGBACKREST_PATH}/bin/restore/start.sh"
28+
output_success "Restore-Job completed"
3629
else
37-
output_info "Basebackup not defined - Skip create basebackup"
30+
output_info "Restore not defined - Skip Restore-Step"
31+
if [ "$RESTORE_BASEBACKUP" == "false" ]; then
32+
output_info "pgBackRest: Backup-Job found"
33+
export SELECTOR="cluster-name=${SCOPE},spilo-role=master"
34+
export COMMAND_OPTS="--type=full --stanza=db --repo=1"
35+
source "${PGBACKREST_PATH}/bin/backup/start.sh"
36+
output_info "pgBackRest: Update Restore-Configmap"
37+
configmap="${SCOPE}-pgbackrest-restore"
38+
kubectl get cm $configmap -o yaml | \
39+
sed -e 's|restore_basebackup: "false"|restore_basebackup: "true"|' | \
40+
kubectl apply -f -
41+
output_success "pgBackRest: Backup-Job completed"
42+
else
43+
output_info "Basebackup not defined - Skip create basebackup"
44+
fi
3845
fi
3946
fi
47+
else
48+
output_info "pgBackRest not used. Skip Container"
4049
fi

‎runit/pgbackrest/run‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
#!/bin/sh -e
2+
3+
exec 2>&1
4+
exec pgbackrest server

‎scripts/configure_spilo.py‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@
3939

4040

4141
def parse_args():
42-
sections = ['all', 'patroni', 'certificate', 'wal-e', 'crontab',
42+
sections = ['all', 'patroni', 'certificate', 'pgbackrest', 'wal-e', 'crontab',
4343
'pam-oauth2', 'pgbouncer', 'bootstrap', 'standby-cluster', 'log']
4444
argp = argparse.ArgumentParser(description='Configures Spilo',
4545
epilog="Choose from the following sections:\n\t{}".format('\n\t'.join(sections)),
@@ -563,14 +563,16 @@ def get_placeholders(provider):
563563
# in Kubernetes with Etcd in a non-default namespace
564564
placeholders.setdefault('NAMESPACE', placeholders.get('POD_NAMESPACE', 'default')
565565
if USE_KUBERNETES and placeholders.get('DCS_ENABLE_KUBERNETES_API') else '')
566+
#pgBackRest
567+
placeholders.setdefault('USE_PGBACKREST', False)
568+
placeholders.setdefault('PGBACKREST_SERVER', False)
566569
# use namespaces to set WAL bucket prefix scope naming the folder namespace-clustername for non-default namespace.
567570
placeholders.setdefault('WAL_BUCKET_SCOPE_PREFIX', '{0}-'.format(placeholders['NAMESPACE'])
568571
if placeholders['NAMESPACE'] not in ('default', '') else '')
569572
placeholders.setdefault('WAL_BUCKET_SCOPE_SUFFIX', '')
570573
placeholders.setdefault('WAL_RESTORE_TIMEOUT', '0')
571574
placeholders.setdefault('WALE_ENV_DIR', os.path.join(placeholders['RW_DIR'], 'etc', 'wal-e.d', 'env'))
572575
placeholders.setdefault('USE_WALE', False)
573-
placeholders.setdefault('USE_PGBACKREST', False)
574576
cpu_count = str(min(psutil.cpu_count(), 10))
575577
placeholders.setdefault('WALG_DOWNLOAD_CONCURRENCY', cpu_count)
576578
placeholders.setdefault('WALG_UPLOAD_CONCURRENCY', cpu_count)
@@ -1131,6 +1133,9 @@ def main():
11311133
elif section == 'log':
11321134
if bool(placeholders.get('LOG_S3_BUCKET')):
11331135
write_log_environment(placeholders)
1136+
elif section == 'pgbackrest':
1137+
if placeholders['PGBACKREST_SERVER']:
1138+
link_runit_service(placeholders, 'pgbackrest')
11341139
elif section == 'wal-e':
11351140
if placeholders['USE_WALE']:
11361141
write_wale_environment(placeholders, '', args['force'])

0 commit comments

Comments
 (0)