33590 - OCP Colin Extract Job Setup

jobs/colin-extract-refresh/.dockerignore

@@ -0,0 +1,19 @@
+.env
+.env.*
+.envrc
+.env.local
+
+.venv
+venv
+__pycache__
+.pytest_cache
+.ruff_cache
+*.pytest_cache
+.git
+.gitignore
+Makefile
+openshift
+README.md
+*.md
+Dockerfile
+.dockerignore

jobs/colin-extract-refresh/.env.sample

@@ -0,0 +1,12 @@
+
+DATABASE_USERNAME_COLIN_ORACLE=
+DATABASE_PASSWORD_COLIN_ORACLE=
+DATABASE_NAME_COLIN_ORACLE=
+DATABASE_HOST_COLIN_ORACLE=
+DATABASE_PORT_COLIN_ORACLE=
+
+DATABASE_USERNAME_COLIN_MIGR=
+DATABASE_PASSWORD_COLIN_MIGR=
+DATABASE_NAME_COLIN_MIGR=
+DATABASE_HOST_COLIN_MIGR=
+DATABASE_PORT_COLIN_MIGR=

jobs/colin-extract-refresh/Dockerfile

@@ -0,0 +1,79 @@
+FROM python:3.12-slim
+
+ARG VCS_REF="missing"
+ARG BUILD_DATE="missing"
+
+ENV VCS_REF=${VCS_REF}
+ENV BUILD_DATE=${BUILD_DATE}
+ENV PYTHONUNBUFFERED=1
+
+
+
+
+
+LABEL org.label-schema.vcs-ref=${VCS_REF} \
+      org.label-schema.build-date=${BUILD_DATE}
+
+
+USER root
+
+ENV ORACLE_CLIENT_LIB_DIR=/opt/oracle/instantclient_21_4
+ENV LD_LIBRARY_PATH=${ORACLE_CLIENT_LIB_DIR}
+
+
+WORKDIR /opt/oracle/
+RUN set -eux; \
+    apt-get update; \
+    apt-get install -y --no-install-recommends unzip wget ca-certificates;\
+    apt-get install -y --no-install-recommends libaio1 || apt-get install -y --no-install-recommends libaio1t64; \
+    ln -sf /usr/lib/x86_64-linux-gnu/libaio.so.1t64 /usr/lib/x86_64-linux-gnu/libaio.so.1 || true; \
+    ln -sf /lib/x86_64-linux-gnu/libaio.so.1t64 /lib/x86_64-linux-gnu/libaio.so.1 || true; \
+    wget -q https://download.oracle.com/otn_software/linux/instantclient/214000/instantclient-basic-linux.x64-21.4.0.0.0dbru.zip; \
+    unzip instantclient-basic-linux.x64-21.4.0.0.0dbru.zip; \
+    rm instantclient-basic-linux.x64-21.4.0.0.0dbru.zip; \
+    cd "${ORACLE_CLIENT_LIB_DIR}"; \
+    rm -f *jdbc* *occi* *mysql* *README *jar uidrvci genezi adrci; \
+    test -e libclntsh.so || ln -sf libclntsh.so.* libclntsh.so; \
+    test -f libclntsh.so; \
+    echo "${ORACLE_CLIENT_LIB_DIR}" > /etc/ld.so.conf.d/oracle-instantclient.conf; \
+    ldconfig; \
+    chmod 755 "${ORACLE_CLIENT_LIB_DIR}"; \
+    rm -rf /var/lib/apt/lists/*
+
+ENV DBSCHEMA_HOME=/opt/DbSchema
+ENV DBSCHEMA_VERSION=9_7_1
+
+RUN set -eux; \
+    apt-get update; \
+    apt-get install -y --no-install-recommends default-jre-headless curl;\
+    curl -fsSL "https://www.dbschema.com/download/dbschema_linux_9_7_1.tar.gz" -o /tmp/dbschema.tar.gz;\
+    tar -xzf /tmp/dbschema.tar.gz -C /opt; \
+    rm /tmp/dbschema.tar.gz; \
+    mkdir -p "${DBSCHEMA_HOME}/lib"; \
+    curl -fsSL "https://repo1.maven.org/maven2/org/testfx/openjfx-monocle/17.0.10/openjfx-monocle-17.0.10.jar" -o "${DBSCHEMA_HOME}/lib/monocle.jar"; \
+    ln -sf "${DBSCHEMA_HOME}/DbSchemaCLI" /usr/local/bin/dbschemacli; \
+    test -x "${DBSCHEMA_HOME}/DbSchemaCLI"; \
+    rm -rf /var/lib/apt/lists/*
+
+
+
+# Create directories with proper permissions
+RUN mkdir -p /opt/app-root && \
+    chmod 755 /opt/app-root
+
+WORKDIR /opt/app-root
+COPY requirements.txt .
+RUN apt-get update; \
+    apt-get install -y --no-install-recommends git;\
+    pip install --no-cache-dir -r requirements.txt;\
+    rm -rf /var/lib/apt/lists/*
+
+ENV PATH="${DBSCHEMA_HOME}:${PATH}"
+
+USER 1001
+
+EXPOSE 8080
+
+COPY src .
+
+CMD [ "python", "src/test_connectivity.py" ]

jobs/colin-extract-refresh/Makefile

@@ -0,0 +1,41 @@
+.PHONY: help setup install build build-nc run
+
+MKFILE_PATH:=$(abspath $(lastword $(MAKEFILE_LIST)))
+CURRENT_ABS_DIR:=$(patsubst %/,%,$(dir $(MKFILE_PATH)))
+
+DOCKER_NAME:=colin-extract-refresh
+TAG_NAME ?= dev
+DOCKER_PLATFORM ?= --platform linux/amd64
+
+#################################################################################
+# COMMANDS -- Setup                                                             #
+#################################################################################
+setup: install  ## Setup the project
+
+install:  ## Install python virtual environment
+	test -f .venv/bin/activate || python3.12 -m venv  $(CURRENT_ABS_DIR)/.venv ;\
+	. .venv/bin/activate ;\
+	pip install --upgrade pip ;\
+	pip install -Ur requirements.txt
+
+run: ## Run the project in local
+	. .venv/bin/activate && python src/test_connectivity.py
+
+build:
+	docker build ${DOCKER_PLATFORM} -t $(DOCKER_NAME):$(TAG_NAME) $(CURRENT_ABS_DIR) \
+		--build-arg VCS_REF=$$(git -C $(CURRENT_ABS_DIR) rev-parse --short HEAD 2>/dev/null || echo missing) \
+		--build-arg BUILD_DATE=$$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+
+build-nc:
+	docker build ${DOCKER_PLATFORM} --no-cache -t $(DOCKER_NAME):$(TAG_NAME) $(CURRENT_ABS_DIR)
+
+run-docker: build-nc
+	docker run  $(DOCKER_NAME):$(TAG_NAME)
+
+#################################################################################
+# Self Documenting Commands                                                     #
+#################################################################################
+help:
+	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
+
+.DEFAULT_GOAL := help

jobs/colin-extract-refresh/openshift/Readme.md

@@ -0,0 +1 @@
+# test extract job

jobs/colin-extract-refresh/openshift/templates/bc.yaml

@@ -0,0 +1,21 @@
+apiVersion: template.openshift.io/v1
+kind: Template
+metadata:
+  labels:
+    app: ${NAME}
+  name: ${NAME}-build
+objects:
+- apiVersion: v1
+  kind: ImageStream
+  metadata:
+    name: ${NAME}
+    labels:
+      app: ${NAME}
+parameters:
+  - description: |
+      The name assigned to all of the objects defined in this template.
+      You should keep this as default unless your know what your doing.
+    displayName: Name
+    name: NAME
+    required: true
+    value: colin-extract-refresh

jobs/colin-extract-refresh/openshift/templates/cronjob.yaml

@@ -0,0 +1,131 @@
+apiVersion: template.openshift.io/v1
+kind: Template
+metadata:
+  labels:
+    name: ${NAME}
+  name: ${NAME}-cronjob
+objects:
+- kind: "CronJob"
+  apiVersion: "batch/v1"
+  metadata:
+    name: "${NAME}-${TAG}"
+    labels:
+      name: "${NAME}"
+      environment: "${TAG}"
+      role: "${ROLE}"
+  spec:
+    schedule: "${SCHEDULE}"
+    concurrencyPolicy: Forbid
+    successfulJobsHistoryLimit: 3
+    failedJobsHistoryLimit: 1
+    jobTemplate:
+      metadata:
+        labels:
+          name: "${NAME}"
+          environment: "${TAG}"
+          role: "${ROLE}"
+      spec:
+        backoffLimit: 0
+        activeDeadlineSeconds: 300
+        template:
+          metadata:
+            labels:
+              name: "${NAME}"
+              environment: "${TAG}"
+              role: "${ROLE}"
+          spec:
+            containers:
+              - name: "${NAME}-${TAG}"
+                image: "${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/${NAME}:${IMAGE_TAG}"
+                imagePullPolicy: Always
+                command:
+                  - python
+                  - /opt/app-root/test_connectivity.py
+                envFrom:
+                  - secretRef:
+                      name: ${APP_SECRET_NAME}
+                    resources:
+                      requests:
+                        cpu: "${CPU_REQUEST}"
+                        memory: "${MEMORY_REQUEST}"
+                      limits:
+                        cpu: "${CPU_LIMIT}"
+                        memory: "${MEMORY_LIMIT}"
+            restartPolicy: Never
+            terminationGracePeriodSeconds: 30
+
+  envFrom:
+    secretKeyRef:
+      name: ${NAME}
+      key: TEST_KEY
+      optional: true  
+
+parameters: 
+
+  - name: NAME
+    displayName: Name
+    description: The name assigned to all of the OpenShift resources associated to the server instance.
+    required: true
+    value: colin-extract-refresh
+
+  - name: TAG
+    displayName: Environment TAG name
+    description: The TAG name for this environment, e.g., dev, test, prod
+    value: dev
+    required: true
+
+  - name: ROLE
+    displayName: Role
+    description: Role
+    required: true
+    value: job
+
+  - name: IMAGE_NAMESPACE
+    displayName: Image Namespace
+    required: true
+    description: The namespace of the OpenShift project containing the imagestream for the application.
+    value: cc892f-tools
+
+  - name: IMAGE_TAG
+    displayName: Environment TAG name
+    description: The TAG name for this environment, e.g., dev, test, prod
+    value: dev
+    required: true
+
+  - name: IMAGE_REGISTRY
+    displayName: Image Registry
+    required: true
+    description: The image registry of the OpenShift project.
+    value: image-registry.openshift-image-registry.svc:5000
+
+  - name: APP_SECRET_NAME
+    displayName: App Secret Name
+    description: Secret with business-api and colin-api
+    required: true
+    value:  colin-extract-refresh-secret
+
+  - name: SCHEDULE
+    displayName: "Cron Schedule"
+    description: "Cron Schedule to Execute the Job (using local cluster system TZ) to run at 0:55 * * TUE-SAT at pacific time so it will be 7:55 at UTC"
+    value: "55 7 * * TUE-SAT"
+    required: true
+
+  - name: CPU_REQUEST
+    displayName: CPU request
+    required: true
+    value: 50m
+
+  - name: CPU_LIMIT
+    displayName: CPU limit
+    required: true
+    value: 200m
+
+  - name: MEMORY_REQUEST
+    displayName: memory request
+    required: true
+    value: 128Mi
+
+  - name: MEMORY_LIMIT
+    displayName: Memory Limit
+    required: true
+    value: 512Mi

jobs/colin-extract-refresh/requirements.txt

@@ -0,0 +1,7 @@
+SQLAlchemy==2.0.36
+psycopg2-binary==2.9.9
+oracledb==3.1.1
+python-dotenv==1.1.1
+pg8000==1.31.4
+
+cloud-sql-connector @  git+https://github.com/bcgov/sbc-connect-common.git@main#subdirectory=python/cloud-sql-connector

jobs/colin-extract-refresh/scripts/transfer_cprd_corps_insert.sql

@@ -0,0 +1,12 @@
+vset cli.settings.ignore_errors=false
+vset cli.settings.transfer_threads=4
+vset format.date=YYYY-MM-dd'T'hh:mm:ss'Z'
+vset format.timestamp=YYYY-MM-dd'T'hh:mm:ss'Z'
+
+connect cdev_pg_test;
+
+learn schema colin_extract_temp;
+
+-- cutoff timestamp
+insert into colin_extract_temp.colin_extract_version (extracted_at)
+values (current_timestamp);

jobs/colin-extract-refresh/setup.cfg

@@ -0,0 +1,3 @@
+[flake8]
+exclude = .git,.venv
+max-line-length = 120

jobs/colin-extract-refresh/setup.py

@@ -0,0 +1,22 @@
+# Copyright © 2019 Province of British Columbia.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Installer and setup for this module."""
+
+from setuptools import find_packages, setup
+
+
+setup(
+    name='colin-extract-refresh',
+    packages=find_packages()
+)

jobs/colin-extract-refresh/src/checks/check_business.py

@@ -0,0 +1,31 @@
+import sys
+
+from sqlalchemy import create_engine, text
+from config import get_colin_mig_engine, get_named_config
+
+def run_check() -> int:
+    cfg = get_named_config()
+    if cfg.CLOUDSQL_INSTANCE_CONNECTION_NAME:
+        if not all([cfg.CLOUDSQL_INSTANCE_CONNECTION_NAME, cfg.DB_NAME_COLIN_MIGR, cfg.DB_USER_COLIN_MIGR]):
+            raise RuntimeError(
+                "Missing business env vars"
+            )
+    print("== running check_business.py ==")
+    if cfg.CLOUDSQL_INSTANCE_CONNECTION_NAME:
+        print(f"connecting via cloud sql connector")
+    engine = get_colin_mig_engine(cfg)
+    with engine.connect() as conn:
+        row = conn.execute(text("SELECT * FROM businesses LIMIT 1")).mappings().first()
+        if row is None:
+            print("no rows in business mig db")
+        else:
+            print(f"row found........")
+        return 0
+
+
+if __name__ == "__main__":
+    try:
+        raise SystemExit(run_check())
+    except Exception as exc:
+        print(f"business db check failed......")
+        raise