diff --git a/jobs/colin-extract-refresh/.dockerignore b/jobs/colin-extract-refresh/.dockerignore new file mode 100644 index 0000000000..7015f69fc3 --- /dev/null +++ b/jobs/colin-extract-refresh/.dockerignore @@ -0,0 +1,19 @@ +.env +.env.* +.envrc +.env.local + +.venv +venv +__pycache__ +.pytest_cache +.ruff_cache +*.pytest_cache +.git +.gitignore +Makefile +openshift +README.md +*.md +Dockerfile +.dockerignore \ No newline at end of file diff --git a/jobs/colin-extract-refresh/.env.sample b/jobs/colin-extract-refresh/.env.sample new file mode 100644 index 0000000000..1e25ada2fc --- /dev/null +++ b/jobs/colin-extract-refresh/.env.sample @@ -0,0 +1,12 @@ + +DATABASE_USERNAME_COLIN_ORACLE= +DATABASE_PASSWORD_COLIN_ORACLE= +DATABASE_NAME_COLIN_ORACLE= +DATABASE_HOST_COLIN_ORACLE= +DATABASE_PORT_COLIN_ORACLE= + +DATABASE_USERNAME_COLIN_MIGR= +DATABASE_PASSWORD_COLIN_MIGR= +DATABASE_NAME_COLIN_MIGR= +DATABASE_HOST_COLIN_MIGR= +DATABASE_PORT_COLIN_MIGR= \ No newline at end of file diff --git a/jobs/colin-extract-refresh/Dockerfile b/jobs/colin-extract-refresh/Dockerfile new file mode 100644 index 0000000000..8105b2e9cd --- /dev/null +++ b/jobs/colin-extract-refresh/Dockerfile @@ -0,0 +1,79 @@ +FROM python:3.12-slim + +ARG VCS_REF="missing" +ARG BUILD_DATE="missing" + +ENV VCS_REF=${VCS_REF} +ENV BUILD_DATE=${BUILD_DATE} +ENV PYTHONUNBUFFERED=1 + + + + + +LABEL org.label-schema.vcs-ref=${VCS_REF} \ + org.label-schema.build-date=${BUILD_DATE} + + +USER root + +ENV ORACLE_CLIENT_LIB_DIR=/opt/oracle/instantclient_21_4 +ENV LD_LIBRARY_PATH=${ORACLE_CLIENT_LIB_DIR} + + +WORKDIR /opt/oracle/ +RUN set -eux; \ + apt-get update; \ + apt-get install -y --no-install-recommends unzip wget ca-certificates;\ + apt-get install -y --no-install-recommends libaio1 || apt-get install -y --no-install-recommends libaio1t64; \ + ln -sf /usr/lib/x86_64-linux-gnu/libaio.so.1t64 /usr/lib/x86_64-linux-gnu/libaio.so.1 || true; \ + ln -sf /lib/x86_64-linux-gnu/libaio.so.1t64 /lib/x86_64-linux-gnu/libaio.so.1 || true; \ + wget -q https://download.oracle.com/otn_software/linux/instantclient/214000/instantclient-basic-linux.x64-21.4.0.0.0dbru.zip; \ + unzip instantclient-basic-linux.x64-21.4.0.0.0dbru.zip; \ + rm instantclient-basic-linux.x64-21.4.0.0.0dbru.zip; \ + cd "${ORACLE_CLIENT_LIB_DIR}"; \ + rm -f *jdbc* *occi* *mysql* *README *jar uidrvci genezi adrci; \ + test -e libclntsh.so || ln -sf libclntsh.so.* libclntsh.so; \ + test -f libclntsh.so; \ + echo "${ORACLE_CLIENT_LIB_DIR}" > /etc/ld.so.conf.d/oracle-instantclient.conf; \ + ldconfig; \ + chmod 755 "${ORACLE_CLIENT_LIB_DIR}"; \ + rm -rf /var/lib/apt/lists/* + +ENV DBSCHEMA_HOME=/opt/DbSchema +ENV DBSCHEMA_VERSION=9_7_1 + +RUN set -eux; \ + apt-get update; \ + apt-get install -y --no-install-recommends default-jre-headless curl;\ + curl -fsSL "https://www.dbschema.com/download/dbschema_linux_9_7_1.tar.gz" -o /tmp/dbschema.tar.gz;\ + tar -xzf /tmp/dbschema.tar.gz -C /opt; \ + rm /tmp/dbschema.tar.gz; \ + mkdir -p "${DBSCHEMA_HOME}/lib"; \ + curl -fsSL "https://repo1.maven.org/maven2/org/testfx/openjfx-monocle/17.0.10/openjfx-monocle-17.0.10.jar" -o "${DBSCHEMA_HOME}/lib/monocle.jar"; \ + ln -sf "${DBSCHEMA_HOME}/DbSchemaCLI" /usr/local/bin/dbschemacli; \ + test -x "${DBSCHEMA_HOME}/DbSchemaCLI"; \ + rm -rf /var/lib/apt/lists/* + + + +# Create directories with proper permissions +RUN mkdir -p /opt/app-root && \ + chmod 755 /opt/app-root + +WORKDIR /opt/app-root +COPY requirements.txt . +RUN apt-get update; \ + apt-get install -y --no-install-recommends git;\ + pip install --no-cache-dir -r requirements.txt;\ + rm -rf /var/lib/apt/lists/* + +ENV PATH="${DBSCHEMA_HOME}:${PATH}" + +USER 1001 + +EXPOSE 8080 + +COPY src . + +CMD [ "python", "src/test_connectivity.py" ] diff --git a/jobs/colin-extract-refresh/Makefile b/jobs/colin-extract-refresh/Makefile new file mode 100644 index 0000000000..fd41bbcdd9 --- /dev/null +++ b/jobs/colin-extract-refresh/Makefile @@ -0,0 +1,41 @@ +.PHONY: help setup install build build-nc run + +MKFILE_PATH:=$(abspath $(lastword $(MAKEFILE_LIST))) +CURRENT_ABS_DIR:=$(patsubst %/,%,$(dir $(MKFILE_PATH))) + +DOCKER_NAME:=colin-extract-refresh +TAG_NAME ?= dev +DOCKER_PLATFORM ?= --platform linux/amd64 + +################################################################################# +# COMMANDS -- Setup # +################################################################################# +setup: install ## Setup the project + +install: ## Install python virtual environment + test -f .venv/bin/activate || python3.12 -m venv $(CURRENT_ABS_DIR)/.venv ;\ + . .venv/bin/activate ;\ + pip install --upgrade pip ;\ + pip install -Ur requirements.txt + +run: ## Run the project in local + . .venv/bin/activate && python src/test_connectivity.py + +build: + docker build ${DOCKER_PLATFORM} -t $(DOCKER_NAME):$(TAG_NAME) $(CURRENT_ABS_DIR) \ + --build-arg VCS_REF=$$(git -C $(CURRENT_ABS_DIR) rev-parse --short HEAD 2>/dev/null || echo missing) \ + --build-arg BUILD_DATE=$$(date -u +"%Y-%m-%dT%H:%M:%SZ") + +build-nc: + docker build ${DOCKER_PLATFORM} --no-cache -t $(DOCKER_NAME):$(TAG_NAME) $(CURRENT_ABS_DIR) + +run-docker: build-nc + docker run $(DOCKER_NAME):$(TAG_NAME) + +################################################################################# +# Self Documenting Commands # +################################################################################# +help: + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' + +.DEFAULT_GOAL := help diff --git a/jobs/colin-extract-refresh/README.md b/jobs/colin-extract-refresh/README.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/jobs/colin-extract-refresh/openshift/Readme.md b/jobs/colin-extract-refresh/openshift/Readme.md new file mode 100644 index 0000000000..c8cd8e76bd --- /dev/null +++ b/jobs/colin-extract-refresh/openshift/Readme.md @@ -0,0 +1 @@ +# test extract job \ No newline at end of file diff --git a/jobs/colin-extract-refresh/openshift/templates/bc.yaml b/jobs/colin-extract-refresh/openshift/templates/bc.yaml new file mode 100644 index 0000000000..e0426c7dcb --- /dev/null +++ b/jobs/colin-extract-refresh/openshift/templates/bc.yaml @@ -0,0 +1,21 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + labels: + app: ${NAME} + name: ${NAME}-build +objects: +- apiVersion: v1 + kind: ImageStream + metadata: + name: ${NAME} + labels: + app: ${NAME} +parameters: + - description: | + The name assigned to all of the objects defined in this template. + You should keep this as default unless your know what your doing. + displayName: Name + name: NAME + required: true + value: colin-extract-refresh diff --git a/jobs/colin-extract-refresh/openshift/templates/cronjob.yaml b/jobs/colin-extract-refresh/openshift/templates/cronjob.yaml new file mode 100644 index 0000000000..f702c6182b --- /dev/null +++ b/jobs/colin-extract-refresh/openshift/templates/cronjob.yaml @@ -0,0 +1,131 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + labels: + name: ${NAME} + name: ${NAME}-cronjob +objects: +- kind: "CronJob" + apiVersion: "batch/v1" + metadata: + name: "${NAME}-${TAG}" + labels: + name: "${NAME}" + environment: "${TAG}" + role: "${ROLE}" + spec: + schedule: "${SCHEDULE}" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 1 + jobTemplate: + metadata: + labels: + name: "${NAME}" + environment: "${TAG}" + role: "${ROLE}" + spec: + backoffLimit: 0 + activeDeadlineSeconds: 300 + template: + metadata: + labels: + name: "${NAME}" + environment: "${TAG}" + role: "${ROLE}" + spec: + containers: + - name: "${NAME}-${TAG}" + image: "${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/${NAME}:${IMAGE_TAG}" + imagePullPolicy: Always + command: + - python + - /opt/app-root/test_connectivity.py + envFrom: + - secretRef: + name: ${APP_SECRET_NAME} + resources: + requests: + cpu: "${CPU_REQUEST}" + memory: "${MEMORY_REQUEST}" + limits: + cpu: "${CPU_LIMIT}" + memory: "${MEMORY_LIMIT}" + restartPolicy: Never + terminationGracePeriodSeconds: 30 + + envFrom: + secretKeyRef: + name: ${NAME} + key: TEST_KEY + optional: true + +parameters: + + - name: NAME + displayName: Name + description: The name assigned to all of the OpenShift resources associated to the server instance. + required: true + value: colin-extract-refresh + + - name: TAG + displayName: Environment TAG name + description: The TAG name for this environment, e.g., dev, test, prod + value: dev + required: true + + - name: ROLE + displayName: Role + description: Role + required: true + value: job + + - name: IMAGE_NAMESPACE + displayName: Image Namespace + required: true + description: The namespace of the OpenShift project containing the imagestream for the application. + value: cc892f-tools + + - name: IMAGE_TAG + displayName: Environment TAG name + description: The TAG name for this environment, e.g., dev, test, prod + value: dev + required: true + + - name: IMAGE_REGISTRY + displayName: Image Registry + required: true + description: The image registry of the OpenShift project. + value: image-registry.openshift-image-registry.svc:5000 + + - name: APP_SECRET_NAME + displayName: App Secret Name + description: Secret with business-api and colin-api + required: true + value: colin-extract-refresh-secret + + - name: SCHEDULE + displayName: "Cron Schedule" + description: "Cron Schedule to Execute the Job (using local cluster system TZ) to run at 0:55 * * TUE-SAT at pacific time so it will be 7:55 at UTC" + value: "55 7 * * TUE-SAT" + required: true + + - name: CPU_REQUEST + displayName: CPU request + required: true + value: 50m + + - name: CPU_LIMIT + displayName: CPU limit + required: true + value: 200m + + - name: MEMORY_REQUEST + displayName: memory request + required: true + value: 128Mi + + - name: MEMORY_LIMIT + displayName: Memory Limit + required: true + value: 512Mi \ No newline at end of file diff --git a/jobs/colin-extract-refresh/requirements.txt b/jobs/colin-extract-refresh/requirements.txt new file mode 100644 index 0000000000..d8dc4d73b4 --- /dev/null +++ b/jobs/colin-extract-refresh/requirements.txt @@ -0,0 +1,7 @@ +SQLAlchemy==2.0.36 +psycopg2-binary==2.9.9 +oracledb==3.1.1 +python-dotenv==1.1.1 +pg8000==1.31.4 + +cloud-sql-connector @ git+https://github.com/bcgov/sbc-connect-common.git@main#subdirectory=python/cloud-sql-connector \ No newline at end of file diff --git a/jobs/colin-extract-refresh/scripts/transfer_cprd_corps_insert.sql b/jobs/colin-extract-refresh/scripts/transfer_cprd_corps_insert.sql new file mode 100644 index 0000000000..a81d9189d3 --- /dev/null +++ b/jobs/colin-extract-refresh/scripts/transfer_cprd_corps_insert.sql @@ -0,0 +1,12 @@ +vset cli.settings.ignore_errors=false +vset cli.settings.transfer_threads=4 +vset format.date=YYYY-MM-dd'T'hh:mm:ss'Z' +vset format.timestamp=YYYY-MM-dd'T'hh:mm:ss'Z' + +connect cdev_pg_test; + +learn schema colin_extract_temp; + +-- cutoff timestamp +insert into colin_extract_temp.colin_extract_version (extracted_at) +values (current_timestamp); diff --git a/jobs/colin-extract-refresh/setup.cfg b/jobs/colin-extract-refresh/setup.cfg new file mode 100644 index 0000000000..95a2b3967e --- /dev/null +++ b/jobs/colin-extract-refresh/setup.cfg @@ -0,0 +1,3 @@ +[flake8] +exclude = .git,.venv +max-line-length = 120 diff --git a/jobs/colin-extract-refresh/setup.py b/jobs/colin-extract-refresh/setup.py new file mode 100644 index 0000000000..3f51a3d92b --- /dev/null +++ b/jobs/colin-extract-refresh/setup.py @@ -0,0 +1,22 @@ +# Copyright © 2019 Province of British Columbia. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Installer and setup for this module.""" + +from setuptools import find_packages, setup + + +setup( + name='colin-extract-refresh', + packages=find_packages() +) diff --git a/jobs/colin-extract-refresh/src/checks/__init__.py b/jobs/colin-extract-refresh/src/checks/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/jobs/colin-extract-refresh/src/checks/check_business.py b/jobs/colin-extract-refresh/src/checks/check_business.py new file mode 100644 index 0000000000..4b6c5d604a --- /dev/null +++ b/jobs/colin-extract-refresh/src/checks/check_business.py @@ -0,0 +1,31 @@ +import sys + +from sqlalchemy import create_engine, text +from config import get_colin_mig_engine, get_named_config + +def run_check() -> int: + cfg = get_named_config() + if cfg.CLOUDSQL_INSTANCE_CONNECTION_NAME: + if not all([cfg.CLOUDSQL_INSTANCE_CONNECTION_NAME, cfg.DB_NAME_COLIN_MIGR, cfg.DB_USER_COLIN_MIGR]): + raise RuntimeError( + "Missing business env vars" + ) + print("== running check_business.py ==") + if cfg.CLOUDSQL_INSTANCE_CONNECTION_NAME: + print(f"connecting via cloud sql connector") + engine = get_colin_mig_engine(cfg) + with engine.connect() as conn: + row = conn.execute(text("SELECT * FROM businesses LIMIT 1")).mappings().first() + if row is None: + print("no rows in business mig db") + else: + print(f"row found........") + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(run_check()) + except Exception as exc: + print(f"business db check failed......") + raise diff --git a/jobs/colin-extract-refresh/src/checks/check_colin.py b/jobs/colin-extract-refresh/src/checks/check_colin.py new file mode 100644 index 0000000000..e2da3fbb50 --- /dev/null +++ b/jobs/colin-extract-refresh/src/checks/check_colin.py @@ -0,0 +1,37 @@ +import os +import sys +import oracledb +from sqlalchemy import create_engine, text +from config import get_named_config + +def _colin_oracle_init() -> None: + lib_dir = os.environ.get("ORACLE_CLIENT_LIB_DIR", "") + oracledb.init_oracle_client(lib_dir=lib_dir) + print('👷 Enable thick mode:', not oracledb.is_thin_mode()) + print('👷 Instant Client version:', oracledb.clientversion()) + + +def run_check() -> int: + cfg = get_named_config() + if not all([cfg.DB_USER_COLIN_ORACLE, cfg.DB_PASSWORD_COLIN_ORACLE, cfg.DB_NAME_COLIN_ORACLE, cfg.DB_HOST_COLIN_ORACLE, cfg.DB_PORT_COLIN_ORACLE]): + raise RuntimeError( + "Missing colin env vars" + ) + print("== running check_colin.py ==") + _colin_oracle_init() + engine = create_engine(cfg.SQLALCHEMY_DATABASE_URI_COLIN_ORACLE) + with engine.connect() as conn: + row = conn.execute(text("SELECT * FROM corporation FETCH FIRST 1 ROWS ONLY")).mappings().first() + if row is None: + print("no rows in COLIN db.") + else: + print(f"COLIN sample row found.") + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(run_check()) + except Exception as exc: + print(f"business db check failed.") + raise diff --git a/jobs/colin-extract-refresh/src/config.py b/jobs/colin-extract-refresh/src/config.py new file mode 100644 index 0000000000..8072acd992 --- /dev/null +++ b/jobs/colin-extract-refresh/src/config.py @@ -0,0 +1,173 @@ +# Copyright © 2026 Province of British Columbia +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""All of the configuration for the service is captured here. + +All items are loaded, or have Constants defined here that +are loaded into the Flask configuration. +All modules and lookups get their configuration from the +Flask config, rather than reading environment variables directly +or by accessing this configuration directly. +""" +import os +import sys + +from dotenv import find_dotenv, load_dotenv +from sqlalchemy import create_engine + +# this will load all the envars from a .env file located in the project root (api) +load_dotenv(find_dotenv()) + + +def _get_int(name: str, default: int = 0) -> int: + """Safe int env parsing that avoids None.isnumeric() crashes.""" + val = os.getenv(name) + return int(val) if (val and val.isnumeric()) else default + + +def _get_strict_int(name: str, default: int = 0) -> int: + """Parse an integer env var, raising when a non-blank value is invalid.""" + val = os.getenv(name) + if val is None or val.strip() == '': + return default + try: + return int(val) + except ValueError as exc: + raise ValueError(f'{name} must be a valid integer') from exc + + +def _get_bool(name: str, default: bool = False) -> bool: + """Safe bool env parsing (case-insensitive).""" + val = os.getenv(name) + if val is None: + return default + return val.strip().lower() == 'true' + + +def get_named_config(config_name: str = 'production'): + """Return the configuration object based on the name. + + :raise: KeyError: if an unknown configuration is requested + """ + if config_name in ['production', 'staging', 'default']: + config = ProdConfig() + elif config_name == 'testing': + config = TestConfig() + elif config_name == 'development': + config = DevConfig() + else: + raise KeyError(f'Unknown configuration: {config_name}') + return config + + +class _Config(): # pylint: disable=too-few-public-methods + """Base class configuration that should set reasonable defaults. + + Used as the base for all the other configurations. + """ + + DATA_LOAD_ENV = os.getenv('DATA_LOAD_ENV', '') + + # POSTGRESQL COLIN MIGRATION DB + DB_USER_COLIN_MIGR = os.getenv('DATABASE_USERNAME_COLIN_MIGR', '') + DB_PASSWORD_COLIN_MIGR = os.getenv('DATABASE_PASSWORD_COLIN_MIGR', '') + DB_NAME_COLIN_MIGR = os.getenv('DATABASE_NAME_COLIN_MIGR', '') + DB_HOST_COLIN_MIGR = os.getenv('DATABASE_HOST_COLIN_MIGR', '') + DB_PORT_COLIN_MIGR = os.getenv('DATABASE_PORT_COLIN_MIGR', '5432') + CLOUDSQL_INSTANCE_CONNECTION_NAME = os.getenv('CLOUDSQL_INSTANCE_CONNECTION_NAME', '') + DATABASE_IP_TYPE = os.getenv('DATABASE_IP_TYPE', 'private').lower() + if CLOUDSQL_INSTANCE_CONNECTION_NAME: + SQLALCHEMY_DATABASE_URI_COLIN_MIGR = "postgresql+pg8000://" + + elif DB_HOST_COLIN_MIGR: + SQLALCHEMY_DATABASE_URI_COLIN_MIGR = 'postgresql://{user}:{password}@{host}:{port}/{name}'.format( + user=DB_USER_COLIN_MIGR, + password=DB_PASSWORD_COLIN_MIGR, + host=DB_HOST_COLIN_MIGR, + port=int(DB_PORT_COLIN_MIGR), + name=DB_NAME_COLIN_MIGR, + ) + else: + SQLALCHEMY_DATABASE_URI_COLIN_MIGR = '' + SQLALCHEMY_TRACK_MODIFICATIONS = os.getenv('SQLALCHEMY_TRACK_MODIFICATIONS', False) + + + DATABASE_POOL_PRE_PING = os.getenv('DATABASE_POOL_PRE_PING', 'True') == 'True' + DATABASE_POOL_SIZE = os.getenv('DATABASE_POOL_SIZE', '5') + DATABASE_MAX_OVERFLOW = os.getenv('DATABASE_MAX_OVERFLOW', '10') + + SQLALCHEMY_ENGINE_OPTIONS = { + "pool_pre_ping": DATABASE_POOL_PRE_PING, + "pool_size": int(DATABASE_POOL_SIZE), + "max_overflow": int(DATABASE_MAX_OVERFLOW) + } + + # ORACLE COLIN DB + DB_USER_COLIN_ORACLE = os.getenv('DATABASE_USERNAME_COLIN_ORACLE', '') + DB_PASSWORD_COLIN_ORACLE = os.getenv('DATABASE_PASSWORD_COLIN_ORACLE', '') + DB_NAME_COLIN_ORACLE = os.getenv('DATABASE_NAME_COLIN_ORACLE', '') + DB_HOST_COLIN_ORACLE = os.getenv('DATABASE_HOST_COLIN_ORACLE', '') + DB_PORT_COLIN_ORACLE = os.getenv('DATABASE_PORT_COLIN_ORACLE', '1521') + SQLALCHEMY_DATABASE_URI_COLIN_ORACLE = 'oracle+oracledb://{user}:{password}@{host}:{port}/{name}'.format( + user=DB_USER_COLIN_ORACLE, + password=DB_PASSWORD_COLIN_ORACLE, + host=DB_HOST_COLIN_ORACLE, + port=int(DB_PORT_COLIN_ORACLE), + name=DB_NAME_COLIN_ORACLE, + ) + + TESTING = False + DEBUG = False + + +class DevConfig(_Config): # pylint: disable=too-few-public-methods + """Creates the Development Config object.""" + + TESTING = False + DEBUG = True + + +class TestConfig(_Config): # pylint: disable=too-few-public-methods + """In support of testing only. + + Used by the py.test suite + """ + + DEBUG = True + TESTING = True + + +class ProdConfig(_Config): # pylint: disable=too-few-public-methods + """Production environment configuration.""" + + TESTING = False + DEBUG = False + + +def get_colin_mig_conn(cfg: _Config): + from cloud_sql_connector import DBConfig, getconn + config = DBConfig( + instance_name=cfg.CLOUDSQL_INSTANCE_CONNECTION_NAME, + database=cfg.DB_NAME_COLIN_MIGR, + user=cfg.DB_USER_COLIN_MIGR, + ip_type=cfg.DATABASE_IP_TYPE, + schema="public" + ) + return getconn(config) + +def get_colin_mig_engine(cfg: _Config | None = None): + cfg = cfg or get_named_config() + if cfg.CLOUDSQL_INSTANCE_CONNECTION_NAME and cfg.DB_NAME_COLIN_MIGR and cfg.DB_USER_COLIN_MIGR: + return create_engine("postgresql+pg8000://", creator=lambda: get_colin_mig_conn(cfg)) + else: + return create_engine(cfg.SQLALCHEMY_DATABASE_URI_COLIN_MIGR, **cfg.SQLALCHEMY_ENGINE_OPTIONS) diff --git a/jobs/colin-extract-refresh/src/dbschemacli_init.py b/jobs/colin-extract-refresh/src/dbschemacli_init.py new file mode 100644 index 0000000000..fff6b33e84 --- /dev/null +++ b/jobs/colin-extract-refresh/src/dbschemacli_init.py @@ -0,0 +1,12 @@ +from pathlib import Path +from config import _Config + +def write_dbschema_init(cfg: _Config) -> Path: + init_dir = Path.home() / '.DbSchema' / 'cli' + init_dir.mkdir(parents=True, exist_ok=True) + init_path = init_dir / 'init.sql' + db_user = cfg.DB_USER_COLIN_MIGR + + lines = [ + 'register driver PostgreSql org.postgresql.Driver jdbc:postgresql://:/ "port=5432"' + ] diff --git a/jobs/colin-extract-refresh/src/generate_gc_token.py b/jobs/colin-extract-refresh/src/generate_gc_token.py new file mode 100644 index 0000000000..48ee566429 --- /dev/null +++ b/jobs/colin-extract-refresh/src/generate_gc_token.py @@ -0,0 +1,21 @@ + +import subprocess +import os + +def connect_to_db(): + print("Generating fresh GCP IAM token...") + try: + token = subprocess.check_output( + ["gcloud", "sql", "generate-login-token"], + text=True + ).strip() + except subprocess.CalledProcessError as e: + print("Error: Make sure you are authenticated with 'gcloud auth login'") + return + + os.environ["GCP_IAM_TOKEN"] = token + + print("Connecting to test_dev via DbSchemaCLI...") + +if __name__ == "__main__": + connect_to_db() diff --git a/jobs/colin-extract-refresh/src/test_connectivity.py b/jobs/colin-extract-refresh/src/test_connectivity.py new file mode 100644 index 0000000000..d7d356a1d9 --- /dev/null +++ b/jobs/colin-extract-refresh/src/test_connectivity.py @@ -0,0 +1,18 @@ +import sys + +from checks.check_business import run_check as run_business_check +from checks.check_colin import run_check as run_colin_check + +def main() -> int: + print("== running test_connectivity.py ==") + run_business_check() + run_colin_check() + print("== done test_connectivity.py ==") + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except Exception as exc: + print(f"[test-connectivity] failed") \ No newline at end of file