Skip to content

Commit eff3e39

Browse files
haobibohaobibo
authored
refine web app multi-process (#16)
1 parent abb341a commit eff3e39

14 files changed

Lines changed: 428 additions & 44 deletions

File tree

‎demo/app_common/api/api_common_query_postgres.py‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
from typing import Optional
2+
3+
import pandas as pd
4+
from sqlalchemy import text
5+
6+
from aloha.base import BaseModule
7+
from aloha.db.postgres import PostgresOperator
8+
from aloha.logger import LOG
9+
from aloha.service.api.v0 import APIHandler
10+
11+
12+
class ApiQueryPostgres(APIHandler):
13+
def response(self, sql: str, orient: str = 'columns', config_profile: str = None,
14+
params=None, *args, **kwargs) -> str:
15+
op_query_db = QueryDb()
16+
df = op_query_db.query_db(sql=sql, config_profile=config_profile, params=params)
17+
ret = df.to_json(orient=orient, force_ascii=False)
18+
return ret
19+
20+
21+
class QueryDb(BaseModule):
22+
"""Read Data"""
23+
24+
def get_operator(self, config_profile: str, *args, **kwargs):
25+
config_dict = self.config[config_profile]
26+
return PostgresOperator(config_dict)
27+
28+
def query_db(self, sql: str, config_profile: str = None, params=None, *args, **kwargs) -> Optional[pd.DataFrame]:
29+
op = self.get_operator(config_profile or 'pg_rec_readonly')
30+
return pd.read_sql(sql=text(sql), con=op.engine, params=params)
31+
32+
33+
default_handlers = [
34+
# internal API: QueryDB Postgres with sql directly
35+
(r"/api_internal/query_postgres", ApiQueryPostgres),
36+
]
37+
38+
39+
def main():
40+
import sys
41+
import argparse
42+
sys.argv.pop(0)
43+
parser = argparse.ArgumentParser()
44+
parser.add_argument("--config-profile")
45+
parser.add_argument("--sql", nargs='?')
46+
args = parser.parse_args()
47+
dict_params = vars(args)
48+
49+
query = QueryDb()
50+
op = query.get_operator(**dict_params)
51+
LOG.info('Connection string: %s' % op.connection_str)
52+
53+
if dict_params.get('sql', None) is not None:
54+
from tabulate import tabulate
55+
LOG.info('Query result for: %s' % dict_params['sql'])
56+
df = query.query_db(**dict_params)
57+
table = tabulate(df, headers='keys', tablefmt='psql')
58+
print(table)

‎demo/app_common/debug.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,8 @@ def main():
44
from aloha.settings import SETTINGS
55

66
modules_to_load = [
7-
'app_common.api.api_common_sys_info'
7+
"app_common.api.api_common_sys_info",
8+
"app_common.api.api_common_query_postgres",
89
]
910

1011
if 'service' not in SETTINGS.config:

‎demo/resource/config/deploy-DEV.conf‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
deploy = {
2+
postgres_db0 = {
3+
"host": "localhost",
4+
"port": 5432,
5+
"user": "postgres",
6+
"password": "postgres",
7+
"dbname": "postgres"
8+
}
9+
}

‎demo/resource/config/main.conf‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
include required("deploy-DEV.conf")
2+
3+
APP_MODUEL = "Aloha"
4+
5+
APP_DOMAIN = {
6+
LOCAL = "http://localhost:9999"
7+
}
8+
9+
service = {
10+
# num_process = 1
11+
num_process = ${?NUM_PROCESS}
12+
13+
port = ${?deploy.port_service}
14+
port = ${?PORT_SVC}
15+
}
16+
17+
postgres_default = ${deploy.postgres_db0}

‎notebook/test-api-query-postgresql.ipynb‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "markdown",
5+
"metadata": {},
6+
"source": [
7+
"# Query Postgresql using API"
8+
]
9+
},
10+
{
11+
"cell_type": "markdown",
12+
"metadata": {},
13+
"source": [
14+
"## Setting up PYTHONPATH and DIR_RESOURCE stuff"
15+
]
16+
},
17+
{
18+
"cell_type": "code",
19+
"execution_count": null,
20+
"metadata": {},
21+
"outputs": [],
22+
"source": [
23+
"import os\n",
24+
"import sys\n",
25+
"\n",
26+
"sys.path.insert(0, '../src/')\n",
27+
"sys.path.insert(0, '../demo/')\n",
28+
"os.environ['DIR_RESOURCE'] = '../demo/resource/'"
29+
]
30+
},
31+
{
32+
"cell_type": "markdown",
33+
"metadata": {},
34+
"source": [
35+
"## Import packages and set URL endpoint"
36+
]
37+
},
38+
{
39+
"cell_type": "code",
40+
"execution_count": null,
41+
"metadata": {},
42+
"outputs": [],
43+
"source": [
44+
"from aloha.service.api.v0 import APICaller\n",
45+
"\n",
46+
"from aloha.settings import SETTINGS\n",
47+
"\n",
48+
"api_environment = 'LOCAL' # DEV | STG | PRD\n",
49+
"\n",
50+
"url_base = SETTINGS.config['APP_DOMAIN'][api_environment]\n",
51+
"caller = APICaller(url_base)"
52+
]
53+
},
54+
{
55+
"cell_type": "markdown",
56+
"metadata": {},
57+
"source": [
58+
"## Function to query remote DB via API"
59+
]
60+
},
61+
{
62+
"cell_type": "code",
63+
"execution_count": null,
64+
"metadata": {},
65+
"outputs": [],
66+
"source": [
67+
"import pandas as pd\n",
68+
"\n",
69+
"\n",
70+
"def query_db_with_api(sql: str, config_profile='postgres_default', **kwargs):\n",
71+
" try:\n",
72+
" resp = caller.call('/api_internal/query_postgres', timeout=(20, 2000), data={\"sql\": sql, \"config_profile\": config_profile, })\n",
73+
" data = resp['data']\n",
74+
" return pd.read_json(data)\n",
75+
" except Exception as e:\n",
76+
" print(resp)\n",
77+
" raise e"
78+
]
79+
},
80+
{
81+
"cell_type": "markdown",
82+
"metadata": {},
83+
"source": [
84+
"## Simulate a time-consuming SQL query with `pg_sleep()`"
85+
]
86+
},
87+
{
88+
"cell_type": "code",
89+
"execution_count": null,
90+
"metadata": {},
91+
"outputs": [],
92+
"source": [
93+
"query_db_with_api(sql=\"\"\"\n",
94+
"SELECT pg_sleep(5) AS slept\n",
95+
"\"\"\")"
96+
]
97+
}
98+
],
99+
"metadata": {
100+
"kernelspec": {
101+
"display_name": "Python 3 (ipykernel)",
102+
"language": "python",
103+
"name": "python3"
104+
},
105+
"language_info": {
106+
"codemirror_mode": {
107+
"name": "ipython",
108+
"version": 3
109+
},
110+
"file_extension": ".py",
111+
"mimetype": "text/x-python",
112+
"name": "python",
113+
"nbconvert_exporter": "python",
114+
"pygments_lexer": "ipython3",
115+
"version": "3.10.6"
116+
}
117+
},
118+
"nbformat": 4,
119+
"nbformat_minor": 1
120+
}

‎notebook/test-api-service.ipynb‎

Lines changed: 9 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -7,58 +7,40 @@
77
"metadata": {},
88
"outputs": [],
99
"source": [
10-
"import os, sys\n",
10+
"import sys\n",
1111
"sys.path.insert(0, '../src/')"
1212
]
1313
},
14-
{
15-
"cell_type": "code",
16-
"execution_count": null,
17-
"id": "503e782e",
18-
"metadata": {},
19-
"outputs": [],
20-
"source": [
21-
"from aloha.service.v0 import APICaller"
22-
]
23-
},
2414
{
2515
"cell_type": "code",
2616
"execution_count": null,
2717
"id": "12967663",
2818
"metadata": {},
2919
"outputs": [],
3020
"source": [
31-
"caller = APICaller()"
32-
]
33-
},
34-
{
35-
"cell_type": "code",
36-
"execution_count": null,
37-
"id": "250d9b9c",
38-
"metadata": {},
39-
"outputs": [],
40-
"source": [
41-
"url_base = 'http://localhost:80'"
21+
"from aloha.service.api.v0 import APICaller\n",
22+
"\n",
23+
"caller = APICaller(url_endpoint='http://localhost:9999')"
4224
]
4325
},
4426
{
4527
"cell_type": "code",
4628
"execution_count": null,
47-
"id": "5d227afd",
29+
"id": "c6d390a6",
4830
"metadata": {},
4931
"outputs": [],
5032
"source": [
51-
"caller.call(url_base + '/api/common/sys_info', kind='gpu')"
33+
"caller.call('/api/common/sys_info/gpu')"
5234
]
5335
},
5436
{
5537
"cell_type": "code",
5638
"execution_count": null,
57-
"id": "c6d390a6",
39+
"id": "351a9e14",
5840
"metadata": {},
5941
"outputs": [],
6042
"source": [
61-
"caller.call(url_base + '/api/common/sys_info/gpu')"
43+
"caller.call('/api/common/sys_info', kind='cuda')"
6244
]
6345
}
6446
],
@@ -78,7 +60,7 @@
7860
"name": "python",
7961
"nbconvert_exporter": "python",
8062
"pygments_lexer": "ipython3",
81-
"version": "3.10.2"
63+
"version": "3.10.6"
8264
}
8365
},
8466
"nbformat": 4,

‎notebook/test-db-postgres.ipynb‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "code",
5+
"execution_count": null,
6+
"metadata": {},
7+
"outputs": [],
8+
"source": [
9+
"! /opt/conda/bin/pip install psycopg2-binary sqlalchemy"
10+
]
11+
},
12+
{
13+
"cell_type": "code",
14+
"execution_count": null,
15+
"metadata": {
16+
"collapsed": true
17+
},
18+
"outputs": [],
19+
"source": [
20+
"import os, sys\n",
21+
"sys.path.insert(0, '../src/')\n",
22+
"os.environ['DIR_RESOURCE'] = '../demo/resource'"
23+
]
24+
},
25+
{
26+
"cell_type": "code",
27+
"execution_count": null,
28+
"metadata": {},
29+
"outputs": [],
30+
"source": [
31+
"import aloha\n",
32+
"from aloha.settings import SETTINGS as S\n",
33+
"from aloha.db.postgres import PostgresOperator\n",
34+
"\n",
35+
"print(aloha.__path__)"
36+
]
37+
},
38+
{
39+
"cell_type": "code",
40+
"execution_count": null,
41+
"metadata": {},
42+
"outputs": [],
43+
"source": [
44+
"op_pg = PostgresOperator(S.config.deploy.postgres_db0)"
45+
]
46+
},
47+
{
48+
"cell_type": "code",
49+
"execution_count": null,
50+
"metadata": {},
51+
"outputs": [],
52+
"source": [
53+
"op_pg.engine"
54+
]
55+
},
56+
{
57+
"cell_type": "code",
58+
"execution_count": null,
59+
"metadata": {},
60+
"outputs": [],
61+
"source": [
62+
"cur = op_pg.execute_query(sql=\"SELECT pg_sleep(2*5) AS slept\")"
63+
]
64+
},
65+
{
66+
"cell_type": "code",
67+
"execution_count": null,
68+
"metadata": {},
69+
"outputs": [],
70+
"source": [
71+
"list(cur)"
72+
]
73+
}
74+
],
75+
"metadata": {
76+
"kernelspec": {
77+
"display_name": "Python 3 (ipykernel)",
78+
"language": "python",
79+
"name": "python3"
80+
},
81+
"language_info": {
82+
"codemirror_mode": {
83+
"name": "ipython",
84+
"version": 3
85+
},
86+
"file_extension": ".py",
87+
"mimetype": "text/x-python",
88+
"name": "python",
89+
"nbconvert_exporter": "python",
90+
"pygments_lexer": "ipython3",
91+
"version": "3.10.6"
92+
}
93+
},
94+
"nbformat": 4,
95+
"nbformat_minor": 1
96+
}

‎src/aloha/encrypt/jwt.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
from ..logger import LOG
44

5-
LOG.debug('Version of pyjwt = %s' % jwt.__version__.__str__())
5+
LOG.debug('Using pyjwt == %s' % jwt.__version__.__str__())
66

77

88
def encode(

0 commit comments

Comments
 (0)