update service apis (#9)

Co-authored-by: QPod0 <[email protected]>

Author: haobibo

src/aloha/service/__init__.py

@@ -0,0 +1,7 @@
+import sys
+
+from .api import v0, v1, v2
+
+for module in (v0, v1, v2):
+    full_name = '{}.{}'.format(__package__, module.__name__.rsplit('.')[-1])
+    sys.modules[full_name] = sys.modules[module.__name__]

src/aloha/service/api/__init__.py

@@ -0,0 +1,35 @@
+__all__ = ('APIHandler', 'APICaller',)
+
+import json
+import logging
+from abc import ABC
+
+from ..http import AbstractApiClient, AbstractApiHandler
+
+
+class APIHandler(AbstractApiHandler, ABC):
+    MAP_ERROR_INFO = {
+        'BAD_REQUEST': {'code': '5101', 'message': ['Bad request: fail to parse body as JSON object!']}
+    }
+
+    async def post(self, *args, **kwargs):
+        body_arguments = self.request_body
+        kwargs.update(body_arguments)
+
+        resp = dict(code=5200, message=['success'])
+        try:
+            result = self.response(*args, **kwargs)  # this call may throw TypeError when argument missing
+            resp['data'] = result
+        except Exception as e:
+            if self.LOG.level == logging.DEBUG:
+                self.LOG.error(e, exc_info=True)
+            return self.finish({'code': 5201, 'message': [repr(e)]})
+
+        resp = json.dumps(resp, ensure_ascii=False, default=str, separators=(',', ':'))
+        return self.finish(resp)
+
+
+class APICaller(AbstractApiClient):
+    def wrap_request_data(self, data: dict) -> dict:
+        assert isinstance(data, dict), "Data object must be a dict!"
+        return data

src/aloha/service/api/v0.py

@@ -0,0 +1,121 @@
+__all__ = ('APIHandler', 'APICaller', 'sign_data', 'sign_check')
+
+import json
+import logging
+import uuid
+from abc import ABC
+
+from ..http import AbstractApiClient, AbstractApiHandler
+from ...encrypt.hash import get_md5_of_str, get_sha256_of_str
+from ...settings import SETTINGS
+
+APP_ID_KEYS = SETTINGS.config.get('APP_ID_KEYS', {})
+APP_OPTIONS = SETTINGS.config.get('APP_OPTIONS', {})
+FUNC_SIGN_CHECK = {'md5': get_md5_of_str, 'sha256': get_sha256_of_str}
+func_sign_check_default = FUNC_SIGN_CHECK.get(APP_OPTIONS.get('sign_method', 'md5'))
+
+
+class APIHandler(AbstractApiHandler, ABC):
+    MAP_ERROR_INFO = {
+        'BAD_REQUEST': {'code': '5101', 'message': ['Bad request: fail to parse body as JSON object!']},
+        'MISSING_ARGS': {'code': '5102', 'message': ['Required argument field(s) missing...']},
+        'SIGN_CHECK_FAIL': {'code': '5104', 'message': ['Invalid sign, sign check failed!']},
+    }
+
+    async def post(self):
+        body_arguments = self.request_body
+
+        try:
+            salt_uuid = body_arguments.pop('salt_uuid')
+            app_id = body_arguments.pop('app_id')
+            sign = body_arguments.pop('sign')
+            data = body_arguments.pop('data')
+        except KeyError:  # cannot find default key from parsed body
+            return self.finish(self.MAP_ERROR_INFO['MISSING_ARGS'])
+
+        is_valid_req = sign_check(salt_uuid=salt_uuid, app_id=app_id, sign=sign, data=data)  # , sign_method='sha256'
+        if not is_valid_req:
+            return self.finish(self.MAP_ERROR_INFO['SIGN_CHECK_FAIL'])
+
+        resp = dict(code=5200, message=['success'])
+        try:
+            result = self.response(**data)  # this call may throw TypeError when argument missing
+            resp['data'] = result
+            resp['salt_uuid'] = salt_uuid
+        except Exception as e:
+            if self.LOG.level == logging.DEBUG:
+                self.LOG.error(e, exc_info=True)
+            return self.finish({'code': 5201, 'message': [repr(e)]})
+
+        resp = json.dumps(resp, ensure_ascii=False, default=str, separators=(',', ':'))
+        return self.finish(resp)
+
+
+class APICaller(AbstractApiClient):
+    APP_ID_KEYS = AbstractApiClient.config.get('APP_ID_KEYS', {})
+
+    def wrap_request_data(
+            self, data, app_id: str = None, app_key: str = None, salt_uuid: str = None, sign: str = None, sign_method: str = None
+    ):
+        if app_id is None:
+            # if len(APP_ID_KEYS) != 1:
+            #     raise RuntimeError('Please specify 1 and only 1 in APP_ID_KEYS in configurations!')
+            app_id = list(self.APP_ID_KEYS.keys())[0]
+        salt_uuid = salt_uuid or str(uuid.uuid1())
+        sign = sign or sign_data(
+            salt_uuid=salt_uuid,
+            app_id=app_id,
+            app_key=app_key or self.APP_ID_KEYS.get(app_id),
+            data=data,
+            sign_method=sign_method
+        )
+        return {
+            'salt_uuid': salt_uuid,
+            'app_id': app_id,
+            'sign': sign,
+            'data': data
+        }
+
+
+def sign_data(salt_uuid: str, app_id: str, app_key: str, data, sign_method: str = None):
+    data_str = str(json.dumps(data, ensure_ascii=False, sort_keys=True, separators=(',', ':')))
+    public_key = app_id + salt_uuid + data_str + app_key
+
+    func_sign_check = func_sign_check_default if sign_method is None else FUNC_SIGN_CHECK.get(sign_method)
+    if func_sign_check is None:
+        raise ValueError('Invalid `sign_method`: %s' % sign_method)
+    sign = func_sign_check(public_key)
+    return sign
+
+
+def sign_check(salt_uuid: str, app_id: str, sign: str, data, sign_method: str = None, date_time=None):
+    """Sign Validation
+    :param salt_uuid: Universal Unified ID for 1) Signature, 2) Log tracing
+    :param app_id: APP ID
+    :param sign: sing = hash(app_id + salt_uuid + data + app_key)
+    :param data: data object, will be serialized to JSON string
+    :param date_time: not used for now
+    :param sign_method: Sign method, one of the following: md5, sha256
+    :return: If the signature passed validation
+    """
+
+    func_sign_check = func_sign_check_default if sign_method is None else FUNC_SIGN_CHECK.get(sign_method)
+    if func_sign_check is None:
+        raise ValueError('Invalid `sign_method`: %s' % sign_method)
+
+    app_key = APP_ID_KEYS.get(app_id)
+    if app_key is None:  # APP_ID not in the dict, unknown APP_ID
+        return False
+
+    data_str = str(json.dumps(data, ensure_ascii=False, sort_keys=True, separators=(',', ':')))
+
+    # --> Compatible with older version API
+    right_sign = func_sign_check(app_id + salt_uuid + app_key)
+    if sign == right_sign:
+        return True
+    # <--
+
+    public_key = str(json.dumps(data, ensure_ascii=False, sort_keys=True, separators=(',', ':')))
+    public_key = app_id + salt_uuid + public_key + app_key
+    right_sign = func_sign_check(public_key)
+    return sign == right_sign

src/aloha/service/api/v1.py

@@ -1,83 +1,41 @@
-import abc
+__all__ = ('APIHandler', 'APICaller',)
+
 import json
 import logging
-from datetime import datetime
+from abc import ABC
+from datetime import datetime, timedelta
 from typing import Optional, Awaitable
 
-from tornado import web
-
+from ..http import AbstractApiClient, AbstractApiHandler
 from ...encrypt import jwt
-from ...logger import LOG
 from ...settings import SETTINGS
 
-_RESP_BAD_REQUEST = {'code': '5101', 'message': ['Bad request: fail to parse body as JSON object!']}
-
-
-class APIHandler(web.RequestHandler):
-    LOG = LOG
-
-    def __init__(self, *args, **kwargs):
-        self.api_args: Optional[tuple] = None
-        self.api_kwargs: Optional[dict] = None
-        super().__init__(*args, **kwargs)
-
-    @abc.abstractmethod
-    def response(self, *args, **kwargs) -> dict:
-        raise NotImplementedError()
-
-    @property
-    def request_id(self):
-        if 'Request-ID' not in self.request.headers:
-            self.request.headers['Request-ID'] = datetime.now().strftime('%Y%m%d-%H%M%S-%f')
-        return self.request.headers.get('Request-ID')
-
-    def set_default_headers(self) -> None:
-        self.set_header('Content-Type', 'application/json; charset=utf-8')
-
-    def data_received(self, chunk: bytes) -> Optional[Awaitable[None]]:
-        pass
-
-    def on_finish(self) -> None:
-        func_callback = getattr(self, 'callback', None)
-        if not callable(func_callback):
-            return
-
-        return func_callback(*self.api_args, **self.api_kwargs)
 
+class APIHandler(AbstractApiHandler, ABC):
     async def prepare(self, ) -> Optional[Awaitable[None]]:
         access_token = self.request.headers.get('Access-Token')
         if access_token is None:
             return self.finish({
                 'msg': 'Invalid Access-Token in request header!'
             })
         else:
-            secret_key = SETTINGS.config['APP_SECRET_KEY']  # 'HCTECH-ASKBOB-REC:10062462'
+            secret_key = SETTINGS.config['APP_SECRET_KEY']
             # options = None
             # TODO: if not validate expiration
             options = {"verify_exp": False}
             access_token = jwt.decode(secret_key, access_token, options=options)
             if not isinstance(access_token, dict):
-                LOG.error('Invalid Access-Token found in request for [%s]: %s' % (
+                self.LOG.error('Invalid Access-Token found in request for [%s]: %s' % (
                     str(self.request.full_url()), access_token
                 ))
                 return self.finish({
                     'msg': access_token
                 })
-
         self.set_header('Request-ID', self.request_id)
 
     async def post(self, *args, **kwargs):
-        content_type: str = self.request.headers.get('Content-Type', 'application/json; charset=utf-8')
-        if content_type.startswith('multipart/form-data'):  # only parse files when 'Content-Type' starts with 'multipart/form-data'
-            body_arguments = self.request.body_arguments
-        else:
-            try:
-                body = self.request.body.decode('utf-8')
-                body_arguments = json.loads(body)
-            except (UnicodeDecodeError, json.decoder.JSONDecodeError):  # invalid request body, cannot be parsed as JSON
-                return self.finish(_RESP_BAD_REQUEST)
+        body_arguments = self.request_body
         kwargs.update(body_arguments)
-
         try:
             if self.LOG.level == logging.DEBUG:
                 s_kwargs = json.dumps(kwargs, ensure_ascii=False)
@@ -91,13 +49,11 @@ async def post(self, *args, **kwargs):
 
         if isinstance(resp, (dict, list)):
             resp = json.dumps(resp, ensure_ascii=False, default=str, separators=(',', ':'))
-        elif isinstance(resp, str):
-            pass
         return self.finish(resp)
 
     async def get(self, *args, **kwargs):
-        query = {k: v[0].decode('utf-8') for k, v in self.request.arguments.items()}
-        kwargs.update(query)
+        query_arguments = self.request_param
+        kwargs.update(query_arguments)
         try:
             self.LOG.debug('GET Request [%s]: %s' % (self.request_id, kwargs))
             self.api_args, self.api_kwargs = args or (), kwargs or {}
@@ -109,6 +65,33 @@ async def get(self, *args, **kwargs):
 
         if isinstance(resp, (dict, list)):
             resp = json.dumps(resp, ensure_ascii=False, default=str, separators=(',', ':'))
-        elif isinstance(resp, str):
-            pass
         return self.finish(resp)
+
+
+class APICaller(AbstractApiClient):
+    APP_ID_KEYS = AbstractApiClient.config.get('APP_ID_KEYS', {})
+    APP_SECRET_KEY = AbstractApiClient.config['APP_SECRET_KEY']
+
+    def wrap_request_data(self, data: dict) -> dict:
+        assert isinstance(data, dict), "Data object must be a dict!"
+        return data
+
+    def get_headers(self, app_id: str = None, app_key: str = None) -> dict:
+        if app_id is None:
+            # if len(APP_ID_KEYS) != 1:
+            #     raise RuntimeError('Please specify 1 and only 1 in APP_ID_KEYS in configurations!')
+            app_id = list(self.APP_ID_KEYS.keys())[0]
+
+        expire_time = datetime.now() + timedelta(days=1)
+
+        access_token = jwt.encode(
+            secret_key=self.APP_SECRET_KEY,
+            payload={
+                'exp': int(expire_time.timestamp()),
+                'aid': app_id
+            }
+        )
+
+        headers = super().get_headers()
+        headers.update({'Access-Token': access_token})
+        return headers

src/aloha/service/v2/api_handler.py src/aloha/service/api/v2.pysrc/aloha/service/v2/api_handler.py renamed to src/aloha/service/api/v2.py

@@ -1 +1,3 @@
+from .base_api_handler import AbstractApiHandler
 from .plain_http_handler import PlainHttpHandler
+from .base_api_client import AbstractApiClient

src/aloha/service/http/__init__.py

@@ -1,42 +1,58 @@
+import uuid
+from abc import ABC, abstractmethod
+
 import requests
 from requests.adapters import HTTPAdapter, Retry
 
 from ...logger import LOG
+from ...settings import SETTINGS
 
 
-class APICaller:
-    retry_method_whitelist = frozenset(['GET', 'POST'])
-    retry_status_forcelist = frozenset({413, 429, 503, 502, 504})
+class AbstractApiClient(ABC):
+    LOG = LOG
+    RETRY_METHOD_WHITELIST: frozenset = frozenset(['GET', 'POST'])
+    RETRY_STATUS_FORCELIST: frozenset = frozenset({413, 429, 503, 502, 504})
+    config = SETTINGS.config
 
     @classmethod
     def get_request_session(cls, total_retries: int = 3, *args, **kwargs) -> requests.Session:
         session = requests.Session()
         # https://urllib3.readthedocs.io/en/latest/reference/urllib3.util.html#urllib3.util.Retry.DEFAULT_ALLOWED_METHODS
         retries = Retry(
-            total=total_retries, backoff_factor=0.1, method_whitelist=cls.retry_method_whitelist, status_forcelist=cls.retry_status_forcelist
+            total=total_retries, backoff_factor=0.1, method_whitelist=cls.RETRY_METHOD_WHITELIST, status_forcelist=cls.RETRY_STATUS_FORCELIST
         )
         for prefix in ('http://', 'https://'):
             session.mount(prefix, HTTPAdapter(max_retries=retries))
         return session
 
-    @staticmethod
-    def wrap_request_data(data):
+    def get_headers(self, *args, **kwargs) -> dict:
+        headers = {
+            'Content-Type': 'application/json',
+            'Request-ID': str(uuid.uuid1()),
+        }
+        return headers
+
+    @abstractmethod
+    def wrap_request_data(self, data: dict) -> dict:
         assert isinstance(data, dict), "Data object must be a dict!"
-        return data
+        raise NotImplementedError()
+        # return data
 
-    def call(self, api_url, timeout=5, **kwargs):
+    def call(self, api_url: str, data: dict = None, timeout=5, **kwargs):
         """Trigger API call
-
         :param api_url: do NOT start with slash (/)
+        :param data: a dictionary which includes the request data
         :param timeout: requests timeout in seconds
-        :param kwargs: request data
+        :param kwargs: keywords arguments which will be updated to data
         :return:
         """
-        payload = APICaller.wrap_request_data(data=kwargs)
-        LOG.debug('Calling API: %s' % api_url)
+        body = data or dict()
+        body.update(kwargs)
+        payload = self.wrap_request_data(data=body)
+        LOG.debug('Calling api: %s' % api_url)
         session = self.get_request_session()
         resp = session.post(
-            api_url, json=payload, timeout=timeout, headers={'Content-Type': 'application/json'}
+            api_url, json=payload, timeout=timeout, headers=self.get_headers()
         )
 
         try: