Merge branch 'develop' into main

Author: penyuan

oshminer/GitHub.py

@@ -3,5 +3,14 @@
 # SPDX-FileCopyrightText: 2022 Pen-Yuan Hsing
 # SPDX-License-Identifier: AGPL-3.0-or-later
 
-def make_GitHub_request(url: str, data: list) -> str: 
-    print(f"Constructing an API request to GitHub for repository {url} for the following data {data}")
+async def make_GitHub_request(url: str, data: list) -> dict: 
+    print(f"Constructing an API request to GitHub for repository {url} for the following data {data}")
+
+    # Create a dictionary to hold results from GitHub API query
+    results: dict = {
+        "repository": str(url), 
+        "platform": "GitHub", 
+        "requested_data": {}
+    }
+
+    return results

oshminer/Wikifactory.py

@@ -99,12 +99,19 @@ async def get_files_info(project: dict, session) -> dict:
             - result["files_info"]["data_files"]\
             - result["files_info"]["document_files"]
         # Calculate proportion of each file type
-        result["files_info"]["ecad_proportion"]: float = result["files_info"]["ecad_files"]/result["files_info"]["total_files"]
-        result["files_info"]["mcad_proportion"]: float = result["files_info"]["mcad_files"]/result["files_info"]["total_files"]
-        result["files_info"]["image_proportion"]: float = result["files_info"]["image_files"]/result["files_info"]["total_files"]
-        result["files_info"]["data_proportion"]: float = result["files_info"]["data_files"]/result["files_info"]["total_files"]
-        result["files_info"]["document_proportion"]: float = result["files_info"]["document_files"]/result["files_info"]["total_files"]
-        result["files_info"]["other_proportion"]: float = result["files_info"]["other_files"]/result["files_info"]["total_files"]
+        # Limit to three decimal places to reduce size of response
+        ecad_proportion: float = result["files_info"]["ecad_files"]/result["files_info"]["total_files"]
+        mcad_proportion: float = result["files_info"]["mcad_files"]/result["files_info"]["total_files"]
+        image_proportion: float = result["files_info"]["image_files"]/result["files_info"]["total_files"]
+        data_proportion: float = result["files_info"]["data_files"]/result["files_info"]["total_files"]
+        document_proportion: float = result["files_info"]["document_files"]/result["files_info"]["total_files"]
+        other_proportion: float = result["files_info"]["other_files"]/result["files_info"]["total_files"]
+        result["files_info"]["ecad_proportion"]: float = float(f"{ecad_proportion:.3f}")
+        result["files_info"]["mcad_proportion"]: float = float(f"{mcad_proportion:.3f}")
+        result["files_info"]["image_proportion"]: float = float(f"{image_proportion:.3f}")
+        result["files_info"]["data_proportion"]: float = float(f"{data_proportion:.3f}")
+        result["files_info"]["document_proportion"]: float = float(f"{document_proportion:.3f}")
+        result["files_info"]["other_proportion"]: float = float(f"{other_proportion:.3f}")
     else: # If there are no files, return 0 for everything
         result: dict = {
             "files_info": {
@@ -398,7 +405,7 @@ def parse_url(url: str) -> dict:
     }
     return repo
 
-async def make_Wikifactory_request(url: str, data: list) -> str: 
+async def make_Wikifactory_request(url: str, data: list, api_url: str = "https://wikifactory.com/api/graphql") -> str: 
     print(
         f"Constructing and making an API request to Wikifactory for repository {url} for the following data {data}", 
         file = sys.stderr
@@ -412,7 +419,7 @@ async def make_Wikifactory_request(url: str, data: list) -> str:
     }
 
     # Select transport with the Wikifactory API endpoint URL
-    transport = AIOHTTPTransport(url = "https://wikifactory.com/api/graphql")
+    transport = AIOHTTPTransport(url = api_url)
 
     # Get "space" and "slug" components from this repository's URL
     space_slug: dict = parse_url(url)

oshminer/errors/exceptions.py

@@ -4,4 +4,7 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 
 class BadRepoError(Exception): 
+    pass
+
+class BadWIFAPIError(Exception):
     pass

oshminer/main.py

@@ -6,7 +6,9 @@
 # Python Standard Library imports
 import asyncio
 import json
+import http.client
 import sys
+import urllib
 
 # External library imports
 from fastapi import FastAPI, status
@@ -17,9 +19,12 @@
 from oshminer.supported_platforms import supported_domains
 from oshminer.errors import exceptions
 
+# Default Wikifactory API URL
+WIF_API_DEFAULT: str = "https://wikifactory.com/api/graphql"
 class MiningRequest(BaseModel): 
     repo_urls: list[HttpUrl] = set()
     requested_data: list[str] = set()
+    wikifactory_API_URL: str = WIF_API_DEFAULT
 
 # Supported data-mining request types. Items in `required_data` must
 # be from this list.
@@ -44,23 +49,58 @@ class MiningRequest(BaseModel):
 async def root(): 
     return {"message": "Dashboard data-mining backend is on"}
 
-async def process_repo(repo: HttpUrl, requests: list[str], responses: list): 
+async def process_repo(repo: HttpUrl, requests: list[str], responses: list, WIF_API: str = WIF_API_DEFAULT): 
     platform: str = repo.host.replace("www.", "")
-    try: 
-        repo_info: dict = await supported_domains[platform](repo, requests)
-    except exceptions.BadRepoError: 
-        return JSONResponse(
-            status_code = status.HTTP_400_BAD_REQUEST, 
-            content = f"Error with repository: {repo}"
-        )
-    responses.append(repo_info)
+    # If a custom Wikifactory API URL is provided, then use it.
+    if WIF_API != WIF_API_DEFAULT: 
+        try: 
+            repo_info: dict = await supported_domains[platform](repo, requests, WIF_API)
+        except exceptions.BadRepoError: 
+            return JSONResponse(
+                status_code = status.HTTP_400_BAD_REQUEST, 
+                content = f"Error with repository: {repo}"
+            )
+        responses.append(repo_info)
+    else: 
+        try: 
+            repo_info: dict = await supported_domains[platform](repo, requests)
+        except exceptions.BadRepoError: 
+            return JSONResponse(
+                status_code = status.HTTP_400_BAD_REQUEST, 
+                content = f"Error with repository: {repo}"
+            )
+        responses.append(repo_info)
 
-@app.get(
+@app.post(
     "/data/", 
     name = "API endpoint", 
     description = "Primary endpoint for requesting data."
     )
 async def mining_request(request_body: MiningRequest): 
+    #
+    # Check if custom Wikifactory API URL is provided and test it
+    #
+    
+    if request_body.wikifactory_API_URL != WIF_API_DEFAULT: 
+        print(
+            f"Custom Wikifactory API URL detected: {request_body.wikifactory_API_URL}", 
+            file = sys.stderr
+            )
+        try: 
+            api_url_response = urllib.request.urlopen(
+                request_body.wikifactory_API_URL, 
+                timeout=10
+                )
+            if api_url_response.status != 200: 
+                raise exceptions.BadWIFAPIError
+        except (exceptions.BadWIFAPIError, urllib.error.URLError, http.client.BadStatusLine) as err: 
+            return JSONResponse(
+                status_code = status.HTTP_400_BAD_REQUEST, 
+                content = f"Error reaching Wikifactory API URL: {request_body.wikifactory_API_URL} {err}"
+            )
+    elif request_body.wikifactory_API_URL is None: 
+        request_body.wikifactory_API_URL = WIF_API_DEFAULT
+
     # 
     # Check API client's request body
     #
@@ -103,7 +143,16 @@ async def mining_request(request_body: MiningRequest):
     # Construct, send API requests, and get results
     #
 
-    await asyncio.gather(*[process_repo(repo, request_body.requested_data, response_list) for repo in request_body.repo_urls])
+    await asyncio.gather(
+        *[
+            process_repo(
+                repo, 
+                request_body.requested_data, 
+                response_list, 
+                WIF_API=request_body.wikifactory_API_URL
+                ) for repo in request_body.repo_urls
+            ]
+        )
 
     # for repo in request_body.repo_urls: 
     #     platform = repo.host.replace("www.", "")