Merge branch 'env_var_#95' into develop

Author: penyuan

README.md

@@ -1,4 +1,4 @@
-'# OSD status dashboard _(wp2.2_dev)_
+# OSD status dashboard _(wp2.2_dev)_
 
 [![Demo backend API](https://img.shields.io/badge/Demo-CLICK%20HERE-red.svg?style=flat)](https://wp22dev.herokuapp.com/)
 [![Python version](https://img.shields.io/badge/Python-3.10-blue.svg?style=flat&logo=Python&logoColor=white)](https://docs.python.org/3.8/)
@@ -17,19 +17,21 @@ Targeted at hosters of version control platforms (such as [Wikifactory](https://
 **Please see the [Install](#install) and [Usage](#usage) sections to get up and running with this tool**. For more details on its background and design considerations, please see the [Background](#background), ~~[Design notes](#design-notes), and [Future work](#future-work) sections. There is also a detailed [step-by-step walkthrough](docs/usage-example.md).~~
 ## Table of Contents
 
-- [Table of Contents](#table-of-contents)
-- [Background](#background)
-- [Install](#install)
-  - [Running from source](#running-from-source)
-  - [Deploy as container](#deploy-as-container)
-- [Usage](#usage)
-  - [Making requests to the REST API](#making-requests-to-the-rest-api)
-  - [API response format](#api-response-format)
-- [Design notes](#design-notes)
-- [Maintainers](#maintainers)
-- [Contributing](#contributing)
-- [Acknowledgements](#acknowledgements)
-- [License](#license)
+- [OSD status dashboard _(wp2.2_dev)_](#osd-status-dashboard-wp22_dev)
+  - [Table of Contents](#table-of-contents)
+  - [Background](#background)
+  - [Install](#install)
+    - [Running from source](#running-from-source)
+    - [Deploy as container](#deploy-as-container)
+  - [Usage](#usage)
+    - [Making requests to the REST API](#making-requests-to-the-rest-api)
+    - [API response format](#api-response-format)
+    - [Custom Wikifactory URLs](#custom-wikifactory-urls)
+  - [Design notes](#design-notes)
+  - [Maintainers](#maintainers)
+  - [Contributing](#contributing)
+  - [Acknowledgements](#acknowledgements)
+  - [License](#license)
 
 ## Background
 
@@ -259,6 +261,18 @@ Notes:
 * For `files_info` above, filetypes are identified by file extensions. The categories and mapping are located in [`oshminer/filetypes.py`](./oshminer/filetypes.py).
 * The `license` information and formatting is largely based on that from the GitHub-managed [choosealicense.com repository](https://github.com/github/choosealicense.com), with the exception of some open source hardware licenses which were manually added.
 
+### Custom Wikifactory URLs
+
+By default, this tool will: 
+
+1. Identify whether a provided repository URL in the JSON request body as a Wikifactory project if it is under the domain `wikifactory.com`
+2. Use the public Wikifactory GraphQL API endpoint at `https://wikifactory.com/api/graphql`
+
+Both can be customised with the following environmental variables: 
+
+1. `WIF_BASE_URL` - (default: `wikifactory.com`) The base domain used for pattern-matching and identifying Wikifactory project URLs in the JSON request body in the form of `example.com`. If this is customised, then the requested Wikifactory project URLs passed to this tool should also use that domain instead of `wikifactory.com`. Otherwise, an "Repository URL domain not supported" error will be returned.
+2. `WIF_API_URL` - (default: `https://wikifactory.com/api/graphql`) The full URL of the GraphQL API endpoint to make queries regarding Wikifactory projects in the form of `https://example.com[:port]/foo/bar`.
+
 ## Design notes
 
 [to be updated]

oshminer/Wikifactory.py

@@ -6,17 +6,28 @@
 # Python Standard Library imports
 from datetime import datetime
 import json
-import urllib.parse
+import os
+import urllib
 import sys
 
 # External imports
+from fastapi import status
+from fastapi.responses import JSONResponse
 from gql import gql, Client
 from gql.transport.aiohttp import AIOHTTPTransport
 
 # Internal imports
 import oshminer.filetypes as filetypes
 from oshminer.errors import exceptions
 
+# Set Wikifactory API URL
+# Looks for the `WIF_API_URL` environment variable, and if not found, default to: 
+# https://wikifactory.com/api/graphql
+# See: 
+# https://www.twilio.com/blog/environment-variables-python
+WIF_API_URL_DEFAULT: str = "https://wikifactory.com/api/graphql"
+WIF_API_URL: str = os.environ.get("WIF_API_URL", WIF_API_URL_DEFAULT)
+
 async def get_files_info(project: dict, session) -> dict:
     # Provide a GraphQL query
     query = gql(
@@ -405,28 +416,51 @@ def parse_url(url: str) -> dict:
     }
     return repo
 
-async def make_Wikifactory_request(url: str, data: list, api_url: str = "https://wikifactory.com/api/graphql") -> str: 
-    print(
-        f"Constructing and making an API request to Wikifactory for repository {url} for the following data {data}", 
-        file = sys.stderr
-    )
+async def make_Wikifactory_request(url: str, data: list) -> str: 
+    try: 
+        # First, check if there is a custom Wikifactory API URL and if it works
+        if WIF_API_URL != WIF_API_URL_DEFAULT: 
+            print(
+                f"Checking custom Wikifactory API URL: {WIF_API_URL}", 
+                file = sys.stderr
+                )
+            try: 
+                api_url_response = urllib.request.urlopen(
+                    WIF_API_URL, 
+                    timeout=5
+                    )
+            except (urllib.error.URLError): 
+                print(f"Unable to reach Wikifactory API at: {WIF_API_URL}", file=sys.stderr)
+                raise exceptions.BadWIFAPIError
 
-    # Create a dictionary to hold results from Wikifactory API query
-    results: dict = {
-        "repository": str(url), 
-        "platform": "Wikifactory", 
-        "requested_data": {}
-    }
+        print(
+            f"Constructing and making an API request to Wikifactory for repository {url} for the following data {data}", 
+            file = sys.stderr
+        )
+
+        # If the Wikifactory API is reacheable as tested above, then: 
+        # Create a dictionary to hold results from Wikifactory API query
+        results: dict = {
+            "repository": str(url), 
+            "platform": "Wikifactory", 
+            "requested_data": {}
+        }
 
-    # Select transport with the Wikifactory API endpoint URL
-    transport = AIOHTTPTransport(url = api_url)
+        # Select transport with the Wikifactory API endpoint URL
+        transport = AIOHTTPTransport(url = WIF_API_URL)
 
-    # Get "space" and "slug" components from this repository's URL
-    space_slug: dict = parse_url(url)
+        # Get "space" and "slug" components from this repository's URL
+        space_slug: dict = parse_url(url)
 
-    async with Client(transport = transport, fetch_schema_from_transport = True) as session: 
-        for data_type in data: 
-            query_result: dict = await queries[data_type](space_slug, session)
-            results["requested_data"].update(query_result)
+        async with Client(transport = transport, fetch_schema_from_transport = True) as session: 
+            for data_type in data: 
+                query_result: dict = await queries[data_type](space_slug, session)
+                results["requested_data"].update(query_result)
 
-    return results
+        return results
+        
+    except (exceptions.BadWIFAPIError) as err: 
+        return JSONResponse(
+                status_code = status.HTTP_400_BAD_REQUEST, 
+                content = f"Error reaching Wikifactory API URL: {WIF_API_URL} {err}"
+            )

oshminer/supported_platforms.py

@@ -3,14 +3,27 @@
 # SPDX-FileCopyrightText: 2022 Pen-Yuan Hsing
 # SPDX-License-Identifier: AGPL-3.0-or-later
 
+# Python Standard Library imports
+import os
+import sys
+
+# Internal imports
 import oshminer.GitHub
 import oshminer.Wikifactory
 
+# Set Wikifactory base URL for projects
+# Looks for the `WIF_BASE_URL` environment variable, and if not found, default to: 
+# `wikifactory.com`
+# See: 
+# https://www.twilio.com/blog/environment-variables-python
+WIF_BASE_URL: str = os.environ.get("WIF_BASE_URL", "wikifactory.com")
+print(f"Wikifactory base URL: {WIF_BASE_URL}", file=sys.stderr)
+
 # Supported hosting platforms as a dictionary of their domains, which are 
 # matched to names of internal functions that construct API requests to the 
 # associated platforms.
-# `repo_urls` need to include one of these domains.
+# `repo_urls` in the request body needs to include one of these domains.
 supported_domains: dict = { 
     "github.com": oshminer.GitHub.make_GitHub_request, 
-    "wikifactory.com": oshminer.Wikifactory.make_Wikifactory_request
+    WIF_BASE_URL: oshminer.Wikifactory.make_Wikifactory_request
 }