Git workflow: handle URL error 202

.CI/checkHTMLDoc/checkLinks.py

@@ -2,25 +2,29 @@
 # -*- coding: utf-8 -*-
 
 '''
-Copyright (C) 2020-2022, Modelica Association and contributors
+Copyright (C) 2020-2026, Modelica Association and contributors
 All rights reserved.
 
 Check Modelica HTML documentation for link validity
 '''
 
 import re
 import os
-try:
-    import urllib2
-except ImportError:
-    import urllib.request as urllib2
-import ssl
+# try:
+#     import urllib2
+# except ImportError:
+#     import urllib.request as urllib2
+import requests
+# import ssl
 
 from concurrent.futures import ProcessPoolExecutor as PoolExecutor
 
 # See https://haacked.com/archive/2004/10/25/usingregularexpressionstomatchhtml.aspx/
-PATTERN = re.compile(r'</?\w+((\s+\w+(\s*=\s*(?:\\"(.|\n)*?\\"|\'(.|\n)*?\'|[^\'">\s]+))?)+\s*|\s*)/?>',
+PATTERN = re.compile(
+    r'</?\w+((\s+\w+(\s*=\s*(?:\\"(.|\n)*?\\"|\'(.|\n)*?\'|[^\'">\s]+))?)+\s*|\s*)/?>',
     re.IGNORECASE)
+# DOI-Regex: http(s)://doi.org/... (incl. possible suffixes such as #, ?, etc.)
+DOI_PATTERN = re.compile(r'^https?://doi\.org/', re.IGNORECASE)
 
 def _getFileURLs(file_name):
     urls = []
@@ -53,23 +57,36 @@ def _getURLs(path):
 
 def _checkURL(url):
     try:
-        rc = urllib2.urlopen(url).getcode()
+        allow_redirects=False
+        response = requests.head(url, allow_redirects=allow_redirects)
+        rc = response.status_code
+
+        # If requests.head not allowed -> requests.get
+        if rc == 405:
+            response = requests.get(url, allow_redirects=allow_redirects)
+            rc = response.status_code
         return (url, rc)
     except:
-        pass
+        if DOI_PATTERN.match(url):
+            print("WARNING: Skip DOI-Link (per design, because of persistency): {url}")
+            return (url, 200)
+
     try:
+        allow_redirects=True
         headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64)'}
-        rc = urllib2.urlopen(urllib2.Request(url, None, headers), context=ssl._create_unverified_context()).getcode()
-    except urllib2.HTTPError as e:
-        rc = e.code
-        if rc == 429:
-            # Ignore too many requests
-            rc = 200
-        elif rc in (301, 302):
-            # Handle redirect errors
-            rc = urllib2.build_opener(urllib2.HTTPCookieProcessor).open(url).code
-    except:
-        rc = 0
+        response = requests.head(url, allow_redirects=allow_redirects, headers=headers)
+        rc = response.status_code
+
+        # If requests.head not allowed -> requests.get
+        if rc == 405:
+            response = requests.get(url, allow_redirects=allow_redirects, headers=headers)
+            rc = response.status_code
+    except requests.exceptions.TooManyRedirects:
+        # rc = 310
+        rc = 200
+    except requests.exceptions.RequestException as e:
+        print("Error ", e)
+        rc = 0  # unknown error
     return (url, rc)
 
 def checkLinks(path):

.github/workflows/checkCI.yml

@@ -37,6 +37,23 @@ jobs:
           echo "::add-matcher::./.github/checkTags.json"
           python ./.CI/check_html.py checkTags ./
           echo "::remove-matcher owner=checkTags::"
+  html_documentation_link_checks:
+    timeout-minutes: 5
+    if: github.repository_owner == 'DLR-SR'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          fetch-depth: 5
+      - name: Setup python environment
+        uses: actions/setup-python@v6
+        with:
+          python-version: 3.8
+      - name: Install python packages
+        run: pip install --user futures
+      - name: Check links
+        timeout-minutes: 3
+        run: python ./.CI/check_html.py checkLinks ./
   syntax_checks:
     timeout-minutes: 5
     runs-on: ubuntu-latest