From 81d233eff3a65ec352da013f4289e38e8be1bb2a Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 24 Jul 2026 18:25:12 +0800 Subject: [PATCH 01/10] Support bundled and attached short flags in curl parser Browser- and hand-written curl commands routinely glue short flags together (-XPOST, -fsSL) or attach a value directly (-d'a=1', -H'Accept: ...', -uuser:pass). These previously parsed wrong: -XPOST left the method as GET, -sX POST made the URL "POST", and attached values were dropped. Expand short-flag clusters into canonical tokens before consuming them: a value-taking flag ends the cluster and takes any glued remainder as its value, while an unknown short flag leaves the token untouched. Short-flag sets are derived from the existing tables, and -S/--show-error, -q/--disable plus a few value-taking flags (-C/-z/-D/-K) are recognised so common clusters resolve. Also drop a pre-existing unused import from the test module. --- pybreeze/utils/curl_import/curl_parser.py | 57 ++++++++++++++++++++++- test/test_utils/test_curl_import.py | 54 ++++++++++++++++++++- 2 files changed, 108 insertions(+), 3 deletions(-) diff --git a/pybreeze/utils/curl_import/curl_parser.py b/pybreeze/utils/curl_import/curl_parser.py index f593034..23c2bdf 100644 --- a/pybreeze/utils/curl_import/curl_parser.py +++ b/pybreeze/utils/curl_import/curl_parser.py @@ -103,6 +103,7 @@ def header_value(self, name: str) -> str | None: # Value-less flags to accept and skip (they do not affect the generated request). _VALUELESS_FLAGS = frozenset({ "--compressed", "-L", "--location", "-k", "--insecure", "-s", "--silent", + "-S", "--show-error", "-q", "--disable", "-v", "--verbose", "-i", "--include", "-I", "--head", "-f", "--fail", "--fail-with-body", "-g", "--globoff", "-O", "--remote-name", "-J", "--remote-header-name", "-#", "--progress-bar", "-N", "--no-buffer", @@ -124,9 +125,63 @@ def header_value(self, name: str) -> str | None: "-r", "--range", "-c", "--cookie-jar", "--resolve", "--interface", "--dns-servers", "--local-port", "--ciphers", "-y", "--speed-time", "-Y", "--speed-limit", "--keepalive-time", "--oauth2-bearer", "--aws-sigv4", + "-C", "--continue-at", "-z", "--time-cond", "-D", "--dump-header", + "-K", "--config", }) +def _short_flags(*tables: object) -> frozenset[str]: + """Collect the single-dash, single-letter flags found in *tables*.""" + return frozenset( + flag for table in tables for flag in table + if len(flag) == 2 and flag[0] == "-" and flag[1] != "-" + ) + + +# Short flags derived from the tables above so a bundled cluster like +# ``-sXPOST`` can split into ``-s`` and ``-X`` + ``POST`` without a second list. +_SHORT_VALUE_FLAGS = _short_flags(_VALUE_FLAGS, _IGNORED_VALUE_FLAGS) +_SHORT_VALUELESS_FLAGS = _short_flags(_VALUELESS_FLAGS, _GET_FLAGS) + + +def _is_short_flag_cluster(token: str) -> bool: + """Whether *token* is a single-dash flag bundling more than one character.""" + return len(token) > 2 and token[0] == "-" and token[1] != "-" + + +def _expand_short_flag_cluster(token: str) -> list[str] | None: + """Split a bundled short-flag token into canonical tokens. + + ``-sXPOST`` becomes ``["-s", "-X", "POST"]`` and ``-fsSL`` becomes + ``["-f", "-s", "-S", "-L"]``. A value-taking flag ends the cluster: anything + glued after it is that flag's value. Returns ``None`` when an unknown short + flag is met, so the caller keeps the original token untouched. + """ + expanded: list[str] = [] + body = token[1:] + for position, letter in enumerate(body): + flag = f"-{letter}" + if flag in _SHORT_VALUE_FLAGS: + expanded.append(flag) + attached = body[position + 1:] + if attached: # value glued to the flag, e.g. the POST in -XPOST + expanded.append(attached) + return expanded + if flag not in _SHORT_VALUELESS_FLAGS: + return None # an unknown short flag: do not rewrite the cluster + expanded.append(flag) + return expanded + + +def _expand_short_flags(tokens: list[str]) -> list[str]: + """Expand bundled/attached short-flag tokens; pass everything else through.""" + expanded: list[str] = [] + for token in tokens: + cluster = _expand_short_flag_cluster(token) if _is_short_flag_cluster(token) else None + expanded.extend(cluster if cluster is not None else [token]) + return expanded + + def _normalise_command(command: str) -> str: """Strip continuations so a multi-line pasted command tokenises as one.""" return _LINE_CONTINUATION_RE.sub(" ", command.strip()) @@ -268,6 +323,6 @@ def parse_curl(command: str) -> CurlRequest: raise CurlParseException(not_a_curl_command_error) request = CurlRequest() - _consume_tokens(tokens[1:], request) + _consume_tokens(_expand_short_flags(tokens[1:]), request) _finalise_method(request) return request diff --git a/test/test_utils/test_curl_import.py b/test/test_utils/test_curl_import.py index a943742..0a09ded 100644 --- a/test/test_utils/test_curl_import.py +++ b/test/test_utils/test_curl_import.py @@ -1,8 +1,6 @@ """Tests for the cURL command parser and requests-code generator.""" from __future__ import annotations -import json - import pytest from pybreeze.utils.curl_import.curl_parser import ( @@ -170,6 +168,58 @@ def test_multiple_ignored_flags(self): assert request.body == "a=1" +class TestParseCurlShortFlagClusters: + def test_attached_method_value(self): + # Regression: '-XPOST' used to leave the method as GET. + request = parse_curl("curl -XPOST https://x") + assert request.method == "POST" + assert request.url == "https://x" + + def test_bundled_valueless_then_value_flag(self): + # Regression: '-sX POST' used to make the URL "POST". + request = parse_curl("curl -sX POST https://x/api") + assert request.method == "POST" + assert request.url == "https://x/api" + + def test_all_valueless_cluster(self): + request = parse_curl("curl -fsSL https://x") + assert request.url == "https://x" + assert request.method == "GET" + + def test_attached_data_value(self): + request = parse_curl("curl -d'a=1' https://x") + assert request.body == "a=1" + assert request.method == "POST" + + def test_attached_header_value(self): + request = parse_curl("curl -H'Accept: application/json' https://x") + assert request.headers["Accept"] == "application/json" + + def test_attached_auth_value(self): + request = parse_curl("curl -uuser:pass https://x") + assert request.username == "user" + assert request.password == "pass" + + def test_cluster_with_attached_ignored_value(self): + # '-oout.txt' must consume the filename, not leak it to the URL. + request = parse_curl("curl -oout.txt https://x") + assert request.url == "https://x" + + def test_cluster_ending_in_ignored_value_flag(self): + request = parse_curl("curl -sSL -o /dev/null https://x/api") + assert request.url == "https://x/api" + + def test_unknown_short_cluster_left_untouched(self): + # An unrecognised short flag must not corrupt the URL. + request = parse_curl("curl -Wx https://x") + assert request.url == "https://x" + + def test_cluster_feeds_code_generation(self): + code = to_requests_code(parse_curl("curl -XPOST -d'a=1' https://x/api")) + assert '"POST"' in code + assert "data=data" in code + + class TestParseCurlJsonFlag: def test_json_flag_does_not_leak_to_url(self): # Regression: '--json {...}' must not become the URL. From c8e3e9f1fd410d243123a77ba707c2e65654ae33 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 24 Jul 2026 18:27:58 +0800 Subject: [PATCH 02/10] Carry curl --max-time through to generated requests code --max-time / -m was consumed and discarded, so a generated script silently dropped the original timeout. Capture the (numeric-validated) value on CurlRequest and emit timeout= on the requests.request call, so the generated requests and pytest output honour the command's timeout. Non-numeric values are still consumed but ignored, keeping the URL safe. --- pybreeze/utils/curl_import/curl_parser.py | 17 +++++++++- pybreeze/utils/curl_import/request_codegen.py | 3 ++ test/test_utils/test_curl_import.py | 31 +++++++++++++++++++ 3 files changed, 50 insertions(+), 1 deletion(-) diff --git a/pybreeze/utils/curl_import/curl_parser.py b/pybreeze/utils/curl_import/curl_parser.py index 23c2bdf..b9164fc 100644 --- a/pybreeze/utils/curl_import/curl_parser.py +++ b/pybreeze/utils/curl_import/curl_parser.py @@ -46,6 +46,8 @@ class CurlRequest: :param send_data_as_params: ``True`` when ``-G`` moves the body to the query :param form_fields: multipart form fragments from ``-F`` / ``--form`` :param data_file_refs: filenames whose content forms the body (``-d @file``) + :param timeout: request timeout in seconds from ``--max-time`` / ``-m``, or + ``None`` when the command sets none """ method: str = _DEFAULT_METHOD @@ -58,6 +60,7 @@ class CurlRequest: send_data_as_params: bool = False form_fields: list[str] = field(default_factory=list) data_file_refs: list[str] = field(default_factory=list) + timeout: str | None = None @property def has_body(self) -> bool: @@ -95,6 +98,7 @@ def header_value(self, name: str) -> str | None: "-A": "user_agent", "--user-agent": "user_agent", "-e": "referer", "--referer": "referer", "--url": "url", + "-m": "timeout", "--max-time": "timeout", } # Value-less flags that still change behaviour. @@ -118,7 +122,7 @@ def header_value(self, name: str) -> str | None: # Flags that take a value we do not use; the value is consumed so it cannot be # mistaken for the URL (this is what makes ``--max-time 30 https://x`` parse right). _IGNORED_VALUE_FLAGS = frozenset({ - "-o", "--output", "--max-time", "-m", "--connect-timeout", "--retry", + "-o", "--output", "--connect-timeout", "--retry", "--retry-delay", "--retry-max-time", "-w", "--write-out", "-x", "--proxy", "--proxy-user", "-U", "--cacert", "--capath", "-E", "--cert", "--key", "--cert-type", "--key-type", "--pass", "-T", "--upload-file", "--limit-rate", @@ -221,6 +225,15 @@ def _urlencode_data_part(value: str) -> str: return quote(value, safe="") +def _apply_timeout(request: CurlRequest, value: str) -> None: + """Record a numeric timeout (seconds); ignore a non-numeric value.""" + try: + float(value) + except ValueError: + return + request.timeout = value + + def _apply_value_flag(request: CurlRequest, kind: str, value: str) -> None: """Apply one value-taking flag to *request* according to its *kind*.""" if kind == "method": @@ -251,6 +264,8 @@ def _apply_value_flag(request: CurlRequest, kind: str, value: str) -> None: request.headers.setdefault("Referer", value) elif kind == "url": request.url = value + elif kind == "timeout": + _apply_timeout(request, value) elif kind == "user": request.username, _sep, request.password = value.partition(":") diff --git a/pybreeze/utils/curl_import/request_codegen.py b/pybreeze/utils/curl_import/request_codegen.py index 707256e..0da3499 100644 --- a/pybreeze/utils/curl_import/request_codegen.py +++ b/pybreeze/utils/curl_import/request_codegen.py @@ -99,6 +99,9 @@ def _call_keyword_arguments(request: CurlRequest, payload_kwargs: list[str]) -> arguments.extend(payload_kwargs) if request.username is not None: arguments.append("auth=auth") + if request.timeout is not None: + # timeout is validated as numeric by the parser, so it is safe inline. + arguments.append(f"timeout={request.timeout}") return arguments diff --git a/test/test_utils/test_curl_import.py b/test/test_utils/test_curl_import.py index 0a09ded..8b6d13a 100644 --- a/test/test_utils/test_curl_import.py +++ b/test/test_utils/test_curl_import.py @@ -220,6 +220,37 @@ def test_cluster_feeds_code_generation(self): assert "data=data" in code +class TestParseCurlTimeout: + def test_max_time_captured(self): + request = parse_curl("curl --max-time 30 https://x") + assert request.timeout == "30" + assert request.url == "https://x" + + def test_short_max_time_flag(self): + request = parse_curl("curl -m 5 https://x") + assert request.timeout == "5" + + def test_attached_short_max_time(self): + request = parse_curl("curl -m2.5 https://x") + assert request.timeout == "2.5" + + def test_non_numeric_timeout_ignored(self): + request = parse_curl("curl --max-time soon https://x") + assert request.timeout is None + assert request.url == "https://x" # the value is still consumed + + def test_no_timeout_by_default(self): + assert parse_curl("curl https://x").timeout is None + + def test_timeout_emitted_in_code(self): + code = to_requests_code(parse_curl("curl --max-time 30 https://x")) + assert "timeout=30" in code + compile(code, "", "exec") + + def test_no_timeout_kwarg_when_absent(self): + assert "timeout=" not in to_requests_code(parse_curl("curl https://x")) + + class TestParseCurlJsonFlag: def test_json_flag_does_not_leak_to_url(self): # Regression: '--json {...}' must not become the URL. From 36e966e57f04396642fa18234194d3e7ffcb433c Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 24 Jul 2026 18:29:56 +0800 Subject: [PATCH 03/10] Extract curl -d file/inline handling into a helper Adding the timeout branch lifted _apply_value_flag's cognitive complexity to the point where SonarQube S3776 would trip. Move the -d @file vs inline body decision into a small helper so the dispatch stays a flat, low-complexity chain. --- pybreeze/utils/curl_import/curl_parser.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/pybreeze/utils/curl_import/curl_parser.py b/pybreeze/utils/curl_import/curl_parser.py index b9164fc..a911c93 100644 --- a/pybreeze/utils/curl_import/curl_parser.py +++ b/pybreeze/utils/curl_import/curl_parser.py @@ -234,6 +234,14 @@ def _apply_timeout(request: CurlRequest, value: str) -> None: request.timeout = value +def _apply_data_or_file(request: CurlRequest, value: str) -> None: + """Record a ``-d`` value as an ``@file`` reference or an inline body part.""" + if value.startswith("@"): + request.data_file_refs.append(value[1:]) + else: + request.data_parts.append(value) + + def _apply_value_flag(request: CurlRequest, kind: str, value: str) -> None: """Apply one value-taking flag to *request* according to its *kind*.""" if kind == "method": @@ -245,10 +253,7 @@ def _apply_value_flag(request: CurlRequest, kind: str, value: str) -> None: elif kind == "data_urlencode": request.data_parts.append(_urlencode_data_part(value)) elif kind == "data_file": - if value.startswith("@"): - request.data_file_refs.append(value[1:]) - else: - request.data_parts.append(value) + _apply_data_or_file(request, value) elif kind == "json_flag": # curl --json is shorthand for --data + JSON Content-Type and Accept. request.data_parts.append(value) From 6209043e4c48be4944d094f3ce480f729a474e00 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 24 Jul 2026 18:51:24 +0800 Subject: [PATCH 04/10] Split a URL query string into params on curl import Browser "copy as cURL" keeps the query in the URL (e.g. ?q=test&page=2), so params stayed empty and the query was opaque. Parse it into params (URL-decoding values, keeping blanks) without overwriting explicit -G/-d pairs. Add CurlRequest.full_url to rebuild the original address from the base URL plus params, and drive the LoadDensity template by it so query params - from the URL or from -G, which that template previously dropped - survive. The requests, pytest and APITestka templates already emit params, so they gain the URL query automatically. --- pybreeze/utils/curl_import/curl_parser.py | 32 +++++++++++++- .../utils/curl_import/script_templates.py | 3 +- test/test_utils/test_curl_import.py | 42 +++++++++++++++++++ test/test_utils/test_script_templates.py | 8 ++++ 4 files changed, 83 insertions(+), 2 deletions(-) diff --git a/pybreeze/utils/curl_import/curl_parser.py b/pybreeze/utils/curl_import/curl_parser.py index a911c93..8e53b02 100644 --- a/pybreeze/utils/curl_import/curl_parser.py +++ b/pybreeze/utils/curl_import/curl_parser.py @@ -12,7 +12,7 @@ import re import shlex from dataclasses import dataclass, field -from urllib.parse import quote +from urllib.parse import parse_qsl, quote, urlencode from pybreeze.utils.exception.exception_tags import ( empty_curl_command_error, @@ -72,6 +72,19 @@ def body(self) -> str: """Return the body fragments joined the way ``curl`` sends them.""" return "&".join(self.data_parts) + @property + def full_url(self) -> str: + """The URL with any collected query params reattached, as curl sends it. + + After parsing, a query string embedded in the URL is moved into + :attr:`params`; this rebuilds the original address for consumers (such as + the load-test template) that drive purely by URL. + """ + if not self.params: + return self.url + joiner = "&" if "?" in self.url else "?" + return f"{self.url}{joiner}{urlencode(self.params)}" + def header_value(self, name: str) -> str | None: """Return a header's value by case-insensitive *name*, or ``None``.""" lowered = name.lower() @@ -310,6 +323,22 @@ def _finalise_method(request: CurlRequest) -> None: request.data_parts = [] +def _split_url_query(request: CurlRequest) -> None: + """Move a query string embedded in the URL into ``params``. + + Browser "copy as cURL" keeps the query in the URL; splitting it out lets it + show up alongside ``-G`` / ``-d`` query pairs, while + :attr:`CurlRequest.full_url` can still rebuild the original address. Values + are URL-decoded, and existing params are not overwritten. + """ + base, separator, query = request.url.partition("?") + if not separator: + return + request.url = base + for key, value in parse_qsl(query, keep_blank_values=True): + request.params.setdefault(key, value) + + def parse_query_pairs(parts: list[str]) -> dict[str, str]: """Parse ``key=value`` fragments into a dict, ignoring pieces without ``=``. @@ -345,4 +374,5 @@ def parse_curl(command: str) -> CurlRequest: request = CurlRequest() _consume_tokens(_expand_short_flags(tokens[1:]), request) _finalise_method(request) + _split_url_query(request) return request diff --git a/pybreeze/utils/curl_import/script_templates.py b/pybreeze/utils/curl_import/script_templates.py index 94de4ad..6a48a65 100644 --- a/pybreeze/utils/curl_import/script_templates.py +++ b/pybreeze/utils/curl_import/script_templates.py @@ -92,7 +92,8 @@ def to_loaddensity_python(request: CurlRequest) -> str: :return: Python source calling ``start_test`` """ method_key = request.method.lower() - task = f"{{{_inline_json(method_key)}: {{\"request_url\": {_inline_json(request.url)}}}}}" + # Drive by the full URL so query params (from the URL or -G) are not lost. + task = f"{{{_inline_json(method_key)}: {{\"request_url\": {_inline_json(request.full_url)}}}}}" lines = [ "from je_load_density import start_test", "", diff --git a/test/test_utils/test_curl_import.py b/test/test_utils/test_curl_import.py index 8b6d13a..5882a83 100644 --- a/test/test_utils/test_curl_import.py +++ b/test/test_utils/test_curl_import.py @@ -251,6 +251,48 @@ def test_no_timeout_kwarg_when_absent(self): assert "timeout=" not in to_requests_code(parse_curl("curl https://x")) +class TestParseCurlUrlQuery: + def test_query_moved_to_params(self): + request = parse_curl("curl 'https://x/api?a=1&b=2'") + assert request.url == "https://x/api" + assert request.params == {"a": "1", "b": "2"} + + def test_query_values_are_url_decoded(self): + request = parse_curl("curl 'https://x?q=hello%20world'") + assert request.params["q"] == "hello world" + + def test_blank_query_value_kept(self): + request = parse_curl("curl 'https://x?flag='") + assert request.params == {"flag": ""} + + def test_no_query_leaves_url_untouched(self): + request = parse_curl("curl https://x/api") + assert request.url == "https://x/api" + assert request.params == {} + + def test_full_url_reconstructs_query(self): + request = parse_curl("curl 'https://x/api?a=1&b=2'") + assert request.full_url == "https://x/api?a=1&b=2" + + def test_full_url_without_params_is_url(self): + assert parse_curl("curl https://x/api").full_url == "https://x/api" + + def test_full_url_includes_get_flag_params(self): + # -G params never sat in the URL, but full_url should surface them. + request = parse_curl("curl -G https://x/api -d 'a=1'") + assert request.full_url == "https://x/api?a=1" + + def test_explicit_params_not_overwritten_by_url_query(self): + request = parse_curl("curl -G 'https://x?a=fromurl' -d 'a=fromdata'") + assert request.params["a"] == "fromdata" + + def test_url_query_feeds_requests_params(self): + code = to_requests_code(parse_curl("curl 'https://x/api?a=1'")) + assert 'url = "https://x/api"' in code + assert "params=params" in code + compile(code, "", "exec") + + class TestParseCurlJsonFlag: def test_json_flag_does_not_leak_to_url(self): # Regression: '--json {...}' must not become the URL. diff --git a/test/test_utils/test_script_templates.py b/test/test_utils/test_script_templates.py index c1b703e..a917c62 100644 --- a/test/test_utils/test_script_templates.py +++ b/test/test_utils/test_script_templates.py @@ -194,6 +194,14 @@ def test_get_method(self): def test_is_valid_python(self): compile(to_loaddensity_python(parse_curl("curl https://x")), "", "exec") + def test_url_query_kept_in_request_url(self): + code = to_loaddensity_python(parse_curl("curl 'https://x/api?a=1&b=2'")) + assert '"request_url": "https://x/api?a=1&b=2"' in code + + def test_get_flag_params_kept_in_request_url(self): + code = to_loaddensity_python(parse_curl("curl -G https://x/api -d 'a=1'")) + assert '"request_url": "https://x/api?a=1"' in code + class TestTestFunctionName: def test_from_path(self): From 73d73704c06cba0df124e9eb0e673f775a451001 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 24 Jul 2026 19:15:50 +0800 Subject: [PATCH 05/10] Parse curl -b cookies into a cookies dict for generated code -b/--cookie 'a=1; b=2' was stored as one opaque Cookie header string. Parse the name=value pairs into CurlRequest.cookies and emit an idiomatic, editable cookies={...} in the requests, pytest and APITestka output; a bare token with no '=' is a cookie file curl reads, so it still falls back to a Cookie header. LoadDensity drives by URL and ignores cookies, as it does headers and bodies. --- pybreeze/utils/curl_import/curl_parser.py | 19 +++++++++- pybreeze/utils/curl_import/request_codegen.py | 5 +++ .../utils/curl_import/script_templates.py | 4 +++ test/test_utils/test_curl_import.py | 35 ++++++++++++++++++- test/test_utils/test_script_templates.py | 8 +++++ 5 files changed, 69 insertions(+), 2 deletions(-) diff --git a/pybreeze/utils/curl_import/curl_parser.py b/pybreeze/utils/curl_import/curl_parser.py index 8e53b02..78f74d4 100644 --- a/pybreeze/utils/curl_import/curl_parser.py +++ b/pybreeze/utils/curl_import/curl_parser.py @@ -48,6 +48,7 @@ class CurlRequest: :param data_file_refs: filenames whose content forms the body (``-d @file``) :param timeout: request timeout in seconds from ``--max-time`` / ``-m``, or ``None`` when the command sets none + :param cookies: cookies parsed from ``-b`` / ``--cookie`` name=value pairs """ method: str = _DEFAULT_METHOD @@ -61,6 +62,7 @@ class CurlRequest: form_fields: list[str] = field(default_factory=list) data_file_refs: list[str] = field(default_factory=list) timeout: str | None = None + cookies: dict[str, str] = field(default_factory=dict) @property def has_body(self) -> bool: @@ -255,6 +257,21 @@ def _apply_data_or_file(request: CurlRequest, value: str) -> None: request.data_parts.append(value) +def _apply_cookie(request: CurlRequest, value: str) -> None: + """Parse a ``-b`` cookie string into ``name=value`` pairs. + + ``curl -b 'a=1; b=2'`` yields inline cookies; a value with no ``=`` is a + cookie *file* curl would read, which we keep as a ``Cookie`` header instead. + """ + if "=" not in value: + request.headers.setdefault("Cookie", value) + return + for segment in value.split(";"): + name, separator, cookie_value = segment.strip().partition("=") + if separator and name: + request.cookies[name] = cookie_value + + def _apply_value_flag(request: CurlRequest, kind: str, value: str) -> None: """Apply one value-taking flag to *request* according to its *kind*.""" if kind == "method": @@ -275,7 +292,7 @@ def _apply_value_flag(request: CurlRequest, kind: str, value: str) -> None: elif kind == "form": request.form_fields.append(value) elif kind == "cookie": - request.headers.setdefault("Cookie", value) + _apply_cookie(request, value) elif kind == "user_agent": request.headers.setdefault("User-Agent", value) elif kind == "referer": diff --git a/pybreeze/utils/curl_import/request_codegen.py b/pybreeze/utils/curl_import/request_codegen.py index 0da3499..28d706b 100644 --- a/pybreeze/utils/curl_import/request_codegen.py +++ b/pybreeze/utils/curl_import/request_codegen.py @@ -96,6 +96,8 @@ def _call_keyword_arguments(request: CurlRequest, payload_kwargs: list[str]) -> arguments.append("headers=headers") if request.params: arguments.append("params=params") + if request.cookies: + arguments.append("cookies=cookies") arguments.extend(payload_kwargs) if request.username is not None: arguments.append("auth=auth") @@ -123,6 +125,9 @@ def request_statements(request: CurlRequest) -> list[str]: params_block = _format_dict("params", request.params) if params_block is not None: statements.append(params_block) + cookies_block = _format_dict("cookies", request.cookies) + if cookies_block is not None: + statements.append(cookies_block) statements.extend(payload_sections) if request.username is not None: statements.append( diff --git a/pybreeze/utils/curl_import/script_templates.py b/pybreeze/utils/curl_import/script_templates.py index 6a48a65..7b4ee88 100644 --- a/pybreeze/utils/curl_import/script_templates.py +++ b/pybreeze/utils/curl_import/script_templates.py @@ -71,6 +71,8 @@ def to_apitestka_python(request: CurlRequest) -> str: lines.append(f" headers={_inline_json(request.headers)},") if request.params: lines.append(f" params={_inline_json(request.params)},") + if request.cookies: + lines.append(f" cookies={_inline_json(request.cookies)},") lines.extend(_apitestka_payload_lines(request)) if request.username is not None: lines.append( @@ -118,6 +120,8 @@ def _apitestka_action_params(request: CurlRequest) -> dict: params["headers"] = request.headers if request.params: params["params"] = request.params + if request.cookies: + params["cookies"] = request.cookies _apply_action_payload(request, params) if request.username is not None: params["auth"] = [request.username, request.password or ""] diff --git a/test/test_utils/test_curl_import.py b/test/test_utils/test_curl_import.py index 5882a83..83849e2 100644 --- a/test/test_utils/test_curl_import.py +++ b/test/test_utils/test_curl_import.py @@ -68,7 +68,7 @@ def test_user_agent_flag(self): def test_cookie_flag(self): request = parse_curl("curl -b 'session=1' https://x") - assert request.headers["Cookie"] == "session=1" + assert request.cookies == {"session": "1"} class TestParseCurlBodyAndAuth: @@ -293,6 +293,39 @@ def test_url_query_feeds_requests_params(self): compile(code, "", "exec") +class TestParseCurlCookies: + def test_single_cookie(self): + request = parse_curl("curl -b 'session=1' https://x") + assert request.cookies == {"session": "1"} + assert "Cookie" not in request.headers + + def test_multiple_cookies(self): + request = parse_curl("curl -b 'sessionid=abc; csrftoken=xyz' https://x") + assert request.cookies == {"sessionid": "abc", "csrftoken": "xyz"} + + def test_long_cookie_flag(self): + request = parse_curl("curl --cookie 'a=1' https://x") + assert request.cookies == {"a": "1"} + + def test_cookie_file_falls_back_to_header(self): + # A bare token with no '=' is a cookie file curl would read, not pairs. + request = parse_curl("curl -b cookies.txt https://x") + assert request.cookies == {} + assert request.headers["Cookie"] == "cookies.txt" + + def test_no_cookies_by_default(self): + assert parse_curl("curl https://x").cookies == {} + + def test_cookies_emitted_in_requests_code(self): + code = to_requests_code(parse_curl("curl -b 'a=1; b=2' https://x")) + assert "cookies = {" in code + assert "cookies=cookies" in code + compile(code, "", "exec") + + def test_no_cookies_kwarg_when_absent(self): + assert "cookies=cookies" not in to_requests_code(parse_curl("curl https://x")) + + class TestParseCurlJsonFlag: def test_json_flag_does_not_leak_to_url(self): # Regression: '--json {...}' must not become the URL. diff --git a/test/test_utils/test_script_templates.py b/test/test_utils/test_script_templates.py index a917c62..239d136 100644 --- a/test/test_utils/test_script_templates.py +++ b/test/test_utils/test_script_templates.py @@ -122,6 +122,10 @@ def test_auth(self): code = to_apitestka_python(parse_curl("curl -u user:pass https://x")) assert "auth=(" in code + def test_cookies(self): + code = to_apitestka_python(parse_curl("curl -b 'a=1; b=2' https://x")) + assert "cookies=" in code + def test_params(self): code = to_apitestka_python(parse_curl("curl -G https://x -d 'a=1'")) assert "params=" in code @@ -166,6 +170,10 @@ def test_auth_as_list(self): action = json.loads(to_apitestka_action_json(parse_curl("curl -u user:pass https://x"))) assert action[0][1]["auth"] == ["user", "pass"] + def test_cookies_included(self): + action = json.loads(to_apitestka_action_json(parse_curl("curl -b 'a=1' https://x"))) + assert action[0][1]["cookies"] == {"a": "1"} + def test_get_flag_params(self): action = json.loads(to_apitestka_action_json(parse_curl("curl -G https://x -d 'a=1'"))) assert action[0][1]["params"] == {"a": "1"} From 9df557b7fbb8620e31e8287baab57bfe353f9b9b Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 24 Jul 2026 19:56:19 +0800 Subject: [PATCH 06/10] Add a URL parser/builder tool A companion to the query-string and cURL tools: paste a URL to see its scheme, host, port, path, query and fragment (plus credentials) as an editable JSON object, then turn an edited object back into a URL. - url_tools/url_convert.py: pure-logic parse_url/url_to_json and build_url/json_to_url, with IPv6 bracketing, out-of-range-port tolerance and a stable parse->build round trip - UrlBuilderGUI wired as both a tab and a dock, reusing OutputActions - UrlConvertException plus English and Traditional Chinese strings (parity kept) - logic and widget tests --- .../extend_multi_language/extend_english.py | 14 +++ .../extend_traditional_chinese.py | 14 +++ pybreeze/pybreeze_ui/menu/tools/tools_menu.py | 20 +++ .../pybreeze_ui/tools_gui/url_builder_gui.py | 84 +++++++++++++ pybreeze/utils/exception/exception_tags.py | 4 + pybreeze/utils/exception/exceptions.py | 6 + pybreeze/utils/url_tools/__init__.py | 0 pybreeze/utils/url_tools/url_convert.py | 117 ++++++++++++++++++ test/test_utils/test_url_builder_gui.py | 64 ++++++++++ test/test_utils/test_url_convert.py | 99 +++++++++++++++ 10 files changed, 422 insertions(+) create mode 100644 pybreeze/pybreeze_ui/tools_gui/url_builder_gui.py create mode 100644 pybreeze/utils/url_tools/__init__.py create mode 100644 pybreeze/utils/url_tools/url_convert.py create mode 100644 test/test_utils/test_url_builder_gui.py create mode 100644 test/test_utils/test_url_convert.py diff --git a/pybreeze/extend_multi_language/extend_english.py b/pybreeze/extend_multi_language/extend_english.py index 88e64e2..ea5c9e0 100644 --- a/pybreeze/extend_multi_language/extend_english.py +++ b/pybreeze/extend_multi_language/extend_english.py @@ -393,6 +393,20 @@ "query_json_output_label": "Result:", "query_json_error": "Could not convert: {error}", "query_json_empty_hint": "Enter a query string or a JSON object above.", + # URL Parser / Builder — Menu + "extend_tools_menu_url_builder_tab_action": "URL Parser / Builder Tab", + "extend_tools_menu_url_builder_tab_label": "URL Parser / Builder", + "extend_tools_menu_url_builder_dock_action": "URL Parser / Builder Dock", + "extend_tools_menu_url_builder_dock_title": "URL Parser / Builder", + # URL Parser / Builder — Widget + "url_builder_input_label": "A URL, or a JSON object of URL parts:", + "url_builder_input_placeholder": + "https://user@host:8080/path?a=1#frag or {\"scheme\": \"https\", \"host\": \"host\"}", + "url_builder_to_json_button": "URL → JSON", + "url_builder_to_url_button": "JSON → URL", + "url_builder_output_label": "Result:", + "url_builder_error": "Could not build URL: {error}", + "url_builder_empty_hint": "Enter a URL or a JSON object of URL parts above.", # Regex Tester — Menu "extend_tools_menu_regex_tab_action": "Regex Tester Tab", "extend_tools_menu_regex_tab_label": "Regex", diff --git a/pybreeze/extend_multi_language/extend_traditional_chinese.py b/pybreeze/extend_multi_language/extend_traditional_chinese.py index c2390a0..640f776 100644 --- a/pybreeze/extend_multi_language/extend_traditional_chinese.py +++ b/pybreeze/extend_multi_language/extend_traditional_chinese.py @@ -371,6 +371,20 @@ "query_json_output_label": "結果：", "query_json_error": "無法轉換：{error}", "query_json_empty_hint": "請在上方輸入查詢字串或 JSON 物件。", + # URL 解析／組建器 — 選單 + "extend_tools_menu_url_builder_tab_action": "URL 解析／組建器分頁", + "extend_tools_menu_url_builder_tab_label": "URL 解析／組建器", + "extend_tools_menu_url_builder_dock_action": "URL 解析／組建器停駐窗格", + "extend_tools_menu_url_builder_dock_title": "URL 解析／組建器", + # URL 解析／組建器 — 介面 + "url_builder_input_label": "一個 URL，或描述 URL 各部分的 JSON 物件：", + "url_builder_input_placeholder": + "https://user@host:8080/path?a=1#frag 或 {\"scheme\": \"https\", \"host\": \"host\"}", + "url_builder_to_json_button": "URL → JSON", + "url_builder_to_url_button": "JSON → URL", + "url_builder_output_label": "結果：", + "url_builder_error": "無法組建 URL：{error}", + "url_builder_empty_hint": "請在上方輸入 URL 或描述 URL 各部分的 JSON 物件。", # 正規表示式測試器 — 選單 "extend_tools_menu_regex_tab_action": "正規表示式測試器分頁", "extend_tools_menu_regex_tab_label": "Regex", diff --git a/pybreeze/pybreeze_ui/menu/tools/tools_menu.py b/pybreeze/pybreeze_ui/menu/tools/tools_menu.py index a27c5cd..ce55624 100644 --- a/pybreeze/pybreeze_ui/menu/tools/tools_menu.py +++ b/pybreeze/pybreeze_ui/menu/tools/tools_menu.py @@ -25,6 +25,7 @@ from pybreeze.pybreeze_ui.tools_gui.regex_gui import RegexGUI from pybreeze.pybreeze_ui.tools_gui.response_inspector_gui import ResponseInspectorGUI from pybreeze.pybreeze_ui.tools_gui.timestamp_gui import TimestampGUI +from pybreeze.pybreeze_ui.tools_gui.url_builder_gui import UrlBuilderGUI if TYPE_CHECKING: from pybreeze.pybreeze_ui.editor_main.main_ui import PyBreezeMainWindow @@ -156,6 +157,17 @@ def build_tools_menu(ui_we_want_to_set: PyBreezeMainWindow): )) ui_we_want_to_set.tools_menu.addAction(ui_we_want_to_set.tools_query_json_action) + # URL Parser / Builder + ui_we_want_to_set.tools_url_builder_action = QAction(language_wrapper.language_word_dict.get( + "extend_tools_menu_url_builder_tab_action" + )) + ui_we_want_to_set.tools_url_builder_action.triggered.connect(lambda: ui_we_want_to_set.tab_widget.addTab( + UrlBuilderGUI(ui_we_want_to_set), language_wrapper.language_word_dict.get( + "extend_tools_menu_url_builder_tab_label" + ) + )) + ui_we_want_to_set.tools_menu.addAction(ui_we_want_to_set.tools_url_builder_action) + # Regex Tester ui_we_want_to_set.tools_regex_action = QAction(language_wrapper.language_word_dict.get( "extend_tools_menu_regex_tab_action" @@ -296,6 +308,13 @@ def extend_dock_menu(ui_we_want_to_set: PyBreezeMainWindow): lambda: add_dock(ui_we_want_to_set, "QueryJson")) ui_we_want_to_set.dock_menu.addAction(ui_we_want_to_set.tools_query_json_dock_action) + # URL Parser / Builder Dock + ui_we_want_to_set.tools_url_builder_dock_action = QAction(language_wrapper.language_word_dict.get( + "extend_tools_menu_url_builder_dock_action")) + ui_we_want_to_set.tools_url_builder_dock_action.triggered.connect( + lambda: add_dock(ui_we_want_to_set, "UrlBuilder")) + ui_we_want_to_set.dock_menu.addAction(ui_we_want_to_set.tools_url_builder_dock_action) + # Regex Tester Dock ui_we_want_to_set.tools_regex_dock_action = QAction(language_wrapper.language_word_dict.get( "extend_tools_menu_regex_dock_action")) @@ -352,6 +371,7 @@ def extend_dock_menu(ui_we_want_to_set: PyBreezeMainWindow): "Timestamp": ("extend_tools_menu_timestamp_dock_title", lambda win: TimestampGUI(win)), "Hash": ("extend_tools_menu_hash_dock_title", lambda win: HashGUI(win)), "QueryJson": ("extend_tools_menu_query_json_dock_title", lambda win: QueryJsonGUI(win)), + "UrlBuilder": ("extend_tools_menu_url_builder_dock_title", lambda win: UrlBuilderGUI(win)), "Regex": ("extend_tools_menu_regex_dock_title", lambda win: RegexGUI(win)), "HttpStatus": ( "extend_tools_menu_http_status_dock_title", lambda win: HttpStatusGUI(main_window=win)), diff --git a/pybreeze/pybreeze_ui/tools_gui/url_builder_gui.py b/pybreeze/pybreeze_ui/tools_gui/url_builder_gui.py new file mode 100644 index 0000000..8fe20bf --- /dev/null +++ b/pybreeze/pybreeze_ui/tools_gui/url_builder_gui.py @@ -0,0 +1,84 @@ +"""A tool tab that parses a URL into JSON parts and rebuilds one from them.""" +from __future__ import annotations + +from PySide6.QtWidgets import ( + QHBoxLayout, QLabel, QPushButton, QTextEdit, QVBoxLayout, QWidget +) +from je_editor import language_wrapper + +from pybreeze.pybreeze_ui.tools_gui.output_actions import OutputActions +from pybreeze.utils.exception.exceptions import UrlConvertException +from pybreeze.utils.logging.logger import pybreeze_logger +from pybreeze.utils.url_tools.url_convert import json_to_url, url_to_json + + +class UrlBuilderGUI(QWidget): + """Parse a URL into its JSON parts and build a URL back from them.""" + + def __init__(self, main_window=None) -> None: + """ + :param main_window: window whose ``tab_widget`` "open in editor" uses + """ + super().__init__() + self._valid_output = False + word = language_wrapper.language_word_dict + + self.input_label = QLabel(word.get("url_builder_input_label")) + self.input_edit = QTextEdit() + self.input_edit.setPlaceholderText(word.get("url_builder_input_placeholder")) + self.input_edit.setAcceptRichText(False) + + self.to_json_button = QPushButton(word.get("url_builder_to_json_button")) + self.to_json_button.clicked.connect(self.convert_to_json) + self.to_url_button = QPushButton(word.get("url_builder_to_url_button")) + self.to_url_button.clicked.connect(self.convert_to_url) + + buttons = QHBoxLayout() + buttons.addWidget(self.to_json_button) + buttons.addWidget(self.to_url_button) + + self.output_label = QLabel(word.get("url_builder_output_label")) + self.output_edit = QTextEdit() + self.output_edit.setReadOnly(True) + + self.actions = OutputActions( + self, self.output_edit, main_window=main_window, + basename="url", extension="txt", is_valid=lambda: self._valid_output) + + layout = QVBoxLayout() + layout.addWidget(self.input_label) + layout.addWidget(self.input_edit) + layout.addLayout(buttons) + layout.addWidget(self.output_label) + layout.addWidget(self.output_edit) + layout.addLayout(self.actions.button_row()) + self.setLayout(layout) + + def convert_to_json(self) -> None: + """Parse the input URL into its JSON parts.""" + text = self.input_edit.toPlainText().strip() + if not text: + self._valid_output = False + self.output_edit.setPlainText( + language_wrapper.language_word_dict.get("url_builder_empty_hint")) + return + self._valid_output = True + self.output_edit.setPlainText(url_to_json(text)) + + def convert_to_url(self) -> None: + """Build a URL from the input JSON parts.""" + word = language_wrapper.language_word_dict + text = self.input_edit.toPlainText().strip() + if not text: + self._valid_output = False + self.output_edit.setPlainText(word.get("url_builder_empty_hint")) + return + try: + result = json_to_url(text) + except UrlConvertException as error: + pybreeze_logger.info("url_builder_gui.py to-url failed: %r", error) + self._valid_output = False + self.output_edit.setPlainText(word.get("url_builder_error").format(error=str(error))) + return + self._valid_output = True + self.output_edit.setPlainText(result) diff --git a/pybreeze/utils/exception/exception_tags.py b/pybreeze/utils/exception/exception_tags.py index ede3f44..d43560b 100644 --- a/pybreeze/utils/exception/exception_tags.py +++ b/pybreeze/utils/exception/exception_tags.py @@ -59,3 +59,7 @@ # Regex testing empty_regex_pattern_error: str = "no pattern provided" invalid_regex_pattern_error: str = "invalid regular expression: {detail}" + +# URL parse / build +invalid_json_for_url_error: str = "can't parse the input as JSON" +invalid_url_components_error: str = "the input must be a JSON object of URL parts" diff --git a/pybreeze/utils/exception/exceptions.py b/pybreeze/utils/exception/exceptions.py index 1d0788e..69375b1 100644 --- a/pybreeze/utils/exception/exceptions.py +++ b/pybreeze/utils/exception/exceptions.py @@ -80,3 +80,9 @@ class QueryConvertException(ITEException): class RegexTesterException(ITEException): pass + + +# URL parse / build + +class UrlConvertException(ITEException): + pass diff --git a/pybreeze/utils/url_tools/__init__.py b/pybreeze/utils/url_tools/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pybreeze/utils/url_tools/url_convert.py b/pybreeze/utils/url_tools/url_convert.py new file mode 100644 index 0000000..fe4d193 --- /dev/null +++ b/pybreeze/utils/url_tools/url_convert.py @@ -0,0 +1,117 @@ +"""Parse a URL into its parts and rebuild one from those parts. + +A companion to the query-string and cURL tools: paste a URL to see its scheme, +host, port, path, query and fragment as an editable JSON object, tweak any part, +then turn it back into a URL. Pure logic — no Qt and no network access. +""" +from __future__ import annotations + +import json +from urllib.parse import SplitResult, parse_qsl, urlencode, urlsplit, urlunsplit + +from pybreeze.utils.exception.exception_tags import ( + invalid_json_for_url_error, + invalid_url_components_error, +) +from pybreeze.utils.exception.exceptions import UrlConvertException +from pybreeze.utils.logging.logger import pybreeze_logger + + +def _safe_port(split: SplitResult) -> int | None: + """Return the URL's port, or ``None`` when it is absent or out of range.""" + try: + return split.port + except ValueError: + return None + + +def parse_url(url: str) -> dict: + """Split *url* into its components as a JSON-friendly dict. + + :param url: the URL to parse (leading/trailing whitespace is ignored) + :return: a dict with scheme, host, port, path, query and fragment, plus + username/password when the URL carries credentials + """ + split = urlsplit(url.strip()) + components: dict = { + "scheme": split.scheme, + "host": split.hostname or "", + "port": _safe_port(split), + "path": split.path, + "query": dict(parse_qsl(split.query, keep_blank_values=True)), + "fragment": split.fragment, + } + if split.username is not None: + components["username"] = split.username + if split.password is not None: + components["password"] = split.password + return components + + +def url_to_json(url: str) -> str: + """Parse *url* and render its components as pretty-printed JSON. + + :param url: the URL to parse + :return: a formatted JSON object of the URL's parts + """ + return json.dumps(parse_url(url), indent=4, ensure_ascii=False) + + +def _build_netloc(components: dict, host: str) -> str: + """Assemble the ``user:pass@host:port`` authority from *components*.""" + if ":" in host and not host.startswith("["): + host = f"[{host}]" # bracket an IPv6 literal so the port stays separable + netloc = host + port = components.get("port") + if port is not None and port != "": + netloc = f"{netloc}:{port}" + username = components.get("username") + if username is not None: + password = components.get("password") + credentials = username if password is None else f"{username}:{password}" + netloc = f"{credentials}@{netloc}" + return netloc + + +def _build_query(query: object) -> str: + """Render the query component from a dict of pairs (or a raw string).""" + if not query: + return "" + if isinstance(query, dict): + return urlencode(query) + return str(query) + + +def build_url(components: dict) -> str: + """Rebuild a URL from a components dict (the inverse of :func:`parse_url`). + + Missing parts default to empty, so a partial object still yields a URL. + + :param components: URL parts as produced by :func:`parse_url` + :return: the assembled URL + """ + scheme = str(components.get("scheme", "")) + host = str(components.get("host", "")) + netloc = _build_netloc(components, host) + path = str(components.get("path", "")) + query = _build_query(components.get("query")) + fragment = str(components.get("fragment", "")) + return urlunsplit((scheme, netloc, path, query, fragment)) + + +def json_to_url(json_text: str) -> str: + """Build a URL from a JSON object of URL parts. + + :param json_text: a JSON object as produced by :func:`url_to_json` + :return: the assembled URL + :raises UrlConvertException: when the input is not valid JSON or not an object + """ + try: + components = json.loads(json_text) + except ValueError as error: + pybreeze_logger.error(invalid_json_for_url_error) + raise UrlConvertException(invalid_json_for_url_error) from error + if not isinstance(components, dict): + pybreeze_logger.error(invalid_url_components_error) + raise UrlConvertException(invalid_url_components_error) + return build_url(components) diff --git a/test/test_utils/test_url_builder_gui.py b/test/test_utils/test_url_builder_gui.py new file mode 100644 index 0000000..e080b12 --- /dev/null +++ b/test/test_utils/test_url_builder_gui.py @@ -0,0 +1,64 @@ +"""Tests for the URL parser/builder tool widget.""" +from __future__ import annotations + +import json +import os + +os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") + +import pytest +from PySide6.QtWidgets import QApplication + +from pybreeze.extend_multi_language.update_language_dict import update_language_dict + + +@pytest.fixture(scope="module") +def app(): + instance = QApplication.instance() or QApplication([]) + update_language_dict() + return instance + + +@pytest.fixture() +def widget(app): + from pybreeze.pybreeze_ui.tools_gui.url_builder_gui import UrlBuilderGUI + gui = UrlBuilderGUI() + yield gui + gui.close() + gui.deleteLater() + + +class TestUrlBuilderGUI: + def test_url_to_json(self, widget): + widget.input_edit.setPlainText("https://example.com/api?a=1") + widget.convert_to_json() + data = json.loads(widget.output_edit.toPlainText()) + assert data["host"] == "example.com" + assert data["query"] == {"a": "1"} + + def test_json_to_url(self, widget): + widget.input_edit.setPlainText('{"scheme": "https", "host": "x", "path": "/a"}') + widget.convert_to_url() + assert widget.output_edit.toPlainText() == "https://x/a" + + def test_invalid_json_shows_error(self, widget): + widget.input_edit.setPlainText("not json") + widget.convert_to_url() + assert widget.output_edit.toPlainText() != "" + assert "https://" not in widget.output_edit.toPlainText() + + def test_empty_to_json_shows_hint(self, widget): + widget.input_edit.setPlainText(" ") + widget.convert_to_json() + assert widget.output_edit.toPlainText() != "" + + def test_empty_to_url_shows_hint(self, widget): + widget.input_edit.setPlainText(" ") + widget.convert_to_url() + assert widget.output_edit.toPlainText() != "" + + def test_copy_output(self, app, widget): + widget.input_edit.setPlainText("https://example.com/api?a=1") + widget.convert_to_json() + widget.actions.copy() + assert "example.com" in QApplication.clipboard().text() diff --git a/test/test_utils/test_url_convert.py b/test/test_utils/test_url_convert.py new file mode 100644 index 0000000..2f79730 --- /dev/null +++ b/test/test_utils/test_url_convert.py @@ -0,0 +1,99 @@ +"""Tests for URL <-> JSON parsing and building.""" +from __future__ import annotations + +import json + +import pytest + +from pybreeze.utils.exception.exceptions import UrlConvertException +from pybreeze.utils.url_tools.url_convert import ( + build_url, + json_to_url, + parse_url, + url_to_json, +) + + +class TestParseUrl: + def test_full_url(self): + parts = parse_url("https://user:pass@example.com:8080/api/v1?a=1&b=2#frag") + assert parts["scheme"] == "https" + assert parts["host"] == "example.com" + assert parts["port"] == 8080 + assert parts["path"] == "/api/v1" + assert parts["query"] == {"a": "1", "b": "2"} + assert parts["fragment"] == "frag" + assert parts["username"] == "user" + assert parts["password"] == "pass" + + def test_minimal_url(self): + parts = parse_url("https://example.com") + assert parts["scheme"] == "https" + assert parts["host"] == "example.com" + assert parts["port"] is None + assert parts["path"] == "" + assert parts["query"] == {} + assert "username" not in parts + + def test_query_values_url_decoded(self): + parts = parse_url("https://x/?q=hello%20world") + assert parts["query"]["q"] == "hello world" + + def test_whitespace_is_stripped(self): + assert parse_url(" https://x/api ")["path"] == "/api" + + def test_out_of_range_port_is_none(self): + assert parse_url("https://x:99999/")["port"] is None + + +class TestUrlToJson: + def test_returns_valid_json(self): + data = json.loads(url_to_json("https://x/api?a=1")) + assert data["host"] == "x" + assert data["query"] == {"a": "1"} + + +class TestBuildUrl: + def test_full_components(self): + url = build_url({ + "scheme": "https", "host": "example.com", "port": 8080, + "path": "/api", "query": {"a": "1"}, "fragment": "f", + "username": "user", "password": "pass", + }) + assert url == "https://user:pass@example.com:8080/api?a=1#f" + + def test_minimal_components(self): + assert build_url({"scheme": "https", "host": "x"}) == "https://x" + + def test_missing_parts_default_empty(self): + assert build_url({"host": "x"}) == "//x" + + def test_ipv6_host_is_bracketed(self): + assert build_url({"scheme": "http", "host": "::1", "port": 8080}) == "http://[::1]:8080" + + def test_username_without_password(self): + assert build_url({"scheme": "https", "host": "x", "username": "user"}) == "https://user@x" + + +class TestRoundTrip: + @pytest.mark.parametrize("url", [ + "https://example.com/api/v1?a=1&b=2#section", + "http://user:pass@host:8080/p", + "https://example.com", + "https://x/?flag=", + ]) + def test_parse_then_build_is_stable(self, url): + assert build_url(parse_url(url)) == url + + +class TestJsonToUrl: + def test_builds_from_json(self): + assert json_to_url('{"scheme": "https", "host": "x", "path": "/a"}') == "https://x/a" + + def test_invalid_json_raises(self): + with pytest.raises(UrlConvertException): + json_to_url("not json") + + def test_non_object_raises(self): + with pytest.raises(UrlConvertException): + json_to_url('["a", "b"]') From 8d3bc4573c0fb7158d73cdee83a1f965dca7feaf Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Sat, 25 Jul 2026 14:10:27 +0800 Subject: [PATCH 07/10] Combine repeated curl -H headers instead of overwriting A second -H with the same name replaced the first, silently dropping a value curl would have sent. Repeated headers are now joined the way the receiver joins field lines (", ", or "; " for cookies), and header names are matched case-insensitively throughout, so an explicit -H beats a default implied by --json, -A or -e regardless of its casing. --- README.md | 2 +- pybreeze/utils/curl_import/curl_parser.py | 70 ++++++++++++++++++----- test/test_utils/test_curl_import.py | 41 +++++++++++++ 3 files changed, 99 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 950f77b..42aa361 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,7 @@ PyBreeze is not just a code editor — it is a command center for the automation - Mermaid `flowchart` / `graph` import - Save/Open as `.diagram.json`, export to PNG or SVG - Undo/redo, align, distribute, grid, snap, and zoom controls -- **cURL Import** — Paste a `curl` command copied from your browser's dev tools and generate a ready-to-run script for the target of your choice: **Python `requests`**, an **APITestka Python** snippet (`test_api_method_requests(...)`), an **APITestka JSON action** (`[["AT_test_api_method", {...}]]`) that `execute_files` runs directly, or a **LoadDensity** Locust load-test (`start_test(...)`). (These are the HTTP-oriented modules; a curl request has no meaningful mapping to the browser or desktop-GUI automation modules.) Parses the method, URL, headers, body, basic auth, `-G` query parameters, `-F` multipart form fields (file uploads become `files=open(...)`), the `--json` shortcut (which also sets the JSON `Content-Type`/`Accept`), and `-d @file` bodies (which become `open(...).read()`, while `--data-raw` stays literal) — all with multi-line `\` / `^` continuations. It knows the arity of curl's common flags, so value-taking options like `--max-time 30` or `-o out.json` never leak into the URL; it URL-encodes `--data-urlencode` values the way curl does; and it infers `POST` when a body or form is present and sends JSON bodies as JSON. Targets include Python `requests`, a ready-to-run **pytest test** (a named `test_...` function that sends the request and asserts the status), APITestka (Python and JSON action), and a LoadDensity load test. Pick one from the dropdown, then copy the result, **open it straight into a new editor tab**, or **save it to a `.py` / `.json` file** (the extension follows the chosen target). The parser is pure logic and never executes the command. +- **cURL Import** — Paste a `curl` command copied from your browser's dev tools and generate a ready-to-run script for the target of your choice: **Python `requests`**, an **APITestka Python** snippet (`test_api_method_requests(...)`), an **APITestka JSON action** (`[["AT_test_api_method", {...}]]`) that `execute_files` runs directly, or a **LoadDensity** Locust load-test (`start_test(...)`). (These are the HTTP-oriented modules; a curl request has no meaningful mapping to the browser or desktop-GUI automation modules.) Parses the method, URL, headers, body, basic auth, `-G` query parameters, `-F` multipart form fields (file uploads become `files=open(...)`), the `--json` shortcut (which also sets the JSON `Content-Type`/`Accept`), and `-d @file` bodies (which become `open(...).read()`, while `--data-raw` stays literal) — all with multi-line `\` / `^` continuations. It knows the arity of curl's common flags, so value-taking options like `--max-time 30` or `-o out.json` never leak into the URL; it URL-encodes `--data-urlencode` values the way curl does; and it infers `POST` when a body or form is present and sends JSON bodies as JSON. Targets include Python `requests`, a ready-to-run **pytest test** (a named `test_...` function that sends the request and asserts the status), APITestka (Python and JSON action), and a LoadDensity load test. Pick one from the dropdown, then copy the result, **open it straight into a new editor tab**, or **save it to a `.py` / `.json` file** (the extension follows the chosen target). A repeated `-H` is handled the way HTTP handles it — the values are combined into one header (with `; ` for cookies) rather than the last one silently winning, and names are matched case-insensitively so an explicit header always beats one implied by another flag. The parser is pure logic and never executes the command. - **JWT Decoder** — Paste a JSON Web Token and read its header and payload as pretty-printed JSON, with standard timestamp claims (`exp`, `iat`, `nbf`, `auth_time`) rendered as readable UTC times. Inspection only — the signature is never verified and the token is never trusted. - **Timestamp Converter** — Enter a Unix epoch value (seconds or milliseconds, auto-detected) or an ISO-8601 date-time (with `Z` or an offset) and get every representation back in UTC. Deterministic and independent of the local time zone. - **Hash Generator** — Compute SHA-256, SHA-512, SHA-1 and MD5 digests of any text at once. A general-purpose checksum tool (MD5/SHA-1 are provided for interoperability with `usedforsecurity=False`, never for security decisions). diff --git a/pybreeze/utils/curl_import/curl_parser.py b/pybreeze/utils/curl_import/curl_parser.py index 78f74d4..e49aa21 100644 --- a/pybreeze/utils/curl_import/curl_parser.py +++ b/pybreeze/utils/curl_import/curl_parser.py @@ -28,6 +28,11 @@ _METHOD_WITH_BODY = "POST" # Header name that carries a request body's media type _CONTENT_TYPE_HEADER = "content-type" +# Header whose repeated values are joined with "; " instead of ", " +_COOKIE_HEADER = "cookie" +# How repeated header lines are combined into one value (RFC 9110 field order) +_HEADER_JOINER = ", " +_COOKIE_JOINER = "; " # Matches a backslash or caret line continuation before a newline _LINE_CONTINUATION_RE = re.compile(r"[\\^]\r?\n") @@ -89,11 +94,8 @@ def full_url(self) -> str: def header_value(self, name: str) -> str | None: """Return a header's value by case-insensitive *name*, or ``None``.""" - lowered = name.lower() - for key, value in self.headers.items(): - if key.lower() == lowered: - return value - return None + stored = _stored_header_name(self, name) + return None if stored is None else self.headers[stored] # Flags that take a value and how each value is applied to the request. @@ -215,11 +217,53 @@ def _tokenize(command: str) -> list[str]: raise CurlParseException(malformed_curl_command_error) from error +def _stored_header_name(request: CurlRequest, name: str) -> str | None: + """Return the already-stored spelling of *name*, matched case-insensitively.""" + lowered = name.lower() + for stored in request.headers: + if stored.lower() == lowered: + return stored + return None + + +def _set_default_header(request: CurlRequest, name: str, value: str) -> None: + """Set a header only when no header of that name is present. + + Case-insensitive, so an explicit ``-H 'content-type: ...'`` still wins over a + default implied by another flag instead of producing a second header line. + """ + if _stored_header_name(request, name) is None: + request.headers[name] = value + + +def _join_header_values(name: str, previous: str, value: str) -> str: + """Combine a repeated header's values the way HTTP combines field lines.""" + if not previous: + return value + if not value: + return previous + joiner = _COOKIE_JOINER if name.lower() == _COOKIE_HEADER else _HEADER_JOINER + return f"{previous}{joiner}{value}" + + def _apply_header(request: CurlRequest, raw_header: str) -> None: - """Record a ``Name: Value`` header, ignoring malformed fragments.""" + """Record a ``Name: Value`` header, combining repeats of the same name. + + curl sends every ``-H`` it is given, so a repeated name is not a mistake: the + receiver combines those field lines into one comma-separated value (cookies + use ``; ``), which is what the generated code should carry. Names are matched + case-insensitively, as HTTP header names are. Fragments with no colon or an + empty name are ignored. + """ name, separator, value = raw_header.partition(":") - if separator: - request.headers[name.strip()] = value.strip() + name, value = name.strip(), value.strip() + if not separator or not name: + return + stored = _stored_header_name(request, name) + if stored is None: + request.headers[name] = value + return + request.headers[stored] = _join_header_values(stored, request.headers[stored], value) def _urlencode_data_part(value: str) -> str: @@ -264,7 +308,7 @@ def _apply_cookie(request: CurlRequest, value: str) -> None: cookie *file* curl would read, which we keep as a ``Cookie`` header instead. """ if "=" not in value: - request.headers.setdefault("Cookie", value) + _set_default_header(request, "Cookie", value) return for segment in value.split(";"): name, separator, cookie_value = segment.strip().partition("=") @@ -287,16 +331,16 @@ def _apply_value_flag(request: CurlRequest, kind: str, value: str) -> None: elif kind == "json_flag": # curl --json is shorthand for --data + JSON Content-Type and Accept. request.data_parts.append(value) - request.headers.setdefault("Content-Type", "application/json") - request.headers.setdefault("Accept", "application/json") + _set_default_header(request, "Content-Type", "application/json") + _set_default_header(request, "Accept", "application/json") elif kind == "form": request.form_fields.append(value) elif kind == "cookie": _apply_cookie(request, value) elif kind == "user_agent": - request.headers.setdefault("User-Agent", value) + _set_default_header(request, "User-Agent", value) elif kind == "referer": - request.headers.setdefault("Referer", value) + _set_default_header(request, "Referer", value) elif kind == "url": request.url = value elif kind == "timeout": diff --git a/test/test_utils/test_curl_import.py b/test/test_utils/test_curl_import.py index 83849e2..f8f176a 100644 --- a/test/test_utils/test_curl_import.py +++ b/test/test_utils/test_curl_import.py @@ -71,6 +71,47 @@ def test_cookie_flag(self): assert request.cookies == {"session": "1"} +class TestParseCurlRepeatedHeaders: + def test_repeated_header_values_are_combined(self): + request = parse_curl( + "curl -H 'Accept: text/html' -H 'Accept: application/json' https://x") + assert request.headers == {"Accept": "text/html, application/json"} + + def test_repeat_with_different_casing_keeps_one_entry(self): + request = parse_curl("curl -H 'Accept: text/html' -H 'accept: application/xml' https://x") + # The first spelling is kept; a second entry would send the header twice. + assert request.headers == {"Accept": "text/html, application/xml"} + + def test_repeated_cookie_header_uses_cookie_separator(self): + request = parse_curl("curl -H 'Cookie: a=1' -H 'Cookie: b=2' https://x") + assert request.headers == {"Cookie": "a=1; b=2"} + + def test_repeat_with_empty_value_keeps_previous(self): + request = parse_curl("curl -H 'Accept: text/html' -H 'Accept:' https://x") + assert request.headers == {"Accept": "text/html"} + + def test_empty_first_value_is_replaced_by_the_repeat(self): + request = parse_curl("curl -H 'Accept:' -H 'Accept: text/html' https://x") + assert request.headers == {"Accept": "text/html"} + + def test_header_with_empty_name_is_ignored(self): + request = parse_curl("curl -H ': value' https://x") + assert request.headers == {} + + def test_explicit_header_wins_over_user_agent_flag_case_insensitively(self): + request = parse_curl("curl -H 'user-agent: mine' -A 'flag-agent' https://x") + assert request.headers == {"user-agent": "mine"} + + def test_explicit_lowercase_content_type_wins_over_json_flag(self): + request = parse_curl("curl -H 'content-type: text/plain' --json '{}' https://x") + assert request.headers["content-type"] == "text/plain" + assert "Content-Type" not in request.headers + + def test_cookie_file_does_not_add_a_second_cookie_header(self): + request = parse_curl("curl -H 'cookie: a=1' -b cookies.txt https://x") + assert request.headers == {"cookie": "a=1"} + + class TestParseCurlBodyAndAuth: def test_multiple_data_joined(self): request = parse_curl("curl https://x -d 'a=1' -d 'b=2'") From 5164b7eac2307f15e32a7d819c88f80c47963dbd Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Sat, 25 Jul 2026 14:12:24 +0800 Subject: [PATCH 08/10] Add an HTTP header analyzer tool Paste a request or response header block and get every field back plus what is worth knowing about it: names sent more than once, Set-Cookie entries missing Secure/HttpOnly/SameSite, a wildcard CORS policy (and the wildcard-with-credentials combination browsers reject), a short HSTS max-age, CSP unsafe-inline/unsafe-eval, product banners, deprecated headers, and the response security headers that are absent. Headers that carry credentials are reported by name only, never by value. Findings carry a stable code that the UI translates, so the analysis stays language-neutral. The response inspector can hand its raw text over in one click, which keeps repeated headers intact, and the shared open-tool-tab helper it used for the status/JWT hand-offs now lives in tool_tabs so other tools can reuse it. The header-line pattern moves to the new module and the response analyzer imports it. --- README.md | 4 +- .../extend_multi_language/extend_english.py | 56 +++ .../extend_traditional_chinese.py | 56 +++ pybreeze/pybreeze_ui/menu/tools/tools_menu.py | 21 ++ .../tools_gui/header_analyzer_gui.py | 108 ++++++ .../tools_gui/response_inspector_gui.py | 40 ++- pybreeze/pybreeze_ui/tools_gui/tool_tabs.py | 31 ++ pybreeze/utils/header_tools/__init__.py | 0 .../utils/header_tools/header_analyzer.py | 318 ++++++++++++++++++ .../response_inspector/response_analyzer.py | 5 +- test/test_utils/test_fuzz_pure_logic.py | 26 ++ test/test_utils/test_header_analyzer.py | 190 +++++++++++ test/test_utils/test_header_analyzer_gui.py | 122 +++++++ .../test_utils/test_response_inspector_gui.py | 40 +++ test/test_utils/test_tools_menu_docks.py | 4 + 15 files changed, 1004 insertions(+), 17 deletions(-) create mode 100644 pybreeze/pybreeze_ui/tools_gui/header_analyzer_gui.py create mode 100644 pybreeze/pybreeze_ui/tools_gui/tool_tabs.py create mode 100644 pybreeze/utils/header_tools/__init__.py create mode 100644 pybreeze/utils/header_tools/header_analyzer.py create mode 100644 test/test_utils/test_header_analyzer.py create mode 100644 test/test_utils/test_header_analyzer_gui.py diff --git a/README.md b/README.md index 42aa361..2b8719e 100644 --- a/README.md +++ b/README.md @@ -81,11 +81,12 @@ PyBreeze is not just a code editor — it is a command center for the automation - **Timestamp Converter** — Enter a Unix epoch value (seconds or milliseconds, auto-detected) or an ISO-8601 date-time (with `Z` or an offset) and get every representation back in UTC. Deterministic and independent of the local time zone. - **Hash Generator** — Compute SHA-256, SHA-512, SHA-1 and MD5 digests of any text at once. A general-purpose checksum tool (MD5/SHA-1 are provided for interoperability with `usedforsecurity=False`, never for security decisions). - **Query ⇄ JSON** — Convert an `application/x-www-form-urlencoded` query string to pretty JSON and back, URL-decoding and -encoding as needed. Repeated keys become JSON arrays and vice versa. Copy the result, open it in an editor tab, or save it to a file. +- **HTTP Header Analyzer** — Paste a request or response header block (from dev tools, `curl -i`, or a proxy log) and read back every field plus what is worth knowing about it: names sent more than once, `Set-Cookie` entries missing `Secure` / `HttpOnly` / `SameSite`, a wildcard CORS policy (and the wildcard-plus-credentials combination browsers reject outright), an HSTS `max-age` too short to survive a restart, CSP `unsafe-inline` / `unsafe-eval`, product banners, deprecated headers, and — for responses — the security headers that are absent. Headers carrying credentials are reported by name only; their values are never copied into the report. The Response Inspector hands its headers here in one click. - **Regex Tester** — Try a regular expression against sample text with `IGNORECASE` / `MULTILINE` / `DOTALL` / `VERBOSE` flags, and see every match with its offsets, numbered groups and named groups. Invalid patterns report a friendly error instead of crashing. - **HTTP Status Reference** — Search the full HTTP status code table (sourced from the standard library, so it stays current) by code prefix or keyword, with the reason phrase, description and status class for each. - **Text Diff** — Compare two pieces of text (for example an expected vs. actual API response) side by side and get a unified diff plus a one-line summary of how many lines were added or removed. Copy the diff, open it in an editor tab, or save it to a file. - **JSON Format** — Pretty-print or minify a JSON payload, with a clear validation error when the input is not valid JSON. Copy the result, open it in a new editor tab, or save it to a file. -- **Response Inspector** — Paste a raw HTTP response and get a decoded summary in one step: the status code is looked up in the HTTP reference, headers are parsed, a JSON body is pretty-printed, and any JWTs anywhere in the text (for example an `Authorization: Bearer` header) are decoded with their timestamp claims shown in UTC. Ties the HTTP-status, JSON-format and JWT tools together, and can open a found status code or JWT directly in its dedicated tool tab, pre-filled. +- **Response Inspector** — Paste a raw HTTP response and get a decoded summary in one step: the status code is looked up in the HTTP reference, headers are parsed, a JSON body is pretty-printed, and any JWTs anywhere in the text (for example an `Authorization: Bearer` header) are decoded with their timestamp claims shown in UTC. Ties the HTTP-status, JSON-format and JWT tools together, and can open a found status code, JWT, or the whole header block directly in its dedicated tool tab, pre-filled. - **File Tree Context Menu** — Right-click any file or folder in the project tree to create files/folders, rename, delete, copy absolute or relative paths, or reveal the item in your platform file manager. Renaming or deleting a file that is currently open in an editor tab keeps the tab in sync. - **Package Manager** — Install automation modules and build tools directly from the IDE menu without leaving the editor. - **Integrated Documentation** — Quick access to documentation and GitHub pages for each automation module directly from the menu bar. @@ -221,6 +222,7 @@ PyBreeze UI (PySide6) │ ├── Timestamp Converter (epoch ⇄ ISO-8601) │ ├── Hash Generator (SHA-256/512, SHA-1, MD5) │ ├── Query ⇄ JSON Converter +│ ├── HTTP Header Analyzer (duplicates, cookie flags, security headers) │ ├── Regex Tester │ ├── HTTP Status Reference │ ├── Text Diff diff --git a/pybreeze/extend_multi_language/extend_english.py b/pybreeze/extend_multi_language/extend_english.py index ea5c9e0..0a73690 100644 --- a/pybreeze/extend_multi_language/extend_english.py +++ b/pybreeze/extend_multi_language/extend_english.py @@ -331,6 +331,7 @@ "curl_import_target_apitestka_python": "APITestka (Python)", "curl_import_target_apitestka_action": "APITestka (JSON action)", "curl_import_target_loaddensity_python": "LoadDensity (Python)", + "curl_import_open_url_button": "Open URL in parser / builder", # Shared output actions (copy / open in editor / save to file) "output_actions_copy": "Copy", "output_actions_open_editor": "Open in editor tab", @@ -467,11 +468,66 @@ "response_output_label": "Analysis:", "response_open_jwt_button": "Open JWT in decoder", "response_open_status_button": "Open status in reference", + "response_open_headers_button": "Open headers in analyzer", "response_empty_hint": "Paste a response above, then click analyze.", "response_status_label": "== Status ==", "response_headers_label": "== Headers ==", "response_jwt_label": "== JWT(s) found ==", "response_body_label": "== Body ==", + # HTTP Header Analyzer — Menu + "extend_tools_menu_header_analyzer_tab_action": "HTTP Header Analyzer Tab", + "extend_tools_menu_header_analyzer_tab_label": "Header Analyzer", + "extend_tools_menu_header_analyzer_dock_action": "HTTP Header Analyzer Dock", + "extend_tools_menu_header_analyzer_dock_title": "Header Analyzer", + # HTTP Header Analyzer — Widget + "header_analyzer_input_label": "Paste request or response headers:", + "header_analyzer_input_placeholder": + "HTTP/1.1 200 OK\nContent-Type: text/html\nSet-Cookie: sid=abc; Path=/", + "header_analyzer_analyze_button": "Analyze headers", + "header_analyzer_output_label": "Analysis:", + "header_analyzer_headers_label": "== Headers ({count}) ==", + "header_analyzer_duplicates_label": "== Repeated names ==", + "header_analyzer_findings_label": "== Findings ==", + "header_analyzer_no_findings": "Nothing worth reporting.", + "header_analyzer_no_headers": "No header lines found in that text.", + "header_analyzer_empty_hint": "Paste a header block above, then click analyze.", + "header_analyzer_level_warning": "WARNING", + "header_analyzer_level_info": "INFO", + # HTTP Header Analyzer — Findings ({header} is the header, {detail} its value) + "header_finding_duplicate_header": + "{header}: sent {detail} times; the receiver joins the values into one.", + "header_finding_content_type_options_not_nosniff": + "{header}: '{detail}' has no effect, only 'nosniff' stops MIME sniffing.", + "header_finding_hsts_weak_max_age": + "{header}: max-age={detail} is short; 15552000 (180 days) is the usual minimum.", + "header_finding_csp_unsafe_directive": + "{header}: contains '{detail}', which re-allows what the policy should block.", + "header_finding_cors_wildcard_origin": "{header}: every origin is allowed (*).", + "header_finding_cors_wildcard_with_credentials": + "{header}: '*' with Access-Control-Allow-Credentials: true is rejected by browsers.", + "header_finding_cookie_not_secure": + "{header}: cookie '{detail}' has no Secure attribute, so it can travel over plain HTTP.", + "header_finding_cookie_not_httponly": + "{header}: cookie '{detail}' has no HttpOnly attribute, so scripts can read it.", + "header_finding_cookie_no_samesite": + "{header}: cookie '{detail}' has no SameSite attribute; browsers default it to Lax.", + "header_finding_content_type_no_charset": + "{header}: '{detail}' names no charset, so the client has to guess the encoding.", + "header_finding_server_banner": "{header}: '{detail}' reveals the software in use.", + "header_finding_deprecated_header": + "{header}: '{detail}' is deprecated and ignored by current browsers.", + "header_finding_sensitive_header": + "{header}: carries a credential; mask it before sharing this output.", + "header_finding_missing_hsts": + "{header}: not set, so a browser may fall back to plain HTTP.", + "header_finding_missing_csp": + "{header}: not set, so nothing limits where scripts may be loaded from.", + "header_finding_missing_content_type_options": + "{header}: not set, so a browser may MIME-sniff the response.", + "header_finding_missing_frame_options": + "{header}: not set; it (or CSP frame-ancestors) controls who may frame the page.", + "header_finding_missing_referrer_policy": + "{header}: not set, so full URLs may leak to other sites.", # Diagram Editor — Tools "diagram_editor_tool_select": "Select", "diagram_editor_tool_rect": "Rect", diff --git a/pybreeze/extend_multi_language/extend_traditional_chinese.py b/pybreeze/extend_multi_language/extend_traditional_chinese.py index 640f776..68981aa 100644 --- a/pybreeze/extend_multi_language/extend_traditional_chinese.py +++ b/pybreeze/extend_multi_language/extend_traditional_chinese.py @@ -309,6 +309,7 @@ "curl_import_target_apitestka_python": "APITestka（Python）", "curl_import_target_apitestka_action": "APITestka（JSON action）", "curl_import_target_loaddensity_python": "LoadDensity（Python）", + "curl_import_open_url_button": "在 URL 解析／組建器開啟網址", # 共用輸出動作（複製 / 開成分頁 / 存檔） "output_actions_copy": "複製", "output_actions_open_editor": "開成編輯器分頁", @@ -445,11 +446,66 @@ "response_output_label": "分析結果：", "response_open_jwt_button": "在解碼器開啟 JWT", "response_open_status_button": "在參考開啟狀態碼", + "response_open_headers_button": "在分析器開啟標頭", "response_empty_hint": "請先在上方貼上回應，再按分析。", "response_status_label": "== 狀態 Status ==", "response_headers_label": "== 標頭 Headers ==", "response_jwt_label": "== 找到的 JWT ==", "response_body_label": "== 內容 Body ==", + # HTTP 標頭分析器 — 選單 + "extend_tools_menu_header_analyzer_tab_action": "HTTP 標頭分析器分頁", + "extend_tools_menu_header_analyzer_tab_label": "標頭分析器", + "extend_tools_menu_header_analyzer_dock_action": "HTTP 標頭分析器停駐窗格", + "extend_tools_menu_header_analyzer_dock_title": "標頭分析器", + # HTTP 標頭分析器 — 介面 + "header_analyzer_input_label": "貼上請求或回應的標頭：", + "header_analyzer_input_placeholder": + "HTTP/1.1 200 OK\nContent-Type: text/html\nSet-Cookie: sid=abc; Path=/", + "header_analyzer_analyze_button": "分析標頭", + "header_analyzer_output_label": "分析結果：", + "header_analyzer_headers_label": "== 標頭（{count}）==", + "header_analyzer_duplicates_label": "== 重複出現的名稱 ==", + "header_analyzer_findings_label": "== 分析發現 ==", + "header_analyzer_no_findings": "沒有需要注意的項目。", + "header_analyzer_no_headers": "這段文字中找不到標頭。", + "header_analyzer_empty_hint": "請先在上方貼上標頭，再按分析。", + "header_analyzer_level_warning": "警告", + "header_analyzer_level_info": "資訊", + # HTTP 標頭分析器 — 發現（{header} 為標頭名稱，{detail} 為其值） + "header_finding_duplicate_header": + "{header}：送出 {detail} 次，接收端會把這些值合併成一個。", + "header_finding_content_type_options_not_nosniff": + "{header}：'{detail}' 沒有作用，只有 'nosniff' 能阻止 MIME 探測。", + "header_finding_hsts_weak_max_age": + "{header}：max-age={detail} 太短，一般至少要 15552000（180 天）。", + "header_finding_csp_unsafe_directive": + "{header}：含有 '{detail}'，等於把此政策原本要擋下的行為又放行了。", + "header_finding_cors_wildcard_origin": "{header}：允許任何來源（*）。", + "header_finding_cors_wildcard_with_credentials": + "{header}：'*' 搭配 Access-Control-Allow-Credentials: true，瀏覽器會直接拒絕。", + "header_finding_cookie_not_secure": + "{header}：cookie '{detail}' 沒有 Secure 屬性，可能以純 HTTP 傳送。", + "header_finding_cookie_not_httponly": + "{header}：cookie '{detail}' 沒有 HttpOnly 屬性，指令碼可以讀取。", + "header_finding_cookie_no_samesite": + "{header}：cookie '{detail}' 沒有 SameSite 屬性，瀏覽器會預設為 Lax。", + "header_finding_content_type_no_charset": + "{header}：'{detail}' 沒有指定 charset，用戶端只能自行猜測編碼。", + "header_finding_server_banner": "{header}：'{detail}' 洩漏了所使用的軟體。", + "header_finding_deprecated_header": + "{header}：'{detail}' 已淘汰，現行瀏覽器不再處理。", + "header_finding_sensitive_header": + "{header}：內含憑證，分享這份輸出前請先遮蔽。", + "header_finding_missing_hsts": + "{header}：未設定，瀏覽器可能退回純 HTTP。", + "header_finding_missing_csp": + "{header}：未設定，沒有任何規則限制指令碼的載入來源。", + "header_finding_missing_content_type_options": + "{header}：未設定，瀏覽器可能對回應做 MIME 探測。", + "header_finding_missing_frame_options": + "{header}：未設定；此標頭（或 CSP frame-ancestors）決定誰能將頁面內嵌成框架。", + "header_finding_missing_referrer_policy": + "{header}：未設定，完整網址可能外洩給其他網站。", # Diagram Editor — 工具 "diagram_editor_tool_select": "選取", "diagram_editor_tool_rect": "矩形", diff --git a/pybreeze/pybreeze_ui/menu/tools/tools_menu.py b/pybreeze/pybreeze_ui/menu/tools/tools_menu.py index ce55624..a3eed66 100644 --- a/pybreeze/pybreeze_ui/menu/tools/tools_menu.py +++ b/pybreeze/pybreeze_ui/menu/tools/tools_menu.py @@ -20,6 +20,7 @@ from pybreeze.pybreeze_ui.tools_gui.json_format_gui import JsonFormatGUI from pybreeze.pybreeze_ui.tools_gui.jwt_decoder_gui import JwtDecoderGUI from pybreeze.pybreeze_ui.tools_gui.hash_gui import HashGUI +from pybreeze.pybreeze_ui.tools_gui.header_analyzer_gui import HeaderAnalyzerGUI from pybreeze.pybreeze_ui.tools_gui.http_status_gui import HttpStatusGUI from pybreeze.pybreeze_ui.tools_gui.query_json_gui import QueryJsonGUI from pybreeze.pybreeze_ui.tools_gui.regex_gui import RegexGUI @@ -212,6 +213,17 @@ def build_tools_menu(ui_we_want_to_set: PyBreezeMainWindow): )) ui_we_want_to_set.tools_menu.addAction(ui_we_want_to_set.tools_json_format_action) + # HTTP Header Analyzer + ui_we_want_to_set.tools_header_analyzer_action = QAction(language_wrapper.language_word_dict.get( + "extend_tools_menu_header_analyzer_tab_action" + )) + ui_we_want_to_set.tools_header_analyzer_action.triggered.connect(lambda: ui_we_want_to_set.tab_widget.addTab( + HeaderAnalyzerGUI(ui_we_want_to_set), language_wrapper.language_word_dict.get( + "extend_tools_menu_header_analyzer_tab_label" + ) + )) + ui_we_want_to_set.tools_menu.addAction(ui_we_want_to_set.tools_header_analyzer_action) + # Response Inspector ui_we_want_to_set.tools_response_action = QAction(language_wrapper.language_word_dict.get( "extend_tools_menu_response_tab_action" @@ -343,6 +355,13 @@ def extend_dock_menu(ui_we_want_to_set: PyBreezeMainWindow): lambda: add_dock(ui_we_want_to_set, "JsonFormat")) ui_we_want_to_set.dock_menu.addAction(ui_we_want_to_set.tools_json_format_dock_action) + # HTTP Header Analyzer Dock + ui_we_want_to_set.tools_header_analyzer_dock_action = QAction(language_wrapper.language_word_dict.get( + "extend_tools_menu_header_analyzer_dock_action")) + ui_we_want_to_set.tools_header_analyzer_dock_action.triggered.connect( + lambda: add_dock(ui_we_want_to_set, "HeaderAnalyzer")) + ui_we_want_to_set.dock_menu.addAction(ui_we_want_to_set.tools_header_analyzer_dock_action) + # Response Inspector Dock ui_we_want_to_set.tools_response_dock_action = QAction(language_wrapper.language_word_dict.get( "extend_tools_menu_response_dock_action")) @@ -377,6 +396,8 @@ def extend_dock_menu(ui_we_want_to_set: PyBreezeMainWindow): "extend_tools_menu_http_status_dock_title", lambda win: HttpStatusGUI(main_window=win)), "Diff": ("extend_tools_menu_diff_dock_title", lambda win: DiffGUI(win)), "JsonFormat": ("extend_tools_menu_json_format_dock_title", lambda win: JsonFormatGUI(win)), + "HeaderAnalyzer": ( + "extend_tools_menu_header_analyzer_dock_title", lambda win: HeaderAnalyzerGUI(win)), "ResponseInspector": ( "extend_tools_menu_response_dock_title", lambda win: ResponseInspectorGUI(win)), } diff --git a/pybreeze/pybreeze_ui/tools_gui/header_analyzer_gui.py b/pybreeze/pybreeze_ui/tools_gui/header_analyzer_gui.py new file mode 100644 index 0000000..960fb3e --- /dev/null +++ b/pybreeze/pybreeze_ui/tools_gui/header_analyzer_gui.py @@ -0,0 +1,108 @@ +"""A tool tab that analyses a pasted block of HTTP headers. + +Paste request or response headers and read back every field, the names that were +sent more than once, and what is worth knowing about them — cookie flags, CORS +and HSTS policies, headers carrying credentials, and the response security +headers that are missing. +""" +from __future__ import annotations + +from PySide6.QtWidgets import QLabel, QPushButton, QTextEdit, QVBoxLayout, QWidget +from je_editor import language_wrapper + +from pybreeze.pybreeze_ui.tools_gui.output_actions import OutputActions +from pybreeze.utils.header_tools.header_analyzer import ( + HeaderAnalysis, HeaderFinding, analyze_headers +) + + +def _finding_line(finding: HeaderFinding) -> str: + """Render one finding as a translated, level-prefixed line.""" + word = language_wrapper.language_word_dict + level = word.get(f"header_analyzer_level_{finding.level}") or finding.level + message = word.get(f"header_finding_{finding.code}") or finding.code + return f"[{level}] {message.format(header=finding.header, detail=finding.detail)}" + + +def build_header_report(analysis: HeaderAnalysis) -> str: + """Render a header analysis into a readable, sectioned report. + + :param analysis: the analysed header block + :return: display text, or the "no headers" message when nothing was found + """ + word = language_wrapper.language_word_dict + if not analysis.fields: + return word.get("header_analyzer_no_headers") + + lines = [word.get("header_analyzer_headers_label").format(count=len(analysis.fields))] + lines.extend(f"{header.name}: {header.value}" for header in analysis.fields) + + if analysis.duplicates: + lines.extend(["", word.get("header_analyzer_duplicates_label")]) + lines.extend(f"{name} × {count}" for name, count in analysis.duplicates.items()) + + lines.extend(["", word.get("header_analyzer_findings_label")]) + if analysis.findings: + lines.extend(_finding_line(finding) for finding in analysis.findings) + else: + lines.append(word.get("header_analyzer_no_findings")) + return "\n".join(lines) + + +class HeaderAnalyzerGUI(QWidget): + """Paste HTTP headers and read what they say about the request or response.""" + + def __init__(self, main_window=None, initial_headers: str | None = None) -> None: + """ + :param main_window: window whose ``tab_widget`` "open in editor" uses + :param initial_headers: a header block to pre-fill and analyse on open + """ + super().__init__() + self._analysis: HeaderAnalysis | None = None + word = language_wrapper.language_word_dict + + self.input_label = QLabel(word.get("header_analyzer_input_label")) + self.input_edit = QTextEdit() + self.input_edit.setPlaceholderText(word.get("header_analyzer_input_placeholder")) + self.input_edit.setAcceptRichText(False) + + self.analyze_button = QPushButton(word.get("header_analyzer_analyze_button")) + self.analyze_button.clicked.connect(self.analyze) + + self.output_label = QLabel(word.get("header_analyzer_output_label")) + self.output_edit = QTextEdit() + self.output_edit.setReadOnly(True) + + self.actions = OutputActions( + self, self.output_edit, main_window=main_window, + basename="headers", extension="txt", + is_valid=lambda: self._analysis is not None) + + layout = QVBoxLayout() + for widget in ( + self.input_label, self.input_edit, self.analyze_button, + self.output_label, self.output_edit, + ): + layout.addWidget(widget) + layout.addLayout(self.actions.button_row()) + self.setLayout(layout) + + if initial_headers: + self.input_edit.setPlainText(initial_headers) + self.analyze() + + def analyze(self) -> None: + """Analyse the pasted headers and show the report.""" + word = language_wrapper.language_word_dict + text = self.input_edit.toPlainText().strip() + if not text: + self._analysis = None + self.output_edit.setPlainText(word.get("header_analyzer_empty_hint")) + return + analysis = analyze_headers(text) + if not analysis.fields: + self._analysis = None + self.output_edit.setPlainText(word.get("header_analyzer_no_headers")) + return + self._analysis = analysis + self.output_edit.setPlainText(build_header_report(analysis)) diff --git a/pybreeze/pybreeze_ui/tools_gui/response_inspector_gui.py b/pybreeze/pybreeze_ui/tools_gui/response_inspector_gui.py index 0ac03a4..7e97b84 100644 --- a/pybreeze/pybreeze_ui/tools_gui/response_inspector_gui.py +++ b/pybreeze/pybreeze_ui/tools_gui/response_inspector_gui.py @@ -12,11 +12,12 @@ ) from je_editor import language_wrapper +from pybreeze.pybreeze_ui.tools_gui.header_analyzer_gui import HeaderAnalyzerGUI from pybreeze.pybreeze_ui.tools_gui.http_status_gui import HttpStatusGUI from pybreeze.pybreeze_ui.tools_gui.jwt_decoder_gui import JwtDecoderGUI from pybreeze.pybreeze_ui.tools_gui.output_actions import OutputActions +from pybreeze.pybreeze_ui.tools_gui.tool_tabs import open_tool_tab from pybreeze.utils.jwt_tools.jwt_decoder import humanized_timestamp_claims -from pybreeze.utils.logging.logger import pybreeze_logger from pybreeze.utils.response_inspector.response_analyzer import ( ResponseAnalysis, analyze_response ) @@ -101,6 +102,9 @@ def __init__(self, main_window=None) -> None: self.open_status_button = QPushButton(word.get("response_open_status_button")) self.open_status_button.clicked.connect(self.open_status_in_reference) self.open_status_button.setEnabled(False) + self.open_headers_button = QPushButton(word.get("response_open_headers_button")) + self.open_headers_button.clicked.connect(self.open_headers_in_analyzer) + self.open_headers_button.setEnabled(False) # Shared copy / open-in-editor / save actions, valid once analysed. self.actions = OutputActions( @@ -110,6 +114,7 @@ def __init__(self, main_window=None) -> None: cross_tool = QHBoxLayout() cross_tool.addWidget(self.open_status_button) + cross_tool.addWidget(self.open_headers_button) cross_tool.addWidget(self.open_jwt_button) layout = QVBoxLayout() @@ -130,29 +135,22 @@ def analyze(self) -> None: self._analysis = None self.open_jwt_button.setEnabled(False) self.open_status_button.setEnabled(False) + self.open_headers_button.setEnabled(False) self.output_edit.setPlainText(word.get("response_empty_hint")) return self._analysis = analyze_response(text) self.output_edit.setPlainText(build_report_text(self._analysis)) self.open_jwt_button.setEnabled(bool(self._analysis.jwt_findings)) self.open_status_button.setEnabled(self._analysis.status is not None) - - def _open_tool_tab(self, widget: QWidget, tab_label_key: str) -> QWidget | None: - """Add *widget* as a new tab in the main window, if there is one.""" - tab_widget = getattr(self._main_window, "tab_widget", None) - if tab_widget is None: - pybreeze_logger.info("response_inspector_gui.py no tab_widget to open tool in") - return None - tab_widget.addTab(widget, language_wrapper.language_word_dict.get(tab_label_key)) - tab_widget.setCurrentWidget(widget) - return widget + self.open_headers_button.setEnabled(bool(self._analysis.headers)) def open_jwt_in_decoder(self) -> QWidget | None: """Open the first found JWT in the JWT decoder tool, pre-filled.""" if self._analysis is None or not self._analysis.jwt_findings: return None token = self._analysis.jwt_findings[0].token - return self._open_tool_tab( + return open_tool_tab( + self._main_window, JwtDecoderGUI(initial_token=token, main_window=self._main_window), "extend_tools_menu_jwt_decoder_tab_label") @@ -160,8 +158,24 @@ def open_status_in_reference(self) -> QWidget | None: """Open the detected status code in the HTTP status reference, pre-filled.""" if self._analysis is None or self._analysis.status is None: return None - return self._open_tool_tab( + return open_tool_tab( + self._main_window, HttpStatusGUI( initial_search=str(self._analysis.status.code), main_window=self._main_window), "extend_tools_menu_http_status_tab_label") + + def open_headers_in_analyzer(self) -> QWidget | None: + """Open the response's headers in the header analyzer, pre-filled. + + The pasted text is handed over as written, so repeated headers (several + ``Set-Cookie`` lines, say) survive the hand-off instead of being merged. + """ + if self._analysis is None or not self._analysis.headers: + return None + return open_tool_tab( + self._main_window, + HeaderAnalyzerGUI( + main_window=self._main_window, + initial_headers=self.input_edit.toPlainText()), + "extend_tools_menu_header_analyzer_tab_label") diff --git a/pybreeze/pybreeze_ui/tools_gui/tool_tabs.py b/pybreeze/pybreeze_ui/tools_gui/tool_tabs.py new file mode 100644 index 0000000..e1a1f6c --- /dev/null +++ b/pybreeze/pybreeze_ui/tools_gui/tool_tabs.py @@ -0,0 +1,31 @@ +"""Open one tool from another as a new, pre-filled tab. + +Several tools hand a finding over to the tool that specialises in it — a status +code to the HTTP reference, a URL to the URL parser/builder, a header block to +the header analyzer. They all need the same three steps (find the main window's +tab widget, add the tab, focus it), so it lives here instead of in each tool. +""" +from __future__ import annotations + +from PySide6.QtWidgets import QWidget +from je_editor import language_wrapper + +from pybreeze.utils.logging.logger import pybreeze_logger + + +def open_tool_tab(main_window, widget: QWidget, tab_label_key: str) -> QWidget | None: + """Add *widget* as a new tab of *main_window* and make it the current tab. + + :param main_window: the window whose ``tab_widget`` the tab is added to; + when it has none (or is ``None``) nothing happens + :param widget: the tool widget to show, already pre-filled by the caller + :param tab_label_key: language key of the new tab's label + :return: *widget* when it was opened, otherwise ``None`` + """ + tab_widget = getattr(main_window, "tab_widget", None) + if tab_widget is None: + pybreeze_logger.info("tool_tabs.py no tab_widget to open the tool in") + return None + tab_widget.addTab(widget, language_wrapper.language_word_dict.get(tab_label_key)) + tab_widget.setCurrentWidget(widget) + return widget diff --git a/pybreeze/utils/header_tools/__init__.py b/pybreeze/utils/header_tools/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pybreeze/utils/header_tools/header_analyzer.py b/pybreeze/utils/header_tools/header_analyzer.py new file mode 100644 index 0000000..e133d60 --- /dev/null +++ b/pybreeze/utils/header_tools/header_analyzer.py @@ -0,0 +1,318 @@ +"""Analyse a block of HTTP headers. + +Paste the headers of a request or a response — copied from browser dev tools, a +``curl -i`` run, or a proxy log — and this module reports what is in them: every +header line, names that appear more than once, and the things worth knowing +about while debugging an API (cookie flags, a wildcard CORS policy, a weak HSTS +policy, headers that carry credentials, missing response security headers). + +It is the companion of the response inspector, which decodes one whole response; +this looks only at the headers, and looks harder. Pure logic — no Qt, no network. +Findings carry a stable *code* rather than a sentence, so the UI can translate +them; values that could be secrets are never copied into a finding's detail. +""" +from __future__ import annotations + +import re +from collections.abc import Callable +from dataclasses import dataclass, field + +# Severity of a finding: something to fix, or something merely worth knowing. +LEVEL_WARNING = "warning" +LEVEL_INFO = "info" + +# Matches one header line, e.g. "Content-Type: application/json" +HEADER_LINE_RE = re.compile(r"^([A-Za-z0-9!#$%&'*+.^_`|~-]+):[ \t]?(.*)$") +# Matches a leading response status line, e.g. "HTTP/1.1 200 OK" +_STATUS_LINE_RE = re.compile(r"^\s*HTTP/\d(?:\.\d)?\s+\d{3}\b") +# Matches the max-age directive of an HSTS policy +_MAX_AGE_RE = re.compile(r"max-age\s*=\s*(\d+)", re.IGNORECASE) + +# Header names referenced from more than one table below +_HSTS_HEADER = "strict-transport-security" +_CSP_HEADER = "content-security-policy" +_CONTENT_TYPE_OPTIONS_HEADER = "x-content-type-options" +_CORS_ORIGIN_HEADER = "access-control-allow-origin" +_CORS_CREDENTIALS_HEADER = "access-control-allow-credentials" +_SET_COOKIE_HEADER = "set-cookie" + +# An HSTS policy shorter than 180 days is too short to survive a browser restart +# cycle and is below what the preload list accepts. +_MIN_HSTS_MAX_AGE = 15_552_000 + +# Headers that are legitimately sent more than once, so a repeat is not a finding +_REPEATABLE_HEADERS = frozenset({ + _SET_COOKIE_HEADER, "www-authenticate", "proxy-authenticate", "via", "link", "warning", +}) + +# Headers only a server sends. Without one of these (or a status line) the block +# is treated as a request, where the missing-response-header checks make no sense. +_RESPONSE_ONLY_HEADERS = frozenset({ + _SET_COOKIE_HEADER, _CSP_HEADER, _HSTS_HEADER, _CONTENT_TYPE_OPTIONS_HEADER, + _CORS_ORIGIN_HEADER, "server", "x-powered-by", "x-frame-options", "referrer-policy", + "permissions-policy", "www-authenticate", "location", "etag", "last-modified", + "age", "retry-after", "content-encoding", +}) + +# Response security headers, as (lower-case name, canonical name, finding code) +_RESPONSE_SECURITY_HEADERS: tuple[tuple[str, str, str], ...] = ( + (_HSTS_HEADER, "Strict-Transport-Security", "missing_hsts"), + (_CSP_HEADER, "Content-Security-Policy", "missing_csp"), + (_CONTENT_TYPE_OPTIONS_HEADER, "X-Content-Type-Options", "missing_content_type_options"), + ("x-frame-options", "X-Frame-Options", "missing_frame_options"), + ("referrer-policy", "Referrer-Policy", "missing_referrer_policy"), +) + +# Headers whose value is a credential; their presence is reported, never the value +_SENSITIVE_HEADERS = frozenset({ + "authorization", "proxy-authorization", "cookie", "x-api-key", "api-key", "x-auth-token", +}) + +# CSP directives that give back much of what the policy is meant to take away +_UNSAFE_CSP_DIRECTIVES = ("unsafe-inline", "unsafe-eval") + + +@dataclass(frozen=True) +class HeaderField: + """One header line as it was written. + + :param name: the header name, in its original casing + :param value: the header value, stripped of surrounding whitespace + """ + + name: str + value: str + + +@dataclass(frozen=True) +class HeaderFinding: + """Something worth reporting about a header block. + + :param code: stable slug the UI turns into a translated message + :param level: :data:`LEVEL_WARNING` or :data:`LEVEL_INFO` + :param header: the header the finding is about + :param detail: an untranslated fragment for the message (a value, a count, + a cookie name); never a credential + """ + + code: str + level: str + header: str + detail: str = "" + + +@dataclass +class HeaderAnalysis: + """The result of analysing a header block. + + :param fields: every header line found, in the order they appeared + :param duplicates: lower-case name -> how often it appeared (repeats only) + :param findings: what was noticed, warnings first + :param looks_like_response: whether the block came from a server response, + which is when the missing-security-header checks apply + """ + + fields: list[HeaderField] = field(default_factory=list) + duplicates: dict[str, int] = field(default_factory=dict) + findings: list[HeaderFinding] = field(default_factory=list) + looks_like_response: bool = False + + +def parse_headers(text: str) -> list[HeaderField]: + """Pull the header lines out of *text*. + + Accepts a bare header block or a whole request/response: a start line does + not parse as a header and is skipped, and a blank line after the headers ends + the block so a pasted body is not read as more headers. + + :param text: the pasted text + :return: the headers found, in order, duplicates kept + """ + fields: list[HeaderField] = [] + for line in text.replace("\r\n", "\n").replace("\r", "\n").split("\n"): + if not line.strip(): + if fields: + break # the blank line between the headers and the body + continue + match = HEADER_LINE_RE.match(line) + if match is not None: + fields.append(HeaderField(name=match.group(1), value=match.group(2).strip())) + return fields + + +def _first_value(fields: list[HeaderField], name: str) -> str | None: + """Return the first value of the header called *name*, or ``None``.""" + for header in fields: + if header.name.lower() == name: + return header.value + return None + + +def _duplicate_counts(fields: list[HeaderField]) -> dict[str, int]: + """Count the header names that appear more than once, case-insensitively.""" + counts: dict[str, int] = {} + for header in fields: + lowered = header.name.lower() + counts[lowered] = counts.get(lowered, 0) + 1 + return {name: count for name, count in counts.items() if count > 1} + + +def _duplicate_findings(duplicates: dict[str, int]) -> list[HeaderFinding]: + """Report repeated headers, except the ones HTTP expects to repeat.""" + return [ + HeaderFinding("duplicate_header", LEVEL_WARNING, name, str(count)) + for name, count in duplicates.items() + if name not in _REPEATABLE_HEADERS + ] + + +def _check_content_type_options(header: HeaderField) -> list[HeaderFinding]: + """``X-Content-Type-Options`` does nothing unless its value is ``nosniff``.""" + if header.value.strip().lower() == "nosniff": + return [] + return [HeaderFinding( + "content_type_options_not_nosniff", LEVEL_WARNING, header.name, header.value)] + + +def _check_hsts(header: HeaderField) -> list[HeaderFinding]: + """Flag an HSTS policy whose ``max-age`` is missing or too short to matter.""" + match = _MAX_AGE_RE.search(header.value) + max_age = int(match.group(1)) if match is not None else 0 + if max_age >= _MIN_HSTS_MAX_AGE: + return [] + return [HeaderFinding("hsts_weak_max_age", LEVEL_WARNING, header.name, str(max_age))] + + +def _check_csp(header: HeaderField) -> list[HeaderFinding]: + """Flag the CSP directives that re-allow what the policy is meant to block.""" + lowered = header.value.lower() + return [ + HeaderFinding("csp_unsafe_directive", LEVEL_WARNING, header.name, directive) + for directive in _UNSAFE_CSP_DIRECTIVES if directive in lowered + ] + + +def _check_cors_origin(header: HeaderField) -> list[HeaderFinding]: + """Note a CORS policy that allows every origin.""" + if header.value.strip() != "*": + return [] + return [HeaderFinding("cors_wildcard_origin", LEVEL_INFO, header.name)] + + +def _check_set_cookie(header: HeaderField) -> list[HeaderFinding]: + """Check one ``Set-Cookie`` for the attributes that keep it from leaking.""" + segments = header.value.split(";") + attributes = [segment.strip().lower() for segment in segments[1:]] + cookie_name = segments[0].split("=", 1)[0].strip() or header.name + findings: list[HeaderFinding] = [] + if "secure" not in attributes: + findings.append(HeaderFinding("cookie_not_secure", LEVEL_WARNING, header.name, cookie_name)) + if "httponly" not in attributes: + findings.append( + HeaderFinding("cookie_not_httponly", LEVEL_WARNING, header.name, cookie_name)) + if not any(attribute.startswith("samesite") for attribute in attributes): + findings.append(HeaderFinding("cookie_no_samesite", LEVEL_INFO, header.name, cookie_name)) + return findings + + +def _check_content_type(header: HeaderField) -> list[HeaderFinding]: + """Note a textual media type left without a charset, where it still matters.""" + lowered = header.value.lower() + if not lowered.startswith("text/") or "charset=" in lowered: + return [] + return [HeaderFinding("content_type_no_charset", LEVEL_INFO, header.name, header.value)] + + +def _check_banner(header: HeaderField) -> list[HeaderFinding]: + """Note a product banner: it tells a visitor what software to look up.""" + if not header.value.strip(): + return [] + return [HeaderFinding("server_banner", LEVEL_INFO, header.name, header.value)] + + +def _check_deprecated(header: HeaderField) -> list[HeaderFinding]: + """Note a header browsers no longer honour.""" + return [HeaderFinding("deprecated_header", LEVEL_INFO, header.name, header.value)] + + +# lower-case header name -> the check that applies to it. A table keeps +# _field_findings flat instead of a branch per header. +_FIELD_CHECKS: dict[str, Callable[[HeaderField], list[HeaderFinding]]] = { + _CONTENT_TYPE_OPTIONS_HEADER: _check_content_type_options, + _HSTS_HEADER: _check_hsts, + _CSP_HEADER: _check_csp, + _CORS_ORIGIN_HEADER: _check_cors_origin, + _SET_COOKIE_HEADER: _check_set_cookie, + "content-type": _check_content_type, + "server": _check_banner, + "x-powered-by": _check_banner, + "x-xss-protection": _check_deprecated, +} + + +def _field_findings(fields: list[HeaderField]) -> list[HeaderFinding]: + """Run the per-header checks over every field.""" + findings: list[HeaderFinding] = [] + for header in fields: + lowered = header.name.lower() + check = _FIELD_CHECKS.get(lowered) + if check is not None: + findings.extend(check(header)) + if lowered in _SENSITIVE_HEADERS: + # The value is a credential, so it is deliberately not reported. + findings.append(HeaderFinding("sensitive_header", LEVEL_INFO, header.name)) + return findings + + +def _cors_credentials_findings(fields: list[HeaderField]) -> list[HeaderFinding]: + """Flag a wildcard origin combined with credentials. + + Browsers reject that combination outright, so a policy that sends both never + worked the way its author expected. + """ + origin = _first_value(fields, _CORS_ORIGIN_HEADER) + credentials = _first_value(fields, _CORS_CREDENTIALS_HEADER) + if origin is None or origin.strip() != "*": + return [] + if credentials is None or credentials.strip().lower() != "true": + return [] + return [HeaderFinding("cors_wildcard_with_credentials", LEVEL_WARNING, _CORS_ORIGIN_HEADER)] + + +def _missing_security_findings(fields: list[HeaderField]) -> list[HeaderFinding]: + """Report the response security headers that are not present.""" + present = {header.name.lower() for header in fields} + return [ + HeaderFinding(code, LEVEL_INFO, canonical) + for name, canonical, code in _RESPONSE_SECURITY_HEADERS if name not in present + ] + + +def _looks_like_response(text: str, fields: list[HeaderField]) -> bool: + """Whether the block came from a response rather than a request.""" + if _STATUS_LINE_RE.match(text): + return True + return any(header.name.lower() in _RESPONSE_ONLY_HEADERS for header in fields) + + +def analyze_headers(text: str) -> HeaderAnalysis: + """Analyse a pasted block of HTTP headers. + + :param text: the headers, optionally with a start line and a body after them + :return: the parsed fields, repeated names, and the findings, warnings first + """ + fields = parse_headers(text) + duplicates = _duplicate_counts(fields) + from_response = _looks_like_response(text, fields) + + findings = _duplicate_findings(duplicates) + findings.extend(_field_findings(fields)) + findings.extend(_cors_credentials_findings(fields)) + if from_response: + findings.extend(_missing_security_findings(fields)) + # Stable sort: warnings first, each group still in the order it was found. + findings.sort(key=lambda finding: 0 if finding.level == LEVEL_WARNING else 1) + + return HeaderAnalysis( + fields=fields, duplicates=duplicates, findings=findings, + looks_like_response=from_response) diff --git a/pybreeze/utils/response_inspector/response_analyzer.py b/pybreeze/utils/response_inspector/response_analyzer.py index 791940e..fef4500 100644 --- a/pybreeze/utils/response_inspector/response_analyzer.py +++ b/pybreeze/utils/response_inspector/response_analyzer.py @@ -14,14 +14,13 @@ import re from dataclasses import dataclass, field +from pybreeze.utils.header_tools.header_analyzer import HEADER_LINE_RE from pybreeze.utils.http_reference.status_codes import StatusInfo, lookup from pybreeze.utils.jwt_tools.jwt_decoder import DecodedJwt, decode_jwt from pybreeze.utils.exception.exceptions import JwtDecodeException # Matches the response status line, e.g. "HTTP/1.1 200 OK" _STATUS_LINE_RE = re.compile(r"^HTTP/\d(?:\.\d)?\s+(\d{3})\b") -# Matches a header line, e.g. "Content-Type: application/json" -_HEADER_RE = re.compile(r"^([A-Za-z0-9!#$%&'*+.^_`|~-]+):[ \t]?(.*)$") # Matches a JWT-looking token (three base64url segments; the signature may be empty) _JWT_RE = re.compile(r"eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]*") @@ -82,7 +81,7 @@ def _parse_head_and_body(text: str) -> tuple[int | None, dict[str, str], str]: if line.strip() == "": index += 1 break - match = _HEADER_RE.match(line) + match = HEADER_LINE_RE.match(line) if match is None: break headers[match.group(1)] = match.group(2).strip() diff --git a/test/test_utils/test_fuzz_pure_logic.py b/test/test_utils/test_fuzz_pure_logic.py index 0e7e1ed..74b53e7 100644 --- a/test/test_utils/test_fuzz_pure_logic.py +++ b/test/test_utils/test_fuzz_pure_logic.py @@ -15,6 +15,11 @@ _parse_content_length, ) from pybreeze.utils.exception.exceptions import ITEJsonException +from pybreeze.utils.header_tools.header_analyzer import ( + LEVEL_INFO, + LEVEL_WARNING, + analyze_headers, +) from pybreeze.utils.json_format.json_process import reformat_json from pybreeze.utils.network.http_client import truncate_for_display from pybreeze.utils.network.url_validation import _embedded_ipv4, _is_blocked_ip @@ -86,6 +91,27 @@ def test_is_text_content_type(self, raw): assert isinstance(_is_text_content_type(raw), bool) +class TestHeaderAnalyzerNeverCrashes: + @_FUZZ + @given(st.text()) + def test_arbitrary_text(self, text): + analysis = analyze_headers(text) + assert isinstance(analysis.duplicates, dict) + # Every finding must be renderable by the UI: a code and a known level. + for finding in analysis.findings: + assert finding.code + assert finding.level in {LEVEL_WARNING, LEVEL_INFO} + + @_FUZZ + @given(st.text(alphabet="ABab-:;=,*/ \r\n\"'", max_size=120)) + def test_header_like_text(self, text): + analysis = analyze_headers(text) + # A name counted as duplicated must really appear that often. + for name, count in analysis.duplicates.items(): + assert count > 1 + assert sum(1 for f in analysis.fields if f.name.lower() == name) == count + + class TestSsrfLogicNeverCrashes: @_FUZZ @given(st.ip_addresses(v=4)) diff --git a/test/test_utils/test_header_analyzer.py b/test/test_utils/test_header_analyzer.py new file mode 100644 index 0000000..ddb2953 --- /dev/null +++ b/test/test_utils/test_header_analyzer.py @@ -0,0 +1,190 @@ +"""Tests for the HTTP header analyzer's pure logic.""" +from __future__ import annotations + +from pybreeze.utils.header_tools.header_analyzer import ( + LEVEL_INFO, + LEVEL_WARNING, + analyze_headers, + parse_headers, +) + + +def _codes(text: str) -> set[str]: + return {finding.code for finding in analyze_headers(text).findings} + + +def _finding(text: str, code: str): + return next(f for f in analyze_headers(text).findings if f.code == code) + + +class TestParseHeaders: + def test_plain_block(self): + fields = parse_headers("Content-Type: application/json\nAccept: */*") + assert [(f.name, f.value) for f in fields] == [ + ("Content-Type", "application/json"), ("Accept", "*/*")] + + def test_status_line_is_skipped(self): + fields = parse_headers("HTTP/1.1 200 OK\nServer: nginx") + assert [f.name for f in fields] == ["Server"] + + def test_request_line_is_skipped(self): + fields = parse_headers("GET /api?a=1 HTTP/1.1\nHost: example.com") + assert [f.name for f in fields] == ["Host"] + + def test_body_after_blank_line_is_not_parsed(self): + fields = parse_headers("Content-Type: text/plain\n\nnot-a-header: really") + assert [f.name for f in fields] == ["Content-Type"] + + def test_value_with_colon_is_kept_whole(self): + fields = parse_headers("Host: example.com:8080") + assert fields[0].value == "example.com:8080" + + def test_crlf_line_endings(self): + fields = parse_headers("A: 1\r\nB: 2\r\n") + assert [f.name for f in fields] == ["A", "B"] + + def test_duplicates_are_kept(self): + fields = parse_headers("Set-Cookie: a=1\nSet-Cookie: b=2") + assert len(fields) == 2 + + def test_empty_text(self): + assert parse_headers("") == [] + + +class TestDuplicates: + def test_repeated_name_is_counted_and_reported(self): + analysis = analyze_headers("X-Trace: 1\nx-trace: 2") + assert analysis.duplicates == {"x-trace": 2} + assert "duplicate_header" in {f.code for f in analysis.findings} + + def test_repeatable_header_is_counted_but_not_reported(self): + analysis = analyze_headers("Set-Cookie: a=1; Secure; HttpOnly; SameSite=Lax\n" + "Set-Cookie: b=2; Secure; HttpOnly; SameSite=Lax") + assert analysis.duplicates == {"set-cookie": 2} + assert "duplicate_header" not in {f.code for f in analysis.findings} + + def test_duplicate_detail_carries_the_count(self): + assert _finding("A: 1\nA: 2\nA: 3", "duplicate_header").detail == "3" + + +class TestSecurityHeaderValues: + def test_content_type_options_nosniff_is_clean(self): + assert "content_type_options_not_nosniff" not in _codes( + "X-Content-Type-Options: nosniff") + + def test_content_type_options_other_value_is_flagged(self): + assert "content_type_options_not_nosniff" in _codes("X-Content-Type-Options: sniff") + + def test_long_hsts_is_clean(self): + assert "hsts_weak_max_age" not in _codes( + "Strict-Transport-Security: max-age=31536000; includeSubDomains") + + def test_short_hsts_is_flagged(self): + assert _finding("Strict-Transport-Security: max-age=600", "hsts_weak_max_age").detail == "600" + + def test_hsts_without_max_age_is_flagged_as_zero(self): + assert _finding( + "Strict-Transport-Security: includeSubDomains", "hsts_weak_max_age").detail == "0" + + def test_csp_unsafe_inline_is_flagged(self): + finding = _finding("Content-Security-Policy: default-src 'self' 'unsafe-inline'", + "csp_unsafe_directive") + assert finding.detail == "unsafe-inline" + + def test_csp_without_unsafe_directives_is_clean(self): + assert "csp_unsafe_directive" not in _codes("Content-Security-Policy: default-src 'self'") + + def test_wildcard_cors_is_noted(self): + assert "cors_wildcard_origin" in _codes("Access-Control-Allow-Origin: *") + + def test_specific_cors_origin_is_clean(self): + assert "cors_wildcard_origin" not in _codes("Access-Control-Allow-Origin: https://a.com") + + def test_wildcard_cors_with_credentials_is_a_warning(self): + finding = _finding( + "Access-Control-Allow-Origin: *\nAccess-Control-Allow-Credentials: true", + "cors_wildcard_with_credentials") + assert finding.level == LEVEL_WARNING + + def test_specific_origin_with_credentials_is_clean(self): + assert "cors_wildcard_with_credentials" not in _codes( + "Access-Control-Allow-Origin: https://a.com\nAccess-Control-Allow-Credentials: true") + + +class TestCookieFlags: + def test_hardened_cookie_is_clean(self): + codes = _codes("Set-Cookie: sid=abc; Path=/; Secure; HttpOnly; SameSite=Strict") + assert not codes & {"cookie_not_secure", "cookie_not_httponly", "cookie_no_samesite"} + + def test_bare_cookie_reports_every_missing_flag(self): + codes = _codes("Set-Cookie: sid=abc; Path=/") + assert {"cookie_not_secure", "cookie_not_httponly", "cookie_no_samesite"} <= codes + + def test_finding_names_the_cookie_not_its_value(self): + finding = _finding("Set-Cookie: sid=secret-value; Path=/", "cookie_not_secure") + assert finding.detail == "sid" + + def test_samesite_is_only_informational(self): + assert _finding( + "Set-Cookie: sid=a; Secure; HttpOnly", "cookie_no_samesite").level == LEVEL_INFO + + +class TestOtherFindings: + def test_sensitive_header_never_reports_its_value(self): + finding = _finding("Authorization: Bearer super-secret", "sensitive_header") + assert finding.detail == "" + assert "super-secret" not in str(finding) + + def test_server_banner_is_noted(self): + assert _finding("Server: nginx/1.25.3", "server_banner").detail == "nginx/1.25.3" + + def test_empty_banner_is_ignored(self): + assert "server_banner" not in _codes("Server:") + + def test_deprecated_header_is_noted(self): + assert "deprecated_header" in _codes("X-XSS-Protection: 1; mode=block") + + def test_text_type_without_charset_is_noted(self): + assert "content_type_no_charset" in _codes("Content-Type: text/html") + + def test_text_type_with_charset_is_clean(self): + assert "content_type_no_charset" not in _codes("Content-Type: text/html; charset=utf-8") + + def test_json_type_is_not_asked_for_a_charset(self): + assert "content_type_no_charset" not in _codes("Content-Type: application/json") + + +class TestMissingResponseHeaders: + def test_response_reports_missing_security_headers(self): + codes = _codes("HTTP/1.1 200 OK\nContent-Type: text/html; charset=utf-8") + assert {"missing_hsts", "missing_csp", "missing_content_type_options", + "missing_frame_options", "missing_referrer_policy"} <= codes + + def test_request_block_is_not_audited_for_response_headers(self): + codes = _codes("GET /api HTTP/1.1\nHost: example.com\nAccept: */*") + assert not {code for code in codes if code.startswith("missing_")} + + def test_response_only_header_marks_the_block_as_a_response(self): + assert analyze_headers("Set-Cookie: a=1").looks_like_response + + def test_present_header_is_not_reported_missing(self): + assert "missing_csp" not in _codes( + "HTTP/1.1 200 OK\nContent-Security-Policy: default-src 'self'") + + +class TestAnalysisShape: + def test_warnings_come_before_info(self): + analysis = analyze_headers( + "HTTP/1.1 200 OK\nServer: nginx\nX-Content-Type-Options: sniff") + levels = [finding.level for finding in analysis.findings] + assert levels == sorted(levels, key=lambda level: 0 if level == LEVEL_WARNING else 1) + assert LEVEL_WARNING in levels + + def test_empty_input_yields_empty_analysis(self): + analysis = analyze_headers("") + assert analysis.fields == [] + assert analysis.findings == [] + assert not analysis.looks_like_response + + def test_text_without_headers_yields_no_fields(self): + assert analyze_headers("just some prose").fields == [] diff --git a/test/test_utils/test_header_analyzer_gui.py b/test/test_utils/test_header_analyzer_gui.py new file mode 100644 index 0000000..b498b93 --- /dev/null +++ b/test/test_utils/test_header_analyzer_gui.py @@ -0,0 +1,122 @@ +"""Tests for the HTTP header analyzer tool widget.""" +from __future__ import annotations + +import os + +os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") + +import pytest +from PySide6.QtWidgets import QApplication + +from pybreeze.extend_multi_language.extend_english import pybreeze_english_word_dict as EN +from pybreeze.extend_multi_language.update_language_dict import update_language_dict +from pybreeze.utils.header_tools.header_analyzer import analyze_headers + +_RESPONSE = ( + "HTTP/1.1 200 OK\n" + "Server: nginx/1.25.3\n" + "Content-Type: text/html\n" + "Set-Cookie: sid=abc; Path=/\n" +) + + +@pytest.fixture(scope="module") +def app(): + instance = QApplication.instance() or QApplication([]) + update_language_dict() + return instance + + +@pytest.fixture() +def widget(app): + from pybreeze.pybreeze_ui.tools_gui.header_analyzer_gui import HeaderAnalyzerGUI + gui = HeaderAnalyzerGUI() + yield gui + gui.close() + gui.deleteLater() + + +class TestHeaderAnalyzerGUI: + def test_headers_are_listed(self, widget): + widget.input_edit.setPlainText(_RESPONSE) + widget.analyze() + output = widget.output_edit.toPlainText() + assert "Server: nginx/1.25.3" in output + assert "Content-Type: text/html" in output + + def test_findings_are_translated_not_raw_codes(self, widget): + widget.input_edit.setPlainText(_RESPONSE) + widget.analyze() + output = widget.output_edit.toPlainText() + assert "cookie_not_secure" not in output + assert "Secure attribute" in output + + def test_finding_line_carries_its_level(self, widget): + widget.input_edit.setPlainText(_RESPONSE) + widget.analyze() + assert "[WARNING]" in widget.output_edit.toPlainText() + + def test_duplicate_section_shows_the_count(self, widget): + widget.input_edit.setPlainText("X-Trace: 1\nX-Trace: 2") + widget.analyze() + assert "x-trace × 2" in widget.output_edit.toPlainText() + + def test_clean_request_reports_nothing(self, widget): + widget.input_edit.setPlainText("GET / HTTP/1.1\nHost: example.com") + widget.analyze() + assert EN["header_analyzer_no_findings"] in widget.output_edit.toPlainText() + + def test_empty_input_shows_hint(self, widget): + widget.input_edit.setPlainText(" ") + widget.analyze() + assert widget.output_edit.toPlainText() == EN["header_analyzer_empty_hint"] + + def test_text_without_headers_reports_so(self, widget): + widget.input_edit.setPlainText("just some prose") + widget.analyze() + assert widget.output_edit.toPlainText() == EN["header_analyzer_no_headers"] + + def test_initial_headers_are_analysed_on_open(self, app): + from pybreeze.pybreeze_ui.tools_gui.header_analyzer_gui import HeaderAnalyzerGUI + gui = HeaderAnalyzerGUI(initial_headers=_RESPONSE) + assert "nginx/1.25.3" in gui.output_edit.toPlainText() + gui.close() + gui.deleteLater() + + def test_copy_output(self, app, widget): + widget.input_edit.setPlainText(_RESPONSE) + widget.analyze() + widget.actions.copy() + assert "nginx/1.25.3" in QApplication.clipboard().text() + + def test_save_after_no_headers_is_noop(self, widget): + widget.input_edit.setPlainText("just some prose") + widget.analyze() + assert widget.actions.save_to_file() is None + + +class TestEveryFindingCodeIsTranslated: + def test_report_never_falls_back_to_a_code(self, app): + # Any finding code without a language key would leak the slug into the UI. + from pybreeze.pybreeze_ui.tools_gui.header_analyzer_gui import build_header_report + text = ( + "HTTP/1.1 200 OK\n" + "Server: nginx\n" + "X-Powered-By: PHP\n" + "X-XSS-Protection: 1\n" + "X-Content-Type-Options: sniff\n" + "Strict-Transport-Security: max-age=1\n" + "Content-Security-Policy: default-src 'unsafe-inline' 'unsafe-eval'\n" + "Access-Control-Allow-Origin: *\n" + "Access-Control-Allow-Credentials: true\n" + "Set-Cookie: sid=abc\n" + "Authorization: Bearer secret\n" + "Content-Type: text/html\n" + "X-Trace: 1\n" + "X-Trace: 2\n" + ) + analysis = analyze_headers(text) + report = build_header_report(analysis) + for finding in analysis.findings: + assert finding.code not in report + assert "secret" not in report.split("== Findings ==")[1] diff --git a/test/test_utils/test_response_inspector_gui.py b/test/test_utils/test_response_inspector_gui.py index 8ecf2b3..0f5ed97 100644 --- a/test/test_utils/test_response_inspector_gui.py +++ b/test/test_utils/test_response_inspector_gui.py @@ -161,3 +161,43 @@ def test_open_before_analyze_is_noop(self, widget_with_window): gui, window = widget_with_window assert gui.open_status_in_reference() is None assert window.tab_widget.added == [] + + +class TestResponseInspectorHeaderHandOff: + def test_headers_button_disabled_without_headers(self, widget_with_window): + gui, _window = widget_with_window + gui.input_edit.setPlainText('{"a": 1}') + gui.analyze() + assert not gui.open_headers_button.isEnabled() + + def test_headers_button_enabled_after_headers(self, widget_with_window): + gui, _window = widget_with_window + gui.input_edit.setPlainText("HTTP/1.1 200 OK\nServer: nginx\n\n{}") + gui.analyze() + assert gui.open_headers_button.isEnabled() + + def test_open_headers_opens_prefilled_analyzer(self, widget_with_window): + from pybreeze.pybreeze_ui.tools_gui.header_analyzer_gui import HeaderAnalyzerGUI + gui, window = widget_with_window + gui.input_edit.setPlainText("HTTP/1.1 200 OK\nServer: nginx/1.25.3\n\n{}") + gui.analyze() + gui.open_headers_in_analyzer() + opened = window.tab_widget.added[0][0] + assert isinstance(opened, HeaderAnalyzerGUI) + assert "nginx/1.25.3" in opened.output_edit.toPlainText() + + def test_repeated_headers_survive_the_hand_off(self, widget_with_window): + # The inspector keeps headers in a dict; handing the raw text over is what + # lets the analyzer still see both Set-Cookie lines. + gui, window = widget_with_window + gui.input_edit.setPlainText( + "HTTP/1.1 200 OK\nSet-Cookie: a=1\nSet-Cookie: b=2\n\n{}") + gui.analyze() + gui.open_headers_in_analyzer() + opened = window.tab_widget.added[0][0] + assert "set-cookie × 2" in opened.output_edit.toPlainText() + + def test_open_headers_before_analyze_is_noop(self, widget_with_window): + gui, window = widget_with_window + assert gui.open_headers_in_analyzer() is None + assert window.tab_widget.added == [] diff --git a/test/test_utils/test_tools_menu_docks.py b/test/test_utils/test_tools_menu_docks.py index 8455dde..cccce89 100644 --- a/test/test_utils/test_tools_menu_docks.py +++ b/test/test_utils/test_tools_menu_docks.py @@ -15,6 +15,7 @@ from pybreeze.pybreeze_ui.tools_gui.curl_import_gui import CurlImportGUI from pybreeze.pybreeze_ui.tools_gui.diff_gui import DiffGUI from pybreeze.pybreeze_ui.tools_gui.hash_gui import HashGUI +from pybreeze.pybreeze_ui.tools_gui.header_analyzer_gui import HeaderAnalyzerGUI from pybreeze.pybreeze_ui.tools_gui.http_status_gui import HttpStatusGUI from pybreeze.pybreeze_ui.tools_gui.json_format_gui import JsonFormatGUI from pybreeze.pybreeze_ui.tools_gui.jwt_decoder_gui import JwtDecoderGUI @@ -22,6 +23,7 @@ from pybreeze.pybreeze_ui.tools_gui.regex_gui import RegexGUI from pybreeze.pybreeze_ui.tools_gui.response_inspector_gui import ResponseInspectorGUI from pybreeze.pybreeze_ui.tools_gui.timestamp_gui import TimestampGUI +from pybreeze.pybreeze_ui.tools_gui.url_builder_gui import UrlBuilderGUI @pytest.fixture(scope="module") @@ -39,10 +41,12 @@ def app(): ("Timestamp", TimestampGUI), ("Hash", HashGUI), ("QueryJson", QueryJsonGUI), + ("UrlBuilder", UrlBuilderGUI), ("Regex", RegexGUI), ("HttpStatus", HttpStatusGUI), ("Diff", DiffGUI), ("JsonFormat", JsonFormatGUI), + ("HeaderAnalyzer", HeaderAnalyzerGUI), ("ResponseInspector", ResponseInspectorGUI), ], ) From bf67e27d7b6ddef9fc0db0c74e3a793c5b4ffc9c Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Sat, 25 Jul 2026 14:13:38 +0800 Subject: [PATCH 09/10] Send the imported curl URL to the URL parser/builder The importer already knows the request URL, so a button now opens it in the URL parser/builder, parsed into its parts and ready to edit. The full URL is handed over, query string included, so the address matches what curl would send even though the parser keeps the query in params. The builder gained an initial_url argument for that hand-off, matching how the JWT decoder and status reference are pre-filled. --- README.md | 4 +- .../pybreeze_ui/tools_gui/curl_import_gui.py | 45 +++++++++++++--- .../pybreeze_ui/tools_gui/url_builder_gui.py | 7 ++- test/test_utils/test_curl_import_gui.py | 51 +++++++++++++++++++ test/test_utils/test_url_builder_gui.py | 22 ++++++++ 5 files changed, 121 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 2b8719e..4a0cf68 100644 --- a/README.md +++ b/README.md @@ -76,11 +76,12 @@ PyBreeze is not just a code editor — it is a command center for the automation - Mermaid `flowchart` / `graph` import - Save/Open as `.diagram.json`, export to PNG or SVG - Undo/redo, align, distribute, grid, snap, and zoom controls -- **cURL Import** — Paste a `curl` command copied from your browser's dev tools and generate a ready-to-run script for the target of your choice: **Python `requests`**, an **APITestka Python** snippet (`test_api_method_requests(...)`), an **APITestka JSON action** (`[["AT_test_api_method", {...}]]`) that `execute_files` runs directly, or a **LoadDensity** Locust load-test (`start_test(...)`). (These are the HTTP-oriented modules; a curl request has no meaningful mapping to the browser or desktop-GUI automation modules.) Parses the method, URL, headers, body, basic auth, `-G` query parameters, `-F` multipart form fields (file uploads become `files=open(...)`), the `--json` shortcut (which also sets the JSON `Content-Type`/`Accept`), and `-d @file` bodies (which become `open(...).read()`, while `--data-raw` stays literal) — all with multi-line `\` / `^` continuations. It knows the arity of curl's common flags, so value-taking options like `--max-time 30` or `-o out.json` never leak into the URL; it URL-encodes `--data-urlencode` values the way curl does; and it infers `POST` when a body or form is present and sends JSON bodies as JSON. Targets include Python `requests`, a ready-to-run **pytest test** (a named `test_...` function that sends the request and asserts the status), APITestka (Python and JSON action), and a LoadDensity load test. Pick one from the dropdown, then copy the result, **open it straight into a new editor tab**, or **save it to a `.py` / `.json` file** (the extension follows the chosen target). A repeated `-H` is handled the way HTTP handles it — the values are combined into one header (with `; ` for cookies) rather than the last one silently winning, and names are matched case-insensitively so an explicit header always beats one implied by another flag. The parser is pure logic and never executes the command. +- **cURL Import** — Paste a `curl` command copied from your browser's dev tools and generate a ready-to-run script for the target of your choice: **Python `requests`**, an **APITestka Python** snippet (`test_api_method_requests(...)`), an **APITestka JSON action** (`[["AT_test_api_method", {...}]]`) that `execute_files` runs directly, or a **LoadDensity** Locust load-test (`start_test(...)`). (These are the HTTP-oriented modules; a curl request has no meaningful mapping to the browser or desktop-GUI automation modules.) Parses the method, URL, headers, body, basic auth, `-G` query parameters, `-F` multipart form fields (file uploads become `files=open(...)`), the `--json` shortcut (which also sets the JSON `Content-Type`/`Accept`), and `-d @file` bodies (which become `open(...).read()`, while `--data-raw` stays literal) — all with multi-line `\` / `^` continuations. It knows the arity of curl's common flags, so value-taking options like `--max-time 30` or `-o out.json` never leak into the URL; it URL-encodes `--data-urlencode` values the way curl does; and it infers `POST` when a body or form is present and sends JSON bodies as JSON. Targets include Python `requests`, a ready-to-run **pytest test** (a named `test_...` function that sends the request and asserts the status), APITestka (Python and JSON action), and a LoadDensity load test. Pick one from the dropdown, then copy the result, **open it straight into a new editor tab**, or **save it to a `.py` / `.json` file** (the extension follows the chosen target). A repeated `-H` is handled the way HTTP handles it — the values are combined into one header (with `; ` for cookies) rather than the last one silently winning, and names are matched case-insensitively so an explicit header always beats one implied by another flag. One click also **hands the parsed URL to the URL parser/builder**, query string and all. The parser is pure logic and never executes the command. - **JWT Decoder** — Paste a JSON Web Token and read its header and payload as pretty-printed JSON, with standard timestamp claims (`exp`, `iat`, `nbf`, `auth_time`) rendered as readable UTC times. Inspection only — the signature is never verified and the token is never trusted. - **Timestamp Converter** — Enter a Unix epoch value (seconds or milliseconds, auto-detected) or an ISO-8601 date-time (with `Z` or an offset) and get every representation back in UTC. Deterministic and independent of the local time zone. - **Hash Generator** — Compute SHA-256, SHA-512, SHA-1 and MD5 digests of any text at once. A general-purpose checksum tool (MD5/SHA-1 are provided for interoperability with `usedforsecurity=False`, never for security decisions). - **Query ⇄ JSON** — Convert an `application/x-www-form-urlencoded` query string to pretty JSON and back, URL-decoding and -encoding as needed. Repeated keys become JSON arrays and vice versa. Copy the result, open it in an editor tab, or save it to a file. +- **URL Parser / Builder** — Paste a URL to see its scheme, host, port, path, query, fragment and credentials as an editable JSON object, tweak any part, and turn it back into a URL. Round-trips both ways, brackets IPv6 literals, and re-encodes query parameters for you. The cURL importer can send a request's URL straight here. - **HTTP Header Analyzer** — Paste a request or response header block (from dev tools, `curl -i`, or a proxy log) and read back every field plus what is worth knowing about it: names sent more than once, `Set-Cookie` entries missing `Secure` / `HttpOnly` / `SameSite`, a wildcard CORS policy (and the wildcard-plus-credentials combination browsers reject outright), an HSTS `max-age` too short to survive a restart, CSP `unsafe-inline` / `unsafe-eval`, product banners, deprecated headers, and — for responses — the security headers that are absent. Headers carrying credentials are reported by name only; their values are never copied into the report. The Response Inspector hands its headers here in one click. - **Regex Tester** — Try a regular expression against sample text with `IGNORECASE` / `MULTILINE` / `DOTALL` / `VERBOSE` flags, and see every match with its offsets, numbered groups and named groups. Invalid patterns report a friendly error instead of crashing. - **HTTP Status Reference** — Search the full HTTP status code table (sourced from the standard library, so it stays current) by code prefix or keyword, with the reason phrase, description and status class for each. @@ -222,6 +223,7 @@ PyBreeze UI (PySide6) │ ├── Timestamp Converter (epoch ⇄ ISO-8601) │ ├── Hash Generator (SHA-256/512, SHA-1, MD5) │ ├── Query ⇄ JSON Converter +│ ├── URL Parser / Builder (URL ⇄ JSON parts) │ ├── HTTP Header Analyzer (duplicates, cookie flags, security headers) │ ├── Regex Tester │ ├── HTTP Status Reference diff --git a/pybreeze/pybreeze_ui/tools_gui/curl_import_gui.py b/pybreeze/pybreeze_ui/tools_gui/curl_import_gui.py index 9087907..3ea371d 100644 --- a/pybreeze/pybreeze_ui/tools_gui/curl_import_gui.py +++ b/pybreeze/pybreeze_ui/tools_gui/curl_import_gui.py @@ -4,7 +4,8 @@ This widget parses that command and generates a runnable script for a chosen target (Python ``requests``, APITestka, LoadDensity), which can then be copied, opened straight into a new editor tab, or saved to a ``.py`` / ``.json`` file -via the shared output actions. +via the shared output actions. The parsed URL can also be handed straight to the +URL parser/builder tool for further editing. """ from __future__ import annotations @@ -14,7 +15,9 @@ from je_editor import language_wrapper from pybreeze.pybreeze_ui.tools_gui.output_actions import OutputActions -from pybreeze.utils.curl_import.curl_parser import parse_curl +from pybreeze.pybreeze_ui.tools_gui.tool_tabs import open_tool_tab +from pybreeze.pybreeze_ui.tools_gui.url_builder_gui import UrlBuilderGUI +from pybreeze.utils.curl_import.curl_parser import CurlRequest, parse_curl from pybreeze.utils.curl_import.script_templates import TEMPLATE_TARGETS, generate_template from pybreeze.utils.exception.exceptions import CurlParseException from pybreeze.utils.logging.logger import pybreeze_logger @@ -31,8 +34,11 @@ def __init__(self, main_window=None) -> None: :param main_window: the window whose ``tab_widget`` "open in editor" uses """ super().__init__() + self._main_window = main_window # The last successfully generated code (not an error or hint message). self._generated_code: str | None = None + # The request behind that code, so other tools can be handed its parts. + self._request: CurlRequest | None = None word = language_wrapper.language_word_dict self.input_label = QLabel(word.get("curl_import_input_label")) @@ -54,6 +60,11 @@ def __init__(self, main_window=None) -> None: self.output_edit = QTextEdit() self.output_edit.setReadOnly(True) + # Cross-tool action: send the parsed URL to the URL parser/builder. + self.open_url_button = QPushButton(word.get("curl_import_open_url_button")) + self.open_url_button.clicked.connect(self.open_url_in_builder) + self.open_url_button.setEnabled(False) + # Shared copy / open-in-editor / save actions. The extension and basename # follow the selected target; open/save are no-ops until a valid template. self.actions = OutputActions( @@ -66,7 +77,7 @@ def __init__(self, main_window=None) -> None: for widget in ( self.input_label, self.input_edit, self.target_label, self.target_select, self.convert_button, - self.output_label, self.output_edit, + self.output_label, self.output_edit, self.open_url_button, ): layout.addWidget(widget) layout.addLayout(self.actions.button_row()) @@ -81,21 +92,43 @@ def _on_target_changed(self, _index: int) -> None: if self.input_edit.toPlainText().strip(): self.convert() + def _clear_result(self) -> None: + """Forget the last result, so the output and cross-tool actions go inactive.""" + self._generated_code = None + self._request = None + self.open_url_button.setEnabled(False) + def convert(self) -> None: """Parse the input command and show the template for the chosen target.""" word = language_wrapper.language_word_dict command = self.input_edit.toPlainText().strip() if not command: - self._generated_code = None + self._clear_result() self.output_edit.setPlainText(word.get("curl_import_empty_hint")) return try: - code = generate_template(self.selected_target(), parse_curl(command)) + request = parse_curl(command) + code = generate_template(self.selected_target(), request) except CurlParseException as error: pybreeze_logger.info("curl_import_gui.py convert failed: %r", error) - self._generated_code = None + self._clear_result() self.output_edit.setPlainText( word.get("curl_import_error").format(error=str(error))) return self._generated_code = code + self._request = request + self.open_url_button.setEnabled(bool(request.url)) self.output_edit.setPlainText(code) + + def open_url_in_builder(self) -> QWidget | None: + """Open the parsed URL in the URL parser/builder tool, already parsed. + + The full URL is handed over (query string included), so the builder shows + the address exactly as curl would send it. + """ + if self._request is None or not self._request.url: + return None + return open_tool_tab( + self._main_window, + UrlBuilderGUI(main_window=self._main_window, initial_url=self._request.full_url), + "extend_tools_menu_url_builder_tab_label") diff --git a/pybreeze/pybreeze_ui/tools_gui/url_builder_gui.py b/pybreeze/pybreeze_ui/tools_gui/url_builder_gui.py index 8fe20bf..ee726da 100644 --- a/pybreeze/pybreeze_ui/tools_gui/url_builder_gui.py +++ b/pybreeze/pybreeze_ui/tools_gui/url_builder_gui.py @@ -15,9 +15,10 @@ class UrlBuilderGUI(QWidget): """Parse a URL into its JSON parts and build a URL back from them.""" - def __init__(self, main_window=None) -> None: + def __init__(self, main_window=None, initial_url: str | None = None) -> None: """ :param main_window: window whose ``tab_widget`` "open in editor" uses + :param initial_url: a URL to pre-fill and parse on open, if given """ super().__init__() self._valid_output = False @@ -54,6 +55,10 @@ def __init__(self, main_window=None) -> None: layout.addLayout(self.actions.button_row()) self.setLayout(layout) + if initial_url: + self.input_edit.setPlainText(initial_url) + self.convert_to_json() + def convert_to_json(self) -> None: """Parse the input URL into its JSON parts.""" text = self.input_edit.toPlainText().strip() diff --git a/test/test_utils/test_curl_import_gui.py b/test/test_utils/test_curl_import_gui.py index 2a4a6a7..2bf5305 100644 --- a/test/test_utils/test_curl_import_gui.py +++ b/test/test_utils/test_curl_import_gui.py @@ -1,6 +1,7 @@ """Tests for the cURL import tool widget.""" from __future__ import annotations +import json import os from unittest.mock import patch @@ -196,3 +197,53 @@ def test_save_after_parse_error_is_noop(self, widget): widget.input_edit.setPlainText("wget https://x") widget.convert() assert widget.actions.save_to_file() is None + + +class TestCurlImportOpenUrlInBuilder: + def test_button_disabled_before_convert(self, widget): + assert not widget.open_url_button.isEnabled() + + def test_button_enabled_after_convert(self, widget): + widget.input_edit.setPlainText("curl https://example.com/api") + widget.convert() + assert widget.open_url_button.isEnabled() + + def test_button_disabled_again_after_parse_error(self, widget): + widget.input_edit.setPlainText("curl https://example.com/api") + widget.convert() + widget.input_edit.setPlainText("wget https://example.com") + widget.convert() + assert not widget.open_url_button.isEnabled() + + def test_open_url_opens_prefilled_builder_tab(self, widget_with_window): + from pybreeze.pybreeze_ui.tools_gui.url_builder_gui import UrlBuilderGUI + gui, window = widget_with_window + gui.input_edit.setPlainText("curl 'https://example.com/api?a=1' -H 'Accept: */*'") + gui.convert() + gui.open_url_in_builder() + assert len(window.tab_widget.added) == 1 + opened = window.tab_widget.added[0][0] + assert isinstance(opened, UrlBuilderGUI) + parsed = json.loads(opened.output_edit.toPlainText()) + assert parsed["host"] == "example.com" + assert parsed["path"] == "/api" + + def test_query_string_survives_the_hand_off(self, widget_with_window): + # The curl parser moves a URL's query into params; the builder must still + # receive the address as curl would send it. + gui, window = widget_with_window + gui.input_edit.setPlainText("curl 'https://example.com/api?a=1&b=2'") + gui.convert() + gui.open_url_in_builder() + opened = window.tab_widget.added[0][0] + assert json.loads(opened.output_edit.toPlainText())["query"] == {"a": "1", "b": "2"} + + def test_open_before_convert_is_noop(self, widget_with_window): + gui, window = widget_with_window + assert gui.open_url_in_builder() is None + assert window.tab_widget.added == [] + + def test_open_without_window_is_safe(self, widget): + widget.input_edit.setPlainText("curl https://example.com/api") + widget.convert() + assert widget.open_url_in_builder() is None diff --git a/test/test_utils/test_url_builder_gui.py b/test/test_utils/test_url_builder_gui.py index e080b12..1a5c63a 100644 --- a/test/test_utils/test_url_builder_gui.py +++ b/test/test_utils/test_url_builder_gui.py @@ -62,3 +62,25 @@ def test_copy_output(self, app, widget): widget.convert_to_json() widget.actions.copy() assert "example.com" in QApplication.clipboard().text() + + +class TestUrlBuilderInitialUrl: + def test_initial_url_is_parsed_on_open(self, app): + from pybreeze.pybreeze_ui.tools_gui.url_builder_gui import UrlBuilderGUI + gui = UrlBuilderGUI(initial_url="https://example.com:8443/api?a=1#frag") + data = json.loads(gui.output_edit.toPlainText()) + assert data["host"] == "example.com" + assert data["port"] == 8443 + assert data["fragment"] == "frag" + gui.close() + gui.deleteLater() + + def test_initial_url_is_kept_in_the_input(self, app): + from pybreeze.pybreeze_ui.tools_gui.url_builder_gui import UrlBuilderGUI + gui = UrlBuilderGUI(initial_url="https://example.com/api") + assert gui.input_edit.toPlainText() == "https://example.com/api" + gui.close() + gui.deleteLater() + + def test_without_initial_url_the_output_stays_empty(self, widget): + assert widget.output_edit.toPlainText() == "" From eb0082842e03d972f10c5647f11716efdaf66e85 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Sat, 25 Jul 2026 14:32:11 +0800 Subject: [PATCH 10/10] Join repeated header values with str.join The value joiner returned an interpolated f-string, which Codacy's Semgrep rule for Flask views returning formatted strings matches. No part of this module serves HTTP, so the finding is a false positive, but joining on the separator states the intent more plainly anyway. --- pybreeze/utils/curl_import/curl_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pybreeze/utils/curl_import/curl_parser.py b/pybreeze/utils/curl_import/curl_parser.py index e49aa21..1764a54 100644 --- a/pybreeze/utils/curl_import/curl_parser.py +++ b/pybreeze/utils/curl_import/curl_parser.py @@ -243,7 +243,7 @@ def _join_header_values(name: str, previous: str, value: str) -> str: if not value: return previous joiner = _COOKIE_JOINER if name.lower() == _COOKIE_HEADER else _HEADER_JOINER - return f"{previous}{joiner}{value}" + return joiner.join((previous, value)) def _apply_header(request: CurlRequest, raw_header: str) -> None: