| # Copyright Joyent, Inc. and other Node contributors. |
| # |
| # Permission is hereby granted, free of charge, to any person obtaining a |
| # copy of this software and associated documentation files (the |
| # "Software"), to deal in the Software without restriction, including |
| # without limitation the rights to use, copy, modify, merge, publish, |
| # distribute, sublicense, and/or sell copies of the Software, and to permit |
| # persons to whom the Software is furnished to do so, subject to the |
| # following conditions: |
| # |
| # The above copyright notice and this permission notice shall be included |
| # in all copies or substantial portions of the Software. |
| # |
| # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS |
| # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
| # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN |
| # NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, |
| # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR |
| # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE |
| # USE OR OTHER DEALINGS IN THE SOFTWARE. |
| |
| |
| # Changes from joyent/node: |
| # |
| # 1. No leading slash in paths, |
| # e.g. in `url.parse('http://foo?bar')` pathname is ``, not `/` |
| # |
| # 2. Backslashes are not replaced with slashes, |
| # so `http:\\example.org\` is treated like a relative path |
| # |
| # 3. Trailing colon is treated like a part of the path, |
| # i.e. in `http://example.org:foo` pathname is `:foo` |
| # |
| # 4. Nothing is URL-encoded in the resulting object, |
| # (in joyent/node some chars in auth and paths are encoded) |
| # |
| # 5. `url.parse()` does not have `parseQueryString` argument |
| # |
| # 6. Removed extraneous result properties: `host`, `path`, `query`, etc., |
| # which can be constructed using other parts of the url. |
| |
| from __future__ import annotations |
| |
| from collections import defaultdict |
| import re |
| |
| from mdurl._url import URL |
| |
| # Reference: RFC 3986, RFC 1808, RFC 2396 |
| |
| # define these here so at least they only have to be |
| # compiled once on the first module load. |
| PROTOCOL_PATTERN = re.compile(r"^([a-z0-9.+-]+:)", flags=re.IGNORECASE) |
| PORT_PATTERN = re.compile(r":[0-9]*$") |
| |
| # Special case for a simple path URL |
| SIMPLE_PATH_PATTERN = re.compile(r"^(//?(?!/)[^?\s]*)(\?[^\s]*)?$") |
| |
| # RFC 2396: characters reserved for delimiting URLs. |
| # We actually just auto-escape these. |
| DELIMS = ("<", ">", '"', "`", " ", "\r", "\n", "\t") |
| |
| # RFC 2396: characters not allowed for various reasons. |
| UNWISE = ("{", "}", "|", "\\", "^", "`") + DELIMS |
| |
| # Allowed by RFCs, but cause of XSS attacks. Always escape these. |
| AUTO_ESCAPE = ("'",) + UNWISE |
| # Characters that are never ever allowed in a hostname. |
| # Note that any invalid chars are also handled, but these |
| # are the ones that are *expected* to be seen, so we fast-path |
| # them. |
| NON_HOST_CHARS = ("%", "/", "?", ";", "#") + AUTO_ESCAPE |
| HOST_ENDING_CHARS = ("/", "?", "#") |
| HOSTNAME_MAX_LEN = 255 |
| HOSTNAME_PART_PATTERN = re.compile(r"^[+a-z0-9A-Z_-]{0,63}$") |
| HOSTNAME_PART_START = re.compile(r"^([+a-z0-9A-Z_-]{0,63})(.*)$") |
| # protocols that can allow "unsafe" and "unwise" chars. |
| |
| # protocols that never have a hostname. |
| HOSTLESS_PROTOCOL = defaultdict( |
| bool, |
| { |
| "javascript": True, |
| "javascript:": True, |
| }, |
| ) |
| # protocols that always contain a // bit. |
| SLASHED_PROTOCOL = defaultdict( |
| bool, |
| { |
| "http": True, |
| "https": True, |
| "ftp": True, |
| "gopher": True, |
| "file": True, |
| "http:": True, |
| "https:": True, |
| "ftp:": True, |
| "gopher:": True, |
| "file:": True, |
| }, |
| ) |
| |
| |
| class MutableURL: |
| def __init__(self) -> None: |
| self.protocol: str | None = None |
| self.slashes: bool = False |
| self.auth: str | None = None |
| self.port: str | None = None |
| self.hostname: str | None = None |
| self.hash: str | None = None |
| self.search: str | None = None |
| self.pathname: str | None = None |
| |
| def parse(self, url: str, slashes_denote_host: bool) -> "MutableURL": |
| lower_proto = "" |
| slashes = False |
| rest = url |
| |
| # trim before proceeding. |
| # This is to support parse stuff like " http://foo.com \n" |
| rest = rest.strip() |
| |
| if not slashes_denote_host and len(url.split("#")) == 1: |
| # Try fast path regexp |
| simple_path = SIMPLE_PATH_PATTERN.match(rest) |
| if simple_path: |
| self.pathname = simple_path.group(1) |
| if simple_path.group(2): |
| self.search = simple_path.group(2) |
| return self |
| |
| proto = "" |
| proto_match = PROTOCOL_PATTERN.match(rest) |
| if proto_match: |
| proto = proto_match.group() |
| lower_proto = proto.lower() |
| self.protocol = proto |
| rest = rest[len(proto) :] |
| |
| # figure out if it's got a host |
| # user@server is *always* interpreted as a hostname, and url |
| # resolution will treat //foo/bar as host=foo,path=bar because that's |
| # how the browser resolves relative URLs. |
| if slashes_denote_host or proto or re.search(r"^//[^@/]+@[^@/]+", rest): |
| slashes = rest.startswith("//") |
| if slashes and not (proto and HOSTLESS_PROTOCOL[proto]): |
| rest = rest[2:] |
| self.slashes = True |
| |
| if not HOSTLESS_PROTOCOL[proto] and ( |
| slashes or (proto and not SLASHED_PROTOCOL[proto]) |
| ): |
| |
| # there's a hostname. |
| # the first instance of /, ?, ;, or # ends the host. |
| # |
| # If there is an @ in the hostname, then non-host chars *are* allowed |
| # to the left of the last @ sign, unless some host-ending character |
| # comes *before* the @-sign. |
| # URLs are obnoxious. |
| # |
| # ex: |
| # http://a@b@c/ => user:a@b host:c |
| # http://a@b?@c => user:a host:c path:/?@c |
| |
| # v0.12 TODO(isaacs): This is not quite how Chrome does things. |
| # Review our test case against browsers more comprehensively. |
| |
| # find the first instance of any hostEndingChars |
| host_end = -1 |
| for i in range(len(HOST_ENDING_CHARS)): |
| hec = rest.find(HOST_ENDING_CHARS[i]) |
| if hec != -1 and (host_end == -1 or hec < host_end): |
| host_end = hec |
| |
| # at this point, either we have an explicit point where the |
| # auth portion cannot go past, or the last @ char is the decider. |
| if host_end == -1: |
| # atSign can be anywhere. |
| at_sign = rest.rfind("@") |
| else: |
| # atSign must be in auth portion. |
| # http://a@b/c@d => host:b auth:a path:/c@d |
| at_sign = rest.rfind("@", 0, host_end + 1) |
| |
| # Now we have a portion which is definitely the auth. |
| # Pull that off. |
| if at_sign != -1: |
| auth = rest[:at_sign] |
| rest = rest[at_sign + 1 :] |
| self.auth = auth |
| |
| # the host is the remaining to the left of the first non-host char |
| host_end = -1 |
| for i in range(len(NON_HOST_CHARS)): |
| hec = rest.find(NON_HOST_CHARS[i]) |
| if hec != -1 and (host_end == -1 or hec < host_end): |
| host_end = hec |
| # if we still have not hit it, then the entire thing is a host. |
| if host_end == -1: |
| host_end = len(rest) |
| |
| if host_end > 0 and rest[host_end - 1] == ":": |
| host_end -= 1 |
| host = rest[:host_end] |
| rest = rest[host_end:] |
| |
| # pull out port. |
| self.parse_host(host) |
| |
| # we've indicated that there is a hostname, |
| # so even if it's empty, it has to be present. |
| self.hostname = self.hostname or "" |
| |
| # if hostname begins with [ and ends with ] |
| # assume that it's an IPv6 address. |
| ipv6_hostname = self.hostname.startswith("[") and self.hostname.endswith( |
| "]" |
| ) |
| |
| # validate a little. |
| if not ipv6_hostname: |
| hostparts = self.hostname.split(".") |
| l = len(hostparts) # noqa: E741 |
| i = 0 |
| while i < l: |
| part = hostparts[i] |
| if not part: |
| i += 1 # emulate statement3 in JS for loop |
| continue |
| if not HOSTNAME_PART_PATTERN.search(part): |
| newpart = "" |
| k = len(part) |
| j = 0 |
| while j < k: |
| if ord(part[j]) > 127: |
| # we replace non-ASCII char with a temporary placeholder |
| # we need this to make sure size of hostname is not |
| # broken by replacing non-ASCII by nothing |
| newpart += "x" |
| else: |
| newpart += part[j] |
| j += 1 # emulate statement3 in JS for loop |
| |
| # we test again with ASCII char only |
| if not HOSTNAME_PART_PATTERN.search(newpart): |
| valid_parts = hostparts[:i] |
| not_host = hostparts[i + 1 :] |
| bit = HOSTNAME_PART_START.search(part) |
| if bit: |
| valid_parts.append(bit.group(1)) |
| not_host.insert(0, bit.group(2)) |
| if not_host: |
| rest = ".".join(not_host) + rest |
| self.hostname = ".".join(valid_parts) |
| break |
| i += 1 # emulate statement3 in JS for loop |
| |
| if len(self.hostname) > HOSTNAME_MAX_LEN: |
| self.hostname = "" |
| |
| # strip [ and ] from the hostname |
| # the host field still retains them, though |
| if ipv6_hostname: |
| self.hostname = self.hostname[1:-1] |
| |
| # chop off from the tail first. |
| hash = rest.find("#") # noqa: A001 |
| if hash != -1: |
| # got a fragment string. |
| self.hash = rest[hash:] |
| rest = rest[:hash] |
| qm = rest.find("?") |
| if qm != -1: |
| self.search = rest[qm:] |
| rest = rest[:qm] |
| if rest: |
| self.pathname = rest |
| if SLASHED_PROTOCOL[lower_proto] and self.hostname and not self.pathname: |
| self.pathname = "" |
| |
| return self |
| |
| def parse_host(self, host: str) -> None: |
| port_match = PORT_PATTERN.search(host) |
| if port_match: |
| port = port_match.group() |
| if port != ":": |
| self.port = port[1:] |
| host = host[: -len(port)] |
| if host: |
| self.hostname = host |
| |
| |
| def url_parse(url: URL | str, *, slashes_denote_host: bool = False) -> URL: |
| if isinstance(url, URL): |
| return url |
| u = MutableURL() |
| u.parse(url, slashes_denote_host) |
| return URL( |
| u.protocol, u.slashes, u.auth, u.port, u.hostname, u.hash, u.search, u.pathname |
| ) |
