"""HTTP header value parsing utility functions. from ClientCookie._HeadersUtil import split_header_words values = split_header_words(h.headers["Content-Type"]) This module provides a few functions that help parsing and construction of valid HTTP header values. Copyright 1997-1998, Gisle Aas Copyright 2002-2003, John J. Lee This code is free software; you can redistribute it and/or modify it under the terms of the BSD License (see the file COPYING included with the distribution). """ import re, string from types import StringType try: from types import UnicodeType STRING_TYPES = StringType, UnicodeType except: STRING_TYPES = StringType, from _Util import startswith, endswith, http2time try: True except NameError: True = 1 False = 0 def unmatched(match): """Return unmatched part of re.Match object.""" start, end = match.span(0) return match.string[:start]+match.string[end:] # XXX I really can't see what this =* was for (came from LWP, I guess) #token_re = re.compile(r"^\s*(=*[^\s=;,]+)") token_re = re.compile(r"^\s*([^=\s;,]+)") quoted_value_re = re.compile(r"^\s*=\s*\"([^\"\\]*(?:\\.[^\"\\]*)*)\"") value_re = re.compile(r"^\s*=\s*([^\s;,]*)") escape_re = re.compile(r"\\(.)") def split_header_words(header_values): r"""Parse header values into a list of lists containing key,value pairs. The function knows how to deal with ",", ";" and "=" as well as quoted values after "=". A list of space separated tokens are parsed as if they were separated by ";". If the header_values passed as argument contains multiple values, then they are treated as if they were a single value separated by comma ",". This means that this function is useful for parsing header fields that follow this syntax (BNF as from the HTTP/1.1 specification, but we relax the requirement for tokens). headers = #header header = (token | parameter) *( [";"] (token | parameter)) token = 1* separators = "(" | ")" | "<" | ">" | "@" | "," | ";" | ":" | "\" | <"> | "/" | "[" | "]" | "?" | "=" | "{" | "}" | SP | HT quoted-string = ( <"> *(qdtext | quoted-pair ) <"> ) qdtext = > quoted-pair = "\" CHAR parameter = attribute "=" value attribute = token value = token | quoted-string Each header is represented by a list of key/value pairs. The value for a simple token (not part of a parameter) is None. Syntactically incorrect headers will not necessarily be parsed as you would want. This is easier to describe with some examples: >>> split_header_words(['foo="bar"; port="80,81"; discard, bar=baz']) [[('foo', 'bar'), ('port', '80,81'), ('discard', None)], [('bar', 'baz')]] >>> split_header_words(['text/html; charset="iso-8859-1"']) [[('text/html', None), ('charset', 'iso-8859-1')]] >>> split_header_words([r'Basic realm="\"foo\bar\""']) [[('Basic', None), ('realm', '"foobar"')]] """ assert type(header_values) not in STRING_TYPES result = [] for text in header_values: orig_text = text pairs = [] while text: m = token_re.search(text) if m: text = unmatched(m) name = m.group(1) m = quoted_value_re.search(text) if m: # quoted value text = unmatched(m) value = m.group(1) value = escape_re.sub(r"\1", value) else: m = value_re.search(text) if m: # unquoted value text = unmatched(m) value = m.group(1) value = string.rstrip(value) else: # no value, a lone token value = None pairs.append((name, value)) elif startswith(string.lstrip(text), ","): # concatenated headers, as per RFC 2616 section 4.2 text = string.lstrip(text)[1:] if pairs: result.append(pairs) pairs = [] else: # skip junk non_junk, nr_junk_chars = re.subn("^[=\s;]*", "", text) assert nr_junk_chars > 0, ( "split_header_words bug: '%s', '%s', %s" % (orig_text, text, pairs)) text = non_junk if pairs: result.append(pairs) return result join_escape_re = re.compile(r"([\"\\])") def join_header_words(lists): """Do the inverse of the conversion done by split_header_words. Takes a list of lists of (key, value) pairs and produces a single header value. Attribute values are quoted if needed. >>> join_header_words([[("text/plain", None), ("charset", "iso-8859/1")]]) 'text/plain; charset="iso-8859/1"' >>> join_header_words([[("text/plain", None)], [("charset", "iso-8859/1")]]) 'text/plain, charset="iso-8859/1"' """ headers = [] for pairs in lists: attr = [] for k, v in pairs: if v is not None: if not re.search(r"^\w+$", v): v = join_escape_re.sub(r"\\\1", v) # escape " and \ v = '"%s"' % v if k is None: # Netscape cookies may have no name k = v else: k = "%s=%s" % (k, v) attr.append(k) if attr: headers.append(string.join(attr, "; ")) return string.join(headers, ", ") def parse_ns_headers(ns_headers): """Ad-hoc parser for Netscape protocol cookie-attributes. The old Netscape cookie format for Set-Cookie can for instance contain an unquoted "," in the expires field, so we have to use this ad-hoc parser instead of split_header_words. XXX This may not make the best possible effort to parse all the crap that Netscape Cookie headers contain. Ronald Tschalar's HTTPClient parser is probably better, so could do worse than following that if this ever gives any trouble. Currently, this is also used for parsing RFC 2109 cookies. """ known_attrs = ("expires", "domain", "path", "secure", # RFC 2109 attrs (may turn up in Netscape cookies, too) "port", "max-age") result = [] for ns_header in ns_headers: pairs = [] version_set = False for param in re.split(r";\s*", ns_header): param = string.rstrip(param) if param == "": continue if "=" not in param: if string.lower(param) in known_attrs: k, v = param, None else: # cookie with missing name k, v = None, param else: k, v = re.split(r"\s*=\s*", param, 1) k = string.lstrip(k) if k is not None: lc = string.lower(k) if lc in known_attrs: k = lc if k == "version": # This is an RFC 2109 cookie. Will be treated as RFC 2965 # cookie in rest of code. # Probably it should be parsed with split_header_words, but # that's too much hassle. version_set = True if k == "expires": # convert expires date to seconds since epoch if startswith(v, '"'): v = v[1:] if endswith(v, '"'): v = v[:-1] v = http2time(v) # None if invalid pairs.append((k, v)) if pairs: if not version_set: pairs.append(("version", "0")) result.append(pairs) return result def _test(): import doctest, _HeadersUtil return doctest.testmod(_HeadersUtil) if __name__ == "__main__": _test()