diff --git a/.gitignore b/.gitignore index ecb98b18..603c63e8 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,7 @@ test-results.xml # production for react-app build + +# python bytecode +.pyc +__pycache__ diff --git a/src/Utils/Logger.ts b/src/Utils/Logger.ts index 490e163b..090c6151 100644 --- a/src/Utils/Logger.ts +++ b/src/Utils/Logger.ts @@ -33,26 +33,38 @@ const isDebugMode = process.env.VSCODE_DEBUG_MODE === "true"; * - message: watch out */ function _logStr(severity: string, tag: string, ...msgs: MsgList) { - let logStrList = []; - if (msgs.length === 0) { // Do not print return ""; } - for (let m of msgs) { - if (m instanceof Error) { - const err = m as Error; - logStrList.push( - `\nError was thrown:\n- name: ${err.name}\n- message: ${err.message}` - ); - } else if (typeof m === "object") { - logStrList.push(`\n${m.constructor.name}: ${JSON.stringify(m)}`); - } else { - logStrList.push(`${m}`); + const flatten = (msgs: MsgList) => { + let logStrList = []; + for (let m of msgs) { + if (m instanceof Error) { + const err = m as Error; + logStrList.push( + `\nError was thrown:\n- name: ${err.name}\n- message: ${err.message}` + ); + } else if (typeof m === "object") { + logStrList.push(`\n${m.constructor.name}: ${JSON.stringify(m)}`); + } else { + logStrList.push(`${m}`); + } } - } - const msg = logStrList.join(" "); + return logStrList.join(" "); + }; + + const redact = (msg: string) => { + // Replace Github Personal Access Tokens with ******** + const classicPAT = "ghp_[a-zA-Z0-9]+"; + const findGrainedPAT = "github_pat_[a-zA-Z0-9_]+"; + const regex = new RegExp(`(${classicPAT})|(${findGrainedPAT})`, "g"); + + return msg.replace(regex, "*********************"); + }; + + const msg = redact(flatten(msgs)); const time = new Date().toLocaleString(); return `[${time}][${tag}][${severity}] ${msg}`; @@ -116,6 +128,8 @@ export class Logger { * @brief Print msg and a line feed character without adding '[time][tag][severity]' * @detail When log is long and need to be splitted into many chunks, append() could be used * after the first chunk. + * + * @todo streamify logger to format consistently (ex. redact is not applied to this function) */ public static appendLine(msg: string) { Logger.checkShow(); @@ -126,6 +140,8 @@ export class Logger { * @brief Print msg without adding '[time][tag][severity]' * @detail When log is long and need to be splitted into many chunks, append() could be used * after the first chunk. + * + * @todo streamify logger to format consistently (ex. redact is not applied to this function) */ public static append(msg: string) { Logger.checkShow(); diff --git a/third_party/catapult/common/py_vulcanize/README.chromium b/third_party/catapult/common/py_vulcanize/README.chromium new file mode 100644 index 00000000..128566e4 --- /dev/null +++ b/third_party/catapult/common/py_vulcanize/README.chromium @@ -0,0 +1,9 @@ +Name: py_vulcanize +URL: N/A +Version: N/A +Shipped: yes + +Description: +Py-vulcanize, formerly known as TVCM (trace-viewer component model). +This code doesn't actually live anywhere else currently, but it may +be split out into a separate repository in the future. diff --git a/third_party/catapult/common/py_vulcanize/py_vulcanize/__init__.py b/third_party/catapult/common/py_vulcanize/py_vulcanize/__init__.py new file mode 100644 index 00000000..087a104e --- /dev/null +++ b/third_party/catapult/common/py_vulcanize/py_vulcanize/__init__.py @@ -0,0 +1,12 @@ +# Copyright (c) 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Trace-viewer component model. + +This module implements trace-viewer's component model. +""" + +from __future__ import absolute_import +from py_vulcanize.generate import * # pylint: disable=wildcard-import +from py_vulcanize.project import Project diff --git a/third_party/catapult/common/py_vulcanize/py_vulcanize/fake_fs.py b/third_party/catapult/common/py_vulcanize/py_vulcanize/fake_fs.py new file mode 100644 index 00000000..a26b92fc --- /dev/null +++ b/third_party/catapult/common/py_vulcanize/py_vulcanize/fake_fs.py @@ -0,0 +1,168 @@ +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import builtins +import codecs +import collections +import os +import six + +from io import BytesIO + + +class WithableStringIO(six.StringIO): + + def __enter__(self, *args): + return self + + def __exit__(self, *args): + pass + +class WithableBytesIO(BytesIO): + + def __enter__(self, *args): + return self + + def __exit__(self, *args): + pass + +class FakeFS(object): + + def __init__(self, initial_filenames_and_contents=None): + self._file_contents = {} + if initial_filenames_and_contents: + for k, v in six.iteritems(initial_filenames_and_contents): + self._file_contents[k] = v + + self._bound = False + self._real_codecs_open = codecs.open + self._real_open = builtins.open + + self._real_abspath = os.path.abspath + self._real_exists = os.path.exists + self._real_walk = os.walk + self._real_listdir = os.listdir + + def __enter__(self): + self.Bind() + return self + + def __exit__(self, *args): + self.Unbind() + + def Bind(self): + assert not self._bound + codecs.open = self._FakeCodecsOpen + builtins.open = self._FakeOpen + os.path.abspath = self._FakeAbspath + os.path.exists = self._FakeExists + os.walk = self._FakeWalk + os.listdir = self._FakeListDir + self._bound = True + + def Unbind(self): + assert self._bound + codecs.open = self._real_codecs_open + builtins.open = self._real_open + os.path.abspath = self._real_abspath + os.path.exists = self._real_exists + os.walk = self._real_walk + os.listdir = self._real_listdir + self._bound = False + + def AddFile(self, path, contents): + assert path not in self._file_contents + path = os.path.normpath(path) + self._file_contents[path] = contents + + def _FakeOpen(self, path, mode=None): + if mode is None: + mode = 'r' + if mode == 'r' or mode == 'rU' or mode == 'rb': + if path not in self._file_contents: + return self._real_open(path, mode) + + if mode == 'rb': + return WithableBytesIO(self._file_contents[path]) + else: + return WithableStringIO(self._file_contents[path]) + + raise NotImplementedError() + + def _FakeCodecsOpen(self, path, mode=None, + encoding=None): # pylint: disable=unused-argument + if mode is None: + mode = 'r' + if mode == 'r' or mode == 'rU' or mode == 'rb': + if path not in self._file_contents: + return self._real_open(path, mode) + + if mode == 'rb': + return WithableBytesIO(self._file_contents[path]) + else: + return WithableStringIO(self._file_contents[path]) + + raise NotImplementedError() + + def _FakeAbspath(self, path): + """Normalize the path and ensure it starts with os.path.sep. + + The tests all assume paths start with things like '/my/project', + and this abspath implementaion makes that assumption work correctly + on Windows. + """ + normpath = os.path.normpath(path) + if not normpath.startswith(os.path.sep): + normpath = os.path.sep + normpath + return normpath + + def _FakeExists(self, path): + if path in self._file_contents: + return True + return self._real_exists(path) + + def _FakeWalk(self, top): + assert os.path.isabs(top) + all_filenames = list(self._file_contents.keys()) + pending_prefixes = collections.deque() + pending_prefixes.append(top) + visited_prefixes = set() + while len(pending_prefixes): + prefix = pending_prefixes.popleft() + if prefix in visited_prefixes: + continue + visited_prefixes.add(prefix) + if prefix.endswith(os.path.sep): + prefix_with_trailing_sep = prefix + else: + prefix_with_trailing_sep = prefix + os.path.sep + + dirs = set() + files = [] + for filename in all_filenames: + if not filename.startswith(prefix_with_trailing_sep): + continue + relative_to_prefix = os.path.relpath(filename, prefix) + + dirpart = os.path.dirname(relative_to_prefix) + if len(dirpart) == 0: + files.append(relative_to_prefix) + continue + parts = dirpart.split(os.sep) + if len(parts) == 0: + dirs.add(dirpart) + else: + pending = os.path.join(prefix, parts[0]) + dirs.add(parts[0]) + pending_prefixes.appendleft(pending) + + dirs = sorted(dirs) + yield prefix, dirs, files + + def _FakeListDir(self, dirname): + raise NotImplementedError() diff --git a/third_party/catapult/common/py_vulcanize/py_vulcanize/fake_fs_unittest.py b/third_party/catapult/common/py_vulcanize/py_vulcanize/fake_fs_unittest.py new file mode 100644 index 00000000..b8516f71 --- /dev/null +++ b/third_party/catapult/common/py_vulcanize/py_vulcanize/fake_fs_unittest.py @@ -0,0 +1,54 @@ +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +from __future__ import absolute_import +import os +import unittest + +from py_vulcanize import fake_fs + + +class FakeFSUnittest(unittest.TestCase): + + def testBasic(self): + fs = fake_fs.FakeFS() + fs.AddFile('/blah/x', 'foobar') + with fs: + assert os.path.exists(os.path.normpath('/blah/x')) + self.assertEquals( + 'foobar', + open(os.path.normpath('/blah/x'), 'r').read()) + + def testWithableOpen(self): + fs = fake_fs.FakeFS() + fs.AddFile('/blah/x', 'foobar') + with fs: + with open(os.path.normpath('/blah/x'), 'r') as f: + self.assertEquals('foobar', f.read()) + + def testWalk(self): + fs = fake_fs.FakeFS() + fs.AddFile('/x/w2/w3/z3.txt', '') + fs.AddFile('/x/w/z.txt', '') + fs.AddFile('/x/y.txt', '') + fs.AddFile('/a.txt', 'foobar') + with fs: + gen = os.walk(os.path.normpath('/')) + r = next(gen) + self.assertEquals((os.path.normpath('/'), ['x'], ['a.txt']), r) + + r = next(gen) + self.assertEquals((os.path.normpath('/x'), ['w', 'w2'], ['y.txt']), r) + + r = next(gen) + self.assertEquals((os.path.normpath('/x/w'), [], ['z.txt']), r) + + r = next(gen) + self.assertEquals((os.path.normpath('/x/w2'), ['w3'], []), r) + + r = next(gen) + self.assertEquals((os.path.normpath('/x/w2/w3'), [], ['z3.txt']), r) + + with self.assertRaises(StopIteration): + next(gen) diff --git a/third_party/catapult/common/py_vulcanize/py_vulcanize/generate.py b/third_party/catapult/common/py_vulcanize/py_vulcanize/generate.py new file mode 100644 index 00000000..7d7e6303 --- /dev/null +++ b/third_party/catapult/common/py_vulcanize/py_vulcanize/generate.py @@ -0,0 +1,301 @@ +# Copyright (c) 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import subprocess +import sys +import tempfile + +from py_vulcanize import html_generation_controller + +try: + from six import StringIO +except ImportError: + from io import StringIO + + + +html_warning_message = """ + + + +""" + +js_warning_message = """ +// Copyright 2015 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +/* WARNING: This file is auto generated. + * + * Do not edit directly. + */ +""" + +css_warning_message = """ +/* Copyright 2015 The Chromium Authors. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. */ + +/* WARNING: This file is auto-generated. + * + * Do not edit directly. + */ +""" + +origin_trial_tokens = [ + # WebComponent V0 origin trial token for googleusercontent.com + subdomains. + # This is the domain from which traces in cloud storage are served. + # Expires Nov 5, 2020. See https://crbug.com/1021137 + "AnYuQDtUf6OrWCmR9Okd67JhWVTbmnRedvPi1TEvAxac8+1p6o9q08FoDO6oCbLD0xEqev+SkZFiIhFSzlY9HgUAAABxeyJvcmlnaW4iOiJodHRwczovL2dvb2dsZXVzZXJjb250ZW50LmNvbTo0NDMiLCJmZWF0dXJlIjoiV2ViQ29tcG9uZW50c1YwIiwiZXhwaXJ5IjoxNjA0NjE0NTM4LCJpc1N1YmRvbWFpbiI6dHJ1ZX0=", + # This is for chromium-build-stats.appspot.com (ukai@) + # Expires Feb 2, 2021. see https://crbug.com/1050215 + "AkFXw3wHnOs/XXYqFXpc3diDLrRFd9PTgGs/gs43haZmngI/u1g8L4bDnSKLZkB6fecjmjTwcAMQFCpWMAoHSQEAAAB8eyJvcmlnaW4iOiJodHRwczovL2Nocm9taXVtLWJ1aWxkLXN0YXRzLmFwcHNwb3QuY29tOjQ0MyIsImZlYXR1cmUiOiJXZWJDb21wb25lbnRzVjAiLCJleHBpcnkiOjE2MTIyMjM5OTksImlzU3ViZG9tYWluIjp0cnVlfQ==", + # This is for chromium-build-stats-staging.appspot.com (ukai@) + # Expires Feb 2, 2021, see https://crbug.com/1050215 + "AtQY4wpX9+nj+Vn27cTgygzIPbtB2WoAoMQR5jK9mCm/H2gRIDH6MmGVAaziv9XnYTDKjhBnQYtecbTiIHCQiAIAAACEeyJvcmlnaW4iOiJodHRwczovL2Nocm9taXVtLWJ1aWxkLXN0YXRzLXN0YWdpbmcuYXBwc3BvdC5jb206NDQzIiwiZmVhdHVyZSI6IldlYkNvbXBvbmVudHNWMCIsImV4cGlyeSI6MTYxMjIyMzk5OSwiaXNTdWJkb21haW4iOnRydWV9" + # + # Add more tokens here if traces are served from other domains. + # WebComponent V0 origin tiral token is generated on + # https://developers.chrome.com/origintrials/#/trials/active +] + +def _AssertIsUTF8(f): + if isinstance(f, StringIO): + return + assert f.encoding == 'utf-8' + + +def _MinifyJS(input_js): + py_vulcanize_path = os.path.abspath(os.path.join( + os.path.dirname(__file__), '..')) + rjsmin_path = os.path.abspath( + os.path.join(py_vulcanize_path, 'third_party', 'rjsmin', 'rjsmin.py')) + + with tempfile.NamedTemporaryFile() as _: + args = [ + sys.executable, + rjsmin_path + ] + p = subprocess.Popen(args, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + res = p.communicate(input=input_js.encode('utf-8')) + errorcode = p.wait() + if errorcode != 0: + sys.stderr.write('rJSmin exited with error code %d' % errorcode) + sys.stderr.write(res[1].decode('utf-8')) + raise Exception('Failed to minify, omgah') + return res[0].decode('utf-8') + + +def GenerateJS(load_sequence, + use_include_tags_for_scripts=False, + dir_for_include_tag_root=None, + minify=False, + report_sizes=False): + f = StringIO() + GenerateJSToFile(f, + load_sequence, + use_include_tags_for_scripts, + dir_for_include_tag_root, + minify=minify, + report_sizes=report_sizes) + + return f.getvalue() + + +def GenerateJSToFile(f, + load_sequence, + use_include_tags_for_scripts=False, + dir_for_include_tag_root=None, + minify=False, + report_sizes=False): + _AssertIsUTF8(f) + if use_include_tags_for_scripts and dir_for_include_tag_root is None: + raise Exception('Must provide dir_for_include_tag_root') + + f.write(js_warning_message) + f.write('\n') + + if not minify: + flatten_to_file = f + else: + flatten_to_file = StringIO() + + for module in load_sequence: + module.AppendJSContentsToFile(flatten_to_file, + use_include_tags_for_scripts, + dir_for_include_tag_root) + if minify: + js = flatten_to_file.getvalue() + minified_js = _MinifyJS(js) + f.write(minified_js) + f.write('\n') + + if report_sizes: + for module in load_sequence: + s = StringIO() + module.AppendJSContentsToFile(s, + use_include_tags_for_scripts, + dir_for_include_tag_root) + + # Add minified size info. + js = s.getvalue() + min_js_size = str(len(_MinifyJS(js))) + + # Print names for this module. Some domain-specific simplifications + # are included to make pivoting more obvious. + parts = module.name.split('.') + if parts[:2] == ['base', 'ui']: + parts = ['base_ui'] + parts[2:] + if parts[:2] == ['tracing', 'importer']: + parts = ['importer'] + parts[2:] + tln = parts[0] + sln = '.'.join(parts[:2]) + + # Output + print(('%i\t%s\t%s\t%s\t%s' % + (len(js), min_js_size, module.name, tln, sln))) + sys.stdout.flush() + + +class ExtraScript(object): + + def __init__(self, script_id=None, text_content=None, content_type=None): + if script_id is not None: + assert script_id[0] != '#' + self.script_id = script_id + self.text_content = text_content + self.content_type = content_type + + def WriteToFile(self, output_file): + _AssertIsUTF8(output_file) + attrs = [] + if self.script_id: + attrs.append('id="%s"' % self.script_id) + if self.content_type: + attrs.append('content-type="%s"' % self.content_type) + + if len(attrs) > 0: + output_file.write('\n') + + +def _MinifyCSS(css_text): + py_vulcanize_path = os.path.abspath(os.path.join( + os.path.dirname(__file__), '..')) + rcssmin_path = os.path.abspath( + os.path.join(py_vulcanize_path, 'third_party', 'rcssmin', 'rcssmin.py')) + + with tempfile.NamedTemporaryFile() as _: + rcssmin_args = [sys.executable, rcssmin_path] + p = subprocess.Popen(rcssmin_args, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + res = p.communicate(input=css_text.encode('utf-8')) + errorcode = p.wait() + if errorcode != 0: + sys.stderr.write('rCSSmin exited with error code %d' % errorcode) + sys.stderr.write(res[1]) + raise Exception('Failed to generate css for %s.' % css_text) + return res[0].decode('utf-8') + + +def GenerateStandaloneHTMLAsString(*args, **kwargs): + f = StringIO() + GenerateStandaloneHTMLToFile(f, *args, **kwargs) + return f.getvalue() + +def _WriteOriginTrialTokens(output_file): + for token in origin_trial_tokens: + output_file.write(' \n') + +def GenerateStandaloneHTMLToFile(output_file, + load_sequence, + title=None, + flattened_js_url=None, + extra_scripts=None, + minify=False, + report_sizes=False, + output_html_head_and_body=True): + """Writes a HTML file with the content of all modules in a load sequence. + + The load_sequence is a list of (HTML or JS) Module objects; the order that + they're inserted into the file depends on their type and position in the load + sequence. + """ + _AssertIsUTF8(output_file) + extra_scripts = extra_scripts or [] + + if output_html_head_and_body: + output_file.write( + '\n' + '\n' + ' \n' + ' \n') + _WriteOriginTrialTokens(output_file) + if title: + output_file.write(' %s\n ' % title) + else: + assert title is None + + loader = load_sequence[0].loader + + written_style_sheets = set() + + class HTMLGenerationController( + html_generation_controller.HTMLGenerationController): + + def __init__(self, module): + self.module = module + + def GetHTMLForStylesheetHRef(self, href): + resource = self.module.HRefToResource( + href, '' % href) + style_sheet = loader.LoadStyleSheet(resource.name) + + if style_sheet in written_style_sheets: + return None + written_style_sheets.add(style_sheet) + + text = style_sheet.contents_with_inlined_images + if minify: + text = _MinifyCSS(text) + return '' % text + + for module in load_sequence: + controller = HTMLGenerationController(module) + module.AppendHTMLContentsToFile(output_file, controller, minify=minify) + + if flattened_js_url: + output_file.write('\n' % flattened_js_url) + else: + output_file.write('\n') + + for extra_script in extra_scripts: + extra_script.WriteToFile(output_file) + + if output_html_head_and_body: + output_file.write('\n \n \n\n') diff --git a/third_party/catapult/common/py_vulcanize/py_vulcanize/generate_unittest.py b/third_party/catapult/common/py_vulcanize/py_vulcanize/generate_unittest.py new file mode 100644 index 00000000..942df9d7 --- /dev/null +++ b/third_party/catapult/common/py_vulcanize/py_vulcanize/generate_unittest.py @@ -0,0 +1,90 @@ +# Copyright (c) 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +from __future__ import absolute_import +import os +import unittest + +from py_vulcanize import generate +from py_vulcanize import fake_fs +from py_vulcanize import project as project_module + + +class GenerateTests(unittest.TestCase): + + def setUp(self): + self.fs = fake_fs.FakeFS() + self.fs.AddFile( + '/x/foo/my_module.html', + ('\n' + '\n')) + self.fs.AddFile( + '/x/foo/other_module.html', + ('\n' + '\n' + '\n')) + self.fs.AddFile('/x/foo/raw/raw_script.js', '\n/* raw script */\n') + self.fs.AddFile('/x/components/polymer/polymer.min.js', '\n') + + self.fs.AddFile('/x/foo/external_script.js', 'External()') + self.fs.AddFile('/x/foo/inline_and_external_module.html', + ('\n' + '' + '' + '')) + + self.project = project_module.Project([os.path.normpath('/x')]) + + def testJSGeneration(self): + with self.fs: + load_sequence = self.project.CalcLoadSequenceForModuleNames( + [os.path.normpath('foo.my_module')]) + generate.GenerateJS(load_sequence) + + def testHTMLGeneration(self): + with self.fs: + load_sequence = self.project.CalcLoadSequenceForModuleNames( + [os.path.normpath('foo.my_module')]) + result = generate.GenerateStandaloneHTMLAsString(load_sequence) + self.assertIn('HelloWorld();', result) + + def testExtraScriptWithWriteContentsFunc(self): + with self.fs: + load_sequence = self.project.CalcLoadSequenceForModuleNames( + [os.path.normpath('foo.my_module')]) + + class ExtraScript(generate.ExtraScript): + + def WriteToFile(self, f): + f.write('') + + result = generate.GenerateStandaloneHTMLAsString( + load_sequence, title='Title', extra_scripts=[ExtraScript()]) + self.assertIn('ExtraScript', result) + + def testScriptOrdering(self): + with self.fs: + load_sequence = self.project.CalcLoadSequenceForModuleNames( + [os.path.normpath('foo.inline_and_external_module')]) + result = generate.GenerateStandaloneHTMLAsString(load_sequence) + script1_pos = result.index('Script1()') + script2_pos = result.index('Script2()') + external_pos = result.index('External()') + self.assertTrue(script1_pos < external_pos < script2_pos) + + def testScriptOrderingWithIncludeTag(self): + with self.fs: + load_sequence = self.project.CalcLoadSequenceForModuleNames( + [os.path.normpath('foo.inline_and_external_module')]) + result = generate.GenerateJS(load_sequence, + use_include_tags_for_scripts = True, + dir_for_include_tag_root='/x/') + script1_pos = result.index('Script1()') + script2_pos = result.index('Script2()') + external_path = os.path.join('foo', 'external_script.js') + external_pos = result.index(''.format(external_path)) + self.assertTrue(script1_pos < external_pos < script2_pos) diff --git a/third_party/catapult/common/py_vulcanize/py_vulcanize/html_generation_controller.py b/third_party/catapult/common/py_vulcanize/py_vulcanize/html_generation_controller.py new file mode 100644 index 00000000..991652cc --- /dev/null +++ b/third_party/catapult/common/py_vulcanize/py_vulcanize/html_generation_controller.py @@ -0,0 +1,29 @@ +# Copyright 2013 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +from __future__ import absolute_import +import os +import re +from py_vulcanize import style_sheet + + +class HTMLGenerationController(object): + + def __init__(self): + self.current_module = None + + def GetHTMLForStylesheetHRef(self, href): # pylint: disable=unused-argument + return None + + def GetHTMLForInlineStylesheet(self, contents): + if self.current_module is None: + if re.search('url\(.+\)', contents): + raise Exception( + 'Default HTMLGenerationController cannot handle inline style urls') + return contents + + module_dirname = os.path.dirname(self.current_module.resource.absolute_path) + ss = style_sheet.ParsedStyleSheet( + self.current_module.loader, module_dirname, contents) + return ss.contents_with_inlined_images diff --git a/third_party/catapult/common/py_vulcanize/py_vulcanize/html_module.py b/third_party/catapult/common/py_vulcanize/py_vulcanize/html_module.py new file mode 100644 index 00000000..1cc53f4b --- /dev/null +++ b/third_party/catapult/common/py_vulcanize/py_vulcanize/html_module.py @@ -0,0 +1,155 @@ +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +from __future__ import absolute_import +import os +import re + +from py_vulcanize import js_utils +from py_vulcanize import module +from py_vulcanize import parse_html_deps +from py_vulcanize import style_sheet + + +def IsHTMLResourceTheModuleGivenConflictingResourceNames( + js_resource, html_resource): # pylint: disable=unused-argument + return 'polymer-element' in html_resource.contents + + +class HTMLModule(module.Module): + + @property + def _module_dir_name(self): + return os.path.dirname(self.resource.absolute_path) + + def Parse(self, excluded_scripts): + try: + parser_results = parse_html_deps.HTMLModuleParser().Parse(self.contents) + except Exception as ex: + raise Exception('While parsing %s: %s' % (self.name, str(ex))) + + self.dependency_metadata = Parse(self.loader, + self.name, + self._module_dir_name, + self.IsThirdPartyComponent(), + parser_results, + excluded_scripts) + self._parser_results = parser_results + self.scripts = parser_results.scripts + + def Load(self, excluded_scripts): + super(HTMLModule, self).Load(excluded_scripts=excluded_scripts) + + reachable_names = set([m.name + for m in self.all_dependent_modules_recursive]) + if 'tr.exportTo' in self.contents: + if 'tracing.base.base' not in reachable_names: + raise Exception('%s: Does not have a dependency on base' % + os.path.relpath(self.resource.absolute_path)) + + for script in self.scripts: + if script.is_external: + if excluded_scripts and any(re.match(pattern, script.src) for + pattern in excluded_scripts): + continue + + resource = _HRefToResource(self.loader, self.name, self._module_dir_name, + script.src, + tag_for_err_msg='', '<\/script>') diff --git a/third_party/catapult/common/py_vulcanize/py_vulcanize/js_utils_unittest.py b/third_party/catapult/common/py_vulcanize/py_vulcanize/js_utils_unittest.py new file mode 100644 index 00000000..359eec4e --- /dev/null +++ b/third_party/catapult/common/py_vulcanize/py_vulcanize/js_utils_unittest.py @@ -0,0 +1,19 @@ +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +from __future__ import absolute_import +import unittest + +from py_vulcanize import js_utils + + +class ValidateStrictModeTests(unittest.TestCase): + + def testEscapeJSIfNeeded(self): + self.assertEqual( + '')) + self.assertEqual( + ' +""") + fs.AddFile('/src/y.html', """ + + +""") + fs.AddFile('/src/z.html', """ + +""") + fs.AddFile('/src/py_vulcanize.html', '') + with fs: + project = project_module.Project([os.path.normpath('/src/')]) + loader = resource_loader.ResourceLoader(project) + x_module = loader.LoadModule('x') + + self.assertEquals([loader.loaded_modules['y'], + loader.loaded_modules['z']], + x_module.dependent_modules) + + already_loaded_set = set() + load_sequence = [] + x_module.ComputeLoadSequenceRecursive(load_sequence, already_loaded_set) + + self.assertEquals([loader.loaded_modules['z'], + loader.loaded_modules['y'], + x_module], + load_sequence) + + def testBasic(self): + fs = fake_fs.FakeFS() + fs.AddFile('/x/src/my_module.html', """ + + +}); +""") + fs.AddFile('/x/py_vulcanize/foo.html', """ + +}); +""") + project = project_module.Project([os.path.normpath('/x')]) + loader = resource_loader.ResourceLoader(project) + with fs: + my_module = loader.LoadModule(module_name='src.my_module') + dep_names = [x.name for x in my_module.dependent_modules] + self.assertEquals(['py_vulcanize.foo'], dep_names) + + def testDepsExceptionContext(self): + fs = fake_fs.FakeFS() + fs.AddFile('/x/src/my_module.html', """ + + +""") + fs.AddFile('/x/py_vulcanize/foo.html', """ + + +""") + project = project_module.Project([os.path.normpath('/x')]) + loader = resource_loader.ResourceLoader(project) + with fs: + exc = None + try: + loader.LoadModule(module_name='src.my_module') + assert False, 'Expected an exception' + except module.DepsException as e: + exc = e + self.assertEquals( + ['src.my_module', 'py_vulcanize.foo'], + exc.context) + + def testGetAllDependentFilenamesRecursive(self): + fs = fake_fs.FakeFS() + fs.AddFile('/x/y/z/foo.html', """ + + + + +""") + fs.AddFile('/x/y/z/foo.css', """ +.x .y { + background-image: url(foo.jpeg); +} +""") + fs.AddFile('/x/y/z/foo.jpeg', '') + fs.AddFile('/x/y/z/foo2.html', """ + +""") + fs.AddFile('/x/raw/bar.js', 'hello') + project = project_module.Project([ + os.path.normpath('/x/y'), os.path.normpath('/x/raw/')]) + loader = resource_loader.ResourceLoader(project) + with fs: + my_module = loader.LoadModule(module_name='z.foo') + self.assertEquals(1, len(my_module.dependent_raw_scripts)) + + dependent_filenames = my_module.GetAllDependentFilenamesRecursive() + self.assertEquals( + [ + os.path.normpath('/x/y/z/foo.html'), + os.path.normpath('/x/raw/bar.js'), + os.path.normpath('/x/y/z/foo.css'), + os.path.normpath('/x/y/z/foo.jpeg'), + os.path.normpath('/x/y/z/foo2.html'), + ], + dependent_filenames) diff --git a/third_party/catapult/common/py_vulcanize/py_vulcanize/parse_html_deps.py b/third_party/catapult/common/py_vulcanize/py_vulcanize/parse_html_deps.py new file mode 100644 index 00000000..9c462672 --- /dev/null +++ b/third_party/catapult/common/py_vulcanize/py_vulcanize/parse_html_deps.py @@ -0,0 +1,298 @@ +# Copyright (c) 2013 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import sys +import warnings + +from py_vulcanize import html_generation_controller +from py_vulcanize import js_utils +from py_vulcanize import module +from py_vulcanize import strip_js_comments +import six + + +def _AddToPathIfNeeded(path): + if path not in sys.path: + sys.path.insert(0, path) + + +def _InitBeautifulSoup(): + catapult_path = os.path.abspath( + os.path.join(os.path.dirname(__file__), + os.path.pardir, os.path.pardir, os.path.pardir)) + # Filter out warnings related to soupsieve from beautifulsoup. + # We do not need it and it generates unnecessary warnings during build. + warnings.filterwarnings('ignore', message='.*soupsieve.*', + category=UserWarning, module='bs4') + bs_path = os.path.join(catapult_path, 'third_party', 'beautifulsoup4-4.9.3', + 'py3k') + _AddToPathIfNeeded(bs_path) + + html5lib_path = os.path.join(catapult_path, 'third_party', 'html5lib-1.1') + _AddToPathIfNeeded(html5lib_path) + + webencodings_path = os.path.join( + catapult_path, 'third_party', 'webencodings-0.5.1') + _AddToPathIfNeeded(webencodings_path) + + six_path = os.path.join(catapult_path, 'third_party', 'six') + _AddToPathIfNeeded(six_path) + + +_InitBeautifulSoup() +import bs4 + +class Script(object): + + def __init__(self, soup): + if not soup: + raise module.DepsException('Script object created without soup') + self._soup = soup + + def AppendJSContentsToFile(self, f, *args, **kwargs): + raise NotImplementedError() + +class InlineScript(Script): + + def __init__(self, soup): + super(InlineScript, self).__init__(soup) + self._stripped_contents = None + self._open_tags = None + self.is_external = False + + @property + def contents(self): + return six.text_type(self._soup.string) + + @property + def stripped_contents(self): + if not self._stripped_contents: + self._stripped_contents = strip_js_comments.StripJSComments( + self.contents) + return self._stripped_contents + + @property + def open_tags(self): + if self._open_tags: + return self._open_tags + open_tags = [] + cur = self._soup.parent + while cur: + if isinstance(cur, bs4.BeautifulSoup): + break + + open_tags.append(_Tag(cur.name, cur.attrs)) + cur = cur.parent + + open_tags.reverse() + assert open_tags[-1].tag == 'script' + del open_tags[-1] + + self._open_tags = open_tags + return self._open_tags + + def AppendJSContentsToFile(self, f, *args, **kwargs): + js = self.contents + escaped_js = js_utils.EscapeJSIfNeeded(js) + f.write(escaped_js) + f.write('\n') + +class ExternalScript(Script): + + def __init__(self, soup): + super(ExternalScript, self).__init__(soup) + if 'src' not in soup.attrs: + raise Exception("{0} is not an external script.".format(soup)) + self.is_external = True + self._loaded_raw_script = None + + @property + def loaded_raw_script(self): + if self._loaded_raw_script: + return self._loaded_raw_script + + return None + + @loaded_raw_script.setter + def loaded_raw_script(self, value): + self._loaded_raw_script = value + + @property + def src(self): + return self._soup.attrs['src'] + + def AppendJSContentsToFile(self, + f, + use_include_tags_for_scripts, + dir_for_include_tag_root): + raw_script = self.loaded_raw_script + if not raw_script: + return + + if use_include_tags_for_scripts: + rel_filename = os.path.relpath(raw_script.filename, + dir_for_include_tag_root) + f.write("""\n""" % rel_filename) + else: + f.write(js_utils.EscapeJSIfNeeded(raw_script.contents)) + f.write('\n') + +def _CreateSoupWithoutHeadOrBody(html): + soupCopy = bs4.BeautifulSoup(html, 'html5lib') + soup = bs4.BeautifulSoup() + soup.reset() + if soupCopy.head: + for n in soupCopy.head.contents: + n.extract() + soup.append(n) + if soupCopy.body: + for n in soupCopy.body.contents: + n.extract() + soup.append(n) + return soup + + +class HTMLModuleParserResults(object): + + def __init__(self, html): + self._soup = bs4.BeautifulSoup(html, 'html5lib') + self._inline_scripts = None + self._scripts = None + + @property + def scripts_external(self): + tags = self._soup.findAll('script', src=True) + return [t['src'] for t in tags] + + @property + def inline_scripts(self): + if not self._inline_scripts: + tags = self._soup.findAll('script', src=None) + self._inline_scripts = [InlineScript(t.string) for t in tags] + return self._inline_scripts + + @property + def scripts(self): + if not self._scripts: + self._scripts = [] + script_elements = self._soup.findAll('script') + for element in script_elements: + if 'src' in element.attrs: + self._scripts.append(ExternalScript(element)) + else: + self._scripts.append(InlineScript(element)) + return self._scripts + + @property + def imports(self): + tags = self._soup.findAll('link', rel='import') + return [t['href'] for t in tags] + + @property + def stylesheets(self): + tags = self._soup.findAll('link', rel='stylesheet') + return [t['href'] for t in tags] + + @property + def inline_stylesheets(self): + tags = self._soup.findAll('style') + return [six.text_type(t.string) for t in tags] + + def YieldHTMLInPieces(self, controller, minify=False): + yield self.GenerateHTML(controller, minify) + + def GenerateHTML(self, controller, minify=False, prettify=False): + soup = _CreateSoupWithoutHeadOrBody(six.text_type(self._soup)) + + # Remove declaration. + for x in soup.contents: + if isinstance(x, bs4.Doctype): + x.extract() + + # Remove declaration. + for x in soup.contents: + if isinstance(x, bs4.Declaration): + x.extract() + + # Remove all imports. + imports = soup.findAll('link', rel='import') + for imp in imports: + imp.extract() + + # Remove all script links. + scripts_external = soup.findAll('script', src=True) + for script in scripts_external: + script.extract() + + # Remove all in-line scripts. + scripts_external = soup.findAll('script', src=None) + for script in scripts_external: + script.extract() + + # Process all in-line styles. + inline_styles = soup.findAll('style') + for style in inline_styles: + html = controller.GetHTMLForInlineStylesheet(six.text_type(style.string)) + if html: + ns = soup.new_tag('style') + ns.append(bs4.NavigableString(html)) + style.replaceWith(ns) + else: + style.extract() + + # Rewrite all external stylesheet hrefs or remove, as needed. + stylesheet_links = soup.findAll('link', rel='stylesheet') + for stylesheet_link in stylesheet_links: + html = controller.GetHTMLForStylesheetHRef(stylesheet_link['href']) + if html: + tmp = bs4.BeautifulSoup(html, 'html5lib').findAll('style') + assert len(tmp) == 1 + stylesheet_link.replaceWith(tmp[0]) + else: + stylesheet_link.extract() + + # Remove comments if minifying. + if minify: + comments = soup.findAll( + text=lambda text: isinstance(text, bs4.Comment)) + for comment in comments: + comment.extract() + if prettify: + return soup.prettify('utf-8').strip() + + # We are done. + return six.text_type(soup).strip() + + @property + def html_contents_without_links_and_script(self): + return self.GenerateHTML( + html_generation_controller.HTMLGenerationController()) + + +class _Tag(object): + + def __init__(self, tag, attrs): + self.tag = tag + self.attrs = attrs + + def __repr__(self): + attr_string = ' '.join('%s="%s"' % (x[0], x[1]) for x in self.attrs) + return '<%s %s>' % (self.tag, attr_string) + + +class HTMLModuleParser(): + + def Parse(self, html): + if html is None: + html = '' + else: + if html.find('< /script>') != -1: + raise Exception('Escape script tags with <\/script>') + + return HTMLModuleParserResults(html) diff --git a/third_party/catapult/common/py_vulcanize/py_vulcanize/project.py b/third_party/catapult/common/py_vulcanize/py_vulcanize/project.py new file mode 100644 index 00000000..7a169882 --- /dev/null +++ b/third_party/catapult/common/py_vulcanize/py_vulcanize/project.py @@ -0,0 +1,239 @@ +# Copyright 2013 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +import collections +import os + +try: + from six import StringIO +except ImportError: + from io import StringIO + +from py_vulcanize import resource_loader +import six + + +def _FindAllFilesRecursive(source_paths): + all_filenames = set() + for source_path in source_paths: + for dirpath, _, filenames in os.walk(source_path): + for f in filenames: + if f.startswith('.'): + continue + x = os.path.abspath(os.path.join(dirpath, f)) + all_filenames.add(x) + return all_filenames + + +class AbsFilenameList(object): + + def __init__(self, willDirtyCallback): + self._willDirtyCallback = willDirtyCallback + self._filenames = [] + self._filenames_set = set() + + def _WillBecomeDirty(self): + if self._willDirtyCallback: + self._willDirtyCallback() + + def append(self, filename): + assert os.path.isabs(filename) + self._WillBecomeDirty() + self._filenames.append(filename) + self._filenames_set.add(filename) + + def extend(self, iterable): + self._WillBecomeDirty() + for filename in iterable: + assert os.path.isabs(filename) + self._filenames.append(filename) + self._filenames_set.add(filename) + + def appendRel(self, basedir, filename): + assert os.path.isabs(basedir) + self._WillBecomeDirty() + n = os.path.abspath(os.path.join(basedir, filename)) + self._filenames.append(n) + self._filenames_set.add(n) + + def extendRel(self, basedir, iterable): + self._WillBecomeDirty() + assert os.path.isabs(basedir) + for filename in iterable: + n = os.path.abspath(os.path.join(basedir, filename)) + self._filenames.append(n) + self._filenames_set.add(n) + + def __contains__(self, x): + return x in self._filenames_set + + def __len__(self): + return self._filenames.__len__() + + def __iter__(self): + return iter(self._filenames) + + def __repr__(self): + return repr(self._filenames) + + def __str__(self): + return str(self._filenames) + + +class Project(object): + + py_vulcanize_path = os.path.abspath(os.path.join( + os.path.dirname(__file__), '..')) + + def __init__(self, source_paths=None): + """ + source_paths: A list of top-level directories in which modules and raw + scripts can be found. Module paths are relative to these directories. + """ + self._loader = None + self._frozen = False + self.source_paths = AbsFilenameList(self._WillPartOfPathChange) + + if source_paths is not None: + self.source_paths.extend(source_paths) + + def Freeze(self): + self._frozen = True + + def _WillPartOfPathChange(self): + if self._frozen: + raise Exception('The project is frozen. You cannot edit it now') + self._loader = None + + @staticmethod + def FromDict(d): + return Project(d['source_paths']) + + def AsDict(self): + return { + 'source_paths': list(self.source_paths) + } + + def __repr__(self): + return "Project(%s)" % repr(self.source_paths) + + def AddSourcePath(self, path): + self.source_paths.append(path) + + @property + def loader(self): + if self._loader is None: + self._loader = resource_loader.ResourceLoader(self) + return self._loader + + def ResetLoader(self): + self._loader = None + + def _Load(self, filenames): + return [self.loader.LoadModule(module_filename=filename) for + filename in filenames] + + def LoadModule(self, module_name=None, module_filename=None): + return self.loader.LoadModule(module_name=module_name, + module_filename=module_filename) + + def CalcLoadSequenceForModuleNames(self, module_names, + excluded_scripts=None): + modules = [self.loader.LoadModule(module_name=name, + excluded_scripts=excluded_scripts) for + name in module_names] + return self.CalcLoadSequenceForModules(modules) + + def CalcLoadSequenceForModules(self, modules): + already_loaded_set = set() + load_sequence = [] + for m in modules: + m.ComputeLoadSequenceRecursive(load_sequence, already_loaded_set) + return load_sequence + + def GetDepsGraphFromModuleNames(self, module_names): + modules = [self.loader.LoadModule(module_name=name) for + name in module_names] + return self.GetDepsGraphFromModules(modules) + + def GetDepsGraphFromModules(self, modules): + load_sequence = self.CalcLoadSequenceForModules(modules) + g = _Graph() + for m in load_sequence: + g.AddModule(m) + + for dep in m.dependent_modules: + g.AddEdge(m, dep.id) + + # FIXME: _GetGraph is not defined. Maybe `return g` is intended? + return _GetGraph(load_sequence) + + def GetDominatorGraphForModulesNamed(self, module_names, load_sequence): + modules = [self.loader.LoadModule(module_name=name) + for name in module_names] + return self.GetDominatorGraphForModules(modules, load_sequence) + + def GetDominatorGraphForModules(self, start_modules, load_sequence): + modules_by_id = {} + for m in load_sequence: + modules_by_id[m.id] = m + + module_referrers = collections.defaultdict(list) + for m in load_sequence: + for dep in m.dependent_modules: + module_referrers[dep].append(m) + + # Now start at the top module and reverse. + visited = set() + g = _Graph() + + pending = collections.deque() + pending.extend(start_modules) + while len(pending): + cur = pending.pop() + + g.AddModule(cur) + visited.add(cur) + + for out_dep in module_referrers[cur]: + if out_dep in visited: + continue + g.AddEdge(out_dep, cur) + visited.add(out_dep) + pending.append(out_dep) + + # Visited -> Dot + return g.GetDot() + + +class _Graph(object): + + def __init__(self): + self.nodes = [] + self.edges = [] + + def AddModule(self, m): + f = StringIO() + m.AppendJSContentsToFile(f, False, None) + + attrs = { + 'label': '%s (%i)' % (m.name, f.tell()) + } + + f.close() + + attr_items = ['%s="%s"' % (x, y) for x, y in six.iteritems(attrs)] + node = 'M%i [%s];' % (m.id, ','.join(attr_items)) + self.nodes.append(node) + + def AddEdge(self, mFrom, mTo): + edge = 'M%i -> M%i;' % (mFrom.id, mTo.id) + self.edges.append(edge) + + def GetDot(self): + return 'digraph deps {\n\n%s\n\n%s\n}\n' % ( + '\n'.join(self.nodes), '\n'.join(self.edges)) diff --git a/third_party/catapult/common/py_vulcanize/py_vulcanize/resource.py b/third_party/catapult/common/py_vulcanize/py_vulcanize/resource.py new file mode 100644 index 00000000..b188e731 --- /dev/null +++ b/third_party/catapult/common/py_vulcanize/py_vulcanize/resource.py @@ -0,0 +1,58 @@ +# Copyright 2013 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""A Resource is a file and its various associated canonical names.""" + +from __future__ import absolute_import +import codecs +import os + + +class Resource(object): + """Represents a file found via a path search.""" + + def __init__(self, toplevel_dir, absolute_path, binary=False): + self.toplevel_dir = toplevel_dir + self.absolute_path = absolute_path + self._contents = None + self._binary = binary + + @property + def relative_path(self): + """The path to the file from the top-level directory""" + return os.path.relpath(self.absolute_path, self.toplevel_dir) + + @property + def unix_style_relative_path(self): + return self.relative_path.replace(os.sep, '/') + + @property + def name(self): + """The dotted name for this resource based on its relative path.""" + return self.name_from_relative_path(self.relative_path) + + @staticmethod + def name_from_relative_path(relative_path): + dirname = os.path.dirname(relative_path) + basename = os.path.basename(relative_path) + modname = os.path.splitext(basename)[0] + if len(dirname): + name = dirname.replace(os.path.sep, '.') + '.' + modname + else: + name = modname + return name + + @property + def contents(self): + if self._contents: + return self._contents + if not os.path.exists(self.absolute_path): + raise Exception('%s not found.' % self.absolute_path) + if self._binary: + f = open(self.absolute_path, mode='rb') + else: + f = codecs.open(self.absolute_path, mode='r', encoding='utf-8') + self._contents = f.read() + f.close() + return self._contents diff --git a/third_party/catapult/common/py_vulcanize/py_vulcanize/resource_loader.py b/third_party/catapult/common/py_vulcanize/py_vulcanize/resource_loader.py new file mode 100644 index 00000000..652fa6c1 --- /dev/null +++ b/third_party/catapult/common/py_vulcanize/py_vulcanize/resource_loader.py @@ -0,0 +1,232 @@ +# Copyright (c) 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""ResourceFinder is a helper class for finding resources given their name.""" + +from __future__ import absolute_import +import codecs +import functools +import os +import six + +from py_vulcanize import module +from py_vulcanize import style_sheet as style_sheet_module +from py_vulcanize import resource as resource_module +from py_vulcanize import html_module +from py_vulcanize import strip_js_comments + + +class ResourceLoader(object): + """Manges loading modules and their dependencies from files. + + Modules handle parsing and the construction of their individual dependency + pointers. The loader deals with bookkeeping of what has been loaded, and + mapping names to file resources. + """ + + def __init__(self, project): + self.project = project + self.stripped_js_by_filename = {} + self.loaded_modules = {} + self.loaded_raw_scripts = {} + self.loaded_style_sheets = {} + self.loaded_images = {} + + @property + def source_paths(self): + """A list of base directories to search for modules under.""" + return self.project.source_paths + + def FindResource(self, some_path, binary=False): + """Finds a Resource for the given path. + + Args: + some_path: A relative or absolute path to a file. + + Returns: + A Resource or None. + """ + if os.path.isabs(some_path): + return self.FindResourceGivenAbsolutePath(some_path, binary) + else: + return self.FindResourceGivenRelativePath(some_path, binary) + + def FindResourceGivenAbsolutePath(self, absolute_path, binary=False): + """Returns a Resource for the given absolute path.""" + candidate_paths = [] + for source_path in self.source_paths: + if absolute_path.startswith(source_path): + candidate_paths.append(source_path) + if len(candidate_paths) == 0: + return None + + # Sort by length. Longest match wins. + sorted(candidate_paths, + key=functools.cmp_to_key(lambda x, y: len(x) - len(y)), reverse=True) + longest_candidate = candidate_paths[-1] + return resource_module.Resource(longest_candidate, absolute_path, binary) + + def FindResourceGivenRelativePath(self, relative_path, binary=False): + """Returns a Resource for the given relative path.""" + absolute_path = None + for script_path in self.source_paths: + absolute_path = os.path.join(script_path, relative_path) + if os.path.exists(absolute_path): + return resource_module.Resource(script_path, absolute_path, binary) + return None + + def _FindResourceGivenNameAndSuffix( + self, requested_name, extension, return_resource=False): + """Searches for a file and reads its contents. + + Args: + requested_name: The name of the resource that was requested. + extension: The extension for this requested resource. + + Returns: + A (path, contents) pair. + """ + pathy_name = requested_name.replace('.', os.sep) + filename = pathy_name + extension + + resource = self.FindResourceGivenRelativePath(filename) + if return_resource: + return resource + if not resource: + return None, None + return _read_file(resource.absolute_path) + + def FindModuleResource(self, requested_module_name): + """Finds a module javascript file and returns a Resource, or none.""" + js_resource = self._FindResourceGivenNameAndSuffix( + requested_module_name, '.js', return_resource=True) + html_resource = self._FindResourceGivenNameAndSuffix( + requested_module_name, '.html', return_resource=True) + if js_resource and html_resource: + if html_module.IsHTMLResourceTheModuleGivenConflictingResourceNames( + js_resource, html_resource): + return html_resource + return js_resource + elif js_resource: + return js_resource + return html_resource + + def LoadModule(self, module_name=None, module_filename=None, + excluded_scripts=None): + assert bool(module_name) ^ bool(module_filename), ( + 'Must provide either module_name or module_filename.') + if module_filename: + resource = self.FindResource(module_filename) + if not resource: + raise Exception('Could not find %s in %s' % ( + module_filename, repr(self.source_paths))) + module_name = resource.name + else: + resource = None # Will be set if we end up needing to load. + + if module_name in self.loaded_modules: + assert self.loaded_modules[module_name].contents + return self.loaded_modules[module_name] + + if not resource: # happens when module_name was given + resource = self.FindModuleResource(module_name) + if not resource: + raise module.DepsException('No resource for module "%s"' % module_name) + + m = html_module.HTMLModule(self, module_name, resource) + self.loaded_modules[module_name] = m + + # Fake it, this is probably either polymer.min.js or platform.js which are + # actually .js files.... + if resource.absolute_path.endswith('.js'): + return m + + m.Parse(excluded_scripts) + m.Load(excluded_scripts) + return m + + def LoadRawScript(self, relative_raw_script_path): + resource = None + for source_path in self.source_paths: + possible_absolute_path = os.path.join( + source_path, os.path.normpath(relative_raw_script_path)) + if os.path.exists(possible_absolute_path): + resource = resource_module.Resource( + source_path, possible_absolute_path) + break + if not resource: + raise module.DepsException( + 'Could not find a file for raw script %s in %s' % + (relative_raw_script_path, self.source_paths)) + assert relative_raw_script_path == resource.unix_style_relative_path, ( + 'Expected %s == %s' % (relative_raw_script_path, + resource.unix_style_relative_path)) + + if resource.absolute_path in self.loaded_raw_scripts: + return self.loaded_raw_scripts[resource.absolute_path] + + raw_script = module.RawScript(resource) + self.loaded_raw_scripts[resource.absolute_path] = raw_script + return raw_script + + def LoadStyleSheet(self, name): + if name in self.loaded_style_sheets: + return self.loaded_style_sheets[name] + + resource = self._FindResourceGivenNameAndSuffix( + name, '.css', return_resource=True) + if not resource: + raise module.DepsException( + 'Could not find a file for stylesheet %s' % name) + + style_sheet = style_sheet_module.StyleSheet(self, name, resource) + style_sheet.load() + self.loaded_style_sheets[name] = style_sheet + return style_sheet + + def LoadImage(self, abs_path): + if abs_path in self.loaded_images: + return self.loaded_images[abs_path] + + if not os.path.exists(abs_path): + raise module.DepsException("url('%s') did not exist" % abs_path) + + res = self.FindResourceGivenAbsolutePath(abs_path, binary=True) + if res is None: + raise module.DepsException("url('%s') was not in search path" % abs_path) + + image = style_sheet_module.Image(res) + self.loaded_images[abs_path] = image + return image + + def GetStrippedJSForFilename(self, filename, early_out_if_no_py_vulcanize): + if filename in self.stripped_js_by_filename: + return self.stripped_js_by_filename[filename] + + with open(filename, 'r') as f: + contents = f.read(4096) + if early_out_if_no_py_vulcanize and ('py_vulcanize' not in contents): + return None + + s = strip_js_comments.StripJSComments(contents) + self.stripped_js_by_filename[filename] = s + return s + + +def _read_file(absolute_path): + """Reads a file and returns a (path, contents) pair. + + Args: + absolute_path: Absolute path to a file. + + Raises: + Exception: The given file doesn't exist. + IOError: There was a problem opening or reading the file. + """ + if not os.path.exists(absolute_path): + raise Exception('%s not found.' % absolute_path) + f = codecs.open(absolute_path, mode='r', encoding='utf-8') + contents = f.read() + f.close() + return absolute_path, contents diff --git a/third_party/catapult/common/py_vulcanize/py_vulcanize/resource_unittest.py b/third_party/catapult/common/py_vulcanize/py_vulcanize/resource_unittest.py new file mode 100644 index 00000000..ef186409 --- /dev/null +++ b/third_party/catapult/common/py_vulcanize/py_vulcanize/resource_unittest.py @@ -0,0 +1,18 @@ +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +from __future__ import absolute_import +import os +import unittest + +from py_vulcanize import resource + + +class ResourceUnittest(unittest.TestCase): + + def testBasic(self): + r = resource.Resource('/a', '/a/b/c.js') + self.assertEquals('b.c', r.name) + self.assertEquals(os.path.join('b', 'c.js'), r.relative_path) + self.assertEquals('b/c.js', r.unix_style_relative_path) diff --git a/third_party/catapult/common/py_vulcanize/py_vulcanize/strip_js_comments.py b/third_party/catapult/common/py_vulcanize/py_vulcanize/strip_js_comments.py new file mode 100644 index 00000000..a3ce8779 --- /dev/null +++ b/third_party/catapult/common/py_vulcanize/py_vulcanize/strip_js_comments.py @@ -0,0 +1,82 @@ +# Copyright 2013 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Utility function for stripping comments out of JavaScript source code.""" + +from __future__ import absolute_import +import re + + +def _TokenizeJS(text): + """Splits source code text into segments in preparation for comment stripping. + + Note that this doesn't tokenize for parsing. There is no notion of statements, + variables, etc. The only tokens of interest are comment-related tokens. + + Args: + text: The contents of a JavaScript file. + + Yields: + A succession of strings in the file, including all comment-related symbols. + """ + rest = text + tokens = ['//', '/*', '*/', '\n'] + next_tok = re.compile('|'.join(re.escape(x) for x in tokens)) + while len(rest): + m = next_tok.search(rest) + if not m: + # end of string + yield rest + return + min_index = m.start() + end_index = m.end() + + if min_index > 0: + yield rest[:min_index] + + yield rest[min_index:end_index] + rest = rest[end_index:] + + +def StripJSComments(text): + """Strips comments out of JavaScript source code. + + Args: + text: JavaScript source text. + + Returns: + JavaScript source text with comments stripped out. + """ + result_tokens = [] + token_stream = _TokenizeJS(text).__iter__() + while True: + try: + t = next(token_stream) + except StopIteration: + break + + if t == '//': + while True: + try: + t2 = next(token_stream) + if t2 == '\n': + break + except StopIteration: + break + elif t == '/*': + nesting = 1 + while True: + try: + t2 = next(token_stream) + if t2 == '/*': + nesting += 1 + elif t2 == '*/': + nesting -= 1 + if nesting == 0: + break + except StopIteration: + break + else: + result_tokens.append(t) + return ''.join(result_tokens) diff --git a/third_party/catapult/common/py_vulcanize/py_vulcanize/style_sheet.py b/third_party/catapult/common/py_vulcanize/py_vulcanize/style_sheet.py new file mode 100644 index 00000000..2ffc4ccf --- /dev/null +++ b/third_party/catapult/common/py_vulcanize/py_vulcanize/style_sheet.py @@ -0,0 +1,139 @@ +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +from __future__ import absolute_import +import base64 +import os +import re + + +class Image(object): + + def __init__(self, resource): + self.resource = resource + self.aliases = [] + + @property + def relative_path(self): + return self.resource.relative_path + + @property + def absolute_path(self): + return self.resource.absolute_path + + @property + def contents(self): + return self.resource.contents + + +class ParsedStyleSheet(object): + + def __init__(self, loader, containing_dirname, contents): + self.loader = loader + self.contents = contents + self._images = None + self._Load(containing_dirname) + + @property + def images(self): + return self._images + + def AppendDirectlyDependentFilenamesTo(self, dependent_filenames): + for i in self.images: + dependent_filenames.append(i.resource.absolute_path) + + @property + def contents_with_inlined_images(self): + images_by_url = {} + for i in self.images: + for a in i.aliases: + images_by_url[a] = i + + def InlineUrl(m): + url = m.group('url') + image = images_by_url[url] + + ext = os.path.splitext(image.absolute_path)[1] + data = base64.standard_b64encode(image.contents) + + return 'url(data:image/%s;base64,%s)' % (ext[1:], data.decode('utf-8')) + + # I'm assuming we only have url()'s associated with images + return re.sub('url\((?P"|\'|)(?P[^"\'()]*)(?P=quote)\)', + InlineUrl, self.contents) + + def AppendDirectlyDependentFilenamesTo(self, dependent_filenames): + for i in self.images: + dependent_filenames.append(i.resource.absolute_path) + + def _Load(self, containing_dirname): + if self.contents.find('@import') != -1: + raise Exception('@imports are not supported') + + matches = re.findall( + 'url\((?:["|\']?)([^"\'()]*)(?:["|\']?)\)', + self.contents) + + def resolve_url(url): + if os.path.isabs(url): + # FIXME: module is used here, but py_vulcanize.module is never imported. + # However, py_vulcanize.module cannot be imported since py_vulcanize.module may import + # style_sheet, leading to an import loop. + raise module.DepsException('URL references must be relative') + # URLS are relative to this module's directory + abs_path = os.path.abspath(os.path.join(containing_dirname, url)) + image = self.loader.LoadImage(abs_path) + image.aliases.append(url) + return image + + self._images = [resolve_url(x) for x in matches] + + +class StyleSheet(object): + """Represents a stylesheet resource referenced by a module via the + base.requireStylesheet(xxx) directive.""" + + def __init__(self, loader, name, resource): + self.loader = loader + self.name = name + self.resource = resource + self._parsed_style_sheet = None + + @property + def filename(self): + return self.resource.absolute_path + + @property + def contents(self): + return self.resource.contents + + def __repr__(self): + return 'StyleSheet(%s)' % self.name + + @property + def images(self): + self._InitParsedStyleSheetIfNeeded() + return self._parsed_style_sheet.images + + def AppendDirectlyDependentFilenamesTo(self, dependent_filenames): + self._InitParsedStyleSheetIfNeeded() + + dependent_filenames.append(self.resource.absolute_path) + self._parsed_style_sheet.AppendDirectlyDependentFilenamesTo( + dependent_filenames) + + @property + def contents_with_inlined_images(self): + self._InitParsedStyleSheetIfNeeded() + return self._parsed_style_sheet.contents_with_inlined_images + + def load(self): + self._InitParsedStyleSheetIfNeeded() + + def _InitParsedStyleSheetIfNeeded(self): + if self._parsed_style_sheet: + return + module_dirname = os.path.dirname(self.resource.absolute_path) + self._parsed_style_sheet = ParsedStyleSheet( + self.loader, module_dirname, self.contents) diff --git a/third_party/catapult/common/py_vulcanize/py_vulcanize/style_sheet_unittest.py b/third_party/catapult/common/py_vulcanize/py_vulcanize/style_sheet_unittest.py new file mode 100644 index 00000000..89bc0f47 --- /dev/null +++ b/third_party/catapult/common/py_vulcanize/py_vulcanize/style_sheet_unittest.py @@ -0,0 +1,68 @@ +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +from __future__ import absolute_import +import base64 +import os +import unittest + +from py_vulcanize import project as project_module +from py_vulcanize import resource_loader +from py_vulcanize import fake_fs +from py_vulcanize import module + + +class StyleSheetUnittest(unittest.TestCase): + + def testImages(self): + fs = fake_fs.FakeFS() + fs.AddFile('/src/foo/x.css', """ +.x .y { + background-image: url(../images/bar.jpeg); +} +""") + fs.AddFile('/src/images/bar.jpeg', b'hello world') + with fs: + project = project_module.Project([os.path.normpath('/src/')]) + loader = resource_loader.ResourceLoader(project) + + foo_x = loader.LoadStyleSheet('foo.x') + self.assertEquals(1, len(foo_x.images)) + + r0 = foo_x.images[0] + self.assertEquals(os.path.normpath('/src/images/bar.jpeg'), + r0.absolute_path) + + inlined = foo_x.contents_with_inlined_images + self.assertEquals(""" +.x .y { + background-image: url(data:image/jpeg;base64,%s); +} +""" % base64.standard_b64encode(b'hello world').decode('utf-8'), inlined) + + def testURLResolveFails(self): + fs = fake_fs.FakeFS() + fs.AddFile('/src/foo/x.css', """ +.x .y { + background-image: url(../images/missing.jpeg); +} +""") + with fs: + project = project_module.Project([os.path.normpath('/src')]) + loader = resource_loader.ResourceLoader(project) + + self.assertRaises(module.DepsException, + lambda: loader.LoadStyleSheet('foo.x')) + + def testImportsCauseFailure(self): + fs = fake_fs.FakeFS() + fs.AddFile('/src/foo/x.css', """ +@import url(awesome.css); +""") + with fs: + project = project_module.Project([os.path.normpath('/src')]) + loader = resource_loader.ResourceLoader(project) + + self.assertRaises(Exception, + lambda: loader.LoadStyleSheet('foo.x')) diff --git a/third_party/catapult/third_party/beautifulsoup4-4.9.3/COPYING.txt b/third_party/catapult/third_party/beautifulsoup4-4.9.3/COPYING.txt new file mode 100644 index 00000000..fb6ae69c --- /dev/null +++ b/third_party/catapult/third_party/beautifulsoup4-4.9.3/COPYING.txt @@ -0,0 +1,27 @@ +Beautiful Soup is made available under the MIT license: + + Copyright (c) 2004-2017 Leonard Richardson + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, subject to + the following conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. + +Beautiful Soup incorporates code from the html5lib library, which is +also made available under the MIT license. Copyright (c) 2006-2013 +James Graham and other contributors diff --git a/third_party/catapult/third_party/beautifulsoup4-4.9.3/LICENSE b/third_party/catapult/third_party/beautifulsoup4-4.9.3/LICENSE new file mode 100644 index 00000000..4c068bab --- /dev/null +++ b/third_party/catapult/third_party/beautifulsoup4-4.9.3/LICENSE @@ -0,0 +1,30 @@ +Beautiful Soup is made available under the MIT license: + + Copyright (c) 2004-2019 Leonard Richardson + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, subject to + the following conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. + +Beautiful Soup incorporates code from the html5lib library, which is +also made available under the MIT license. Copyright (c) 2006-2013 +James Graham and other contributors + +Beautiful Soup depends on the soupsieve library, which is also made +available under the MIT license. Copyright (c) 2018 Isaac Muse diff --git a/third_party/catapult/third_party/beautifulsoup4-4.9.3/MANIFEST.in b/third_party/catapult/third_party/beautifulsoup4-4.9.3/MANIFEST.in new file mode 100644 index 00000000..33821b97 --- /dev/null +++ b/third_party/catapult/third_party/beautifulsoup4-4.9.3/MANIFEST.in @@ -0,0 +1,10 @@ +include test-all-versions +include convert-py3k +include LICENSE +include *.txt +include doc*/Makefile +include doc*/source/*.py +include doc*/source/*.rst +include doc*/source/*.jpg +include scripts/*.py +include scripts/*.txt diff --git a/third_party/catapult/third_party/beautifulsoup4-4.9.3/NEWS.txt b/third_party/catapult/third_party/beautifulsoup4-4.9.3/NEWS.txt new file mode 100644 index 00000000..625bb34c --- /dev/null +++ b/third_party/catapult/third_party/beautifulsoup4-4.9.3/NEWS.txt @@ -0,0 +1,1547 @@ += 4.9.3 (20201003) + +* Implemented a significant performance optimization to the process of + searching the parse tree. Patch by Morotti. [bug=1898212] + += 4.9.2 (20200926) + +* Fixed a bug that caused too many tags to be popped from the tag + stack during tree building, when encountering a closing tag that had + no matching opening tag. [bug=1880420] + +* Fixed a bug that inconsistently moved elements over when passing + a Tag, rather than a list, into Tag.extend(). [bug=1885710] + +* Specify the soupsieve dependency in a way that complies with + PEP 508. Patch by Mike Nerone. [bug=1893696] + +* Change the signatures for BeautifulSoup.insert_before and insert_after + (which are not implemented) to match PageElement.insert_before and + insert_after, quieting warnings in some IDEs. [bug=1897120] + += 4.9.1 (20200517) + +* Added a keyword argument 'on_duplicate_attribute' to the + BeautifulSoupHTMLParser constructor (used by the html.parser tree + builder) which lets you customize the handling of markup that + contains the same attribute more than once, as in: + [bug=1878209] + +* Added a distinct subclass, GuessedAtParserWarning, for the warning + issued when BeautifulSoup is instantiated without a parser being + specified. [bug=1873787] + +* Added a distinct subclass, MarkupResemblesLocatorWarning, for the + warning issued when BeautifulSoup is instantiated with 'markup' that + actually seems to be a URL or the path to a file on + disk. [bug=1873787] + +* The new NavigableString subclasses (Stylesheet, Script, and + TemplateString) can now be imported directly from the bs4 package. + +* If you encode a document with a Python-specific encoding like + 'unicode_escape', that encoding is no longer mentioned in the final + XML or HTML document. Instead, encoding information is omitted or + left blank. [bug=1874955] + +* Fixed test failures when run against soupselect 2.0. Patch by Tomáš + Chvátal. [bug=1872279] + += 4.9.0 (20200405) + +* Added PageElement.decomposed, a new property which lets you + check whether you've already called decompose() on a Tag or + NavigableString. + +* Embedded CSS and Javascript is now stored in distinct Stylesheet and + Script tags, which are ignored by methods like get_text() since most + people don't consider this sort of content to be 'text'. This + feature is not supported by the html5lib treebuilder. [bug=1868861] + +* Added a Russian translation by 'authoress' to the repository. + +* Fixed an unhandled exception when formatting a Tag that had been + decomposed.[bug=1857767] + +* Fixed a bug that happened when passing a Unicode filename containing + non-ASCII characters as markup into Beautiful Soup, on a system that + allows Unicode filenames. [bug=1866717] + +* Added a performance optimization to PageElement.extract(). Patch by + Arthur Darcet. + += 4.8.2 (20191224) + +* Added Python docstrings to all public methods of the most commonly + used classes. + +* Added a Chinese translation by Deron Wang and a Brazilian Portuguese + translation by Cezar Peixeiro to the repository. + +* Fixed two deprecation warnings. Patches by Colin + Watson and Nicholas Neumann. [bug=1847592] [bug=1855301] + +* The html.parser tree builder now correctly handles DOCTYPEs that are + not uppercase. [bug=1848401] + +* PageElement.select() now returns a ResultSet rather than a regular + list, making it consistent with methods like find_all(). + += 4.8.1 (20191006) + +* When the html.parser or html5lib parsers are in use, Beautiful Soup + will, by default, record the position in the original document where + each tag was encountered. This includes line number (Tag.sourceline) + and position within a line (Tag.sourcepos). Based on code by Chris + Mayo. [bug=1742921] + +* When instantiating a BeautifulSoup object, it's now possible to + provide a dictionary ('element_classes') of the classes you'd like to be + instantiated instead of Tag, NavigableString, etc. + +* Fixed the definition of the default XML namespace when using + lxml 4.4. Patch by Isaac Muse. [bug=1840141] + +* Fixed a crash when pretty-printing tags that were not created + during initial parsing. [bug=1838903] + +* Copying a Tag preserves information that was originally obtained from + the TreeBuilder used to build the original Tag. [bug=1838903] + +* Raise an explanatory exception when the underlying parser + completely rejects the incoming markup. [bug=1838877] + +* Avoid a crash when trying to detect the declared encoding of a + Unicode document. [bug=1838877] + +* Avoid a crash when unpickling certain parse trees generated + using html5lib on Python 3. [bug=1843545] + += 4.8.0 (20190720, "One Small Soup") + +This release focuses on making it easier to customize Beautiful Soup's +input mechanism (the TreeBuilder) and output mechanism (the Formatter). + +* You can customize the TreeBuilder object by passing keyword + arguments into the BeautifulSoup constructor. Those keyword + arguments will be passed along into the TreeBuilder constructor. + + The main reason to do this right now is to change how which + attributes are treated as multi-valued attributes (the way 'class' + is treated by default). You can do this with the + 'multi_valued_attributes' argument. [bug=1832978] + +* The role of Formatter objects has been greatly expanded. The Formatter + class now controls the following: + + - The function to call to perform entity substitution. (This was + previously Formatter's only job.) + - Which tags should be treated as containing CDATA and have their + contents exempt from entity substitution. + - The order in which a tag's attributes are output. [bug=1812422] + - Whether or not to put a '/' inside a void element, e.g. '
' vs '
' + + All preexisting code should work as before. + +* Added a new method to the API, Tag.smooth(), which consolidates + multiple adjacent NavigableString elements. [bug=1697296] + +* ' (which is valid in XML, XHTML, and HTML 5, but not HTML 4) is always + recognized as a named entity and converted to a single quote. [bug=1818721] + += 4.7.1 (20190106) + +* Fixed a significant performance problem introduced in 4.7.0. [bug=1810617] + +* Fixed an incorrectly raised exception when inserting a tag before or + after an identical tag. [bug=1810692] + +* Beautiful Soup will no longer try to keep track of namespaces that + are not defined with a prefix; this can confuse soupselect. [bug=1810680] + +* Tried even harder to avoid the deprecation warning originally fixed in + 4.6.1. [bug=1778909] + += 4.7.0 (20181231) + +* Beautiful Soup's CSS Selector implementation has been replaced by a + dependency on Isaac Muse's SoupSieve project (the soupsieve package + on PyPI). The good news is that SoupSieve has a much more robust and + complete implementation of CSS selectors, resolving a large number + of longstanding issues. The bad news is that from this point onward, + SoupSieve must be installed if you want to use the select() method. + + You don't have to change anything lf you installed Beautiful Soup + through pip (SoupSieve will be automatically installed when you + upgrade Beautiful Soup) or if you don't use CSS selectors from + within Beautiful Soup. + + SoupSieve documentation: https://facelessuser.github.io/soupsieve/ + +* Added the PageElement.extend() method, which works like list.append(). + [bug=1514970] + +* PageElement.insert_before() and insert_after() now take a variable + number of arguments. [bug=1514970] + +* Fix a number of problems with the tree builder that caused + trees that were superficially okay, but which fell apart when bits + were extracted. Patch by Isaac Muse. [bug=1782928,1809910] + +* Fixed a problem with the tree builder in which elements that + contained no content (such as empty comments and all-whitespace + elements) were not being treated as part of the tree. Patch by Isaac + Muse. [bug=1798699] + +* Fixed a problem with multi-valued attributes where the value + contained whitespace. Thanks to Jens Svalgaard for the + fix. [bug=1787453] + +* Clarified ambiguous license statements in the source code. Beautiful + Soup is released under the MIT license, and has been since 4.4.0. + +* This file has been renamed from NEWS.txt to CHANGELOG. + += 4.6.3 (20180812) + +* Exactly the same as 4.6.2. Re-released to make the README file + render properly on PyPI. + += 4.6.2 (20180812) + +* Fix an exception when a custom formatter was asked to format a void + element. [bug=1784408] + += 4.6.1 (20180728) + +* Stop data loss when encountering an empty numeric entity, and + possibly in other cases. Thanks to tos.kamiya for the fix. [bug=1698503] + +* Preserve XML namespaces introduced inside an XML document, not just + the ones introduced at the top level. [bug=1718787] + +* Added a new formatter, "html5", which represents void elements + as "" rather than "". [bug=1716272] + +* Fixed a problem where the html.parser tree builder interpreted + a string like "&foo " as the character entity "&foo;" [bug=1728706] + +* Correctly handle invalid HTML numeric character entities like “ + which reference code points that are not Unicode code points. Note + that this is only fixed when Beautiful Soup is used with the + html.parser parser -- html5lib already worked and I couldn't fix it + with lxml. [bug=1782933] + +* Improved the warning given when no parser is specified. [bug=1780571] + +* When markup contains duplicate elements, a select() call that + includes multiple match clauses will match all relevant + elements. [bug=1770596] + +* Fixed code that was causing deprecation warnings in recent Python 3 + versions. Includes a patch from Ville Skyttä. [bug=1778909] [bug=1689496] + +* Fixed a Windows crash in diagnose() when checking whether a long + markup string is a filename. [bug=1737121] + +* Stopped HTMLParser from raising an exception in very rare cases of + bad markup. [bug=1708831] + +* Fixed a bug where find_all() was not working when asked to find a + tag with a namespaced name in an XML document that was parsed as + HTML. [bug=1723783] + +* You can get finer control over formatting by subclassing + bs4.element.Formatter and passing a Formatter instance into (e.g.) + encode(). [bug=1716272] + +* You can pass a dictionary of `attrs` into + BeautifulSoup.new_tag. This makes it possible to create a tag with + an attribute like 'name' that would otherwise be masked by another + argument of new_tag. [bug=1779276] + +* Clarified the deprecation warning when accessing tag.fooTag, to cover + the possibility that you might really have been looking for a tag + called 'fooTag'. + += 4.6.0 (20170507) = + +* Added the `Tag.get_attribute_list` method, which acts like `Tag.get` for + getting the value of an attribute, but which always returns a list, + whether or not the attribute is a multi-value attribute. [bug=1678589] + +* It's now possible to use a tag's namespace prefix when searching, + e.g. soup.find('namespace:tag') [bug=1655332] + +* Improved the handling of empty-element tags like
when using the + html.parser parser. [bug=1676935] + +* HTML parsers treat all HTML4 and HTML5 empty element tags (aka void + element tags) correctly. [bug=1656909] + +* Namespace prefix is preserved when an XML tag is copied. Thanks + to Vikas for a patch and test. [bug=1685172] + += 4.5.3 (20170102) = + +* Fixed foster parenting when html5lib is the tree builder. Thanks to + Geoffrey Sneddon for a patch and test. + +* Fixed yet another problem that caused the html5lib tree builder to + create a disconnected parse tree. [bug=1629825] + += 4.5.2 (20170102) = + +* Apart from the version number, this release is identical to + 4.5.3. Due to user error, it could not be completely uploaded to + PyPI. Use 4.5.3 instead. + += 4.5.1 (20160802) = + +* Fixed a crash when passing Unicode markup that contained a + processing instruction into the lxml HTML parser on Python + 3. [bug=1608048] + += 4.5.0 (20160719) = + +* Beautiful Soup is no longer compatible with Python 2.6. This + actually happened a few releases ago, but it's now official. + +* Beautiful Soup will now work with versions of html5lib greater than + 0.99999999. [bug=1603299] + +* If a search against each individual value of a multi-valued + attribute fails, the search will be run one final time against the + complete attribute value considered as a single string. That is, if + a tag has class="foo bar" and neither "foo" nor "bar" matches, but + "foo bar" does, the tag is now considered a match. + + This happened in previous versions, but only when the value being + searched for was a string. Now it also works when that value is + a regular expression, a list of strings, etc. [bug=1476868] + +* Fixed a bug that deranged the tree when a whitespace element was + reparented into a tag that contained an identical whitespace + element. [bug=1505351] + +* Added support for CSS selector values that contain quoted spaces, + such as tag[style="display: foo"]. [bug=1540588] + +* Corrected handling of XML processing instructions. [bug=1504393] + +* Corrected an encoding error that happened when a BeautifulSoup + object was copied. [bug=1554439] + +* The contents of +
+
This numeric entity is missing the final semicolon:
+
, that attribute value was closed by the subsequent tag
+
a
+
This document contains (do you see it?)
+
This document ends with That attribute value was bogus
+The doctype is invalid because it contains extra whitespace +
That boolean attribute had no value
+
Here's a nonexistent entity: &#foo; (do you see it?)
+
This document ends before the entity finishes: > +

Paragraphs shouldn't contain block display elements, but this one does:

you see?

+Multiple values for the same attribute. +
Here's a table
+
+
This tag contains nothing but whitespace:
+

This p tag is cut off by

the end of the blockquote tag
+
Here's a nested table:
foo
This table contains bare markup
+ +
This document contains a surprise doctype
+ +
Tag name contains Unicode characters
+ + +""" + + +class SoupTest(unittest.TestCase): + + @property + def default_builder(self): + return default_builder + + def soup(self, markup, **kwargs): + """Build a Beautiful Soup object from markup.""" + builder = kwargs.pop('builder', self.default_builder) + return BeautifulSoup(markup, builder=builder, **kwargs) + + def document_for(self, markup, **kwargs): + """Turn an HTML fragment into a document. + + The details depend on the builder. + """ + return self.default_builder(**kwargs).test_fragment_to_document(markup) + + def assertSoupEquals(self, to_parse, compare_parsed_to=None): + builder = self.default_builder + obj = BeautifulSoup(to_parse, builder=builder) + if compare_parsed_to is None: + compare_parsed_to = to_parse + + # Verify that the documents come out the same. + self.assertEqual(obj.decode(), self.document_for(compare_parsed_to)) + + # Also run some checks on the BeautifulSoup object itself: + + # Verify that every tag that was opened was eventually closed. + + # There are no tags in the open tag counter. + assert all(v==0 for v in obj.open_tag_counter.values()) + + # The only tag in the tag stack is the one for the root + # document. + self.assertEqual( + [obj.ROOT_TAG_NAME], [x.name for x in obj.tagStack] + ) + + def assertConnectedness(self, element): + """Ensure that next_element and previous_element are properly + set for all descendants of the given element. + """ + earlier = None + for e in element.descendants: + if earlier: + self.assertEqual(e, earlier.next_element) + self.assertEqual(earlier, e.previous_element) + earlier = e + + def linkage_validator(self, el, _recursive_call=False): + """Ensure proper linkage throughout the document.""" + descendant = None + # Document element should have no previous element or previous sibling. + # It also shouldn't have a next sibling. + if el.parent is None: + assert el.previous_element is None,\ + "Bad previous_element\nNODE: {}\nPREV: {}\nEXPECTED: {}".format( + el, el.previous_element, None + ) + assert el.previous_sibling is None,\ + "Bad previous_sibling\nNODE: {}\nPREV: {}\nEXPECTED: {}".format( + el, el.previous_sibling, None + ) + assert el.next_sibling is None,\ + "Bad next_sibling\nNODE: {}\nNEXT: {}\nEXPECTED: {}".format( + el, el.next_sibling, None + ) + + idx = 0 + child = None + last_child = None + last_idx = len(el.contents) - 1 + for child in el.contents: + descendant = None + + # Parent should link next element to their first child + # That child should have no previous sibling + if idx == 0: + if el.parent is not None: + assert el.next_element is child,\ + "Bad next_element\nNODE: {}\nNEXT: {}\nEXPECTED: {}".format( + el, el.next_element, child + ) + assert child.previous_element is el,\ + "Bad previous_element\nNODE: {}\nPREV: {}\nEXPECTED: {}".format( + child, child.previous_element, el + ) + assert child.previous_sibling is None,\ + "Bad previous_sibling\nNODE: {}\nPREV {}\nEXPECTED: {}".format( + child, child.previous_sibling, None + ) + + # If not the first child, previous index should link as sibling to this index + # Previous element should match the last index or the last bubbled up descendant + else: + assert child.previous_sibling is el.contents[idx - 1],\ + "Bad previous_sibling\nNODE: {}\nPREV {}\nEXPECTED {}".format( + child, child.previous_sibling, el.contents[idx - 1] + ) + assert el.contents[idx - 1].next_sibling is child,\ + "Bad next_sibling\nNODE: {}\nNEXT {}\nEXPECTED {}".format( + el.contents[idx - 1], el.contents[idx - 1].next_sibling, child + ) + + if last_child is not None: + assert child.previous_element is last_child,\ + "Bad previous_element\nNODE: {}\nPREV {}\nEXPECTED {}\nCONTENTS {}".format( + child, child.previous_element, last_child, child.parent.contents + ) + assert last_child.next_element is child,\ + "Bad next_element\nNODE: {}\nNEXT {}\nEXPECTED {}".format( + last_child, last_child.next_element, child + ) + + if isinstance(child, Tag) and child.contents: + descendant = self.linkage_validator(child, True) + # A bubbled up descendant should have no next siblings + assert descendant.next_sibling is None,\ + "Bad next_sibling\nNODE: {}\nNEXT {}\nEXPECTED {}".format( + descendant, descendant.next_sibling, None + ) + + # Mark last child as either the bubbled up descendant or the current child + if descendant is not None: + last_child = descendant + else: + last_child = child + + # If last child, there are non next siblings + if idx == last_idx: + assert child.next_sibling is None,\ + "Bad next_sibling\nNODE: {}\nNEXT {}\nEXPECTED {}".format( + child, child.next_sibling, None + ) + idx += 1 + + child = descendant if descendant is not None else child + if child is None: + child = el + + if not _recursive_call and child is not None: + target = el + while True: + if target is None: + assert child.next_element is None, \ + "Bad next_element\nNODE: {}\nNEXT {}\nEXPECTED {}".format( + child, child.next_element, None + ) + break + elif target.next_sibling is not None: + assert child.next_element is target.next_sibling, \ + "Bad next_element\nNODE: {}\nNEXT {}\nEXPECTED {}".format( + child, child.next_element, target.next_sibling + ) + break + target = target.parent + + # We are done, so nothing to return + return None + else: + # Return the child to the recursive caller + return child + + +class HTMLTreeBuilderSmokeTest(object): + + """A basic test of a treebuilder's competence. + + Any HTML treebuilder, present or future, should be able to pass + these tests. With invalid markup, there's room for interpretation, + and different parsers can handle it differently. But with the + markup in these tests, there's not much room for interpretation. + """ + + def test_empty_element_tags(self): + """Verify that all HTML4 and HTML5 empty element (aka void element) tags + are handled correctly. + """ + for name in [ + 'area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input', 'keygen', 'link', 'menuitem', 'meta', 'param', 'source', 'track', 'wbr', + 'spacer', 'frame' + ]: + soup = self.soup("") + new_tag = soup.new_tag(name) + self.assertEqual(True, new_tag.is_empty_element) + + def test_special_string_containers(self): + soup = self.soup( + "" + ) + assert isinstance(soup.style.string, Stylesheet) + assert isinstance(soup.script.string, Script) + + soup = self.soup( + "" + ) + assert isinstance(soup.style.string, Stylesheet) + # The contents of the style tag resemble an HTML comment, but + # it's not treated as a comment. + self.assertEqual("", soup.style.string) + assert isinstance(soup.style.string, Stylesheet) + + def test_pickle_and_unpickle_identity(self): + # Pickling a tree, then unpickling it, yields a tree identical + # to the original. + tree = self.soup("foo") + dumped = pickle.dumps(tree, 2) + loaded = pickle.loads(dumped) + self.assertEqual(loaded.__class__, BeautifulSoup) + self.assertEqual(loaded.decode(), tree.decode()) + + def assertDoctypeHandled(self, doctype_fragment): + """Assert that a given doctype string is handled correctly.""" + doctype_str, soup = self._document_with_doctype(doctype_fragment) + + # Make sure a Doctype object was created. + doctype = soup.contents[0] + self.assertEqual(doctype.__class__, Doctype) + self.assertEqual(doctype, doctype_fragment) + self.assertEqual( + soup.encode("utf8")[:len(doctype_str)], + doctype_str + ) + + # Make sure that the doctype was correctly associated with the + # parse tree and that the rest of the document parsed. + self.assertEqual(soup.p.contents[0], 'foo') + + def _document_with_doctype(self, doctype_fragment, doctype_string="DOCTYPE"): + """Generate and parse a document with the given doctype.""" + doctype = '' % (doctype_string, doctype_fragment) + markup = doctype + '\n

foo

' + soup = self.soup(markup) + return doctype.encode("utf8"), soup + + def test_normal_doctypes(self): + """Make sure normal, everyday HTML doctypes are handled correctly.""" + self.assertDoctypeHandled("html") + self.assertDoctypeHandled( + 'html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"') + + def test_empty_doctype(self): + soup = self.soup("") + doctype = soup.contents[0] + self.assertEqual("", doctype.strip()) + + def test_mixed_case_doctype(self): + # A lowercase or mixed-case doctype becomes a Doctype. + for doctype_fragment in ("doctype", "DocType"): + doctype_str, soup = self._document_with_doctype( + "html", doctype_fragment + ) + + # Make sure a Doctype object was created and that the DOCTYPE + # is uppercase. + doctype = soup.contents[0] + self.assertEqual(doctype.__class__, Doctype) + self.assertEqual(doctype, "html") + self.assertEqual( + soup.encode("utf8")[:len(doctype_str)], + b"" + ) + + # Make sure that the doctype was correctly associated with the + # parse tree and that the rest of the document parsed. + self.assertEqual(soup.p.contents[0], 'foo') + + def test_public_doctype_with_url(self): + doctype = 'html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"' + self.assertDoctypeHandled(doctype) + + def test_system_doctype(self): + self.assertDoctypeHandled('foo SYSTEM "http://www.example.com/"') + + def test_namespaced_system_doctype(self): + # We can handle a namespaced doctype with a system ID. + self.assertDoctypeHandled('xsl:stylesheet SYSTEM "htmlent.dtd"') + + def test_namespaced_public_doctype(self): + # Test a namespaced doctype with a public id. + self.assertDoctypeHandled('xsl:stylesheet PUBLIC "htmlent.dtd"') + + def test_real_xhtml_document(self): + """A real XHTML document should come out more or less the same as it went in.""" + markup = b""" + + +Hello. +Goodbye. +""" + soup = self.soup(markup) + self.assertEqual( + soup.encode("utf-8").replace(b"\n", b""), + markup.replace(b"\n", b"")) + + def test_namespaced_html(self): + """When a namespaced XML document is parsed as HTML it should + be treated as HTML with weird tag names. + """ + markup = b"""content""" + soup = self.soup(markup) + self.assertEqual(2, len(soup.find_all("ns1:foo"))) + + def test_processing_instruction(self): + # We test both Unicode and bytestring to verify that + # process_markup correctly sets processing_instruction_class + # even when the markup is already Unicode and there is no + # need to process anything. + markup = u"""""" + soup = self.soup(markup) + self.assertEqual(markup, soup.decode()) + + markup = b"""""" + soup = self.soup(markup) + self.assertEqual(markup, soup.encode("utf8")) + + def test_deepcopy(self): + """Make sure you can copy the tree builder. + + This is important because the builder is part of a + BeautifulSoup object, and we want to be able to copy that. + """ + copy.deepcopy(self.default_builder) + + def test_p_tag_is_never_empty_element(self): + """A

tag is never designated as an empty-element tag. + + Even if the markup shows it as an empty-element tag, it + shouldn't be presented that way. + """ + soup = self.soup("

") + self.assertFalse(soup.p.is_empty_element) + self.assertEqual(str(soup.p), "

") + + def test_unclosed_tags_get_closed(self): + """A tag that's not closed by the end of the document should be closed. + + This applies to all tags except empty-element tags. + """ + self.assertSoupEquals("

", "

") + self.assertSoupEquals("", "") + + self.assertSoupEquals("
", "
") + + def test_br_is_always_empty_element_tag(self): + """A
tag is designated as an empty-element tag. + + Some parsers treat

as one
tag, some parsers as + two tags, but it should always be an empty-element tag. + """ + soup = self.soup("

") + self.assertTrue(soup.br.is_empty_element) + self.assertEqual(str(soup.br), "
") + + def test_nested_formatting_elements(self): + self.assertSoupEquals("") + + def test_double_head(self): + html = ''' + + +Ordinary HEAD element test + + + +Hello, world! + + +''' + soup = self.soup(html) + self.assertEqual("text/javascript", soup.find('script')['type']) + + def test_comment(self): + # Comments are represented as Comment objects. + markup = "

foobaz

" + self.assertSoupEquals(markup) + + soup = self.soup(markup) + comment = soup.find(text="foobar") + self.assertEqual(comment.__class__, Comment) + + # The comment is properly integrated into the tree. + foo = soup.find(text="foo") + self.assertEqual(comment, foo.next_element) + baz = soup.find(text="baz") + self.assertEqual(comment, baz.previous_element) + + def test_preserved_whitespace_in_pre_and_textarea(self): + """Whitespace must be preserved in 
 and "
+        self.assertSoupEquals(pre_markup)
+        self.assertSoupEquals(textarea_markup)
+
+        soup = self.soup(pre_markup)
+        self.assertEqual(soup.pre.prettify(), pre_markup)
+
+        soup = self.soup(textarea_markup)
+        self.assertEqual(soup.textarea.prettify(), textarea_markup)
+
+        soup = self.soup("")
+        self.assertEqual(soup.textarea.prettify(), "")
+
+    def test_nested_inline_elements(self):
+        """Inline elements can be nested indefinitely."""
+        b_tag = "Inside a B tag"
+        self.assertSoupEquals(b_tag)
+
+        nested_b_tag = "
A nested tag
"
+        self.assertSoupEquals(nested_b_tag)
+
+        double_nested_b_tag = "
A doubly nested tag
"
+        self.assertSoupEquals(nested_b_tag)
+
+    def test_nested_block_level_elements(self):
+        """Block elements can be nested."""
+        soup = self.soup('
Foo
')
+        blockquote = soup.blockquote
+        self.assertEqual(blockquote.p.b.string, 'Foo')
+        self.assertEqual(blockquote.b.string, 'Foo')
+
+    def test_correctly_nested_tables(self):
+        """One table can go inside another one."""
+        markup = (''
+                  ''
+                  "')
+
+        self.assertSoupEquals(
+            markup,
+            '
Here's another table:"
+                  ''
+                  ''
+                  '
foo
Here\'s another table:'
+            '
foo
'
+            '
')
+
+        self.assertSoupEquals(
+            ""
+            ""
+            "
Foo
Bar
Baz
")
+
+    def test_multivalued_attribute_with_whitespace(self):
+        # Whitespace separating the values of a multi-valued attribute
+        # should be ignored.
+
+        markup = '
'
+        soup = self.soup(markup)
+        self.assertEqual(['foo', 'bar'], soup.div['class'])
+
+        # If you search by the literal name of the class it's like the whitespace
+        # wasn't there.
+        self.assertEqual(soup.div, soup.find('div', class_="foo bar"))
+        
+    def test_deeply_nested_multivalued_attribute(self):
+        # html5lib can set the attributes of the same tag many times
+        # as it rearranges the tree. This has caused problems with
+        # multivalued attributes.
+        markup = '
'
+        soup = self.soup(markup)
+        self.assertEqual(["css"], soup.div.div['class'])
+
+    def test_multivalued_attribute_on_html(self):
+        # html5lib uses a different API to set the attributes ot the
+        #  tag. This has caused problems with multivalued
+        # attributes.
+        markup = ''
+        soup = self.soup(markup)
+        self.assertEqual(["a", "b"], soup.html['class'])
+
+    def test_angle_brackets_in_attribute_values_are_escaped(self):
+        self.assertSoupEquals('', '')
+
+    def test_strings_resembling_character_entity_references(self):
+        # "&T" and "&p" look like incomplete character entities, but they are
+        # not.
+        self.assertSoupEquals(
+            u"
• AT&T is in the s&p 500
",
+            u"
\u2022 AT&T is in the s&p 500
"
+        )
+
+    def test_apos_entity(self):
+        self.assertSoupEquals(
+            u"
Bob's Bar
",
+            u"
Bob's Bar
",
+        )
+        
+    def test_entities_in_foreign_document_encoding(self):
+        # “ and ” are invalid numeric entities referencing
+        # Windows-1252 characters. - references a character common
+        # to Windows-1252 and Unicode, and ☃ references a
+        # character only found in Unicode.
+        #
+        # All of these entities should be converted to Unicode
+        # characters.
+        markup = "
“Hello” -☃
"
+        soup = self.soup(markup)
+        self.assertEquals(u"“Hello†-☃", soup.p.string)
+        
+    def test_entities_in_attributes_converted_to_unicode(self):
+        expect = u'
'
+        self.assertSoupEquals('
', expect)
+        self.assertSoupEquals('
', expect)
+        self.assertSoupEquals('
', expect)
+        self.assertSoupEquals('
', expect)
+
+    def test_entities_in_text_converted_to_unicode(self):
+        expect = u'
pi\N{LATIN SMALL LETTER N WITH TILDE}ata
'
+        self.assertSoupEquals("
piñata
", expect)
+        self.assertSoupEquals("
piñata
", expect)
+        self.assertSoupEquals("
piñata
", expect)
+        self.assertSoupEquals("
piñata
", expect)
+
+    def test_quot_entity_converted_to_quotation_mark(self):
+        self.assertSoupEquals("
I said "good day!"
",
+                              '
I said "good day!"
')
+
+    def test_out_of_range_entity(self):
+        expect = u"\N{REPLACEMENT CHARACTER}"
+        self.assertSoupEquals("�", expect)
+        self.assertSoupEquals("�", expect)
+        self.assertSoupEquals("�", expect)
+        
+    def test_multipart_strings(self):
+        "Mostly to prevent a recurrence of a bug in the html5lib treebuilder."
+        soup = self.soup("
\nfoo
")
+        self.assertEqual("p", soup.h2.string.next_element.name)
+        self.assertEqual("p", soup.p.name)
+        self.assertConnectedness(soup)
+
+    def test_empty_element_tags(self):
+        """Verify consistent handling of empty-element tags,
+        no matter how they come in through the markup.
+        """
+        self.assertSoupEquals('


', "


")
+        self.assertSoupEquals('


', "


")
+        
+    def test_head_tag_between_head_and_body(self):
+        "Prevent recurrence of a bug in the html5lib treebuilder."
+        content = """
+  
+  foo
+
+"""
+        soup = self.soup(content)
+        self.assertNotEqual(None, soup.html.body)
+        self.assertConnectedness(soup)
+
+    def test_multiple_copies_of_a_tag(self):
+        "Prevent recurrence of a bug in the html5lib treebuilder."
+        content = """
+
+ 
+   
+  
+
+"""
+        soup = self.soup(content)
+        self.assertConnectedness(soup.article)
+
+    def test_basic_namespaces(self):
+        """Parsers don't need to *understand* namespaces, but at the
+        very least they should not choke on namespaces or lose
+        data."""
+
+        markup = b'4'
+        soup = self.soup(markup)
+        self.assertEqual(markup, soup.encode())
+        html = soup.html
+        self.assertEqual('http://www.w3.org/1999/xhtml', soup.html['xmlns'])
+        self.assertEqual(
+            'http://www.w3.org/1998/Math/MathML', soup.html['xmlns:mathml'])
+        self.assertEqual(
+            'http://www.w3.org/2000/svg', soup.html['xmlns:svg'])
+
+    def test_multivalued_attribute_value_becomes_list(self):
+        markup = b''
+        soup = self.soup(markup)
+        self.assertEqual(['foo', 'bar'], soup.a['class'])
+
+    #
+    # Generally speaking, tests below this point are more tests of
+    # Beautiful Soup than tests of the tree builders. But parsers are
+    # weird, so we run these tests separately for every tree builder
+    # to detect any differences between them.
+    #
+
+    def test_can_parse_unicode_document(self):
+        # A seemingly innocuous document... but it's in Unicode! And
+        # it contains characters that can't be represented in the
+        # encoding found in the  declaration! The horror!
+        markup = u'Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!'
+        soup = self.soup(markup)
+        self.assertEqual(u'Sacr\xe9 bleu!', soup.body.string)
+
+    def test_soupstrainer(self):
+        """Parsers should be able to work with SoupStrainers."""
+        strainer = SoupStrainer("b")
+        soup = self.soup("A bold  statement",
+                         parse_only=strainer)
+        self.assertEqual(soup.decode(), "bold")
+
+    def test_single_quote_attribute_values_become_double_quotes(self):
+        self.assertSoupEquals("",
+                              '')
+
+    def test_attribute_values_with_nested_quotes_are_left_alone(self):
+        text = """a"""
+        self.assertSoupEquals(text)
+
+    def test_attribute_values_with_double_nested_quotes_get_quoted(self):
+        text = """a"""
+        soup = self.soup(text)
+        soup.foo['attr'] = 'Brawls happen at "Bob\'s Bar"'
+        self.assertSoupEquals(
+            soup.foo.decode(),
+            """a""")
+
+    def test_ampersand_in_attribute_value_gets_escaped(self):
+        self.assertSoupEquals('',
+                              '')
+
+        self.assertSoupEquals(
+            'foo',
+            'foo')
+
+    def test_escaped_ampersand_in_attribute_value_is_left_alone(self):
+        self.assertSoupEquals('')
+
+    def test_entities_in_strings_converted_during_parsing(self):
+        # Both XML and HTML entities are converted to Unicode characters
+        # during parsing.
+        text = "
<<sacré bleu!>>
"
+        expected = u"
<<sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>>
"
+        self.assertSoupEquals(text, expected)
+
+    def test_smart_quotes_converted_on_the_way_in(self):
+        # Microsoft smart quotes are converted to Unicode characters during
+        # parsing.
+        quote = b"
\x91Foo\x92
"
+        soup = self.soup(quote)
+        self.assertEqual(
+            soup.p.string,
+            u"\N{LEFT SINGLE QUOTATION MARK}Foo\N{RIGHT SINGLE QUOTATION MARK}")
+
+    def test_non_breaking_spaces_converted_on_the_way_in(self):
+        soup = self.soup("  ")
+        self.assertEqual(soup.a.string, u"\N{NO-BREAK SPACE}" * 2)
+
+    def test_entities_converted_on_the_way_out(self):
+        text = "
<<sacré bleu!>>
"
+        expected = u"
<<sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>>
".encode("utf-8")
+        soup = self.soup(text)
+        self.assertEqual(soup.p.encode("utf-8"), expected)
+
+    def test_real_iso_latin_document(self):
+        # Smoke test of interrelated functionality, using an
+        # easy-to-understand document.
+
+        # Here it is in Unicode. Note that it claims to be in ISO-Latin-1.
+        unicode_html = u'
Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!
'
+
+        # That's because we're going to encode it into ISO-Latin-1, and use
+        # that to test.
+        iso_latin_html = unicode_html.encode("iso-8859-1")
+
+        # Parse the ISO-Latin-1 HTML.
+        soup = self.soup(iso_latin_html)
+        # Encode it to UTF-8.
+        result = soup.encode("utf-8")
+
+        # What do we expect the result to look like? Well, it would
+        # look like unicode_html, except that the META tag would say
+        # UTF-8 instead of ISO-Latin-1.
+        expected = unicode_html.replace("ISO-Latin-1", "utf-8")
+
+        # And, of course, it would be in UTF-8, not Unicode.
+        expected = expected.encode("utf-8")
+
+        # Ta-da!
+        self.assertEqual(result, expected)
+
+    def test_real_shift_jis_document(self):
+        # Smoke test to make sure the parser can handle a document in
+        # Shift-JIS encoding, without choking.
+        shift_jis_html = (
+            b'
'
+            b'\x82\xb1\x82\xea\x82\xcdShift-JIS\x82\xc5\x83R\x81[\x83f'
+            b'\x83B\x83\x93\x83O\x82\xb3\x82\xea\x82\xbd\x93\xfa\x96{\x8c'
+            b'\xea\x82\xcc\x83t\x83@\x83C\x83\x8b\x82\xc5\x82\xb7\x81B'
+            b'
')
+        unicode_html = shift_jis_html.decode("shift-jis")
+        soup = self.soup(unicode_html)
+
+        # Make sure the parse tree is correctly encoded to various
+        # encodings.
+        self.assertEqual(soup.encode("utf-8"), unicode_html.encode("utf-8"))
+        self.assertEqual(soup.encode("euc_jp"), unicode_html.encode("euc_jp"))
+
+    def test_real_hebrew_document(self):
+        # A real-world test to make sure we can convert ISO-8859-9 (a
+        # Hebrew encoding) to UTF-8.
+        hebrew_document = b'Hebrew (ISO 8859-8) in Visual Directionality
Hebrew (ISO 8859-8) in Visual Directionality
\xed\xe5\xec\xf9'
+        soup = self.soup(
+            hebrew_document, from_encoding="iso8859-8")
+        # Some tree builders call it iso8859-8, others call it iso-8859-9.
+        # That's not a difference we really care about.
+        assert soup.original_encoding in ('iso8859-8', 'iso-8859-8')
+        self.assertEqual(
+            soup.encode('utf-8'),
+            hebrew_document.decode("iso8859-8").encode("utf-8"))
+
+    def test_meta_tag_reflects_current_encoding(self):
+        # Here's the  tag saying that a document is
+        # encoded in Shift-JIS.
+        meta_tag = ('')
+
+        # Here's a document incorporating that meta tag.
+        shift_jis_html = (
+            '\n%s\n'
+            ''
+            'Shift-JIS markup goes here.') % meta_tag
+        soup = self.soup(shift_jis_html)
+
+        # Parse the document, and the charset is seemingly unaffected.
+        parsed_meta = soup.find('meta', {'http-equiv': 'Content-type'})
+        content = parsed_meta['content']
+        self.assertEqual('text/html; charset=x-sjis', content)
+
+        # But that value is actually a ContentMetaAttributeValue object.
+        self.assertTrue(isinstance(content, ContentMetaAttributeValue))
+
+        # And it will take on a value that reflects its current
+        # encoding.
+        self.assertEqual('text/html; charset=utf8', content.encode("utf8"))
+
+        # For the rest of the story, see TestSubstitutions in
+        # test_tree.py.
+
+    def test_html5_style_meta_tag_reflects_current_encoding(self):
+        # Here's the  tag saying that a document is
+        # encoded in Shift-JIS.
+        meta_tag = ('')
+
+        # Here's a document incorporating that meta tag.
+        shift_jis_html = (
+            '\n%s\n'
+            ''
+            'Shift-JIS markup goes here.') % meta_tag
+        soup = self.soup(shift_jis_html)
+
+        # Parse the document, and the charset is seemingly unaffected.
+        parsed_meta = soup.find('meta', id="encoding")
+        charset = parsed_meta['charset']
+        self.assertEqual('x-sjis', charset)
+
+        # But that value is actually a CharsetMetaAttributeValue object.
+        self.assertTrue(isinstance(charset, CharsetMetaAttributeValue))
+
+        # And it will take on a value that reflects its current
+        # encoding.
+        self.assertEqual('utf8', charset.encode("utf8"))
+
+    def test_python_specific_encodings_not_used_in_charset(self):
+        # You can encode an HTML document using a Python-specific
+        # encoding, but that encoding won't be mentioned _inside_ the
+        # resulting document. Instead, the document will appear to
+        # have no encoding.
+        for markup in [
+            b''
+            b''
+        ]:
+            soup = self.soup(markup)
+            for encoding in PYTHON_SPECIFIC_ENCODINGS:
+                if encoding in (
+                    u'idna', u'mbcs', u'oem', u'undefined',
+                    u'string_escape', u'string-escape'
+                ):
+                    # For one reason or another, these will raise an
+                    # exception if we actually try to use them, so don't
+                    # bother.
+                    continue
+                encoded = soup.encode(encoding)
+                assert b'meta charset=""' in encoded
+                assert encoding.encode("ascii") not in encoded
+        
+    def test_tag_with_no_attributes_can_have_attributes_added(self):
+        data = self.soup("text")
+        data.a['foo'] = 'bar'
+        self.assertEqual('text', data.a.decode())
+
+    def test_closing_tag_with_no_opening_tag(self):
+        # Without BeautifulSoup.open_tag_counter, the  tag will
+        # cause _popToTag to be called over and over again as we look
+        # for a  tag that wasn't there. The result is that 'text2'
+        # will show up outside the body of the document.
+        soup = self.soup("
text1
text2
")
+        self.assertEqual(
+            "
text1
text2
", soup.body.decode()
+        )
+        
+    def test_worst_case(self):
+        """Test the worst case (currently) for linking issues."""
+
+        soup = self.soup(BAD_DOCUMENT)
+        self.linkage_validator(soup)
+
+
+class XMLTreeBuilderSmokeTest(object):
+
+    def test_pickle_and_unpickle_identity(self):
+        # Pickling a tree, then unpickling it, yields a tree identical
+        # to the original.
+        tree = self.soup("foo")
+        dumped = pickle.dumps(tree, 2)
+        loaded = pickle.loads(dumped)
+        self.assertEqual(loaded.__class__, BeautifulSoup)
+        self.assertEqual(loaded.decode(), tree.decode())
+
+    def test_docstring_generated(self):
+        soup = self.soup("")
+        self.assertEqual(
+            soup.encode(), b'\n')
+
+    def test_xml_declaration(self):
+        markup = b"""\n"""
+        soup = self.soup(markup)
+        self.assertEqual(markup, soup.encode("utf8"))
+
+    def test_python_specific_encodings_not_used_in_xml_declaration(self):
+        # You can encode an XML document using a Python-specific
+        # encoding, but that encoding won't be mentioned _inside_ the
+        # resulting document.
+        markup = b"""\n"""
+        soup = self.soup(markup)
+        for encoding in PYTHON_SPECIFIC_ENCODINGS:
+            if encoding in (
+                u'idna', u'mbcs', u'oem', u'undefined',
+                u'string_escape', u'string-escape'
+            ):
+                # For one reason or another, these will raise an
+                # exception if we actually try to use them, so don't
+                # bother.
+                continue
+            encoded = soup.encode(encoding)
+            assert b'' in encoded
+            assert encoding.encode("ascii") not in encoded
+
+    def test_processing_instruction(self):
+        markup = b"""\n"""
+        soup = self.soup(markup)
+        self.assertEqual(markup, soup.encode("utf8"))
+
+    def test_real_xhtml_document(self):
+        """A real XHTML document should come out *exactly* the same as it went in."""
+        markup = b"""
+
+
+Hello.
+Goodbye.
+"""
+        soup = self.soup(markup)
+        self.assertEqual(
+            soup.encode("utf-8"), markup)
+       
+    def test_nested_namespaces(self):
+        doc = b"""
+
+
+
+
+
+"""
+        soup = self.soup(doc)
+        self.assertEqual(doc, soup.encode())
+        
+    def test_formatter_processes_script_tag_for_xml_documents(self):
+        doc = """
+  
+"""
+        soup = BeautifulSoup(doc, "lxml-xml")
+        # lxml would have stripped this while parsing, but we can add
+        # it later.
+        soup.script.string = 'console.log("< < hey > > ");'
+        encoded = soup.encode()
+        self.assertTrue(b"< < hey > >" in encoded)
+
+    def test_can_parse_unicode_document(self):
+        markup = u'Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!'
+        soup = self.soup(markup)
+        self.assertEqual(u'Sacr\xe9 bleu!', soup.root.string)
+
+    def test_popping_namespaced_tag(self):
+        markup = 'b2012-07-02T20:33:42Zcd'
+        soup = self.soup(markup)
+        self.assertEqual(
+            unicode(soup.rss), markup)
+
+    def test_docstring_includes_correct_encoding(self):
+        soup = self.soup("")
+        self.assertEqual(
+            soup.encode("latin1"),
+            b'\n')
+
+    def test_large_xml_document(self):
+        """A large XML document should come out the same as it went in."""
+        markup = (b'\n'
+                  + b'0' * (2**12)
+                  + b'')
+        soup = self.soup(markup)
+        self.assertEqual(soup.encode("utf-8"), markup)
+
+
+    def test_tags_are_empty_element_if_and_only_if_they_are_empty(self):
+        self.assertSoupEquals("
", "
")
+        self.assertSoupEquals("
foo
")
+
+    def test_namespaces_are_preserved(self):
+        markup = 'This tag is in the a namespaceThis tag is in the b namespace'
+        soup = self.soup(markup)
+        root = soup.root
+        self.assertEqual("http://example.com/", root['xmlns:a'])
+        self.assertEqual("http://example.net/", root['xmlns:b'])
+
+    def test_closing_namespaced_tag(self):
+        markup = '
20010504
'
+        soup = self.soup(markup)
+        self.assertEqual(unicode(soup.p), markup)
+
+    def test_namespaced_attributes(self):
+        markup = ''
+        soup = self.soup(markup)
+        self.assertEqual(unicode(soup.foo), markup)
+
+    def test_namespaced_attributes_xml_namespace(self):
+        markup = 'bar'
+        soup = self.soup(markup)
+        self.assertEqual(unicode(soup.foo), markup)
+
+    def test_find_by_prefixed_name(self):
+        doc = """
+foo
+    bar
+    baz
+
+"""
+        soup = self.soup(doc)
+
+        # There are three  tags.
+        self.assertEqual(3, len(soup.find_all('tag')))
+
+        # But two of them are ns1:tag and one of them is ns2:tag.
+        self.assertEqual(2, len(soup.find_all('ns1:tag')))
+        self.assertEqual(1, len(soup.find_all('ns2:tag')))
+        
+        self.assertEqual(1, len(soup.find_all('ns2:tag', key='value')))
+        self.assertEqual(3, len(soup.find_all(['ns1:tag', 'ns2:tag'])))
+        
+    def test_copy_tag_preserves_namespace(self):
+        xml = """
+"""
+    
+        soup = self.soup(xml)
+        tag = soup.document
+        duplicate = copy.copy(tag)
+
+        # The two tags have the same namespace prefix.
+        self.assertEqual(tag.prefix, duplicate.prefix)
+
+    def test_worst_case(self):
+        """Test the worst case (currently) for linking issues."""
+
+        soup = self.soup(BAD_DOCUMENT)
+        self.linkage_validator(soup)
+
+
+class HTML5TreeBuilderSmokeTest(HTMLTreeBuilderSmokeTest):
+    """Smoke test for a tree builder that supports HTML5."""
+
+    def test_real_xhtml_document(self):
+        # Since XHTML is not HTML5, HTML5 parsers are not tested to handle
+        # XHTML documents in any particular way.
+        pass
+
+    def test_html_tags_have_namespace(self):
+        markup = ""
+        soup = self.soup(markup)
+        self.assertEqual("http://www.w3.org/1999/xhtml", soup.a.namespace)
+
+    def test_svg_tags_have_namespace(self):
+        markup = ''
+        soup = self.soup(markup)
+        namespace = "http://www.w3.org/2000/svg"
+        self.assertEqual(namespace, soup.svg.namespace)
+        self.assertEqual(namespace, soup.circle.namespace)
+
+
+    def test_mathml_tags_have_namespace(self):
+        markup = '5'
+        soup = self.soup(markup)
+        namespace = 'http://www.w3.org/1998/Math/MathML'
+        self.assertEqual(namespace, soup.math.namespace)
+        self.assertEqual(namespace, soup.msqrt.namespace)
+
+    def test_xml_declaration_becomes_comment(self):
+        markup = ''
+        soup = self.soup(markup)
+        self.assertTrue(isinstance(soup.contents[0], Comment))
+        self.assertEqual(soup.contents[0], '?xml version="1.0" encoding="utf-8"?')
+        self.assertEqual("html", soup.contents[0].next_element.name)
+
+def skipIf(condition, reason):
+   def nothing(test, *args, **kwargs):
+       return None
+
+   def decorator(test_item):
+       if condition:
+           return nothing
+       else:
+           return test_item
+
+   return decorator
diff --git a/third_party/catapult/third_party/beautifulsoup4-4.9.3/bs4/tests/__init__.py b/third_party/catapult/third_party/beautifulsoup4-4.9.3/bs4/tests/__init__.py
new file mode 100644
index 00000000..142c8cc3
--- /dev/null
+++ b/third_party/catapult/third_party/beautifulsoup4-4.9.3/bs4/tests/__init__.py
@@ -0,0 +1 @@
+"The beautifulsoup tests."
diff --git a/third_party/catapult/third_party/beautifulsoup4-4.9.3/bs4/tests/test_builder_registry.py b/third_party/catapult/third_party/beautifulsoup4-4.9.3/bs4/tests/test_builder_registry.py
new file mode 100644
index 00000000..90cad829
--- /dev/null
+++ b/third_party/catapult/third_party/beautifulsoup4-4.9.3/bs4/tests/test_builder_registry.py
@@ -0,0 +1,147 @@
+"""Tests of the builder registry."""
+
+import unittest
+import warnings
+
+from bs4 import BeautifulSoup
+from bs4.builder import (
+    builder_registry as registry,
+    HTMLParserTreeBuilder,
+    TreeBuilderRegistry,
+)
+
+try:
+    from bs4.builder import HTML5TreeBuilder
+    HTML5LIB_PRESENT = True
+except ImportError:
+    HTML5LIB_PRESENT = False
+
+try:
+    from bs4.builder import (
+        LXMLTreeBuilderForXML,
+        LXMLTreeBuilder,
+        )
+    LXML_PRESENT = True
+except ImportError:
+    LXML_PRESENT = False
+
+
+class BuiltInRegistryTest(unittest.TestCase):
+    """Test the built-in registry with the default builders registered."""
+
+    def test_combination(self):
+        if LXML_PRESENT:
+            self.assertEqual(registry.lookup('fast', 'html'),
+                             LXMLTreeBuilder)
+
+        if LXML_PRESENT:
+            self.assertEqual(registry.lookup('permissive', 'xml'),
+                             LXMLTreeBuilderForXML)
+        self.assertEqual(registry.lookup('strict', 'html'),
+                          HTMLParserTreeBuilder)
+        if HTML5LIB_PRESENT:
+            self.assertEqual(registry.lookup('html5lib', 'html'),
+                              HTML5TreeBuilder)
+
+    def test_lookup_by_markup_type(self):
+        if LXML_PRESENT:
+            self.assertEqual(registry.lookup('html'), LXMLTreeBuilder)
+            self.assertEqual(registry.lookup('xml'), LXMLTreeBuilderForXML)
+        else:
+            self.assertEqual(registry.lookup('xml'), None)
+            if HTML5LIB_PRESENT:
+                self.assertEqual(registry.lookup('html'), HTML5TreeBuilder)
+            else:
+                self.assertEqual(registry.lookup('html'), HTMLParserTreeBuilder)
+
+    def test_named_library(self):
+        if LXML_PRESENT:
+            self.assertEqual(registry.lookup('lxml', 'xml'),
+                             LXMLTreeBuilderForXML)
+            self.assertEqual(registry.lookup('lxml', 'html'),
+                             LXMLTreeBuilder)
+        if HTML5LIB_PRESENT:
+            self.assertEqual(registry.lookup('html5lib'),
+                              HTML5TreeBuilder)
+
+        self.assertEqual(registry.lookup('html.parser'),
+                          HTMLParserTreeBuilder)
+
+    def test_beautifulsoup_constructor_does_lookup(self):
+
+        with warnings.catch_warnings(record=True) as w:
+            # This will create a warning about not explicitly
+            # specifying a parser, but we'll ignore it.
+
+            # You can pass in a string.
+            BeautifulSoup("", features="html")
+            # Or a list of strings.
+            BeautifulSoup("", features=["html", "fast"])
+
+        # You'll get an exception if BS can't find an appropriate
+        # builder.
+        self.assertRaises(ValueError, BeautifulSoup,
+                          "", features="no-such-feature")
+
+class RegistryTest(unittest.TestCase):
+    """Test the TreeBuilderRegistry class in general."""
+
+    def setUp(self):
+        self.registry = TreeBuilderRegistry()
+
+    def builder_for_features(self, *feature_list):
+        cls = type('Builder_' + '_'.join(feature_list),
+                   (object,), {'features' : feature_list})
+
+        self.registry.register(cls)
+        return cls
+
+    def test_register_with_no_features(self):
+        builder = self.builder_for_features()
+
+        # Since the builder advertises no features, you can't find it
+        # by looking up features.
+        self.assertEqual(self.registry.lookup('foo'), None)
+
+        # But you can find it by doing a lookup with no features, if
+        # this happens to be the only registered builder.
+        self.assertEqual(self.registry.lookup(), builder)
+
+    def test_register_with_features_makes_lookup_succeed(self):
+        builder = self.builder_for_features('foo', 'bar')
+        self.assertEqual(self.registry.lookup('foo'), builder)
+        self.assertEqual(self.registry.lookup('bar'), builder)
+
+    def test_lookup_fails_when_no_builder_implements_feature(self):
+        builder = self.builder_for_features('foo', 'bar')
+        self.assertEqual(self.registry.lookup('baz'), None)
+
+    def test_lookup_gets_most_recent_registration_when_no_feature_specified(self):
+        builder1 = self.builder_for_features('foo')
+        builder2 = self.builder_for_features('bar')
+        self.assertEqual(self.registry.lookup(), builder2)
+
+    def test_lookup_fails_when_no_tree_builders_registered(self):
+        self.assertEqual(self.registry.lookup(), None)
+
+    def test_lookup_gets_most_recent_builder_supporting_all_features(self):
+        has_one = self.builder_for_features('foo')
+        has_the_other = self.builder_for_features('bar')
+        has_both_early = self.builder_for_features('foo', 'bar', 'baz')
+        has_both_late = self.builder_for_features('foo', 'bar', 'quux')
+        lacks_one = self.builder_for_features('bar')
+        has_the_other = self.builder_for_features('foo')
+
+        # There are two builders featuring 'foo' and 'bar', but
+        # the one that also features 'quux' was registered later.
+        self.assertEqual(self.registry.lookup('foo', 'bar'),
+                          has_both_late)
+
+        # There is only one builder featuring 'foo', 'bar', and 'baz'.
+        self.assertEqual(self.registry.lookup('foo', 'bar', 'baz'),
+                          has_both_early)
+
+    def test_lookup_fails_when_cannot_reconcile_requested_features(self):
+        builder1 = self.builder_for_features('foo', 'bar')
+        builder2 = self.builder_for_features('foo', 'baz')
+        self.assertEqual(self.registry.lookup('bar', 'baz'), None)
diff --git a/third_party/catapult/third_party/beautifulsoup4-4.9.3/bs4/tests/test_docs.py b/third_party/catapult/third_party/beautifulsoup4-4.9.3/bs4/tests/test_docs.py
new file mode 100644
index 00000000..5b9f6770
--- /dev/null
+++ b/third_party/catapult/third_party/beautifulsoup4-4.9.3/bs4/tests/test_docs.py
@@ -0,0 +1,36 @@
+"Test harness for doctests."
+
+# pylint: disable-msg=E0611,W0142
+
+__metaclass__ = type
+__all__ = [
+    'additional_tests',
+    ]
+
+import atexit
+import doctest
+import os
+#from pkg_resources import (
+#    resource_filename, resource_exists, resource_listdir, cleanup_resources)
+import unittest
+
+DOCTEST_FLAGS = (
+    doctest.ELLIPSIS |
+    doctest.NORMALIZE_WHITESPACE |
+    doctest.REPORT_NDIFF)
+
+
+# def additional_tests():
+#     "Run the doc tests (README.txt and docs/*, if any exist)"
+#     doctest_files = [
+#         os.path.abspath(resource_filename('bs4', 'README.txt'))]
+#     if resource_exists('bs4', 'docs'):
+#         for name in resource_listdir('bs4', 'docs'):
+#             if name.endswith('.txt'):
+#                 doctest_files.append(
+#                     os.path.abspath(
+#                         resource_filename('bs4', 'docs/%s' % name)))
+#     kwargs = dict(module_relative=False, optionflags=DOCTEST_FLAGS)
+#     atexit.register(cleanup_resources)
+#     return unittest.TestSuite((
+#         doctest.DocFileSuite(*doctest_files, **kwargs)))
diff --git a/third_party/catapult/third_party/beautifulsoup4-4.9.3/bs4/tests/test_html5lib.py b/third_party/catapult/third_party/beautifulsoup4-4.9.3/bs4/tests/test_html5lib.py
new file mode 100644
index 00000000..7b0a6d44
--- /dev/null
+++ b/third_party/catapult/third_party/beautifulsoup4-4.9.3/bs4/tests/test_html5lib.py
@@ -0,0 +1,190 @@
+"""Tests to ensure that the html5lib tree builder generates good trees."""
+
+import warnings
+
+try:
+    from bs4.builder import HTML5TreeBuilder
+    HTML5LIB_PRESENT = True
+except ImportError, e:
+    HTML5LIB_PRESENT = False
+from bs4.element import SoupStrainer
+from bs4.testing import (
+    HTML5TreeBuilderSmokeTest,
+    SoupTest,
+    skipIf,
+)
+
+@skipIf(
+    not HTML5LIB_PRESENT,
+    "html5lib seems not to be present, not testing its tree builder.")
+class HTML5LibBuilderSmokeTest(SoupTest, HTML5TreeBuilderSmokeTest):
+    """See ``HTML5TreeBuilderSmokeTest``."""
+
+    @property
+    def default_builder(self):
+        return HTML5TreeBuilder
+
+    def test_soupstrainer(self):
+        # The html5lib tree builder does not support SoupStrainers.
+        strainer = SoupStrainer("b")
+        markup = "
A bold statement.
"
+        with warnings.catch_warnings(record=True) as w:
+            soup = self.soup(markup, parse_only=strainer)
+        self.assertEqual(
+            soup.decode(), self.document_for(markup))
+
+        self.assertTrue(
+            "the html5lib tree builder doesn't support parse_only" in
+            str(w[0].message))
+
+    def test_correctly_nested_tables(self):
+        """html5lib inserts  tags where other parsers don't."""
+        markup = (''
+                  ''
+                  "')
+
+        self.assertSoupEquals(
+            markup,
+            '
Here's another table:"
+                  ''
+                  ''
+                  '
foo
Here\'s another table:'
+            '
foo
'
+            '
')
+
+        self.assertSoupEquals(
+            ""
+            ""
+            "
Foo
Bar
Baz
")
+
+    def test_xml_declaration_followed_by_doctype(self):
+        markup = '''
+
+
+  
+  
+  
+   
foo

+  
+'''
+        soup = self.soup(markup)
+        # Verify that we can reach the 
 tag; this means the tree is connected.
+        self.assertEqual(b"
foo
", soup.p.encode())
+
+    def test_reparented_markup(self):
+        markup = '
foo
\n
bar
'
+        soup = self.soup(markup)
+        self.assertEqual(u"
foo
\n
bar
", soup.body.decode())
+        self.assertEqual(2, len(soup.find_all('p')))
+
+
+    def test_reparented_markup_ends_with_whitespace(self):
+        markup = '
foo
\n
bar
\n'
+        soup = self.soup(markup)
+        self.assertEqual(u"
foo
\n
bar
\n", soup.body.decode())
+        self.assertEqual(2, len(soup.find_all('p')))
+
+    def test_reparented_markup_containing_identical_whitespace_nodes(self):
+        """Verify that we keep the two whitespace nodes in this
+        document distinct when reparenting the adjacent  tags.
+        """
+        markup = '
  
'
+        soup = self.soup(markup)
+        space1, space2 = soup.find_all(string=' ')
+        tbody1, tbody2 = soup.find_all('tbody')
+        assert space1.next_element is tbody1
+        assert tbody2.next_element is space2
+
+    def test_reparented_markup_containing_children(self):
+        markup = ''
+        soup = self.soup(markup)
+        noscript = soup.noscript
+        self.assertEqual("target", noscript.next_element)
+        target = soup.find(string='target')
+
+        # The 'aftermath' string was duplicated; we want the second one.
+        final_aftermath = soup.find_all(string='aftermath')[-1]
+
+        # The