diff options
| author | Vito Graffagnino <vito@graffagnino.xyz> | 2020-09-08 18:10:49 +0100 |
|---|---|---|
| committer | Vito Graffagnino <vito@graffagnino.xyz> | 2020-09-08 18:10:49 +0100 |
| commit | 3b0142cedcde39e4c2097ecd916a870a3ced5ec6 (patch) | |
| tree | 2116c49a845dfc0945778f2aa3e2118d72be428b /.config/qutebrowser/scripts/dev | |
| parent | 8cc927e930d5b6aafe3e9862a61e81705479a1b4 (diff) | |
Added the relevent parts of the .config directory. Alss add ssh config
Diffstat (limited to '.config/qutebrowser/scripts/dev')
31 files changed, 3471 insertions, 0 deletions
diff --git a/.config/qutebrowser/scripts/dev/Makefile-dmg b/.config/qutebrowser/scripts/dev/Makefile-dmg new file mode 100644 index 0000000..1cf4cfb --- /dev/null +++ b/.config/qutebrowser/scripts/dev/Makefile-dmg @@ -0,0 +1,71 @@ +# +# Build file for creating DMG files. +# +# The DMG packager looks for a template.dmg.bz2 for using as its +# DMG template. If it doesn't find one, it generates a clean one. +# +# If you create a DMG template, you should make one containing all +# the files listed in $(SOURCE_FILES) below, and arrange everything to suit +# your style. The contents of the files themselves does not matter, so +# they can be empty (they will be overwritten later). +# +# Remko Tronçon +# https://el-tramo.be +# Licensed under the MIT License. See COPYING for details. + + +################################################################################ +# Customizable variables +################################################################################ + +NAME ?= qutebrowser + +SOURCE_DIR ?= . +SOURCE_FILES ?= dist/qutebrowser.app LICENSE + +TEMPLATE_DMG ?= template.dmg +TEMPLATE_SIZE ?= 300m + +################################################################################ +# DMG building. No editing should be needed beyond this point. +################################################################################ + +MASTER_DMG=$(NAME).dmg +WC_DMG=wc.dmg +WC_DIR=wc + +.PHONY: all +all: $(MASTER_DMG) + +$(TEMPLATE_DMG): + @echo + @echo --------------------- Generating empty template -------------------- + mkdir template + hdiutil create -fs HFSX -layout SPUD -size $(TEMPLATE_SIZE) "$(TEMPLATE_DMG)" -srcfolder template -format UDRW -volname "$(NAME)" -quiet + rmdir template + +$(WC_DMG): $(TEMPLATE_DMG) + cp $< $@ + +$(MASTER_DMG): $(WC_DMG) $(addprefix $(SOURCE_DIR)/,$(SOURCE_FILES)) + @echo + @echo --------------------- Creating Disk Image -------------------- + mkdir -p $(WC_DIR) + hdiutil attach "$(WC_DMG)" -noautoopen -quiet -mountpoint "$(WC_DIR)" + for i in $(SOURCE_FILES); do \ + rm -rf "$(WC_DIR)/$$i"; \ + ditto -rsrc "$(SOURCE_DIR)/$$i" "$(WC_DIR)/$${i##*/}"; \ + done + ln -s /Applications $(WC_DIR) + #rm -f "$@" + #hdiutil create -srcfolder "$(WC_DIR)" -format UDZO -imagekey zlib-level=9 "$@" -volname "$(NAME) $(VERSION)" -scrub -quiet + WC_DEV=`hdiutil info | grep "$(WC_DIR)" | grep "Apple_HFS" | awk '{print $$1}'` && \ + hdiutil detach $$WC_DEV -quiet -force + rm -f "$(MASTER_DMG)" + hdiutil convert "$(WC_DMG)" -quiet -format UDZO -imagekey zlib-level=9 -o "$@" + rm -rf $(WC_DIR) + @echo + +.PHONY: clean +clean: + -rm -rf $(TEMPLATE_DMG) $(MASTER_DMG) $(WC_DMG) diff --git a/.config/qutebrowser/scripts/dev/__init__.py b/.config/qutebrowser/scripts/dev/__init__.py new file mode 100644 index 0000000..7dc0433 --- /dev/null +++ b/.config/qutebrowser/scripts/dev/__init__.py @@ -0,0 +1,3 @@ +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +"""Various scripts used for developing qutebrowser.""" diff --git a/.config/qutebrowser/scripts/dev/build_release.py b/.config/qutebrowser/scripts/dev/build_release.py new file mode 100755 index 0000000..254132b --- /dev/null +++ b/.config/qutebrowser/scripts/dev/build_release.py @@ -0,0 +1,419 @@ +#!/usr/bin/env python3 +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org> +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + +"""Build a new release.""" + + +import os +import os.path +import sys +import time +import glob +import shutil +import plistlib +import subprocess +import argparse +import tarfile +import tempfile +import collections + +try: + import winreg +except ImportError: + pass + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir, + os.pardir)) + +import qutebrowser +from scripts import utils +# from scripts.dev import update_3rdparty + + +def call_script(name, *args, python=sys.executable): + """Call a given shell script. + + Args: + name: The script to call. + *args: The arguments to pass. + python: The python interpreter to use. + """ + path = os.path.join(os.path.dirname(__file__), os.pardir, name) + subprocess.run([python, path] + list(args), check=True) + + +def call_tox(toxenv, *args, python=sys.executable): + """Call tox. + + Args: + toxenv: Which tox environment to use + *args: The arguments to pass. + python: The python interpreter to use. + """ + env = os.environ.copy() + env['PYTHON'] = python + env['PATH'] = os.environ['PATH'] + os.pathsep + os.path.dirname(python) + subprocess.run( + [sys.executable, '-m', 'tox', '-vv', '-e', toxenv] + list(args), + env=env, check=True) + + +def run_asciidoc2html(args): + """Common buildsteps used for all OS'.""" + utils.print_title("Running asciidoc2html.py") + if args.asciidoc is not None: + a2h_args = ['--asciidoc'] + args.asciidoc + else: + a2h_args = [] + call_script('asciidoc2html.py', *a2h_args) + + +def _maybe_remove(path): + """Remove a path if it exists.""" + try: + shutil.rmtree(path) + except FileNotFoundError: + pass + + +def smoke_test(executable): + """Try starting the given qutebrowser executable.""" + subprocess.run([executable, '--no-err-windows', '--nowindow', + '--temp-basedir', 'about:blank', ':later 500 quit'], + check=True) + + +def patch_mac_app(): + """Patch .app to copy missing data and link some libs. + + See https://github.com/pyinstaller/pyinstaller/issues/2276 + """ + app_path = os.path.join('dist', 'qutebrowser.app') + qtwe_core_dir = os.path.join('.tox', 'pyinstaller', 'lib', 'python3.6', + 'site-packages', 'PyQt5', 'Qt', 'lib', + 'QtWebEngineCore.framework') + # Copy QtWebEngineProcess.app + proc_app = 'QtWebEngineProcess.app' + shutil.copytree(os.path.join(qtwe_core_dir, 'Helpers', proc_app), + os.path.join(app_path, 'Contents', 'MacOS', proc_app)) + # Copy resources + for f in glob.glob(os.path.join(qtwe_core_dir, 'Resources', '*')): + dest = os.path.join(app_path, 'Contents', 'Resources') + if os.path.isdir(f): + dir_dest = os.path.join(dest, os.path.basename(f)) + print("Copying directory {} to {}".format(f, dir_dest)) + shutil.copytree(f, dir_dest) + else: + print("Copying {} to {}".format(f, dest)) + shutil.copy(f, dest) + # Link dependencies + for lib in ['QtCore', 'QtWebEngineCore', 'QtQuick', 'QtQml', 'QtNetwork', + 'QtGui', 'QtWebChannel', 'QtPositioning']: + dest = os.path.join(app_path, lib + '.framework', 'Versions', '5') + os.makedirs(dest) + os.symlink(os.path.join(os.pardir, os.pardir, os.pardir, 'Contents', + 'MacOS', lib), + os.path.join(dest, lib)) + # Patch Info.plist - pyinstaller's options are too limiting + plist_path = os.path.join(app_path, 'Contents', 'Info.plist') + with open(plist_path, "rb") as f: + plist_data = plistlib.load(f) + plist_data.update(INFO_PLIST_UPDATES) + with open(plist_path, "wb") as f: + plistlib.dump(plist_data, f) + + +INFO_PLIST_UPDATES = { + 'CFBundleVersion': qutebrowser.__version__, + 'CFBundleShortVersionString': qutebrowser.__version__, + 'NSSupportsAutomaticGraphicsSwitching': True, + 'NSHighResolutionCapable': True, + 'CFBundleURLTypes': [{ + "CFBundleURLName": "http(s) URL", + "CFBundleURLSchemes": ["http", "https"] + }, { + "CFBundleURLName": "local file URL", + "CFBundleURLSchemes": ["file"] + }], + 'CFBundleDocumentTypes': [{ + "CFBundleTypeExtensions": ["html", "htm"], + "CFBundleTypeMIMETypes": ["text/html"], + "CFBundleTypeName": "HTML document", + "CFBundleTypeOSTypes": ["HTML"], + "CFBundleTypeRole": "Viewer", + }, { + "CFBundleTypeExtensions": ["xhtml"], + "CFBundleTypeMIMETypes": ["text/xhtml"], + "CFBundleTypeName": "XHTML document", + "CFBundleTypeRole": "Viewer", + }] +} + + +def build_mac(): + """Build macOS .dmg/.app.""" + utils.print_title("Cleaning up...") + for f in ['wc.dmg', 'template.dmg']: + try: + os.remove(f) + except FileNotFoundError: + pass + for d in ['dist', 'build']: + shutil.rmtree(d, ignore_errors=True) + utils.print_title("Updating 3rdparty content") + # Currently disabled because QtWebEngine has no pdfjs support + # update_3rdparty.run(ace=False, pdfjs=True, fancy_dmg=False) + utils.print_title("Building .app via pyinstaller") + call_tox('pyinstaller', '-r') + utils.print_title("Patching .app") + patch_mac_app() + utils.print_title("Building .dmg") + subprocess.run(['make', '-f', 'scripts/dev/Makefile-dmg'], check=True) + + dmg_name = 'qutebrowser-{}.dmg'.format(qutebrowser.__version__) + os.rename('qutebrowser.dmg', dmg_name) + + utils.print_title("Running smoke test") + + try: + with tempfile.TemporaryDirectory() as tmpdir: + subprocess.run(['hdiutil', 'attach', dmg_name, + '-mountpoint', tmpdir], check=True) + try: + binary = os.path.join(tmpdir, 'qutebrowser.app', 'Contents', + 'MacOS', 'qutebrowser') + smoke_test(binary) + finally: + time.sleep(5) + subprocess.run(['hdiutil', 'detach', tmpdir]) + except PermissionError as e: + print("Failed to remove tempdir: {}".format(e)) + + return [(dmg_name, 'application/x-apple-diskimage', 'macOS .dmg')] + + +def patch_windows(out_dir): + """Copy missing DLLs for windows into the given output.""" + dll_dir = os.path.join('.tox', 'pyinstaller', 'lib', 'site-packages', + 'PyQt5', 'Qt', 'bin') + dlls = ['libEGL.dll', 'libGLESv2.dll', 'libeay32.dll', 'ssleay32.dll'] + for dll in dlls: + shutil.copy(os.path.join(dll_dir, dll), out_dir) + + +def build_windows(): + """Build windows executables/setups.""" + utils.print_title("Updating 3rdparty content") + # Currently disabled because QtWebEngine has no pdfjs support + # update_3rdparty.run(ace=False, pdfjs=True, fancy_dmg=False) + + utils.print_title("Building Windows binaries") + parts = str(sys.version_info.major), str(sys.version_info.minor) + ver = ''.join(parts) + dot_ver = '.'.join(parts) + + # Get python path from registry if possible + try: + reg64_key = winreg.OpenKeyEx(winreg.HKEY_LOCAL_MACHINE, + r'SOFTWARE\Python\PythonCore' + r'\{}\InstallPath'.format(dot_ver)) + python_x64 = winreg.QueryValueEx(reg64_key, 'ExecutablePath')[0] + except FileNotFoundError: + python_x64 = r'C:\Python{}\python.exe'.format(ver) + + out_pyinstaller = os.path.join('dist', 'qutebrowser') + out_64 = os.path.join('dist', + 'qutebrowser-{}-x64'.format(qutebrowser.__version__)) + + artifacts = [] + + from scripts.dev import gen_versioninfo + utils.print_title("Updating VersionInfo file") + gen_versioninfo.main() + + utils.print_title("Running pyinstaller 64bit") + _maybe_remove(out_64) + call_tox('pyinstaller', '-r', python=python_x64) + shutil.move(out_pyinstaller, out_64) + patch_windows(out_64) + + utils.print_title("Building installers") + subprocess.run(['makensis.exe', + '/DX64', + '/DVERSION={}'.format(qutebrowser.__version__), + 'misc/qutebrowser.nsi'], check=True) + + name_64 = 'qutebrowser-{}-amd64.exe'.format(qutebrowser.__version__) + + artifacts += [ + (os.path.join('dist', name_64), + 'application/vnd.microsoft.portable-executable', + 'Windows 64bit installer'), + ] + + utils.print_title("Running 64bit smoke test") + smoke_test(os.path.join(out_64, 'qutebrowser.exe')) + + utils.print_title("Zipping 64bit standalone...") + name = 'qutebrowser-{}-windows-standalone-amd64'.format( + qutebrowser.__version__) + shutil.make_archive(name, 'zip', 'dist', os.path.basename(out_64)) + artifacts.append(('{}.zip'.format(name), + 'application/zip', + 'Windows 64bit standalone')) + + return artifacts + + +def build_sdist(): + """Build an sdist and list the contents.""" + utils.print_title("Building sdist") + + _maybe_remove('dist') + + subprocess.run([sys.executable, 'setup.py', 'sdist'], check=True) + dist_files = os.listdir(os.path.abspath('dist')) + assert len(dist_files) == 1 + + dist_file = os.path.join('dist', dist_files[0]) + subprocess.run(['gpg', '--detach-sign', '-a', dist_file], check=True) + + tar = tarfile.open(dist_file) + by_ext = collections.defaultdict(list) + + for tarinfo in tar.getmembers(): + if not tarinfo.isfile(): + continue + name = os.sep.join(tarinfo.name.split(os.sep)[1:]) + _base, ext = os.path.splitext(name) + by_ext[ext].append(name) + + assert '.pyc' not in by_ext + + utils.print_title("sdist contents") + + for ext, files in sorted(by_ext.items()): + utils.print_subtitle(ext) + print('\n'.join(files)) + + filename = 'qutebrowser-{}.tar.gz'.format(qutebrowser.__version__) + artifacts = [ + (os.path.join('dist', filename), 'application/gzip', 'Source release'), + (os.path.join('dist', filename + '.asc'), 'application/pgp-signature', + 'Source release - PGP signature'), + ] + + return artifacts + + +def test_makefile(): + """Make sure the Makefile works correctly.""" + utils.print_title("Testing makefile") + with tempfile.TemporaryDirectory() as tmpdir: + subprocess.run(['make', '-f', 'misc/Makefile', + 'DESTDIR={}'.format(tmpdir), 'install'], check=True) + + +def read_github_token(): + """Read the GitHub API token from disk.""" + token_file = os.path.join(os.path.expanduser('~'), '.gh_token') + with open(token_file, encoding='ascii') as f: + token = f.read().strip() + return token + + +def github_upload(artifacts, tag): + """Upload the given artifacts to GitHub. + + Args: + artifacts: A list of (filename, mimetype, description) tuples + tag: The name of the release tag + """ + import github3 + utils.print_title("Uploading to github...") + + token = read_github_token() + gh = github3.login(token=token) + repo = gh.repository('qutebrowser', 'qutebrowser') + + release = None # to satisfy pylint + for release in repo.releases(): + if release.tag_name == tag: + break + else: + raise Exception("No release found for {!r}!".format(tag)) + + for filename, mimetype, description in artifacts: + with open(filename, 'rb') as f: + basename = os.path.basename(filename) + asset = release.upload_asset(mimetype, basename, f) + asset.edit(basename, description) + + +def pypi_upload(artifacts): + """Upload the given artifacts to PyPI using twine.""" + filenames = [a[0] for a in artifacts] + subprocess.run(['twine', 'upload'] + filenames, check=True) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--asciidoc', help="Full path to python and " + "asciidoc.py. If not given, it's searched in PATH.", + nargs=2, required=False, + metavar=('PYTHON', 'ASCIIDOC')) + parser.add_argument('--upload', help="Tag to upload the release for", + nargs=1, required=False, metavar='TAG') + args = parser.parse_args() + utils.change_cwd() + + upload_to_pypi = False + + if args.upload is not None: + # Fail early when trying to upload without github3 installed + # or without API token + import github3 # pylint: disable=unused-variable + read_github_token() + + run_asciidoc2html(args) + if os.name == 'nt': + artifacts = build_windows() + elif sys.platform == 'darwin': + artifacts = build_mac() + else: + test_makefile() + artifacts = build_sdist() + upload_to_pypi = True + + if args.upload is not None: + utils.print_title("Press enter to release...") + input() + github_upload(artifacts, args.upload[0]) + if upload_to_pypi: + pypi_upload(artifacts) + else: + print() + utils.print_title("Artifacts") + for artifact in artifacts: + print(artifact) + + +if __name__ == '__main__': + main() diff --git a/.config/qutebrowser/scripts/dev/check_coverage.py b/.config/qutebrowser/scripts/dev/check_coverage.py new file mode 100644 index 0000000..32c5afc --- /dev/null +++ b/.config/qutebrowser/scripts/dev/check_coverage.py @@ -0,0 +1,348 @@ +#!/usr/bin/env python3 +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2015-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org> + +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + +"""Enforce perfect coverage on some files.""" + +import os +import os.path +import sys +import enum +import subprocess +from xml.etree import ElementTree + +import attr + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir, + os.pardir)) + +from scripts import utils as scriptutils +from qutebrowser.utils import utils + + +@attr.s +class Message: + + """A message shown by coverage.py.""" + + typ = attr.ib() + filename = attr.ib() + text = attr.ib() + + +MsgType = enum.Enum('MsgType', 'insufficent_coverage, perfect_file') + + +# A list of (test_file, tested_file) tuples. test_file can be None. +PERFECT_FILES = [ + (None, + 'commands/cmdexc.py'), + ('tests/unit/commands/test_cmdutils.py', + 'commands/cmdutils.py'), + ('tests/unit/commands/test_argparser.py', + 'commands/argparser.py'), + + ('tests/unit/browser/webkit/test_cache.py', + 'browser/webkit/cache.py'), + ('tests/unit/browser/webkit/test_cookies.py', + 'browser/webkit/cookies.py'), + ('tests/unit/browser/test_history.py', + 'browser/history.py'), + ('tests/unit/browser/webkit/http/test_http.py', + 'browser/webkit/http.py'), + ('tests/unit/browser/webkit/http/test_content_disposition.py', + 'browser/webkit/rfc6266.py'), + # ('tests/unit/browser/webkit/test_webkitelem.py', + # 'browser/webkit/webkitelem.py'), + # ('tests/unit/browser/webkit/test_webkitelem.py', + # 'browser/webelem.py'), + ('tests/unit/browser/webkit/network/test_filescheme.py', + 'browser/webkit/network/filescheme.py'), + ('tests/unit/browser/webkit/network/test_networkreply.py', + 'browser/webkit/network/networkreply.py'), + + ('tests/unit/browser/test_signalfilter.py', + 'browser/signalfilter.py'), + (None, + 'browser/webengine/certificateerror.py'), + # ('tests/unit/browser/test_tab.py', + # 'browser/tab.py'), + + ('tests/unit/keyinput/test_basekeyparser.py', + 'keyinput/basekeyparser.py'), + ('tests/unit/keyinput/test_keyutils.py', + 'keyinput/keyutils.py'), + + ('tests/unit/misc/test_autoupdate.py', + 'misc/autoupdate.py'), + ('tests/unit/misc/test_readline.py', + 'misc/readline.py'), + ('tests/unit/misc/test_split.py', + 'misc/split.py'), + ('tests/unit/misc/test_msgbox.py', + 'misc/msgbox.py'), + ('tests/unit/misc/test_checkpyver.py', + 'misc/checkpyver.py'), + ('tests/unit/misc/test_guiprocess.py', + 'misc/guiprocess.py'), + ('tests/unit/misc/test_editor.py', + 'misc/editor.py'), + ('tests/unit/misc/test_cmdhistory.py', + 'misc/cmdhistory.py'), + ('tests/unit/misc/test_ipc.py', + 'misc/ipc.py'), + ('tests/unit/misc/test_keyhints.py', + 'misc/keyhintwidget.py'), + ('tests/unit/misc/test_pastebin.py', + 'misc/pastebin.py'), + (None, + 'misc/objects.py'), + + (None, + 'mainwindow/statusbar/keystring.py'), + ('tests/unit/mainwindow/statusbar/test_percentage.py', + 'mainwindow/statusbar/percentage.py'), + ('tests/unit/mainwindow/statusbar/test_progress.py', + 'mainwindow/statusbar/progress.py'), + ('tests/unit/mainwindow/statusbar/test_tabindex.py', + 'mainwindow/statusbar/tabindex.py'), + ('tests/unit/mainwindow/statusbar/test_textbase.py', + 'mainwindow/statusbar/textbase.py'), + ('tests/unit/mainwindow/statusbar/test_url.py', + 'mainwindow/statusbar/url.py'), + ('tests/unit/mainwindow/statusbar/test_backforward.py', + 'mainwindow/statusbar/backforward.py'), + ('tests/unit/mainwindow/test_messageview.py', + 'mainwindow/messageview.py'), + + ('tests/unit/config/test_config.py', + 'config/config.py'), + ('tests/unit/config/test_configdata.py', + 'config/configdata.py'), + ('tests/unit/config/test_configexc.py', + 'config/configexc.py'), + ('tests/unit/config/test_configfiles.py', + 'config/configfiles.py'), + ('tests/unit/config/test_configtypes.py', + 'config/configtypes.py'), + ('tests/unit/config/test_configinit.py', + 'config/configinit.py'), + ('tests/unit/config/test_configcommands.py', + 'config/configcommands.py'), + ('tests/unit/config/test_configutils.py', + 'config/configutils.py'), + + ('tests/unit/utils/test_qtutils.py', + 'utils/qtutils.py'), + ('tests/unit/utils/test_standarddir.py', + 'utils/standarddir.py'), + ('tests/unit/utils/test_urlutils.py', + 'utils/urlutils.py'), + ('tests/unit/utils/usertypes', + 'utils/usertypes.py'), + ('tests/unit/utils/test_utils.py', + 'utils/utils.py'), + ('tests/unit/utils/test_version.py', + 'utils/version.py'), + ('tests/unit/utils/test_debug.py', + 'utils/debug.py'), + ('tests/unit/utils/test_jinja.py', + 'utils/jinja.py'), + ('tests/unit/utils/test_error.py', + 'utils/error.py'), + ('tests/unit/utils/test_javascript.py', + 'utils/javascript.py'), + ('tests/unit/utils/test_urlmatch.py', + 'utils/urlmatch.py'), + + (None, + 'completion/models/util.py'), + ('tests/unit/completion/test_models.py', + 'completion/models/urlmodel.py'), + ('tests/unit/completion/test_models.py', + 'completion/models/configmodel.py'), + ('tests/unit/completion/test_histcategory.py', + 'completion/models/histcategory.py'), + ('tests/unit/completion/test_listcategory.py', + 'completion/models/listcategory.py'), + + ('tests/unit/browser/webengine/test_spell.py', + 'browser/webengine/spell.py'), + +] + + +# 100% coverage because of end2end tests, but no perfect unit tests yet. +WHITELISTED_FILES = [ + 'browser/webkit/webkitinspector.py', + 'keyinput/macros.py', + 'browser/webkit/webkitelem.py', +] + + +class Skipped(Exception): + + """Exception raised when skipping coverage checks.""" + + def __init__(self, reason): + self.reason = reason + super().__init__("Skipping coverage checks " + reason) + + +def _get_filename(filename): + """Transform the absolute test filenames to relative ones.""" + if os.path.isabs(filename): + basedir = os.path.abspath( + os.path.join(os.path.dirname(__file__), '..', '..')) + common_path = os.path.commonprefix([basedir, filename]) + if common_path: + filename = filename[len(common_path):].lstrip('/') + if filename.startswith('qutebrowser/'): + filename = filename.split('/', maxsplit=1)[1] + + return filename + + +def check(fileobj, perfect_files): + """Main entry point which parses/checks coverage.xml if applicable.""" + if not utils.is_linux: + raise Skipped("on non-Linux system.") + elif '-k' in sys.argv[1:]: + raise Skipped("because -k is given.") + elif '-m' in sys.argv[1:]: + raise Skipped("because -m is given.") + elif '--lf' in sys.argv[1:]: + raise Skipped("because --lf is given.") + + perfect_src_files = [e[1] for e in perfect_files] + + filename_args = [arg for arg in sys.argv[1:] + if arg.startswith('tests' + os.sep)] + filtered_files = [tpl[1] for tpl in perfect_files if tpl[0] in + filename_args] + + if filename_args and not filtered_files: + raise Skipped("because there is nothing to check.") + + tree = ElementTree.parse(fileobj) + classes = tree.getroot().findall('./packages/package/classes/class') + + messages = [] + + for klass in classes: + filename = _get_filename(klass.attrib['filename']) + + line_cov = float(klass.attrib['line-rate']) * 100 + branch_cov = float(klass.attrib['branch-rate']) * 100 + + if filtered_files and filename not in filtered_files: + continue + + assert 0 <= line_cov <= 100, line_cov + assert 0 <= branch_cov <= 100, branch_cov + assert '\\' not in filename, filename + + is_bad = line_cov < 100 or branch_cov < 100 + + if filename in perfect_src_files and is_bad: + text = "{} has {:.2f}% line and {:.2f}% branch coverage!".format( + filename, line_cov, branch_cov) + messages.append(Message(MsgType.insufficent_coverage, filename, + text)) + elif (filename not in perfect_src_files and not is_bad and + filename not in WHITELISTED_FILES): + text = ("{} has 100% coverage but is not in " + "perfect_files!".format(filename)) + messages.append(Message(MsgType.perfect_file, filename, text)) + + return messages + + +def main_check(): + """Check coverage after a test run.""" + try: + with open('coverage.xml', encoding='utf-8') as f: + messages = check(f, PERFECT_FILES) + except Skipped as e: + print(e) + messages = [] + + if messages: + print() + print() + scriptutils.print_title("Coverage check failed") + for msg in messages: + print(msg.text) + print() + filters = ','.join('qutebrowser/' + msg.filename for msg in messages) + subprocess.run([sys.executable, '-m', 'coverage', 'report', + '--show-missing', '--include', filters], check=True) + print() + print("To debug this, run 'tox -e py36-pyqt59-cov' " + "(or py35-pyqt59-cov) locally and check htmlcov/index.html") + print("or check https://codecov.io/github/qutebrowser/qutebrowser") + print() + + if 'CI' in os.environ: + print("Keeping coverage.xml on CI.") + else: + os.remove('coverage.xml') + return 1 if messages else 0 + + +def main_check_all(): + """Check the coverage for all files individually. + + This makes sure the files have 100% coverage without running unrelated + tests. + + This runs pytest with the used executable, so check_coverage.py should be + called with something like ./.tox/py36/bin/python. + """ + for test_file, src_file in PERFECT_FILES: + if test_file is None: + continue + subprocess.run( + [sys.executable, '-m', 'pytest', '--cov', 'qutebrowser', + '--cov-report', 'xml', test_file], check=True) + with open('coverage.xml', encoding='utf-8') as f: + messages = check(f, [(test_file, src_file)]) + os.remove('coverage.xml') + + messages = [msg for msg in messages + if msg.typ == MsgType.insufficent_coverage] + if messages: + for msg in messages: + print(msg.text) + return 1 + else: + print("Check ok!") + return 0 + + +def main(): + scriptutils.change_cwd() + if '--check-all' in sys.argv: + return main_check_all() + else: + return main_check() + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/.config/qutebrowser/scripts/dev/check_doc_changes.py b/.config/qutebrowser/scripts/dev/check_doc_changes.py new file mode 100755 index 0000000..3d90bea --- /dev/null +++ b/.config/qutebrowser/scripts/dev/check_doc_changes.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2016-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org> + +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + +"""Check if docs changed and output an error if so.""" + +import sys +import subprocess +import os + +code = subprocess.run(['git', '--no-pager', 'diff', + '--exit-code', '--stat']).returncode + +if os.environ.get('TRAVIS_PULL_REQUEST', 'false') != 'false': + if code != 0: + print("Docs changed but ignoring change as we're building a PR") + sys.exit(0) + +if code != 0: + print() + print('The autogenerated docs changed, please run this to update them:') + print(' tox -e docs') + print(' git commit -am "Update docs"') + print() + print('(Or you have uncommitted changes, in which case you can ignore ' + 'this.)') + if 'TRAVIS' in os.environ: + print() + print("travis_fold:start:gitdiff") + subprocess.run(['git', '--no-pager', 'diff']) + print("travis_fold:end:gitdiff") +sys.exit(code) diff --git a/.config/qutebrowser/scripts/dev/ci/travis_backtrace.sh b/.config/qutebrowser/scripts/dev/ci/travis_backtrace.sh new file mode 100644 index 0000000..227dde8 --- /dev/null +++ b/.config/qutebrowser/scripts/dev/ci/travis_backtrace.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# +# Find all possible core files under current directory. Attempt +# to determine exe using file(1) and dump stack trace with gdb. +# + +case $TESTENV in + py3*-pyqt*) + exe=$(readlink -f ".tox/$TESTENV/bin/python") + full= + ;; + *) + echo "Skipping coredump analysis in testenv $TESTENV!" + exit 0 + ;; +esac + +find . \( -name "*.core" -o -name core \) -exec gdb --batch --quiet -ex "thread apply all bt $full" "$exe" {} \; diff --git a/.config/qutebrowser/scripts/dev/ci/travis_install.sh b/.config/qutebrowser/scripts/dev/ci/travis_install.sh new file mode 100644 index 0000000..18f5aa9 --- /dev/null +++ b/.config/qutebrowser/scripts/dev/ci/travis_install.sh @@ -0,0 +1,108 @@ +#!/bin/bash +# vim: ft=sh fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org> + +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + +# Stolen from https://github.com/travis-ci/travis-build/blob/master/lib/travis/build/templates/header.sh +# and adjusted to use ((...)) +travis_retry() { + local ANSI_RED='\033[31;1m' + local ANSI_RESET='\033[0m' + local result=0 + local count=1 + while (( count < 3 )); do + if (( result != 0 )); then + echo -e "\\n${ANSI_RED}The command \"$*\" failed. Retrying, $count of 3.${ANSI_RESET}\\n" >&2 + fi + "$@" + result=$? + (( result == 0 )) && break + count=$(( count + 1 )) + sleep 1 + done + + if (( count > 3 )); then + echo -e "\\n${ANSI_RED}The command \"$*\" failed 3 times.${ANSI_RESET}\\n" >&2 + fi + + return $result +} + +pip_install() { + travis_retry python3 -m pip install "$@" +} + +npm_install() { + # Make sure npm is up-to-date first + travis_retry npm install -g npm + travis_retry npm install -g "$@" +} + +check_pyqt() { + python3 <<EOF +import sys +from PyQt5.QtCore import QT_VERSION_STR, PYQT_VERSION_STR, qVersion +try: + from PyQt.sip import SIP_VERSION_STR +except ModuleNotFoundError: + from sip import SIP_VERSION_STR + +print("Python {}".format(sys.version)) +print("PyQt5 {}".format(PYQT_VERSION_STR)) +print("Qt5 {} (runtime {})".format(QT_VERSION_STR, qVersion())) +print("sip {}".format(SIP_VERSION_STR)) +EOF +} + +set -e + +if [[ $DOCKER ]]; then + exit 0 +elif [[ $TRAVIS_OS_NAME == osx ]]; then + # Disable App Nap + defaults write NSGlobalDomain NSAppSleepDisabled -bool YES + + curl -LO https://bootstrap.pypa.io/get-pip.py + sudo -H python get-pip.py + + brew --version + brew update + brew upgrade python libyaml + brew install qt5 pyqt5 + + pip_install -r misc/requirements/requirements-tox.txt + python3 -m pip --version + tox --version + check_pyqt + exit 0 +fi + +case $TESTENV in + eslint) + npm_install eslint + ;; + shellcheck) + ;; + *) + pip_install pip + pip_install -r misc/requirements/requirements-tox.txt + if [[ $TESTENV == *-cov ]]; then + pip_install -r misc/requirements/requirements-codecov.txt + fi + ;; +esac diff --git a/.config/qutebrowser/scripts/dev/ci/travis_run.sh b/.config/qutebrowser/scripts/dev/ci/travis_run.sh new file mode 100644 index 0000000..55ca7c1 --- /dev/null +++ b/.config/qutebrowser/scripts/dev/ci/travis_run.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +if [[ $DOCKER ]]; then + docker run \ + --privileged \ + -v "$PWD:/outside" \ + -e "QUTE_BDD_WEBENGINE=$QUTE_BDD_WEBENGINE" \ + -e "DOCKER=$DOCKER" \ + -e "CI=$CI" \ + -e "TRAVIS=$TRAVIS" \ + "qutebrowser/travis:$DOCKER" +elif [[ $TESTENV == eslint ]]; then + # Can't run this via tox as we can't easily install tox in the javascript + # travis env + cd qutebrowser/javascript || exit 1 + eslint --color --report-unused-disable-directives . +elif [[ $TESTENV == shellcheck ]]; then + SCRIPTS=$( mktemp ) + find scripts/dev/ -name '*.sh' >"$SCRIPTS" + find misc/userscripts/ -type f -exec grep -lE '[/ ][bd]ash$|[/ ]sh$|[/ ]ksh$' {} + >>"$SCRIPTS" + mapfile -t scripts <"$SCRIPTS" + rm -f "$SCRIPTS" + docker run \ + -v "$PWD:/outside" \ + -w /outside \ + koalaman/shellcheck:latest "${scripts[@]}" +else + args=() + [[ $TRAVIS_OS_NAME == osx ]] && args=('--qute-bdd-webengine' '--no-xvfb' 'tests/unit') + + tox -e "$TESTENV" -- "${args[@]}" +fi diff --git a/.config/qutebrowser/scripts/dev/cleanup.py b/.config/qutebrowser/scripts/dev/cleanup.py new file mode 100755 index 0000000..d1bb84a --- /dev/null +++ b/.config/qutebrowser/scripts/dev/cleanup.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org> +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + +"""Script to clean up the mess made by Python/setuptools/PyInstaller.""" + +import os +import os.path +import sys +import glob +import shutil +import fnmatch + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir, + os.pardir)) + +from scripts import utils + + +recursive_lint = ('__pycache__', '*.pyc') +lint = ('build', 'dist', 'pkg/pkg', 'pkg/qutebrowser-*.pkg.tar.xz', 'pkg/src', + 'pkg/qutebrowser', 'qutebrowser.egg-info', 'setuptools-*.egg', + 'setuptools-*.zip', 'doc/qutebrowser.asciidoc', 'doc/*.html', + 'doc/qutebrowser.1', 'README.html', 'qutebrowser/html/doc') + + +def remove(path): + """Remove either a file or directory unless --dry-run is given.""" + if os.path.isdir(path): + print("rm -r '{}'".format(path)) + if '--dry-run' not in sys.argv: + shutil.rmtree(path) + else: + print("rm '{}'".format(path)) + if '--dry-run' not in sys.argv: + os.remove(path) + + +def main(): + """Clean up lint in the current dir.""" + utils.change_cwd() + for elem in lint: + for f in glob.glob(elem): + remove(f) + + for root, _dirs, _files in os.walk(os.getcwd()): + path = os.path.basename(root) + if any(fnmatch.fnmatch(path, e) for e in recursive_lint): + remove(root) + + +if __name__ == '__main__': + main() diff --git a/.config/qutebrowser/scripts/dev/download_release.sh b/.config/qutebrowser/scripts/dev/download_release.sh new file mode 100644 index 0000000..207da21 --- /dev/null +++ b/.config/qutebrowser/scripts/dev/download_release.sh @@ -0,0 +1,34 @@ +#!/bin/bash +set -e + +# This script downloads the given release from GitHub so we can mirror it on +# qutebrowser.org. + +tmpdir=$(mktemp -d) +oldpwd=$PWD + +if [[ $# != 1 ]]; then + echo "Usage: $0 <version>" >&2 + exit 1 +fi + +cd "$tmpdir" +mkdir windows + +base="https://github.com/qutebrowser/qutebrowser/releases/download/v$1" + +wget "$base/qutebrowser-$1.tar.gz" +wget "$base/qutebrowser-$1.tar.gz.asc" +wget "$base/qutebrowser-$1.dmg" +wget "$base/qutebrowser_${1}-1_all.deb" + +cd windows +wget "$base/qutebrowser-${1}-amd64.msi" +wget "$base/qutebrowser-${1}-win32.msi" +wget "$base/qutebrowser-${1}-windows-standalone-amd64.zip" +wget "$base/qutebrowser-${1}-windows-standalone-win32.zip" + +dest="/srv/http/qutebrowser/releases/v$1" +cd "$oldpwd" +sudo mv "$tmpdir" "$dest" +sudo chown -R http:http "$dest" diff --git a/.config/qutebrowser/scripts/dev/gen_resources.py b/.config/qutebrowser/scripts/dev/gen_resources.py new file mode 100644 index 0000000..cbfc69b --- /dev/null +++ b/.config/qutebrowser/scripts/dev/gen_resources.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# copyright 2014 florian bruhin (the compiler) <mail@qutebrowser.org> + +# this file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the gnu general public license as published by +# the free software foundation, either version 3 of the license, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but without any warranty; without even the implied warranty of +# merchantability or fitness for a particular purpose. see the +# gnu general public license for more details. +# +# you should have received a copy of the gnu general public license +# along with qutebrowser. if not, see <http://www.gnu.org/licenses/>. + +"""Generate Qt resources based on source files.""" + +import subprocess + +with open('qutebrowser/resources.py', 'w', encoding='utf-8') as f: + subprocess.run(['pyrcc5', 'qutebrowser.rcc'], stdout=f, check=True) diff --git a/.config/qutebrowser/scripts/dev/gen_versioninfo.py b/.config/qutebrowser/scripts/dev/gen_versioninfo.py new file mode 100644 index 0000000..1aa4b64 --- /dev/null +++ b/.config/qutebrowser/scripts/dev/gen_versioninfo.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org> +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + +"""Generate file_version_info.txt for Pyinstaller use with Windows builds.""" + +import os.path +import sys + +# pylint: disable=import-error,no-member,useless-suppression +from PyInstaller.utils.win32 import versioninfo as vs +# pylint: enable=import-error,no-member,useless-suppression + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir, + os.pardir)) + +import qutebrowser +from scripts import utils + + +def main(): + utils.change_cwd() + out_filename = 'misc/file_version_info.txt' + + filevers = qutebrowser.__version_info__ + (0,) + prodvers = qutebrowser.__version_info__ + (0,) + str_filevers = qutebrowser.__version__ + str_prodvers = qutebrowser.__version__ + + comment_text = qutebrowser.__doc__ + copyright_text = qutebrowser.__copyright__ + trademark_text = ("qutebrowser is free software under the GNU General " + "Public License") + + # https://www.science.co.il/language/Locale-codes.php#definitions + # https://msdn.microsoft.com/en-us/library/windows/desktop/dd317756.aspx + en_us = 1033 # 0x0409 + utf_16 = 1200 # 0x04B0 + + ffi = vs.FixedFileInfo(filevers, prodvers) + + kids = [ + vs.StringFileInfo([ + # 0x0409: MAKELANGID(LANG_ENGLISH, SUBLANG_ENGLISH_US) + # 0x04B0: codepage 1200 (UTF-16LE) + vs.StringTable('040904B0', [ + vs.StringStruct('Comments', comment_text), + vs.StringStruct('CompanyName', "qutebrowser.org"), + vs.StringStruct('FileDescription', "qutebrowser"), + vs.StringStruct('FileVersion', str_filevers), + vs.StringStruct('InternalName', "qutebrowser"), + vs.StringStruct('LegalCopyright', copyright_text), + vs.StringStruct('LegalTrademarks', trademark_text), + vs.StringStruct('OriginalFilename', "qutebrowser.exe"), + vs.StringStruct('ProductName', "qutebrowser"), + vs.StringStruct('ProductVersion', str_prodvers) + ]), + ]), + vs.VarFileInfo([vs.VarStruct('Translation', [en_us, utf_16])]), + ] + + file_version_info = vs.VSVersionInfo(ffi, kids) + + with open(out_filename, 'w', encoding='utf-8') as f: + f.write(str(file_version_info)) + + +if __name__ == '__main__': + main() diff --git a/.config/qutebrowser/scripts/dev/get_coredumpctl_traces.py b/.config/qutebrowser/scripts/dev/get_coredumpctl_traces.py new file mode 100644 index 0000000..d286d38 --- /dev/null +++ b/.config/qutebrowser/scripts/dev/get_coredumpctl_traces.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2015-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org> + +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + +"""Get qutebrowser crash information and stacktraces from coredumpctl.""" + +import os +import os.path +import sys +import argparse +import subprocess +import tempfile + +import attr + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir, + os.pardir)) + +from scripts import utils + + +@attr.s +class Line: + + """A line in "coredumpctl list".""" + + time = attr.ib() + pid = attr.ib() + uid = attr.ib() + gid = attr.ib() + sig = attr.ib() + present = attr.ib() + exe = attr.ib() + + +def _convert_present(data): + """Convert " "/"*" to True/False for parse_coredumpctl_line.""" + if data == '*': + return True + elif data == ' ': + return False + else: + raise ValueError(data) + + +def parse_coredumpctl_line(line): + """Parse a given string coming from coredumpctl and return a Line object. + + Example input: + Mon 2015-09-28 23:22:24 CEST 10606 1000 1000 11 /usr/bin/python3.4 + """ + fields = { + 'time': (0, 28, str), + 'pid': (29, 35, int), + 'uid': (36, 41, int), + 'gid': (42, 47, int), + 'sig': (48, 51, int), + 'present': (52, 53, _convert_present), + 'exe': (54, None, str), + } + + data = {} + for name, (start, end, converter) in fields.items(): + data[name] = converter(line[start:end]) + return Line(**data) + + +def get_info(pid): + """Get and parse "coredumpctl info" output for the given PID.""" + data = {} + output = subprocess.run(['coredumpctl', 'info', str(pid)], check=True, + stdout=subprocess.PIPE).stdout + output = output.decode('utf-8') + for line in output.split('\n'): + if not line.strip(): + continue + try: + key, value = line.split(':', maxsplit=1) + except ValueError: + # systemd stack output + continue + data[key.strip()] = value.strip() + return data + + +def is_qutebrowser_dump(parsed): + """Check if the given Line is a qutebrowser dump.""" + basename = os.path.basename(parsed.exe) + if basename == 'python' or basename.startswith('python3'): + info = get_info(parsed.pid) + try: + cmdline = info['Command Line'] + except KeyError: + return True + else: + return '-m qutebrowser' in cmdline + else: + return basename == 'qutebrowser' + + +def dump_infos_gdb(parsed): + """Dump all needed infos for the given crash using gdb.""" + with tempfile.TemporaryDirectory() as tempdir: + coredump = os.path.join(tempdir, 'dump') + subprocess.run(['coredumpctl', 'dump', '-o', coredump, + str(parsed.pid)], check=True) + subprocess.run(['gdb', parsed.exe, coredump, + '-ex', 'info threads', + '-ex', 'thread apply all bt full', + '-ex', 'quit'], check=True) + + +def dump_infos(parsed): + """Dump all possible infos for the given crash.""" + if not parsed.present: + info = get_info(parsed.pid) + print("{}: Signal {} with no coredump: {}".format( + parsed.time, info.get('Signal', None), + info.get('Command Line', None))) + else: + print('\n\n\n') + utils.print_title('{} - {}'.format(parsed.time, parsed.pid)) + sys.stdout.flush() + dump_infos_gdb(parsed) + + +def check_prerequisites(): + """Check if coredumpctl/gdb are installed.""" + for binary in ['coredumpctl', 'gdb']: + try: + subprocess.run([binary, '--version'], check=True) + except FileNotFoundError: + print("{} is needed to run this script!".format(binary), + file=sys.stderr) + sys.exit(1) + + +def main(): + check_prerequisites() + + parser = argparse.ArgumentParser() + parser.add_argument('--all', help="Also list crashes without coredumps.", + action='store_true') + args = parser.parse_args() + + coredumps = subprocess.run(['coredumpctl', 'list'], check=True, + stdout=subprocess.PIPE).stdout + lines = coredumps.decode('utf-8').split('\n') + for line in lines[1:]: + if not line.strip(): + continue + parsed = parse_coredumpctl_line(line) + if not parsed.present and not args.all: + continue + if is_qutebrowser_dump(parsed): + dump_infos(parsed) + + +if __name__ == '__main__': + main() diff --git a/.config/qutebrowser/scripts/dev/misc_checks.py b/.config/qutebrowser/scripts/dev/misc_checks.py new file mode 100644 index 0000000..2992464 --- /dev/null +++ b/.config/qutebrowser/scripts/dev/misc_checks.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org> +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + +"""Various small code checkers.""" + +import os +import os.path +import re +import sys +import argparse +import subprocess +import tokenize +import traceback +import collections + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir, + os.pardir)) + +from scripts import utils + + +def _get_files(only_py=False): + """Iterate over all python files and yield filenames.""" + for (dirpath, _dirnames, filenames) in os.walk('.'): + parts = dirpath.split(os.sep) + if len(parts) >= 2: + rootdir = parts[1] + if rootdir.startswith('.') or rootdir == 'htmlcov': + # ignore hidden dirs and htmlcov + continue + + if only_py: + endings = {'.py'} + else: + endings = {'.py', '.asciidoc', '.js', '.feature'} + files = (e for e in filenames if os.path.splitext(e)[1] in endings) + for name in files: + yield os.path.join(dirpath, name) + + +def check_git(): + """Check for uncommitted git files..""" + if not os.path.isdir(".git"): + print("No .git dir, ignoring") + print() + return False + untracked = [] + gitst = subprocess.run(['git', 'status', '--porcelain'], check=True, + stdout=subprocess.PIPE).stdout + gitst = gitst.decode('UTF-8').strip() + for line in gitst.splitlines(): + s, name = line.split(maxsplit=1) + if s == '??' and name != '.venv/': + untracked.append(name) + status = True + if untracked: + status = False + utils.print_col("Untracked files:", 'red') + print('\n'.join(untracked)) + print() + return status + + +def check_spelling(): + """Check commonly misspelled words.""" + # Words which I often misspell + words = {'[Bb]ehaviour', '[Qq]uitted', 'Ll]ikelyhood', '[Ss]ucessfully', + '[Oo]ccur[^rs .]', '[Ss]eperator', '[Ee]xplicitely', + '[Aa]uxillary', '[Aa]ccidentaly', '[Aa]mbigious', '[Ll]oosly', + '[Ii]nitialis', '[Cc]onvienence', '[Ss]imiliar', '[Uu]ncommited', + '[Rr]eproducable', '[Aa]n [Uu]ser', '[Cc]onvienience', + '[Ww]ether', '[Pp]rogramatically', '[Ss]plitted', '[Ee]xitted', + '[Mm]ininum', '[Rr]esett?ed', '[Rr]ecieved', '[Rr]egularily', + '[Uu]nderlaying', '[Ii]nexistant', '[Ee]lipsis', 'commiting', + 'existant', '[Rr]esetted', '[Ss]imilarily', '[Ii]nformations'} + + # Words which look better when splitted, but might need some fine tuning. + words |= {'[Ww]ebelements', '[Mm]ouseevent', '[Kk]eysequence', + '[Nn]ormalmode', '[Ee]ventloops', '[Ss]izehint', + '[Ss]tatemachine', '[Mm]etaobject', '[Ll]ogrecord', + '[Ff]iletype'} + + # Files which should be ignored, e.g. because they come from another + # package + ignored = [ + os.path.join('.', 'scripts', 'dev', 'misc_checks.py'), + os.path.join('.', 'qutebrowser', '3rdparty', 'pdfjs'), + os.path.join('.', 'tests', 'end2end', 'data', 'hints', 'ace', + 'ace.js'), + ] + + seen = collections.defaultdict(list) + try: + ok = True + for fn in _get_files(): + with tokenize.open(fn) as f: + if any(fn.startswith(i) for i in ignored): + continue + for line in f: + for w in words: + if (re.search(w, line) and + fn not in seen[w] and + '# pragma: no spellcheck' not in line): + print('Found "{}" in {}!'.format(w, fn)) + seen[w].append(fn) + ok = False + print() + return ok + except Exception: + traceback.print_exc() + return None + + +def check_vcs_conflict(): + """Check VCS conflict markers.""" + try: + ok = True + for fn in _get_files(only_py=True): + with tokenize.open(fn) as f: + for line in f: + if any(line.startswith(c * 7) for c in '<>=|'): + print("Found conflict marker in {}".format(fn)) + ok = False + print() + return ok + except Exception: + traceback.print_exc() + return None + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('checker', choices=('git', 'vcs', 'spelling'), + help="Which checker to run.") + args = parser.parse_args() + if args.checker == 'git': + ok = check_git() + elif args.checker == 'vcs': + ok = check_vcs_conflict() + elif args.checker == 'spelling': + ok = check_spelling() + return 0 if ok else 1 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/.config/qutebrowser/scripts/dev/pylint_checkers/qute_pylint/__init__.py b/.config/qutebrowser/scripts/dev/pylint_checkers/qute_pylint/__init__.py new file mode 100644 index 0000000..1341a93 --- /dev/null +++ b/.config/qutebrowser/scripts/dev/pylint_checkers/qute_pylint/__init__.py @@ -0,0 +1 @@ +"""Custom pylint checkers.""" diff --git a/.config/qutebrowser/scripts/dev/pylint_checkers/qute_pylint/config.py b/.config/qutebrowser/scripts/dev/pylint_checkers/qute_pylint/config.py new file mode 100644 index 0000000..5aa5250 --- /dev/null +++ b/.config/qutebrowser/scripts/dev/pylint_checkers/qute_pylint/config.py @@ -0,0 +1,84 @@ +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org> +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + +"""Custom astroid checker for config calls.""" + +import sys +import pathlib + +import yaml +import astroid +from pylint import interfaces, checkers +from pylint.checkers import utils + + +OPTIONS = None +FAILED_LOAD = False + + +class ConfigChecker(checkers.BaseChecker): + + """Custom astroid checker for config calls.""" + + __implements__ = interfaces.IAstroidChecker + name = 'config' + msgs = { + 'E9998': ('%s is no valid config option.', # flake8: disable=S001 + 'bad-config-option', + None), + } + priority = -1 + printed_warning = False + + @utils.check_messages('bad-config-option') + def visit_attribute(self, node): + """Visit a getattr node.""" + # At the end of a config.val.foo.bar chain + if not isinstance(node.parent, astroid.Attribute): + # FIXME:conf do some proper check for this... + node_str = node.as_string() + prefix = 'config.val.' + if node_str.startswith(prefix): + self._check_config(node, node_str[len(prefix):]) + + def _check_config(self, node, name): + """Check that we're accessing proper config options.""" + if FAILED_LOAD: + if not ConfigChecker.printed_warning: + print("[WARN] Could not find configdata.yml. Please run " + "pylint from qutebrowser root.", file=sys.stderr) + print("Skipping some checks...", file=sys.stderr) + ConfigChecker.printed_warning = True + return + if name not in OPTIONS: + self.add_message('bad-config-option', node=node, args=name) + + +def register(linter): + """Register this checker.""" + linter.register_checker(ConfigChecker(linter)) + global OPTIONS + global FAILED_LOAD + yaml_file = pathlib.Path('qutebrowser') / 'config' / 'configdata.yml' + if not yaml_file.exists(): + OPTIONS = None + FAILED_LOAD = True + return + with yaml_file.open(mode='r', encoding='utf-8') as f: + OPTIONS = list(yaml.load(f)) diff --git a/.config/qutebrowser/scripts/dev/pylint_checkers/qute_pylint/modeline.py b/.config/qutebrowser/scripts/dev/pylint_checkers/qute_pylint/modeline.py new file mode 100644 index 0000000..429974c --- /dev/null +++ b/.config/qutebrowser/scripts/dev/pylint_checkers/qute_pylint/modeline.py @@ -0,0 +1,63 @@ +# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org> +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + +"""Checker for vim modelines in files.""" + +import os.path +import contextlib + +from pylint import interfaces, checkers + + +class ModelineChecker(checkers.BaseChecker): + + """Check for vim modelines in files.""" + + __implements__ = interfaces.IRawChecker + + name = 'modeline' + msgs = {'W9002': ('Does not have vim modeline', 'modeline-missing', None), + 'W9003': ('Modeline is invalid', 'invalid-modeline', None), + 'W9004': ('Modeline position is wrong', 'modeline-position', None)} + options = () + priority = -1 + + def process_module(self, node): + """Process the module.""" + if os.path.basename(os.path.splitext(node.file)[0]) == '__init__': + return + max_lineno = 1 + with contextlib.closing(node.stream()) as stream: + for (lineno, line) in enumerate(stream): + if lineno == 1 and line.startswith(b'#!'): + max_lineno += 1 + continue + elif line.startswith(b'# vim:'): + if lineno > max_lineno: + self.add_message('modeline-position', line=lineno) + if (line.rstrip() != b'# vim: ft=python ' + b'fileencoding=utf-8 sts=4 sw=4 et:'): + self.add_message('invalid-modeline', line=lineno) + break + else: + self.add_message('modeline-missing', line=1) + + +def register(linter): + """Register the checker.""" + linter.register_checker(ModelineChecker(linter)) diff --git a/.config/qutebrowser/scripts/dev/pylint_checkers/qute_pylint/openencoding.py b/.config/qutebrowser/scripts/dev/pylint_checkers/qute_pylint/openencoding.py new file mode 100644 index 0000000..f577011 --- /dev/null +++ b/.config/qutebrowser/scripts/dev/pylint_checkers/qute_pylint/openencoding.py @@ -0,0 +1,83 @@ +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org> + +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + +"""Make sure open() has an encoding set.""" + +import astroid +from pylint import interfaces, checkers +from pylint.checkers import utils + + +class OpenEncodingChecker(checkers.BaseChecker): + + """Checker to check open() has an encoding set.""" + + __implements__ = interfaces.IAstroidChecker + name = 'open-encoding' + + msgs = { + 'W9400': ('open() called without encoding', 'open-without-encoding', + None), + } + + @utils.check_messages('open-without-encoding') + def visit_call(self, node): + """Visit a Call node.""" + if hasattr(node, 'func'): + infer = utils.safe_infer(node.func) + if infer and infer.root().name == '_io': + if getattr(node.func, 'name', None) in ['open', 'file']: + self._check_open_encoding(node) + + def _check_open_encoding(self, node): + """Check that an open() call always has an encoding set.""" + try: + mode_arg = utils.get_argument_from_call(node, position=1, + keyword='mode') + except utils.NoSuchArgumentError: + mode_arg = None + _encoding = None + try: + _encoding = utils.get_argument_from_call(node, position=2) + except utils.NoSuchArgumentError: + try: + _encoding = utils.get_argument_from_call(node, + keyword='encoding') + except utils.NoSuchArgumentError: + pass + if _encoding is None: + if mode_arg is None: + mode = None + else: + mode = utils.safe_infer(mode_arg) + if mode is not None and not isinstance(mode, astroid.Const): + # We can't say what mode is exactly. + return + if mode is None: + self.add_message('open-without-encoding', node=node) + elif 'b' in getattr(mode, 'value', ''): + # Files opened as binary don't need an encoding. + return + else: + self.add_message('open-without-encoding', node=node) + + +def register(linter): + """Register this checker.""" + linter.register_checker(OpenEncodingChecker(linter)) diff --git a/.config/qutebrowser/scripts/dev/pylint_checkers/qute_pylint/settrace.py b/.config/qutebrowser/scripts/dev/pylint_checkers/qute_pylint/settrace.py new file mode 100644 index 0000000..c82d646 --- /dev/null +++ b/.config/qutebrowser/scripts/dev/pylint_checkers/qute_pylint/settrace.py @@ -0,0 +1,49 @@ +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org> +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + +"""Custom astroid checker for set_trace calls.""" + +from pylint.interfaces import IAstroidChecker +from pylint.checkers import BaseChecker, utils + + +class SetTraceChecker(BaseChecker): + + """Custom astroid checker for set_trace calls.""" + + __implements__ = IAstroidChecker + name = 'settrace' + msgs = { + 'E9101': ('set_trace call found', 'set-trace', None), + } + priority = -1 + + @utils.check_messages('set-trace') + def visit_call(self, node): + """Visit a Call node.""" + if hasattr(node, 'func'): + infer = utils.safe_infer(node.func) + if infer: + if getattr(node.func, 'name', None) == 'set_trace': + self.add_message('set-trace', node=node) + + +def register(linter): + """Register this checker.""" + linter.register_checker(SetTraceChecker(linter)) diff --git a/.config/qutebrowser/scripts/dev/pylint_checkers/setup.py b/.config/qutebrowser/scripts/dev/pylint_checkers/setup.py new file mode 100644 index 0000000..7833c7d --- /dev/null +++ b/.config/qutebrowser/scripts/dev/pylint_checkers/setup.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 + +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2016-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org> +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + +"""This is only here so we can install those plugins in tox.ini easily.""" + +from setuptools import setup +setup(name='qute_pylint', packages=['qute_pylint']) diff --git a/.config/qutebrowser/scripts/dev/quit_segfault_test.sh b/.config/qutebrowser/scripts/dev/quit_segfault_test.sh new file mode 100755 index 0000000..389f125 --- /dev/null +++ b/.config/qutebrowser/scripts/dev/quit_segfault_test.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +[[ $PWD == */scripts ]] && cd .. + +echo > crash.log +while :; do + exit=0 + while (( exit == 0 )); do + duration=$(( RANDOM % 10000 )) + python3 -m qutebrowser --debug ":later $duration quit" http://www.heise.de/ + exit=$? + done + echo "$(date) $exit $duration" >> crash.log +done diff --git a/.config/qutebrowser/scripts/dev/recompile_requirements.py b/.config/qutebrowser/scripts/dev/recompile_requirements.py new file mode 100644 index 0000000..6e26145 --- /dev/null +++ b/.config/qutebrowser/scripts/dev/recompile_requirements.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2016-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org> +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + +"""Script to regenerate requirements files in misc/requirements.""" + +import re +import sys +import os.path +import glob +import subprocess +import tempfile + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir, + os.pardir)) + +from scripts import utils + +REPO_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), + '..', '..') # /scripts/dev -> /scripts -> / +REQ_DIR = os.path.join(REPO_DIR, 'misc', 'requirements') + + +def convert_line(line, comments): + """Convert the given requirement line to place into the output.""" + for pattern, repl in comments['replace'].items(): + line = re.sub(pattern, repl, line) + + pkgname = line.split('=')[0] + + if pkgname in comments['ignore']: + line = '# ' + line + + try: + line += ' # ' + comments['comment'][pkgname] + except KeyError: + pass + + try: + line += ' # rq.filter: {}'.format(comments['filter'][pkgname]) + except KeyError: + pass + + return line + + +def read_comments(fobj): + """Find special comments in the config. + + Args: + fobj: A file object for the config. + + Return: + A dict with the parsed comment data. + """ + comments = { + 'filter': {}, + 'comment': {}, + 'ignore': [], + 'replace': {}, + } + for line in fobj: + if line.startswith('#@'): + command, args = line[2:].split(':', maxsplit=1) + command = command.strip() + args = args.strip() + if command == 'filter': + pkg, filt = args.split(' ', maxsplit=1) + comments['filter'][pkg] = filt + elif command == 'comment': + pkg, comment = args.split(' ', maxsplit=1) + comments['comment'][pkg] = comment + elif command == 'ignore': + comments['ignore'] += args.split(', ') + elif command == 'replace': + pattern, replacement = args.split(' ', maxsplit=1) + comments['replace'][pattern] = replacement + return comments + + +def get_all_names(): + """Get all requirement names based on filenames.""" + for filename in glob.glob(os.path.join(REQ_DIR, 'requirements-*.txt-raw')): + basename = os.path.basename(filename) + yield basename[len('requirements-'):-len('.txt-raw')] + + +def main(): + """Re-compile the given (or all) requirement files.""" + names = sys.argv[1:] if len(sys.argv) > 1 else sorted(get_all_names()) + + for name in names: + utils.print_title(name) + filename = os.path.join(REQ_DIR, + 'requirements-{}.txt-raw'.format(name)) + if name == 'qutebrowser': + outfile = os.path.join(REPO_DIR, 'requirements.txt') + else: + outfile = os.path.join(REQ_DIR, 'requirements-{}.txt'.format(name)) + + with tempfile.TemporaryDirectory() as tmpdir: + pip_bin = os.path.join(tmpdir, 'bin', 'pip') + subprocess.run(['virtualenv', tmpdir], check=True) + subprocess.run([pip_bin, 'install', '-r', filename], check=True) + proc = subprocess.run([pip_bin, 'freeze'], check=True, + stdout=subprocess.PIPE) + reqs = proc.stdout.decode('utf-8') + + with open(filename, 'r', encoding='utf-8') as f: + comments = read_comments(f) + + with open(outfile, 'w', encoding='utf-8') as f: + f.write("# This file is automatically generated by " + "scripts/dev/recompile_requirements.py\n\n") + for line in reqs.splitlines(): + f.write(convert_line(line, comments) + '\n') + + +if __name__ == '__main__': + main() diff --git a/.config/qutebrowser/scripts/dev/run_profile.py b/.config/qutebrowser/scripts/dev/run_profile.py new file mode 100755 index 0000000..93e0b61 --- /dev/null +++ b/.config/qutebrowser/scripts/dev/run_profile.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org> +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + +"""Profile qutebrowser.""" + +import sys +import cProfile +import os.path +import os +import tempfile +import subprocess +import shutil +import argparse +import shlex + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir, + os.pardir)) + +import qutebrowser.qutebrowser + + +def parse_args(): + """Parse commandline arguments. + + Return: + A (namespace, remaining_args) tuple from argparse. + """ + parser = argparse.ArgumentParser() + parser.add_argument('--profile-tool', metavar='TOOL', + action='store', choices=['kcachegrind', 'snakeviz', + 'gprof2dot', 'none'], + default='snakeviz', + help="The tool to use to view the profiling data") + parser.add_argument('--profile-file', metavar='FILE', action='store', + help="The filename to use with --profile-tool=none") + return parser.parse_known_args() + + +def main(): + args, remaining = parse_args() + tempdir = tempfile.mkdtemp() + + if args.profile_tool == 'none': + profilefile = os.path.join(os.getcwd(), args.profile_file) + else: + profilefile = os.path.join(tempdir, 'profile') + + sys.argv = [sys.argv[0]] + remaining + + profiler = cProfile.Profile() + profiler.runcall(qutebrowser.qutebrowser.main) + + # If we have an exception after here, we don't want the qutebrowser + # exception hook to take over. + sys.excepthook = sys.__excepthook__ + profiler.dump_stats(profilefile) + + if args.profile_tool == 'none': + pass + elif args.profile_tool == 'gprof2dot': + # yep, shell=True. I know what I'm doing. + subprocess.run( + 'gprof2dot -f pstats {} | dot -Tpng | feh -F -'.format( + shlex.quote(profilefile)), shell=True) + elif args.profile_tool == 'kcachegrind': + callgraphfile = os.path.join(tempdir, 'callgraph') + subprocess.run(['pyprof2calltree', '-k', '-i', profilefile, + '-o', callgraphfile]) + elif args.profile_tool == 'snakeviz': + subprocess.run(['snakeviz', profilefile]) + + shutil.rmtree(tempdir) + + +if __name__ == '__main__': + main() diff --git a/.config/qutebrowser/scripts/dev/run_pylint_on_tests.py b/.config/qutebrowser/scripts/dev/run_pylint_on_tests.py new file mode 100644 index 0000000..7adf45f --- /dev/null +++ b/.config/qutebrowser/scripts/dev/run_pylint_on_tests.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org> + +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + +"""Run pylint on tests. + +This is needed because pylint can't check a folder which isn't a package: +https://bitbucket.org/logilab/pylint/issue/512/ +""" + +import os +import os.path +import sys +import subprocess + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir, + os.pardir)) + +from scripts import utils + + +def main(): + """Main entry point. + + Return: + The pylint exit status. + """ + utils.change_cwd() + files = [] + for dirpath, _dirnames, filenames in os.walk('tests'): + for fn in filenames: + if os.path.splitext(fn)[1] == '.py': + files.append(os.path.join(dirpath, fn)) + + disabled = [ + # pytest fixtures + 'redefined-outer-name', + 'unused-argument', + # things which are okay in tests + 'missing-docstring', + 'protected-access', + 'len-as-condition', + # directories without __init__.py... + 'import-error', + ] + + toxinidir = sys.argv[1] + pythonpath = os.environ.get('PYTHONPATH', '').split(os.pathsep) + [ + toxinidir, + ] + + args = (['--disable={}'.format(','.join(disabled)), + '--ignored-modules=helpers,pytest,PyQt5'] + + sys.argv[2:] + files) + env = os.environ.copy() + env['PYTHONPATH'] = os.pathsep.join(pythonpath) + + ret = subprocess.run(['pylint'] + args, env=env).returncode + return ret + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/.config/qutebrowser/scripts/dev/run_vulture.py b/.config/qutebrowser/scripts/dev/run_vulture.py new file mode 100755 index 0000000..cb19d62 --- /dev/null +++ b/.config/qutebrowser/scripts/dev/run_vulture.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2015-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org> + +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + +"""Run vulture on the source files and filter out false-positives.""" + +import sys +import os +import re +import tempfile +import inspect +import argparse + +import vulture + +import qutebrowser.app # pylint: disable=unused-import +from qutebrowser.commands import cmdutils +from qutebrowser.utils import utils +from qutebrowser.browser.webkit import rfc6266 +# To run the decorators from there +# pylint: disable=unused-import +from qutebrowser.browser.webkit.network import webkitqutescheme +# pylint: enable=unused-import +from qutebrowser.browser import qutescheme +from qutebrowser.config import configtypes + + +def whitelist_generator(): # noqa + """Generator which yields lines to add to a vulture whitelist.""" + # qutebrowser commands + for cmd in cmdutils.cmd_dict.values(): + yield utils.qualname(cmd.handler) + + # pyPEG2 classes + for name, member in inspect.getmembers(rfc6266, inspect.isclass): + for attr in ['grammar', 'regex']: + if hasattr(member, attr): + yield 'qutebrowser.browser.webkit.rfc6266.{}.{}'.format(name, + attr) + + # PyQt properties + yield 'qutebrowser.mainwindow.statusbar.bar.StatusBar.color_flags' + yield 'qutebrowser.mainwindow.statusbar.url.UrlText.urltype' + + # Not used yet, but soon (or when debugging) + yield 'qutebrowser.utils.debug.log_events' + yield 'qutebrowser.utils.debug.log_signals' + yield 'qutebrowser.utils.debug.qflags_key' + yield 'qutebrowser.utils.qtutils.QtOSError.qt_errno' + yield 'scripts.utils.bg_colors' + + # Qt attributes + yield 'PyQt5.QtWebKit.QWebPage.ErrorPageExtensionReturn().baseUrl' + yield 'PyQt5.QtWebKit.QWebPage.ErrorPageExtensionReturn().content' + yield 'PyQt5.QtWebKit.QWebPage.ErrorPageExtensionReturn().encoding' + yield 'PyQt5.QtWebKit.QWebPage.ErrorPageExtensionReturn().fileNames' + yield 'PyQt5.QtWidgets.QStyleOptionViewItem.backgroundColor' + + ## qute://... handlers + for name in qutescheme._HANDLERS: # pylint: disable=protected-access + name = name.replace('-', '_') + yield 'qutebrowser.browser.qutescheme.qute_' + name + + # Other false-positives + yield 'qutebrowser.completion.models.listcategory.ListCategory().lessThan' + yield 'qutebrowser.utils.jinja.Loader.get_source' + yield 'qutebrowser.utils.log.QtWarningFilter.filter' + yield 'qutebrowser.browser.pdfjs.is_available' + yield 'qutebrowser.misc.guiprocess.spawn_output' + yield 'QEvent.posted' + yield 'log_stack' # from message.py + yield 'propagate' # logging.getLogger('...).propagate = False + # vulture doesn't notice the hasattr() and thus thinks netrc_used is unused + # in NetworkManager.on_authentication_required + yield 'PyQt5.QtNetwork.QNetworkReply.netrc_used' + yield 'qutebrowser.browser.downloads.last_used_directory' + yield 'PaintContext.clip' # from completiondelegate.py + yield 'logging.LogRecord.log_color' # from logging.py + yield 'scripts.utils.use_color' # from asciidoc2html.py + for attr in ['pyeval_output', 'log_clipboard', 'fake_clipboard']: + yield 'qutebrowser.misc.utilcmds.' + attr + + for attr in ['fileno', 'truncate', 'closed', 'readable']: + yield 'qutebrowser.utils.qtutils.PyQIODevice.' + attr + + for attr in ['msgs', 'priority', 'visit_attribute']: + yield 'scripts.dev.pylint_checkers.config.' + attr + for attr in ['visit_call', 'process_module']: + yield 'scripts.dev.pylint_checkers.modeline.' + attr + + for name, _member in inspect.getmembers(configtypes, inspect.isclass): + yield 'qutebrowser.config.configtypes.' + name + yield 'qutebrowser.config.configexc.ConfigErrorDesc.traceback' + yield 'qutebrowser.config.configfiles.ConfigAPI.load_autoconfig' + yield 'types.ModuleType.c' # configfiles:read_config_py + for name in ['configdir', 'datadir']: + yield 'qutebrowser.config.configfiles.ConfigAPI.' + name + + yield 'include_aliases' + + for attr in ['_get_default_metavar_for_optional', + '_get_default_metavar_for_positional', '_metavar_formatter']: + yield 'scripts.dev.src2asciidoc.UsageFormatter.' + attr + + # attrs + yield 'qutebrowser.browser.webkit.network.networkmanager.ProxyId.hostname' + yield 'qutebrowser.command.command.ArgInfo._validate_exclusive' + yield 'scripts.get_coredumpctl_traces.Line.uid' + yield 'scripts.get_coredumpctl_traces.Line.gid' + yield 'scripts.importer.import_moz_places.places.row_factory' + + +def filter_func(item): + """Check if a missing function should be filtered or not. + + Return: + True if the missing function should be filtered/ignored, False + otherwise. + """ + return bool(re.fullmatch(r'[a-z]+[A-Z][a-zA-Z]+', item.name)) + + +def report(items): + """Generate a report based on the given vulture.Item's. + + Based on vulture.Vulture.report, but we can't use that as we can't set the + properties which get used for the items. + """ + output = [] + for item in sorted(items, + key=lambda e: (e.filename.lower(), e.first_lineno)): + output.append(item.get_report()) + return output + + +def run(files): + """Run vulture over the given files.""" + with tempfile.NamedTemporaryFile(mode='w', delete=False) as whitelist_file: + for line in whitelist_generator(): + whitelist_file.write(line + '\n') + + whitelist_file.close() + + vult = vulture.Vulture(verbose=False) + vult.scavenge(files + [whitelist_file.name]) + + os.remove(whitelist_file.name) + + filters = { + 'unused_funcs': filter_func, + 'unused_props': lambda item: False, + 'unused_vars': lambda item: False, + 'unused_attrs': lambda item: False, + } + + items = [] + + for attr, func in filters.items(): + sub_items = getattr(vult, attr) + for item in sub_items: + filtered = func(item) + if not filtered: + items.append(item) + + return report(items) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('files', nargs='*', default=['qutebrowser', 'scripts', + 'setup.py']) + args = parser.parse_args() + out = run(args.files) + for line in out: + print(line) + sys.exit(bool(out)) + + +if __name__ == '__main__': + main() diff --git a/.config/qutebrowser/scripts/dev/segfault_test.py b/.config/qutebrowser/scripts/dev/segfault_test.py new file mode 100755 index 0000000..aaf495f --- /dev/null +++ b/.config/qutebrowser/scripts/dev/segfault_test.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org> +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + +"""Tester for Qt segfaults with different harfbuzz engines.""" + +import os +import os.path +import signal +import sys +import subprocess + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir, + os.pardir)) + +from scripts import utils + + +SCRIPT = """ +import sys + +from PyQt5.QtCore import QUrl +from PyQt5.QtWidgets import QApplication +from PyQt5.QtWebKitWidgets import QWebView + +def on_load_finished(ok): + if ok: + app.exit(0) + else: + app.exit(1) + +app = QApplication([]) +wv = QWebView() +wv.loadFinished.connect(on_load_finished) +wv.load(QUrl(sys.argv[1])) +#wv.show() +app.exec_() +""" + + +def print_ret(ret): + """Print information about an exit status.""" + if ret == 0: + utils.print_col("success", 'green') + elif ret == -signal.SIGSEGV: + utils.print_col("segfault", 'red') + else: + utils.print_col("error {}".format(ret), 'yellow') + print() + + +def main(): + retvals = [] + if len(sys.argv) < 2: + # pages which previously caused problems + pages = [ + # ANGLE, https://bugreports.qt.io/browse/QTBUG-39723 + ('http://www.binpress.com/', False), + ('http://david.li/flow/', False), + ('https://imzdl.com/', False), + # not reproducible + # https://bugreports.qt.io/browse/QTBUG-39847 + ('http://www.20min.ch/', True), + # HarfBuzz, https://bugreports.qt.io/browse/QTBUG-39278 + ('http://www.the-compiler.org/', True), + ('http://phoronix.com', True), + ('http://twitter.com', True), + # HarfBuzz #2, https://bugreports.qt.io/browse/QTBUG-36099 + ('http://lenta.ru/', True), + # Unknown, https://bugreports.qt.io/browse/QTBUG-41360 + ('http://salt.readthedocs.org/en/latest/topics/pillar/', True), + ] + else: + pages = [(e, True) for e in sys.argv[1:]] + for page, test_harfbuzz in pages: + utils.print_bold("==== {} ====".format(page)) + if test_harfbuzz: + print("With system harfbuzz:") + ret = subprocess.run([sys.executable, '-c', SCRIPT, page]).returncode + print_ret(ret) + retvals.append(ret) + if test_harfbuzz: + print("With QT_HARFBUZZ=old:") + env = dict(os.environ) + env['QT_HARFBUZZ'] = 'old' + ret = subprocess.run([sys.executable, '-c', SCRIPT, page], + env=env).returncode + print_ret(ret) + retvals.append(ret) + print("With QT_HARFBUZZ=new:") + env = dict(os.environ) + env['QT_HARFBUZZ'] = 'new' + ret = subprocess.run([sys.executable, '-c', SCRIPT, page], + env=env).returncode + print_ret(ret) + retvals.append(ret) + if all(r == 0 for r in retvals): + sys.exit(0) + else: + sys.exit(1) + + +if __name__ == '__main__': + main() diff --git a/.config/qutebrowser/scripts/dev/src2asciidoc.py b/.config/qutebrowser/scripts/dev/src2asciidoc.py new file mode 100755 index 0000000..cc00c37 --- /dev/null +++ b/.config/qutebrowser/scripts/dev/src2asciidoc.py @@ -0,0 +1,561 @@ +#!/usr/bin/env python3 +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org> + +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + +"""Generate asciidoc source for qutebrowser based on docstrings.""" + +import os +import os.path +import sys +import shutil +import inspect +import subprocess +import tempfile +import argparse + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir, + os.pardir)) + +# We import qutebrowser.app so all @cmdutils-register decorators are run. +import qutebrowser.app +from qutebrowser import qutebrowser, commands +from qutebrowser.commands import cmdutils, argparser +from qutebrowser.config import configdata, configtypes +from qutebrowser.utils import docutils, usertypes +from scripts import asciidoc2html, utils + +FILE_HEADER = """ +// DO NOT EDIT THIS FILE DIRECTLY! +// It is autogenerated by running: +// $ python3 scripts/dev/src2asciidoc.py +// vim: readonly: + +""".lstrip() + + +class UsageFormatter(argparse.HelpFormatter): + + """Patched HelpFormatter to include some asciidoc markup in the usage. + + This does some horrible things, but the alternative would be to reimplement + argparse.HelpFormatter while copying 99% of the code :-/ + """ + + def _format_usage(self, usage, actions, groups, _prefix): + """Override _format_usage to not add the 'usage:' prefix.""" + return super()._format_usage(usage, actions, groups, '') + + def _get_default_metavar_for_optional(self, action): + """Do name transforming when getting metavar.""" + return argparser.arg_name(action.dest.upper()) + + def _get_default_metavar_for_positional(self, action): + """Do name transforming when getting metavar.""" + return argparser.arg_name(action.dest) + + def _metavar_formatter(self, action, default_metavar): + """Override _metavar_formatter to add asciidoc markup to metavars. + + Most code here is copied from Python 3.4's argparse.py. + """ + if action.metavar is not None: + result = "'{}'".format(action.metavar) + elif action.choices is not None: + choice_strs = [str(choice) for choice in action.choices] + result = ('{' + ','.join('*{}*'.format(e) for e in choice_strs) + + '}') + else: + result = "'{}'".format(default_metavar) + + def fmt(tuple_size): + """Format the result according to the tuple size.""" + if isinstance(result, tuple): + return result + else: + return (result, ) * tuple_size + return fmt + + def _format_actions_usage(self, actions, groups): + """Override _format_actions_usage to add asciidoc markup to flags. + + Because argparse.py's _format_actions_usage is very complex, we first + monkey-patch the option strings to include the asciidoc markup, then + run the original method, then undo the patching. + """ + old_option_strings = {} + for action in actions: + old_option_strings[action] = action.option_strings[:] + action.option_strings = ['*{}*'.format(s) + for s in action.option_strings] + ret = super()._format_actions_usage(actions, groups) + for action in actions: + action.option_strings = old_option_strings[action] + return ret + + +def _open_file(name, mode='w'): + """Open a file with a preset newline/encoding mode.""" + return open(name, mode, newline='\n', encoding='utf-8') + + +def _get_cmd_syntax(_name, cmd): + """Get the command syntax for a command. + + We monkey-patch the parser's formatter_class here to use our UsageFormatter + which adds some asciidoc markup. + """ + old_fmt_class = cmd.parser.formatter_class + cmd.parser.formatter_class = UsageFormatter + usage = cmd.parser.format_usage().rstrip() + cmd.parser.formatter_class = old_fmt_class + return usage + + +def _get_command_quickref(cmds): + """Generate the command quick reference.""" + out = [] + out.append('[options="header",width="75%",cols="25%,75%"]') + out.append('|==============') + out.append('|Command|Description') + for name, cmd in cmds: + desc = inspect.getdoc(cmd.handler).splitlines()[0] + out.append('|<<{},{}>>|{}'.format(name, name, desc)) + out.append('|==============') + return '\n'.join(out) + + +def _get_setting_quickref(): + """Generate the settings quick reference.""" + out = [] + out.append('') + out.append('[options="header",width="75%",cols="25%,75%"]') + out.append('|==============') + out.append('|Setting|Description') + for opt in sorted(configdata.DATA.values()): + desc = opt.description.splitlines()[0] + out.append('|<<{},{}>>|{}'.format(opt.name, opt.name, desc)) + out.append('|==============') + return '\n'.join(out) + + +def _get_configtypes(): + """Get configtypes classes to document.""" + predicate = lambda e: ( + inspect.isclass(e) and + # pylint: disable=protected-access + e not in [configtypes.BaseType, configtypes.MappingType, + configtypes._Numeric] and + # pylint: enable=protected-access + issubclass(e, configtypes.BaseType)) + yield from inspect.getmembers(configtypes, predicate) + + +def _get_setting_types_quickref(): + """Generate the setting types quick reference.""" + out = [] + out.append('[[types]]') + out.append('[options="header",width="75%",cols="25%,75%"]') + out.append('|==============') + out.append('|Type|Description') + + for name, typ in _get_configtypes(): + parser = docutils.DocstringParser(typ) + desc = parser.short_desc + if parser.long_desc: + desc += '\n\n' + parser.long_desc + out.append('|{}|{}'.format(name, desc)) + + out.append('|==============') + return '\n'.join(out) + + +def _get_command_doc(name, cmd): + """Generate the documentation for a command.""" + output = ['[[{}]]'.format(name)] + output += ['=== {}'.format(name)] + syntax = _get_cmd_syntax(name, cmd) + if syntax != name: + output.append('Syntax: +:{}+'.format(syntax)) + output.append("") + parser = docutils.DocstringParser(cmd.handler) + output.append(parser.short_desc) + if parser.long_desc: + output.append("") + output.append(parser.long_desc) + + output += list(_get_command_doc_args(cmd, parser)) + output += list(_get_command_doc_count(cmd, parser)) + output += list(_get_command_doc_notes(cmd)) + + output.append("") + output.append("") + return '\n'.join(output) + + +def _get_command_doc_args(cmd, parser): + """Get docs for the arguments of a command. + + Args: + cmd: The Command to get the docs for. + parser: The DocstringParser to use. + + Yield: + Strings which should be added to the docs. + """ + if cmd.pos_args: + yield "" + yield "==== positional arguments" + for arg, name in cmd.pos_args: + try: + yield "* +'{}'+: {}".format(name, parser.arg_descs[arg]) + except KeyError as e: + raise KeyError("No description for arg {} of command " + "'{}'!".format(e, cmd.name)) from e + + if cmd.opt_args: + yield "" + yield "==== optional arguments" + for arg, (long_flag, short_flag) in cmd.opt_args.items(): + try: + yield '* +*{}*+, +*{}*+: {}'.format(short_flag, long_flag, + parser.arg_descs[arg]) + except KeyError as e: + raise KeyError("No description for arg {} of command " + "'{}'!".format(e, cmd.name)) from e + + +def _get_command_doc_count(cmd, parser): + """Get docs for the count of a command. + + Args: + cmd: The Command to get the docs for. + parser: The DocstringParser to use. + + Yield: + Strings which should be added to the docs. + """ + for param in inspect.signature(cmd.handler).parameters.values(): + if cmd.get_arg_info(param).count: + yield "" + yield "==== count" + try: + yield parser.arg_descs[param.name] + except KeyError as e: + raise KeyError("No description for count arg {!r} of command " + "{!r}!".format(param.name, cmd.name)) from e + + +def _get_command_doc_notes(cmd): + """Get docs for the notes of a command. + + Args: + cmd: The Command to get the docs for. + parser: The DocstringParser to use. + + Yield: + Strings which should be added to the docs. + """ + if (cmd.maxsplit is not None or cmd.no_cmd_split or + cmd.no_replace_variables and cmd.name != "spawn"): + yield "" + yield "==== note" + if cmd.maxsplit is not None: + yield ("* This command does not split arguments after the last " + "argument and handles quotes literally.") + if cmd.no_cmd_split: + yield ("* With this command, +;;+ is interpreted literally " + "instead of splitting off a second command.") + if cmd.no_replace_variables and cmd.name != "spawn": + yield r"* This command does not replace variables like +\{url\}+." + + +def _get_action_metavar(action, nargs=1): + """Get the metavar to display for an argparse action. + + Args: + action: The argparse action to get the metavar for. + nargs: The nargs setting for the related argument. + """ + if action.metavar is not None: + if isinstance(action.metavar, str): + elems = [action.metavar] * nargs + else: + elems = action.metavar + return ' '.join("'{}'".format(e) for e in elems) + elif action.choices is not None: + choices = ','.join(str(e) for e in action.choices) + return "'{{{}}}'".format(choices) + else: + return "'{}'".format(action.dest.upper()) + + +def _format_action_args(action): + """Get an argument string based on an argparse action.""" + if action.nargs is None: + return _get_action_metavar(action) + elif action.nargs == '?': + return '[{}]'.format(_get_action_metavar(action)) + elif action.nargs == '*': + return '[{mv} [{mv} ...]]'.format(mv=_get_action_metavar(action)) + elif action.nargs == '+': + return '{mv} [{mv} ...]'.format(mv=_get_action_metavar(action)) + elif action.nargs == '...': + return '...' + else: + return _get_action_metavar(action, nargs=action.nargs) + + +def _format_action(action): + """Get an invocation string/help from an argparse action.""" + if action.help == argparse.SUPPRESS: + return None + if not action.option_strings: + invocation = '*{}*::'.format(_get_action_metavar(action)) + else: + parts = [] + if action.nargs == 0: + # Doesn't take a value, so the syntax is -s, --long + parts += ['*{}*'.format(s) for s in action.option_strings] + else: + # Takes a value, so the syntax is -s ARGS or --long ARGS. + args_string = _format_action_args(action) + for opt in action.option_strings: + parts.append('*{}* {}'.format(opt, args_string)) + invocation = ', '.join(parts) + '::' + return '{}\n {}\n'.format(invocation, action.help) + + +def generate_commands(filename): + """Generate the complete commands section.""" + with _open_file(filename) as f: + f.write(FILE_HEADER) + f.write("= Commands\n\n") + f.write(commands.__doc__) + normal_cmds = [] + other_cmds = [] + debug_cmds = [] + for name, cmd in cmdutils.cmd_dict.items(): + if cmd.deprecated: + continue + if usertypes.KeyMode.normal not in cmd.modes: + other_cmds.append((name, cmd)) + elif cmd.debug: + debug_cmds.append((name, cmd)) + else: + normal_cmds.append((name, cmd)) + normal_cmds.sort() + other_cmds.sort() + debug_cmds.sort() + f.write("\n") + f.write("== Normal commands\n") + f.write(".Quick reference\n") + f.write(_get_command_quickref(normal_cmds) + '\n') + for name, cmd in normal_cmds: + f.write(_get_command_doc(name, cmd)) + f.write("\n") + f.write("== Commands not usable in normal mode\n") + f.write(".Quick reference\n") + f.write(_get_command_quickref(other_cmds) + '\n') + for name, cmd in other_cmds: + f.write(_get_command_doc(name, cmd)) + f.write("\n") + f.write("== Debugging commands\n") + f.write("These commands are mainly intended for debugging. They are " + "hidden if qutebrowser was started without the " + "`--debug`-flag.\n") + f.write("\n") + f.write(".Quick reference\n") + f.write(_get_command_quickref(debug_cmds) + '\n') + for name, cmd in debug_cmds: + f.write(_get_command_doc(name, cmd)) + + +def _generate_setting_backend_info(f, opt): + """Generate backend information for the given option.""" + all_backends = [usertypes.Backend.QtWebKit, usertypes.Backend.QtWebEngine] + if opt.raw_backends is not None: + for name, conditional in sorted(opt.raw_backends.items()): + if conditional is True: + pass + elif conditional is False: + f.write("\nOn {}, this setting is unavailable.\n".format(name)) + else: + f.write("\nOn {}, this setting requires {} or newer.\n" + .format(name, conditional)) + elif opt.backends == all_backends: + pass + elif opt.backends == [usertypes.Backend.QtWebKit]: + f.write("\nThis setting is only available with the QtWebKit " + "backend.\n") + elif opt.backends == [usertypes.Backend.QtWebEngine]: + f.write("\nThis setting is only available with the QtWebEngine " + "backend.\n") + else: + raise ValueError("Invalid value {!r} for opt.backends" + .format(opt.backends)) + + +def _generate_setting_option(f, opt): + """Generate documentation for a single section.""" + f.write("\n") + f.write('[[{}]]'.format(opt.name) + "\n") + f.write("=== {}".format(opt.name) + "\n") + f.write(opt.description + "\n") + if opt.restart: + f.write("This setting requires a restart.\n") + if opt.supports_pattern: + f.write("\nThis setting supports URL patterns.\n") + if opt.no_autoconfig: + f.write("\nThis setting can only be set in config.py.\n") + f.write("\n") + typ = opt.typ.get_name().replace(',', ',') + f.write('Type: <<types,{typ}>>\n'.format(typ=typ)) + f.write("\n") + + valid_values = opt.typ.get_valid_values() + if valid_values is not None and valid_values.generate_docs: + f.write("Valid values:\n") + f.write("\n") + for val in valid_values: + try: + desc = valid_values.descriptions[val] + f.write(" * +{}+: {}".format(val, desc) + "\n") + except KeyError: + f.write(" * +{}+".format(val) + "\n") + f.write("\n") + + f.write("Default: {}\n".format(opt.typ.to_doc(opt.default))) + _generate_setting_backend_info(f, opt) + + +def generate_settings(filename): + """Generate the complete settings section.""" + configdata.init() + with _open_file(filename) as f: + f.write(FILE_HEADER) + f.write("= Setting reference\n\n") + f.write("== All settings\n") + f.write(_get_setting_quickref() + "\n") + for opt in sorted(configdata.DATA.values()): + _generate_setting_option(f, opt) + f.write("\n== Setting types\n") + f.write(_get_setting_types_quickref() + "\n") + + +def _format_block(filename, what, data): + """Format a block in a file. + + The block is delimited by markers like these: + // QUTE_*_START + ... + // QUTE_*_END + + The * part is the part which should be given as 'what'. + + Args: + filename: The file to change. + what: What to change (authors, options, etc.) + data; A list of strings which is the new data. + """ + what = what.upper() + oshandle, tmpname = tempfile.mkstemp() + try: + with _open_file(filename, mode='r') as infile, \ + _open_file(oshandle, mode='w') as temp: + found_start = False + found_end = False + for line in infile: + if line.strip() == '// QUTE_{}_START'.format(what): + temp.write(line) + temp.write(''.join(data)) + found_start = True + elif line.strip() == '// QUTE_{}_END'.format(what.upper()): + temp.write(line) + found_end = True + elif (not found_start) or found_end: + temp.write(line) + if not found_start: + raise Exception("Marker '// QUTE_{}_START' not found in " + "'{}'!".format(what, filename)) + elif not found_end: + raise Exception("Marker '// QUTE_{}_END' not found in " + "'{}'!".format(what, filename)) + except: + os.remove(tmpname) + raise + else: + os.remove(filename) + shutil.move(tmpname, filename) + + +def regenerate_manpage(filename): + """Update manpage OPTIONS using an argparse parser.""" + parser = qutebrowser.get_argparser() + groups = [] + # positionals, optionals and user-defined groups + # pylint: disable=protected-access + for group in parser._action_groups: + groupdata = [] + groupdata.append('=== {}'.format(group.title)) + if group.description is not None: + groupdata.append(group.description) + for action in group._group_actions: + action_data = _format_action(action) + if action_data is not None: + groupdata.append(action_data) + groups.append('\n'.join(groupdata)) + # pylint: enable=protected-access + options = '\n'.join(groups) + # epilog + if parser.epilog is not None: + options += parser.epilog + _format_block(filename, 'options', options) + + +def regenerate_cheatsheet(): + """Generate cheatsheet PNGs based on the SVG.""" + files = [ + ('doc/img/cheatsheet-small.png', 300, 185), + ('doc/img/cheatsheet-big.png', 3342, 2060), + ] + + for filename, x, y in files: + subprocess.run(['inkscape', '-e', filename, '-b', 'white', + '-w', str(x), '-h', str(y), + 'misc/cheatsheet.svg'], check=True) + + +def main(): + """Regenerate all documentation.""" + utils.change_cwd() + print("Generating manpage...") + regenerate_manpage('doc/qutebrowser.1.asciidoc') + print("Generating settings help...") + generate_settings('doc/help/settings.asciidoc') + print("Generating command help...") + generate_commands('doc/help/commands.asciidoc') + if '--cheatsheet' in sys.argv: + print("Regenerating cheatsheet .pngs") + regenerate_cheatsheet() + if '--html' in sys.argv: + asciidoc2html.main() + + +if __name__ == '__main__': + main() diff --git a/.config/qutebrowser/scripts/dev/standardpaths_tester.py b/.config/qutebrowser/scripts/dev/standardpaths_tester.py new file mode 100644 index 0000000..27b8382 --- /dev/null +++ b/.config/qutebrowser/scripts/dev/standardpaths_tester.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2017-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org> +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + +"""Show various QStandardPath paths.""" + +import os +import sys + +from PyQt5.QtCore import (QT_VERSION_STR, PYQT_VERSION_STR, qVersion, + QStandardPaths, QCoreApplication) + + +def print_header(): + """Show system information.""" + print("Python {}".format(sys.version)) + print("os.name: {}".format(os.name)) + print("sys.platform: {}".format(sys.platform)) + print() + + print("Qt {}, compiled {}".format(qVersion(), QT_VERSION_STR)) + print("PyQt {}".format(PYQT_VERSION_STR)) + print() + + +def print_paths(): + """Print all QStandardPaths.StandardLocation members.""" + for name, obj in vars(QStandardPaths).items(): + if isinstance(obj, QStandardPaths.StandardLocation): + location = QStandardPaths.writableLocation(obj) + print("{:25} {}".format(name, location)) + + +def main(): + print_header() + + print("No QApplication") + print("===============") + print() + print_paths() + + app = QCoreApplication(sys.argv) + app.setApplicationName("qapp_name") + + print() + print("With QApplication") + print("=================") + print() + print_paths() + + +if __name__ == '__main__': + main() diff --git a/.config/qutebrowser/scripts/dev/strip_whitespace.sh b/.config/qutebrowser/scripts/dev/strip_whitespace.sh new file mode 100644 index 0000000..ee14278 --- /dev/null +++ b/.config/qutebrowser/scripts/dev/strip_whitespace.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +# Strip trailing whitespace from files in this repo + +find qutebrowser scripts tests \ + -type f \( \ + -name '*.py' -o \ + -name '*.feature' -o \ + -name '*.sh' \ + \) -exec sed -i 's/ \+$//' {} + diff --git a/.config/qutebrowser/scripts/dev/ua_fetch.py b/.config/qutebrowser/scripts/dev/ua_fetch.py new file mode 100755 index 0000000..75ce4c2 --- /dev/null +++ b/.config/qutebrowser/scripts/dev/ua_fetch.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2015-2018 lamarpavel +# Copyright 2015-2018 Alexey Nabrodov (Averrin) +# Copyright 2015-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org> +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + + +"""Fetch list of popular user-agents. + +The script is based on a gist posted by github.com/averrin, the output of this +script is formatted to be pasted into configdata.yml +""" + +import requests +from lxml import html # pylint: disable=import-error + + +def fetch(): + """Fetch list of popular user-agents. + + Return: + List of relevant strings. + """ + url = 'https://techblog.willshouse.com/2012/01/03/most-common-user-agents/' + page = requests.get(url) + page = html.fromstring(page.text) + path = '//*[@id="post-2229"]/div[2]/table/tbody' + return page.xpath(path)[0] + + +def filter_list(complete_list, browsers): + """Filter the received list based on a look up table. + + The LUT should be a dictionary of the format {browser: versions}, where + 'browser' is the name of the browser (eg. "Firefox") as string and + 'versions' is a set of different versions of this browser that should be + included when found (eg. {"Linux", "MacOSX"}). This function returns a + dictionary with the same keys as the LUT, but storing lists of tuples + (user_agent, browser_description) as values. + """ + # pylint: disable=too-many-nested-blocks + table = {} + for entry in complete_list: + # Tuple of (user_agent, browser_description) + candidate = (entry[1].text_content(), entry[2].text_content()) + for name in browsers: + found = False + if name.lower() in candidate[1].lower(): + for version in browsers[name]: + if version.lower() in candidate[1].lower(): + if table.get(name) is None: + table[name] = [] + table[name].append(candidate) + browsers[name].remove(version) + found = True + break + if found: + break + return table + + +def add_diversity(table): + """Insert a few additional entries for diversity into the dict. + + (as returned by filter_list()) + """ + table["Obscure"] = [ + ('Mozilla/5.0 (compatible; Googlebot/2.1; ' + '+http://www.google.com/bot.html', + "Google Bot"), + ('Wget/1.16.1 (linux-gnu)', + "wget 1.16.1"), + ('curl/7.40.0', + "curl 7.40.0"), + ('Mozilla/5.0 (Linux; U; Android 7.1.2) AppleWebKit/534.30 ' + '(KHTML, like Gecko) Version/4.0 Mobile Safari/534.30', + "Mobile Generic Android"), + ('Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like ' + 'Gecko', + "IE 11.0 for Desktop Win7 64-bit"), + ] + return table + + +def main(): + """Generate user agent code.""" + fetched = fetch() + lut = { + "Firefox": {"Win", "MacOSX", "Linux", "Android"}, + "Chrome": {"Win", "MacOSX", "Linux"}, + "Safari": {"MacOSX", "iOS"} + } + filtered = filter_list(fetched, lut) + filtered = add_diversity(filtered) + + tab = " " + for browser in ["Firefox", "Safari", "Chrome", "Obscure"]: + for it in filtered[browser]: + print('{}- - "{}"'.format(3 * tab, it[0])) + desc = it[1].replace('\xa0', ' ').replace(' ', ' ') + print("{}- {}".format(4 * tab, desc)) + print("") + + +if __name__ == '__main__': + main() diff --git a/.config/qutebrowser/scripts/dev/update_3rdparty.py b/.config/qutebrowser/scripts/dev/update_3rdparty.py new file mode 100755 index 0000000..c40015d --- /dev/null +++ b/.config/qutebrowser/scripts/dev/update_3rdparty.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2015 Daniel Schadt +# Copyright 2016-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org> +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + +"""Update all third-party-modules.""" + +import argparse +import urllib.request +import urllib.error +import shutil +import json +import os +import sys + +sys.path.insert( + 0, os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)) +from scripts import dictcli +from qutebrowser.config import configdata + + +def get_latest_pdfjs_url(): + """Get the URL of the latest pdf.js prebuilt package. + + Returns a (version, url)-tuple. + """ + github_api = 'https://api.github.com' + endpoint = 'repos/mozilla/pdf.js/releases/latest' + request_url = '{}/{}'.format(github_api, endpoint) + with urllib.request.urlopen(request_url) as fp: + data = json.loads(fp.read().decode('utf-8')) + + download_url = data['assets'][0]['browser_download_url'] + version_name = data['name'] + return (version_name, download_url) + + +def update_pdfjs(target_version=None): + """Download and extract the latest pdf.js version. + + If target_version is not None, download the given version instead. + + Args: + target_version: None or version string ('x.y.z') + """ + if target_version is None: + version, url = get_latest_pdfjs_url() + else: + # We need target_version as x.y.z, without the 'v' prefix, though the + # user might give it on the command line + if target_version.startswith('v'): + target_version = target_version[1:] + # version should have the prefix to be consistent with the return value + # of get_latest_pdfjs_url() + version = 'v' + target_version + url = ('https://github.com/mozilla/pdf.js/releases/download/' + 'v{0}/pdfjs-{0}-dist.zip').format(target_version) + + os.chdir(os.path.join(os.path.dirname(os.path.abspath(__file__)), + '..', '..')) + target_path = os.path.join('qutebrowser', '3rdparty', 'pdfjs') + print("=> Downloading pdf.js {}".format(version)) + try: + (archive_path, _headers) = urllib.request.urlretrieve(url) + except urllib.error.HTTPError as error: + print("Could not retrieve pdfjs {}: {}".format(version, error)) + return + if os.path.isdir(target_path): + print("Removing old version in {}".format(target_path)) + shutil.rmtree(target_path) + os.makedirs(target_path) + print("Extracting new version") + with open(archive_path, 'rb') as archive: + shutil.unpack_archive(archive, target_path, 'zip') + urllib.request.urlcleanup() + + +def update_dmg_makefile(): + """Update fancy-dmg Makefile. + + See https://el-tramo.be/blog/fancy-dmg/ + """ + print("Updating fancy-dmg Makefile...") + url = 'https://raw.githubusercontent.com/remko/fancy-dmg/master/Makefile' + target_path = os.path.join('scripts', 'dev', 'Makefile-dmg') + urllib.request.urlretrieve(url, target_path) + urllib.request.urlcleanup() + + +def update_ace(): + """Update ACE. + + See https://ace.c9.io/ and https://github.com/ajaxorg/ace-builds/ + """ + print("Updating ACE...") + url = 'https://raw.githubusercontent.com/ajaxorg/ace-builds/master/src/ace.js' + target_path = os.path.join('tests', 'end2end', 'data', 'hints', 'ace', + 'ace.js') + urllib.request.urlretrieve(url, target_path) + urllib.request.urlcleanup() + + +def test_dicts(): + """Test available dictionaries.""" + configdata.init() + for lang in dictcli.available_languages(): + print('Testing dictionary {}... '.format(lang.code), end='') + lang_url = urllib.parse.urljoin(dictcli.API_URL, lang.remote_path) + request = urllib.request.Request(lang_url, method='HEAD') + response = urllib.request.urlopen(request) + if response.status == 200: + print('OK') + else: + print('ERROR: {}'.format(response.status)) + + +def run(ace=False, pdfjs=True, fancy_dmg=False, pdfjs_version=None, + dicts=False): + """Update components based on the given arguments.""" + if pdfjs: + update_pdfjs(pdfjs_version) + if ace: + update_ace() + if fancy_dmg: + update_dmg_makefile() + if dicts: + test_dicts() + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + '--pdfjs', '-p', + help='Specify pdfjs version. If not given, ' + 'the latest version is used.', + required=False, metavar='VERSION') + parser.add_argument('--fancy-dmg', help="Update fancy-dmg Makefile", + action='store_true') + parser.add_argument( + '--dicts', '-d', + help='Test whether all available dictionaries ' + 'can be reached at the remote repository.', + required=False, action='store_true') + args = parser.parse_args() + run(ace=True, pdfjs=True, fancy_dmg=args.fancy_dmg, + pdfjs_version=args.pdfjs, dicts=args.dicts) + + +if __name__ == '__main__': + main() |
