summaryrefslogtreecommitdiff
path: root/.config/qutebrowser/scripts/dev
diff options
context:
space:
mode:
authorVito Graffagnino <vito@graffagnino.xyz>2020-09-08 18:10:49 +0100
committerVito Graffagnino <vito@graffagnino.xyz>2020-09-08 18:10:49 +0100
commit3b0142cedcde39e4c2097ecd916a870a3ced5ec6 (patch)
tree2116c49a845dfc0945778f2aa3e2118d72be428b /.config/qutebrowser/scripts/dev
parent8cc927e930d5b6aafe3e9862a61e81705479a1b4 (diff)
Added the relevent parts of the .config directory. Alss add ssh config
Diffstat (limited to '.config/qutebrowser/scripts/dev')
-rw-r--r--.config/qutebrowser/scripts/dev/Makefile-dmg71
-rw-r--r--.config/qutebrowser/scripts/dev/__init__.py3
-rwxr-xr-x.config/qutebrowser/scripts/dev/build_release.py419
-rw-r--r--.config/qutebrowser/scripts/dev/check_coverage.py348
-rwxr-xr-x.config/qutebrowser/scripts/dev/check_doc_changes.py48
-rw-r--r--.config/qutebrowser/scripts/dev/ci/travis_backtrace.sh18
-rw-r--r--.config/qutebrowser/scripts/dev/ci/travis_install.sh108
-rw-r--r--.config/qutebrowser/scripts/dev/ci/travis_run.sh32
-rwxr-xr-x.config/qutebrowser/scripts/dev/cleanup.py69
-rw-r--r--.config/qutebrowser/scripts/dev/download_release.sh34
-rw-r--r--.config/qutebrowser/scripts/dev/gen_resources.py26
-rw-r--r--.config/qutebrowser/scripts/dev/gen_versioninfo.py85
-rw-r--r--.config/qutebrowser/scripts/dev/get_coredumpctl_traces.py176
-rw-r--r--.config/qutebrowser/scripts/dev/misc_checks.py163
-rw-r--r--.config/qutebrowser/scripts/dev/pylint_checkers/qute_pylint/__init__.py1
-rw-r--r--.config/qutebrowser/scripts/dev/pylint_checkers/qute_pylint/config.py84
-rw-r--r--.config/qutebrowser/scripts/dev/pylint_checkers/qute_pylint/modeline.py63
-rw-r--r--.config/qutebrowser/scripts/dev/pylint_checkers/qute_pylint/openencoding.py83
-rw-r--r--.config/qutebrowser/scripts/dev/pylint_checkers/qute_pylint/settrace.py49
-rw-r--r--.config/qutebrowser/scripts/dev/pylint_checkers/setup.py25
-rwxr-xr-x.config/qutebrowser/scripts/dev/quit_segfault_test.sh14
-rw-r--r--.config/qutebrowser/scripts/dev/recompile_requirements.py136
-rwxr-xr-x.config/qutebrowser/scripts/dev/run_profile.py93
-rw-r--r--.config/qutebrowser/scripts/dev/run_pylint_on_tests.py79
-rwxr-xr-x.config/qutebrowser/scripts/dev/run_vulture.py196
-rwxr-xr-x.config/qutebrowser/scripts/dev/segfault_test.py120
-rwxr-xr-x.config/qutebrowser/scripts/dev/src2asciidoc.py561
-rw-r--r--.config/qutebrowser/scripts/dev/standardpaths_tester.py69
-rw-r--r--.config/qutebrowser/scripts/dev/strip_whitespace.sh10
-rwxr-xr-x.config/qutebrowser/scripts/dev/ua_fetch.py122
-rwxr-xr-x.config/qutebrowser/scripts/dev/update_3rdparty.py166
31 files changed, 3471 insertions, 0 deletions
diff --git a/.config/qutebrowser/scripts/dev/Makefile-dmg b/.config/qutebrowser/scripts/dev/Makefile-dmg
new file mode 100644
index 0000000..1cf4cfb
--- /dev/null
+++ b/.config/qutebrowser/scripts/dev/Makefile-dmg
@@ -0,0 +1,71 @@
+#
+# Build file for creating DMG files.
+#
+# The DMG packager looks for a template.dmg.bz2 for using as its
+# DMG template. If it doesn't find one, it generates a clean one.
+#
+# If you create a DMG template, you should make one containing all
+# the files listed in $(SOURCE_FILES) below, and arrange everything to suit
+# your style. The contents of the files themselves does not matter, so
+# they can be empty (they will be overwritten later).
+#
+# Remko Tronçon
+# https://el-tramo.be
+# Licensed under the MIT License. See COPYING for details.
+
+
+################################################################################
+# Customizable variables
+################################################################################
+
+NAME ?= qutebrowser
+
+SOURCE_DIR ?= .
+SOURCE_FILES ?= dist/qutebrowser.app LICENSE
+
+TEMPLATE_DMG ?= template.dmg
+TEMPLATE_SIZE ?= 300m
+
+################################################################################
+# DMG building. No editing should be needed beyond this point.
+################################################################################
+
+MASTER_DMG=$(NAME).dmg
+WC_DMG=wc.dmg
+WC_DIR=wc
+
+.PHONY: all
+all: $(MASTER_DMG)
+
+$(TEMPLATE_DMG):
+ @echo
+ @echo --------------------- Generating empty template --------------------
+ mkdir template
+ hdiutil create -fs HFSX -layout SPUD -size $(TEMPLATE_SIZE) "$(TEMPLATE_DMG)" -srcfolder template -format UDRW -volname "$(NAME)" -quiet
+ rmdir template
+
+$(WC_DMG): $(TEMPLATE_DMG)
+ cp $< $@
+
+$(MASTER_DMG): $(WC_DMG) $(addprefix $(SOURCE_DIR)/,$(SOURCE_FILES))
+ @echo
+ @echo --------------------- Creating Disk Image --------------------
+ mkdir -p $(WC_DIR)
+ hdiutil attach "$(WC_DMG)" -noautoopen -quiet -mountpoint "$(WC_DIR)"
+ for i in $(SOURCE_FILES); do \
+ rm -rf "$(WC_DIR)/$$i"; \
+ ditto -rsrc "$(SOURCE_DIR)/$$i" "$(WC_DIR)/$${i##*/}"; \
+ done
+ ln -s /Applications $(WC_DIR)
+ #rm -f "$@"
+ #hdiutil create -srcfolder "$(WC_DIR)" -format UDZO -imagekey zlib-level=9 "$@" -volname "$(NAME) $(VERSION)" -scrub -quiet
+ WC_DEV=`hdiutil info | grep "$(WC_DIR)" | grep "Apple_HFS" | awk '{print $$1}'` && \
+ hdiutil detach $$WC_DEV -quiet -force
+ rm -f "$(MASTER_DMG)"
+ hdiutil convert "$(WC_DMG)" -quiet -format UDZO -imagekey zlib-level=9 -o "$@"
+ rm -rf $(WC_DIR)
+ @echo
+
+.PHONY: clean
+clean:
+ -rm -rf $(TEMPLATE_DMG) $(MASTER_DMG) $(WC_DMG)
diff --git a/.config/qutebrowser/scripts/dev/__init__.py b/.config/qutebrowser/scripts/dev/__init__.py
new file mode 100644
index 0000000..7dc0433
--- /dev/null
+++ b/.config/qutebrowser/scripts/dev/__init__.py
@@ -0,0 +1,3 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+"""Various scripts used for developing qutebrowser."""
diff --git a/.config/qutebrowser/scripts/dev/build_release.py b/.config/qutebrowser/scripts/dev/build_release.py
new file mode 100755
index 0000000..254132b
--- /dev/null
+++ b/.config/qutebrowser/scripts/dev/build_release.py
@@ -0,0 +1,419 @@
+#!/usr/bin/env python3
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>.
+
+"""Build a new release."""
+
+
+import os
+import os.path
+import sys
+import time
+import glob
+import shutil
+import plistlib
+import subprocess
+import argparse
+import tarfile
+import tempfile
+import collections
+
+try:
+ import winreg
+except ImportError:
+ pass
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir,
+ os.pardir))
+
+import qutebrowser
+from scripts import utils
+# from scripts.dev import update_3rdparty
+
+
+def call_script(name, *args, python=sys.executable):
+ """Call a given shell script.
+
+ Args:
+ name: The script to call.
+ *args: The arguments to pass.
+ python: The python interpreter to use.
+ """
+ path = os.path.join(os.path.dirname(__file__), os.pardir, name)
+ subprocess.run([python, path] + list(args), check=True)
+
+
+def call_tox(toxenv, *args, python=sys.executable):
+ """Call tox.
+
+ Args:
+ toxenv: Which tox environment to use
+ *args: The arguments to pass.
+ python: The python interpreter to use.
+ """
+ env = os.environ.copy()
+ env['PYTHON'] = python
+ env['PATH'] = os.environ['PATH'] + os.pathsep + os.path.dirname(python)
+ subprocess.run(
+ [sys.executable, '-m', 'tox', '-vv', '-e', toxenv] + list(args),
+ env=env, check=True)
+
+
+def run_asciidoc2html(args):
+ """Common buildsteps used for all OS'."""
+ utils.print_title("Running asciidoc2html.py")
+ if args.asciidoc is not None:
+ a2h_args = ['--asciidoc'] + args.asciidoc
+ else:
+ a2h_args = []
+ call_script('asciidoc2html.py', *a2h_args)
+
+
+def _maybe_remove(path):
+ """Remove a path if it exists."""
+ try:
+ shutil.rmtree(path)
+ except FileNotFoundError:
+ pass
+
+
+def smoke_test(executable):
+ """Try starting the given qutebrowser executable."""
+ subprocess.run([executable, '--no-err-windows', '--nowindow',
+ '--temp-basedir', 'about:blank', ':later 500 quit'],
+ check=True)
+
+
+def patch_mac_app():
+ """Patch .app to copy missing data and link some libs.
+
+ See https://github.com/pyinstaller/pyinstaller/issues/2276
+ """
+ app_path = os.path.join('dist', 'qutebrowser.app')
+ qtwe_core_dir = os.path.join('.tox', 'pyinstaller', 'lib', 'python3.6',
+ 'site-packages', 'PyQt5', 'Qt', 'lib',
+ 'QtWebEngineCore.framework')
+ # Copy QtWebEngineProcess.app
+ proc_app = 'QtWebEngineProcess.app'
+ shutil.copytree(os.path.join(qtwe_core_dir, 'Helpers', proc_app),
+ os.path.join(app_path, 'Contents', 'MacOS', proc_app))
+ # Copy resources
+ for f in glob.glob(os.path.join(qtwe_core_dir, 'Resources', '*')):
+ dest = os.path.join(app_path, 'Contents', 'Resources')
+ if os.path.isdir(f):
+ dir_dest = os.path.join(dest, os.path.basename(f))
+ print("Copying directory {} to {}".format(f, dir_dest))
+ shutil.copytree(f, dir_dest)
+ else:
+ print("Copying {} to {}".format(f, dest))
+ shutil.copy(f, dest)
+ # Link dependencies
+ for lib in ['QtCore', 'QtWebEngineCore', 'QtQuick', 'QtQml', 'QtNetwork',
+ 'QtGui', 'QtWebChannel', 'QtPositioning']:
+ dest = os.path.join(app_path, lib + '.framework', 'Versions', '5')
+ os.makedirs(dest)
+ os.symlink(os.path.join(os.pardir, os.pardir, os.pardir, 'Contents',
+ 'MacOS', lib),
+ os.path.join(dest, lib))
+ # Patch Info.plist - pyinstaller's options are too limiting
+ plist_path = os.path.join(app_path, 'Contents', 'Info.plist')
+ with open(plist_path, "rb") as f:
+ plist_data = plistlib.load(f)
+ plist_data.update(INFO_PLIST_UPDATES)
+ with open(plist_path, "wb") as f:
+ plistlib.dump(plist_data, f)
+
+
+INFO_PLIST_UPDATES = {
+ 'CFBundleVersion': qutebrowser.__version__,
+ 'CFBundleShortVersionString': qutebrowser.__version__,
+ 'NSSupportsAutomaticGraphicsSwitching': True,
+ 'NSHighResolutionCapable': True,
+ 'CFBundleURLTypes': [{
+ "CFBundleURLName": "http(s) URL",
+ "CFBundleURLSchemes": ["http", "https"]
+ }, {
+ "CFBundleURLName": "local file URL",
+ "CFBundleURLSchemes": ["file"]
+ }],
+ 'CFBundleDocumentTypes': [{
+ "CFBundleTypeExtensions": ["html", "htm"],
+ "CFBundleTypeMIMETypes": ["text/html"],
+ "CFBundleTypeName": "HTML document",
+ "CFBundleTypeOSTypes": ["HTML"],
+ "CFBundleTypeRole": "Viewer",
+ }, {
+ "CFBundleTypeExtensions": ["xhtml"],
+ "CFBundleTypeMIMETypes": ["text/xhtml"],
+ "CFBundleTypeName": "XHTML document",
+ "CFBundleTypeRole": "Viewer",
+ }]
+}
+
+
+def build_mac():
+ """Build macOS .dmg/.app."""
+ utils.print_title("Cleaning up...")
+ for f in ['wc.dmg', 'template.dmg']:
+ try:
+ os.remove(f)
+ except FileNotFoundError:
+ pass
+ for d in ['dist', 'build']:
+ shutil.rmtree(d, ignore_errors=True)
+ utils.print_title("Updating 3rdparty content")
+ # Currently disabled because QtWebEngine has no pdfjs support
+ # update_3rdparty.run(ace=False, pdfjs=True, fancy_dmg=False)
+ utils.print_title("Building .app via pyinstaller")
+ call_tox('pyinstaller', '-r')
+ utils.print_title("Patching .app")
+ patch_mac_app()
+ utils.print_title("Building .dmg")
+ subprocess.run(['make', '-f', 'scripts/dev/Makefile-dmg'], check=True)
+
+ dmg_name = 'qutebrowser-{}.dmg'.format(qutebrowser.__version__)
+ os.rename('qutebrowser.dmg', dmg_name)
+
+ utils.print_title("Running smoke test")
+
+ try:
+ with tempfile.TemporaryDirectory() as tmpdir:
+ subprocess.run(['hdiutil', 'attach', dmg_name,
+ '-mountpoint', tmpdir], check=True)
+ try:
+ binary = os.path.join(tmpdir, 'qutebrowser.app', 'Contents',
+ 'MacOS', 'qutebrowser')
+ smoke_test(binary)
+ finally:
+ time.sleep(5)
+ subprocess.run(['hdiutil', 'detach', tmpdir])
+ except PermissionError as e:
+ print("Failed to remove tempdir: {}".format(e))
+
+ return [(dmg_name, 'application/x-apple-diskimage', 'macOS .dmg')]
+
+
+def patch_windows(out_dir):
+ """Copy missing DLLs for windows into the given output."""
+ dll_dir = os.path.join('.tox', 'pyinstaller', 'lib', 'site-packages',
+ 'PyQt5', 'Qt', 'bin')
+ dlls = ['libEGL.dll', 'libGLESv2.dll', 'libeay32.dll', 'ssleay32.dll']
+ for dll in dlls:
+ shutil.copy(os.path.join(dll_dir, dll), out_dir)
+
+
+def build_windows():
+ """Build windows executables/setups."""
+ utils.print_title("Updating 3rdparty content")
+ # Currently disabled because QtWebEngine has no pdfjs support
+ # update_3rdparty.run(ace=False, pdfjs=True, fancy_dmg=False)
+
+ utils.print_title("Building Windows binaries")
+ parts = str(sys.version_info.major), str(sys.version_info.minor)
+ ver = ''.join(parts)
+ dot_ver = '.'.join(parts)
+
+ # Get python path from registry if possible
+ try:
+ reg64_key = winreg.OpenKeyEx(winreg.HKEY_LOCAL_MACHINE,
+ r'SOFTWARE\Python\PythonCore'
+ r'\{}\InstallPath'.format(dot_ver))
+ python_x64 = winreg.QueryValueEx(reg64_key, 'ExecutablePath')[0]
+ except FileNotFoundError:
+ python_x64 = r'C:\Python{}\python.exe'.format(ver)
+
+ out_pyinstaller = os.path.join('dist', 'qutebrowser')
+ out_64 = os.path.join('dist',
+ 'qutebrowser-{}-x64'.format(qutebrowser.__version__))
+
+ artifacts = []
+
+ from scripts.dev import gen_versioninfo
+ utils.print_title("Updating VersionInfo file")
+ gen_versioninfo.main()
+
+ utils.print_title("Running pyinstaller 64bit")
+ _maybe_remove(out_64)
+ call_tox('pyinstaller', '-r', python=python_x64)
+ shutil.move(out_pyinstaller, out_64)
+ patch_windows(out_64)
+
+ utils.print_title("Building installers")
+ subprocess.run(['makensis.exe',
+ '/DX64',
+ '/DVERSION={}'.format(qutebrowser.__version__),
+ 'misc/qutebrowser.nsi'], check=True)
+
+ name_64 = 'qutebrowser-{}-amd64.exe'.format(qutebrowser.__version__)
+
+ artifacts += [
+ (os.path.join('dist', name_64),
+ 'application/vnd.microsoft.portable-executable',
+ 'Windows 64bit installer'),
+ ]
+
+ utils.print_title("Running 64bit smoke test")
+ smoke_test(os.path.join(out_64, 'qutebrowser.exe'))
+
+ utils.print_title("Zipping 64bit standalone...")
+ name = 'qutebrowser-{}-windows-standalone-amd64'.format(
+ qutebrowser.__version__)
+ shutil.make_archive(name, 'zip', 'dist', os.path.basename(out_64))
+ artifacts.append(('{}.zip'.format(name),
+ 'application/zip',
+ 'Windows 64bit standalone'))
+
+ return artifacts
+
+
+def build_sdist():
+ """Build an sdist and list the contents."""
+ utils.print_title("Building sdist")
+
+ _maybe_remove('dist')
+
+ subprocess.run([sys.executable, 'setup.py', 'sdist'], check=True)
+ dist_files = os.listdir(os.path.abspath('dist'))
+ assert len(dist_files) == 1
+
+ dist_file = os.path.join('dist', dist_files[0])
+ subprocess.run(['gpg', '--detach-sign', '-a', dist_file], check=True)
+
+ tar = tarfile.open(dist_file)
+ by_ext = collections.defaultdict(list)
+
+ for tarinfo in tar.getmembers():
+ if not tarinfo.isfile():
+ continue
+ name = os.sep.join(tarinfo.name.split(os.sep)[1:])
+ _base, ext = os.path.splitext(name)
+ by_ext[ext].append(name)
+
+ assert '.pyc' not in by_ext
+
+ utils.print_title("sdist contents")
+
+ for ext, files in sorted(by_ext.items()):
+ utils.print_subtitle(ext)
+ print('\n'.join(files))
+
+ filename = 'qutebrowser-{}.tar.gz'.format(qutebrowser.__version__)
+ artifacts = [
+ (os.path.join('dist', filename), 'application/gzip', 'Source release'),
+ (os.path.join('dist', filename + '.asc'), 'application/pgp-signature',
+ 'Source release - PGP signature'),
+ ]
+
+ return artifacts
+
+
+def test_makefile():
+ """Make sure the Makefile works correctly."""
+ utils.print_title("Testing makefile")
+ with tempfile.TemporaryDirectory() as tmpdir:
+ subprocess.run(['make', '-f', 'misc/Makefile',
+ 'DESTDIR={}'.format(tmpdir), 'install'], check=True)
+
+
+def read_github_token():
+ """Read the GitHub API token from disk."""
+ token_file = os.path.join(os.path.expanduser('~'), '.gh_token')
+ with open(token_file, encoding='ascii') as f:
+ token = f.read().strip()
+ return token
+
+
+def github_upload(artifacts, tag):
+ """Upload the given artifacts to GitHub.
+
+ Args:
+ artifacts: A list of (filename, mimetype, description) tuples
+ tag: The name of the release tag
+ """
+ import github3
+ utils.print_title("Uploading to github...")
+
+ token = read_github_token()
+ gh = github3.login(token=token)
+ repo = gh.repository('qutebrowser', 'qutebrowser')
+
+ release = None # to satisfy pylint
+ for release in repo.releases():
+ if release.tag_name == tag:
+ break
+ else:
+ raise Exception("No release found for {!r}!".format(tag))
+
+ for filename, mimetype, description in artifacts:
+ with open(filename, 'rb') as f:
+ basename = os.path.basename(filename)
+ asset = release.upload_asset(mimetype, basename, f)
+ asset.edit(basename, description)
+
+
+def pypi_upload(artifacts):
+ """Upload the given artifacts to PyPI using twine."""
+ filenames = [a[0] for a in artifacts]
+ subprocess.run(['twine', 'upload'] + filenames, check=True)
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--asciidoc', help="Full path to python and "
+ "asciidoc.py. If not given, it's searched in PATH.",
+ nargs=2, required=False,
+ metavar=('PYTHON', 'ASCIIDOC'))
+ parser.add_argument('--upload', help="Tag to upload the release for",
+ nargs=1, required=False, metavar='TAG')
+ args = parser.parse_args()
+ utils.change_cwd()
+
+ upload_to_pypi = False
+
+ if args.upload is not None:
+ # Fail early when trying to upload without github3 installed
+ # or without API token
+ import github3 # pylint: disable=unused-variable
+ read_github_token()
+
+ run_asciidoc2html(args)
+ if os.name == 'nt':
+ artifacts = build_windows()
+ elif sys.platform == 'darwin':
+ artifacts = build_mac()
+ else:
+ test_makefile()
+ artifacts = build_sdist()
+ upload_to_pypi = True
+
+ if args.upload is not None:
+ utils.print_title("Press enter to release...")
+ input()
+ github_upload(artifacts, args.upload[0])
+ if upload_to_pypi:
+ pypi_upload(artifacts)
+ else:
+ print()
+ utils.print_title("Artifacts")
+ for artifact in artifacts:
+ print(artifact)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/.config/qutebrowser/scripts/dev/check_coverage.py b/.config/qutebrowser/scripts/dev/check_coverage.py
new file mode 100644
index 0000000..32c5afc
--- /dev/null
+++ b/.config/qutebrowser/scripts/dev/check_coverage.py
@@ -0,0 +1,348 @@
+#!/usr/bin/env python3
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2015-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>.
+
+"""Enforce perfect coverage on some files."""
+
+import os
+import os.path
+import sys
+import enum
+import subprocess
+from xml.etree import ElementTree
+
+import attr
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir,
+ os.pardir))
+
+from scripts import utils as scriptutils
+from qutebrowser.utils import utils
+
+
+@attr.s
+class Message:
+
+ """A message shown by coverage.py."""
+
+ typ = attr.ib()
+ filename = attr.ib()
+ text = attr.ib()
+
+
+MsgType = enum.Enum('MsgType', 'insufficent_coverage, perfect_file')
+
+
+# A list of (test_file, tested_file) tuples. test_file can be None.
+PERFECT_FILES = [
+ (None,
+ 'commands/cmdexc.py'),
+ ('tests/unit/commands/test_cmdutils.py',
+ 'commands/cmdutils.py'),
+ ('tests/unit/commands/test_argparser.py',
+ 'commands/argparser.py'),
+
+ ('tests/unit/browser/webkit/test_cache.py',
+ 'browser/webkit/cache.py'),
+ ('tests/unit/browser/webkit/test_cookies.py',
+ 'browser/webkit/cookies.py'),
+ ('tests/unit/browser/test_history.py',
+ 'browser/history.py'),
+ ('tests/unit/browser/webkit/http/test_http.py',
+ 'browser/webkit/http.py'),
+ ('tests/unit/browser/webkit/http/test_content_disposition.py',
+ 'browser/webkit/rfc6266.py'),
+ # ('tests/unit/browser/webkit/test_webkitelem.py',
+ # 'browser/webkit/webkitelem.py'),
+ # ('tests/unit/browser/webkit/test_webkitelem.py',
+ # 'browser/webelem.py'),
+ ('tests/unit/browser/webkit/network/test_filescheme.py',
+ 'browser/webkit/network/filescheme.py'),
+ ('tests/unit/browser/webkit/network/test_networkreply.py',
+ 'browser/webkit/network/networkreply.py'),
+
+ ('tests/unit/browser/test_signalfilter.py',
+ 'browser/signalfilter.py'),
+ (None,
+ 'browser/webengine/certificateerror.py'),
+ # ('tests/unit/browser/test_tab.py',
+ # 'browser/tab.py'),
+
+ ('tests/unit/keyinput/test_basekeyparser.py',
+ 'keyinput/basekeyparser.py'),
+ ('tests/unit/keyinput/test_keyutils.py',
+ 'keyinput/keyutils.py'),
+
+ ('tests/unit/misc/test_autoupdate.py',
+ 'misc/autoupdate.py'),
+ ('tests/unit/misc/test_readline.py',
+ 'misc/readline.py'),
+ ('tests/unit/misc/test_split.py',
+ 'misc/split.py'),
+ ('tests/unit/misc/test_msgbox.py',
+ 'misc/msgbox.py'),
+ ('tests/unit/misc/test_checkpyver.py',
+ 'misc/checkpyver.py'),
+ ('tests/unit/misc/test_guiprocess.py',
+ 'misc/guiprocess.py'),
+ ('tests/unit/misc/test_editor.py',
+ 'misc/editor.py'),
+ ('tests/unit/misc/test_cmdhistory.py',
+ 'misc/cmdhistory.py'),
+ ('tests/unit/misc/test_ipc.py',
+ 'misc/ipc.py'),
+ ('tests/unit/misc/test_keyhints.py',
+ 'misc/keyhintwidget.py'),
+ ('tests/unit/misc/test_pastebin.py',
+ 'misc/pastebin.py'),
+ (None,
+ 'misc/objects.py'),
+
+ (None,
+ 'mainwindow/statusbar/keystring.py'),
+ ('tests/unit/mainwindow/statusbar/test_percentage.py',
+ 'mainwindow/statusbar/percentage.py'),
+ ('tests/unit/mainwindow/statusbar/test_progress.py',
+ 'mainwindow/statusbar/progress.py'),
+ ('tests/unit/mainwindow/statusbar/test_tabindex.py',
+ 'mainwindow/statusbar/tabindex.py'),
+ ('tests/unit/mainwindow/statusbar/test_textbase.py',
+ 'mainwindow/statusbar/textbase.py'),
+ ('tests/unit/mainwindow/statusbar/test_url.py',
+ 'mainwindow/statusbar/url.py'),
+ ('tests/unit/mainwindow/statusbar/test_backforward.py',
+ 'mainwindow/statusbar/backforward.py'),
+ ('tests/unit/mainwindow/test_messageview.py',
+ 'mainwindow/messageview.py'),
+
+ ('tests/unit/config/test_config.py',
+ 'config/config.py'),
+ ('tests/unit/config/test_configdata.py',
+ 'config/configdata.py'),
+ ('tests/unit/config/test_configexc.py',
+ 'config/configexc.py'),
+ ('tests/unit/config/test_configfiles.py',
+ 'config/configfiles.py'),
+ ('tests/unit/config/test_configtypes.py',
+ 'config/configtypes.py'),
+ ('tests/unit/config/test_configinit.py',
+ 'config/configinit.py'),
+ ('tests/unit/config/test_configcommands.py',
+ 'config/configcommands.py'),
+ ('tests/unit/config/test_configutils.py',
+ 'config/configutils.py'),
+
+ ('tests/unit/utils/test_qtutils.py',
+ 'utils/qtutils.py'),
+ ('tests/unit/utils/test_standarddir.py',
+ 'utils/standarddir.py'),
+ ('tests/unit/utils/test_urlutils.py',
+ 'utils/urlutils.py'),
+ ('tests/unit/utils/usertypes',
+ 'utils/usertypes.py'),
+ ('tests/unit/utils/test_utils.py',
+ 'utils/utils.py'),
+ ('tests/unit/utils/test_version.py',
+ 'utils/version.py'),
+ ('tests/unit/utils/test_debug.py',
+ 'utils/debug.py'),
+ ('tests/unit/utils/test_jinja.py',
+ 'utils/jinja.py'),
+ ('tests/unit/utils/test_error.py',
+ 'utils/error.py'),
+ ('tests/unit/utils/test_javascript.py',
+ 'utils/javascript.py'),
+ ('tests/unit/utils/test_urlmatch.py',
+ 'utils/urlmatch.py'),
+
+ (None,
+ 'completion/models/util.py'),
+ ('tests/unit/completion/test_models.py',
+ 'completion/models/urlmodel.py'),
+ ('tests/unit/completion/test_models.py',
+ 'completion/models/configmodel.py'),
+ ('tests/unit/completion/test_histcategory.py',
+ 'completion/models/histcategory.py'),
+ ('tests/unit/completion/test_listcategory.py',
+ 'completion/models/listcategory.py'),
+
+ ('tests/unit/browser/webengine/test_spell.py',
+ 'browser/webengine/spell.py'),
+
+]
+
+
+# 100% coverage because of end2end tests, but no perfect unit tests yet.
+WHITELISTED_FILES = [
+ 'browser/webkit/webkitinspector.py',
+ 'keyinput/macros.py',
+ 'browser/webkit/webkitelem.py',
+]
+
+
+class Skipped(Exception):
+
+ """Exception raised when skipping coverage checks."""
+
+ def __init__(self, reason):
+ self.reason = reason
+ super().__init__("Skipping coverage checks " + reason)
+
+
+def _get_filename(filename):
+ """Transform the absolute test filenames to relative ones."""
+ if os.path.isabs(filename):
+ basedir = os.path.abspath(
+ os.path.join(os.path.dirname(__file__), '..', '..'))
+ common_path = os.path.commonprefix([basedir, filename])
+ if common_path:
+ filename = filename[len(common_path):].lstrip('/')
+ if filename.startswith('qutebrowser/'):
+ filename = filename.split('/', maxsplit=1)[1]
+
+ return filename
+
+
+def check(fileobj, perfect_files):
+ """Main entry point which parses/checks coverage.xml if applicable."""
+ if not utils.is_linux:
+ raise Skipped("on non-Linux system.")
+ elif '-k' in sys.argv[1:]:
+ raise Skipped("because -k is given.")
+ elif '-m' in sys.argv[1:]:
+ raise Skipped("because -m is given.")
+ elif '--lf' in sys.argv[1:]:
+ raise Skipped("because --lf is given.")
+
+ perfect_src_files = [e[1] for e in perfect_files]
+
+ filename_args = [arg for arg in sys.argv[1:]
+ if arg.startswith('tests' + os.sep)]
+ filtered_files = [tpl[1] for tpl in perfect_files if tpl[0] in
+ filename_args]
+
+ if filename_args and not filtered_files:
+ raise Skipped("because there is nothing to check.")
+
+ tree = ElementTree.parse(fileobj)
+ classes = tree.getroot().findall('./packages/package/classes/class')
+
+ messages = []
+
+ for klass in classes:
+ filename = _get_filename(klass.attrib['filename'])
+
+ line_cov = float(klass.attrib['line-rate']) * 100
+ branch_cov = float(klass.attrib['branch-rate']) * 100
+
+ if filtered_files and filename not in filtered_files:
+ continue
+
+ assert 0 <= line_cov <= 100, line_cov
+ assert 0 <= branch_cov <= 100, branch_cov
+ assert '\\' not in filename, filename
+
+ is_bad = line_cov < 100 or branch_cov < 100
+
+ if filename in perfect_src_files and is_bad:
+ text = "{} has {:.2f}% line and {:.2f}% branch coverage!".format(
+ filename, line_cov, branch_cov)
+ messages.append(Message(MsgType.insufficent_coverage, filename,
+ text))
+ elif (filename not in perfect_src_files and not is_bad and
+ filename not in WHITELISTED_FILES):
+ text = ("{} has 100% coverage but is not in "
+ "perfect_files!".format(filename))
+ messages.append(Message(MsgType.perfect_file, filename, text))
+
+ return messages
+
+
+def main_check():
+ """Check coverage after a test run."""
+ try:
+ with open('coverage.xml', encoding='utf-8') as f:
+ messages = check(f, PERFECT_FILES)
+ except Skipped as e:
+ print(e)
+ messages = []
+
+ if messages:
+ print()
+ print()
+ scriptutils.print_title("Coverage check failed")
+ for msg in messages:
+ print(msg.text)
+ print()
+ filters = ','.join('qutebrowser/' + msg.filename for msg in messages)
+ subprocess.run([sys.executable, '-m', 'coverage', 'report',
+ '--show-missing', '--include', filters], check=True)
+ print()
+ print("To debug this, run 'tox -e py36-pyqt59-cov' "
+ "(or py35-pyqt59-cov) locally and check htmlcov/index.html")
+ print("or check https://codecov.io/github/qutebrowser/qutebrowser")
+ print()
+
+ if 'CI' in os.environ:
+ print("Keeping coverage.xml on CI.")
+ else:
+ os.remove('coverage.xml')
+ return 1 if messages else 0
+
+
+def main_check_all():
+ """Check the coverage for all files individually.
+
+ This makes sure the files have 100% coverage without running unrelated
+ tests.
+
+ This runs pytest with the used executable, so check_coverage.py should be
+ called with something like ./.tox/py36/bin/python.
+ """
+ for test_file, src_file in PERFECT_FILES:
+ if test_file is None:
+ continue
+ subprocess.run(
+ [sys.executable, '-m', 'pytest', '--cov', 'qutebrowser',
+ '--cov-report', 'xml', test_file], check=True)
+ with open('coverage.xml', encoding='utf-8') as f:
+ messages = check(f, [(test_file, src_file)])
+ os.remove('coverage.xml')
+
+ messages = [msg for msg in messages
+ if msg.typ == MsgType.insufficent_coverage]
+ if messages:
+ for msg in messages:
+ print(msg.text)
+ return 1
+ else:
+ print("Check ok!")
+ return 0
+
+
+def main():
+ scriptutils.change_cwd()
+ if '--check-all' in sys.argv:
+ return main_check_all()
+ else:
+ return main_check()
+
+
+if __name__ == '__main__':
+ sys.exit(main())
diff --git a/.config/qutebrowser/scripts/dev/check_doc_changes.py b/.config/qutebrowser/scripts/dev/check_doc_changes.py
new file mode 100755
index 0000000..3d90bea
--- /dev/null
+++ b/.config/qutebrowser/scripts/dev/check_doc_changes.py
@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2016-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>.
+
+"""Check if docs changed and output an error if so."""
+
+import sys
+import subprocess
+import os
+
+code = subprocess.run(['git', '--no-pager', 'diff',
+ '--exit-code', '--stat']).returncode
+
+if os.environ.get('TRAVIS_PULL_REQUEST', 'false') != 'false':
+ if code != 0:
+ print("Docs changed but ignoring change as we're building a PR")
+ sys.exit(0)
+
+if code != 0:
+ print()
+ print('The autogenerated docs changed, please run this to update them:')
+ print(' tox -e docs')
+ print(' git commit -am "Update docs"')
+ print()
+ print('(Or you have uncommitted changes, in which case you can ignore '
+ 'this.)')
+ if 'TRAVIS' in os.environ:
+ print()
+ print("travis_fold:start:gitdiff")
+ subprocess.run(['git', '--no-pager', 'diff'])
+ print("travis_fold:end:gitdiff")
+sys.exit(code)
diff --git a/.config/qutebrowser/scripts/dev/ci/travis_backtrace.sh b/.config/qutebrowser/scripts/dev/ci/travis_backtrace.sh
new file mode 100644
index 0000000..227dde8
--- /dev/null
+++ b/.config/qutebrowser/scripts/dev/ci/travis_backtrace.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+#
+# Find all possible core files under current directory. Attempt
+# to determine exe using file(1) and dump stack trace with gdb.
+#
+
+case $TESTENV in
+ py3*-pyqt*)
+ exe=$(readlink -f ".tox/$TESTENV/bin/python")
+ full=
+ ;;
+ *)
+ echo "Skipping coredump analysis in testenv $TESTENV!"
+ exit 0
+ ;;
+esac
+
+find . \( -name "*.core" -o -name core \) -exec gdb --batch --quiet -ex "thread apply all bt $full" "$exe" {} \;
diff --git a/.config/qutebrowser/scripts/dev/ci/travis_install.sh b/.config/qutebrowser/scripts/dev/ci/travis_install.sh
new file mode 100644
index 0000000..18f5aa9
--- /dev/null
+++ b/.config/qutebrowser/scripts/dev/ci/travis_install.sh
@@ -0,0 +1,108 @@
+#!/bin/bash
+# vim: ft=sh fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>.
+
+# Stolen from https://github.com/travis-ci/travis-build/blob/master/lib/travis/build/templates/header.sh
+# and adjusted to use ((...))
+travis_retry() {
+ local ANSI_RED='\033[31;1m'
+ local ANSI_RESET='\033[0m'
+ local result=0
+ local count=1
+ while (( count < 3 )); do
+ if (( result != 0 )); then
+ echo -e "\\n${ANSI_RED}The command \"$*\" failed. Retrying, $count of 3.${ANSI_RESET}\\n" >&2
+ fi
+ "$@"
+ result=$?
+ (( result == 0 )) && break
+ count=$(( count + 1 ))
+ sleep 1
+ done
+
+ if (( count > 3 )); then
+ echo -e "\\n${ANSI_RED}The command \"$*\" failed 3 times.${ANSI_RESET}\\n" >&2
+ fi
+
+ return $result
+}
+
+pip_install() {
+ travis_retry python3 -m pip install "$@"
+}
+
+npm_install() {
+ # Make sure npm is up-to-date first
+ travis_retry npm install -g npm
+ travis_retry npm install -g "$@"
+}
+
+check_pyqt() {
+ python3 <<EOF
+import sys
+from PyQt5.QtCore import QT_VERSION_STR, PYQT_VERSION_STR, qVersion
+try:
+ from PyQt.sip import SIP_VERSION_STR
+except ModuleNotFoundError:
+ from sip import SIP_VERSION_STR
+
+print("Python {}".format(sys.version))
+print("PyQt5 {}".format(PYQT_VERSION_STR))
+print("Qt5 {} (runtime {})".format(QT_VERSION_STR, qVersion()))
+print("sip {}".format(SIP_VERSION_STR))
+EOF
+}
+
+set -e
+
+if [[ $DOCKER ]]; then
+ exit 0
+elif [[ $TRAVIS_OS_NAME == osx ]]; then
+ # Disable App Nap
+ defaults write NSGlobalDomain NSAppSleepDisabled -bool YES
+
+ curl -LO https://bootstrap.pypa.io/get-pip.py
+ sudo -H python get-pip.py
+
+ brew --version
+ brew update
+ brew upgrade python libyaml
+ brew install qt5 pyqt5
+
+ pip_install -r misc/requirements/requirements-tox.txt
+ python3 -m pip --version
+ tox --version
+ check_pyqt
+ exit 0
+fi
+
+case $TESTENV in
+ eslint)
+ npm_install eslint
+ ;;
+ shellcheck)
+ ;;
+ *)
+ pip_install pip
+ pip_install -r misc/requirements/requirements-tox.txt
+ if [[ $TESTENV == *-cov ]]; then
+ pip_install -r misc/requirements/requirements-codecov.txt
+ fi
+ ;;
+esac
diff --git a/.config/qutebrowser/scripts/dev/ci/travis_run.sh b/.config/qutebrowser/scripts/dev/ci/travis_run.sh
new file mode 100644
index 0000000..55ca7c1
--- /dev/null
+++ b/.config/qutebrowser/scripts/dev/ci/travis_run.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+
+if [[ $DOCKER ]]; then
+ docker run \
+ --privileged \
+ -v "$PWD:/outside" \
+ -e "QUTE_BDD_WEBENGINE=$QUTE_BDD_WEBENGINE" \
+ -e "DOCKER=$DOCKER" \
+ -e "CI=$CI" \
+ -e "TRAVIS=$TRAVIS" \
+ "qutebrowser/travis:$DOCKER"
+elif [[ $TESTENV == eslint ]]; then
+ # Can't run this via tox as we can't easily install tox in the javascript
+ # travis env
+ cd qutebrowser/javascript || exit 1
+ eslint --color --report-unused-disable-directives .
+elif [[ $TESTENV == shellcheck ]]; then
+ SCRIPTS=$( mktemp )
+ find scripts/dev/ -name '*.sh' >"$SCRIPTS"
+ find misc/userscripts/ -type f -exec grep -lE '[/ ][bd]ash$|[/ ]sh$|[/ ]ksh$' {} + >>"$SCRIPTS"
+ mapfile -t scripts <"$SCRIPTS"
+ rm -f "$SCRIPTS"
+ docker run \
+ -v "$PWD:/outside" \
+ -w /outside \
+ koalaman/shellcheck:latest "${scripts[@]}"
+else
+ args=()
+ [[ $TRAVIS_OS_NAME == osx ]] && args=('--qute-bdd-webengine' '--no-xvfb' 'tests/unit')
+
+ tox -e "$TESTENV" -- "${args[@]}"
+fi
diff --git a/.config/qutebrowser/scripts/dev/cleanup.py b/.config/qutebrowser/scripts/dev/cleanup.py
new file mode 100755
index 0000000..d1bb84a
--- /dev/null
+++ b/.config/qutebrowser/scripts/dev/cleanup.py
@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>.
+
+"""Script to clean up the mess made by Python/setuptools/PyInstaller."""
+
+import os
+import os.path
+import sys
+import glob
+import shutil
+import fnmatch
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir,
+ os.pardir))
+
+from scripts import utils
+
+
+recursive_lint = ('__pycache__', '*.pyc')
+lint = ('build', 'dist', 'pkg/pkg', 'pkg/qutebrowser-*.pkg.tar.xz', 'pkg/src',
+ 'pkg/qutebrowser', 'qutebrowser.egg-info', 'setuptools-*.egg',
+ 'setuptools-*.zip', 'doc/qutebrowser.asciidoc', 'doc/*.html',
+ 'doc/qutebrowser.1', 'README.html', 'qutebrowser/html/doc')
+
+
+def remove(path):
+ """Remove either a file or directory unless --dry-run is given."""
+ if os.path.isdir(path):
+ print("rm -r '{}'".format(path))
+ if '--dry-run' not in sys.argv:
+ shutil.rmtree(path)
+ else:
+ print("rm '{}'".format(path))
+ if '--dry-run' not in sys.argv:
+ os.remove(path)
+
+
+def main():
+ """Clean up lint in the current dir."""
+ utils.change_cwd()
+ for elem in lint:
+ for f in glob.glob(elem):
+ remove(f)
+
+ for root, _dirs, _files in os.walk(os.getcwd()):
+ path = os.path.basename(root)
+ if any(fnmatch.fnmatch(path, e) for e in recursive_lint):
+ remove(root)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/.config/qutebrowser/scripts/dev/download_release.sh b/.config/qutebrowser/scripts/dev/download_release.sh
new file mode 100644
index 0000000..207da21
--- /dev/null
+++ b/.config/qutebrowser/scripts/dev/download_release.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+set -e
+
+# This script downloads the given release from GitHub so we can mirror it on
+# qutebrowser.org.
+
+tmpdir=$(mktemp -d)
+oldpwd=$PWD
+
+if [[ $# != 1 ]]; then
+ echo "Usage: $0 <version>" >&2
+ exit 1
+fi
+
+cd "$tmpdir"
+mkdir windows
+
+base="https://github.com/qutebrowser/qutebrowser/releases/download/v$1"
+
+wget "$base/qutebrowser-$1.tar.gz"
+wget "$base/qutebrowser-$1.tar.gz.asc"
+wget "$base/qutebrowser-$1.dmg"
+wget "$base/qutebrowser_${1}-1_all.deb"
+
+cd windows
+wget "$base/qutebrowser-${1}-amd64.msi"
+wget "$base/qutebrowser-${1}-win32.msi"
+wget "$base/qutebrowser-${1}-windows-standalone-amd64.zip"
+wget "$base/qutebrowser-${1}-windows-standalone-win32.zip"
+
+dest="/srv/http/qutebrowser/releases/v$1"
+cd "$oldpwd"
+sudo mv "$tmpdir" "$dest"
+sudo chown -R http:http "$dest"
diff --git a/.config/qutebrowser/scripts/dev/gen_resources.py b/.config/qutebrowser/scripts/dev/gen_resources.py
new file mode 100644
index 0000000..cbfc69b
--- /dev/null
+++ b/.config/qutebrowser/scripts/dev/gen_resources.py
@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# copyright 2014 florian bruhin (the compiler) <mail@qutebrowser.org>
+
+# this file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the gnu general public license as published by
+# the free software foundation, either version 3 of the license, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but without any warranty; without even the implied warranty of
+# merchantability or fitness for a particular purpose. see the
+# gnu general public license for more details.
+#
+# you should have received a copy of the gnu general public license
+# along with qutebrowser. if not, see <http://www.gnu.org/licenses/>.
+
+"""Generate Qt resources based on source files."""
+
+import subprocess
+
+with open('qutebrowser/resources.py', 'w', encoding='utf-8') as f:
+ subprocess.run(['pyrcc5', 'qutebrowser.rcc'], stdout=f, check=True)
diff --git a/.config/qutebrowser/scripts/dev/gen_versioninfo.py b/.config/qutebrowser/scripts/dev/gen_versioninfo.py
new file mode 100644
index 0000000..1aa4b64
--- /dev/null
+++ b/.config/qutebrowser/scripts/dev/gen_versioninfo.py
@@ -0,0 +1,85 @@
+#!/usr/bin/env python3
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>.
+
+"""Generate file_version_info.txt for Pyinstaller use with Windows builds."""
+
+import os.path
+import sys
+
+# pylint: disable=import-error,no-member,useless-suppression
+from PyInstaller.utils.win32 import versioninfo as vs
+# pylint: enable=import-error,no-member,useless-suppression
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir,
+ os.pardir))
+
+import qutebrowser
+from scripts import utils
+
+
+def main():
+ utils.change_cwd()
+ out_filename = 'misc/file_version_info.txt'
+
+ filevers = qutebrowser.__version_info__ + (0,)
+ prodvers = qutebrowser.__version_info__ + (0,)
+ str_filevers = qutebrowser.__version__
+ str_prodvers = qutebrowser.__version__
+
+ comment_text = qutebrowser.__doc__
+ copyright_text = qutebrowser.__copyright__
+ trademark_text = ("qutebrowser is free software under the GNU General "
+ "Public License")
+
+ # https://www.science.co.il/language/Locale-codes.php#definitions
+ # https://msdn.microsoft.com/en-us/library/windows/desktop/dd317756.aspx
+ en_us = 1033 # 0x0409
+ utf_16 = 1200 # 0x04B0
+
+ ffi = vs.FixedFileInfo(filevers, prodvers)
+
+ kids = [
+ vs.StringFileInfo([
+ # 0x0409: MAKELANGID(LANG_ENGLISH, SUBLANG_ENGLISH_US)
+ # 0x04B0: codepage 1200 (UTF-16LE)
+ vs.StringTable('040904B0', [
+ vs.StringStruct('Comments', comment_text),
+ vs.StringStruct('CompanyName', "qutebrowser.org"),
+ vs.StringStruct('FileDescription', "qutebrowser"),
+ vs.StringStruct('FileVersion', str_filevers),
+ vs.StringStruct('InternalName', "qutebrowser"),
+ vs.StringStruct('LegalCopyright', copyright_text),
+ vs.StringStruct('LegalTrademarks', trademark_text),
+ vs.StringStruct('OriginalFilename', "qutebrowser.exe"),
+ vs.StringStruct('ProductName', "qutebrowser"),
+ vs.StringStruct('ProductVersion', str_prodvers)
+ ]),
+ ]),
+ vs.VarFileInfo([vs.VarStruct('Translation', [en_us, utf_16])]),
+ ]
+
+ file_version_info = vs.VSVersionInfo(ffi, kids)
+
+ with open(out_filename, 'w', encoding='utf-8') as f:
+ f.write(str(file_version_info))
+
+
+if __name__ == '__main__':
+ main()
diff --git a/.config/qutebrowser/scripts/dev/get_coredumpctl_traces.py b/.config/qutebrowser/scripts/dev/get_coredumpctl_traces.py
new file mode 100644
index 0000000..d286d38
--- /dev/null
+++ b/.config/qutebrowser/scripts/dev/get_coredumpctl_traces.py
@@ -0,0 +1,176 @@
+#!/usr/bin/env python3
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2015-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>.
+
+"""Get qutebrowser crash information and stacktraces from coredumpctl."""
+
+import os
+import os.path
+import sys
+import argparse
+import subprocess
+import tempfile
+
+import attr
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir,
+ os.pardir))
+
+from scripts import utils
+
+
+@attr.s
+class Line:
+
+ """A line in "coredumpctl list"."""
+
+ time = attr.ib()
+ pid = attr.ib()
+ uid = attr.ib()
+ gid = attr.ib()
+ sig = attr.ib()
+ present = attr.ib()
+ exe = attr.ib()
+
+
+def _convert_present(data):
+ """Convert " "/"*" to True/False for parse_coredumpctl_line."""
+ if data == '*':
+ return True
+ elif data == ' ':
+ return False
+ else:
+ raise ValueError(data)
+
+
+def parse_coredumpctl_line(line):
+ """Parse a given string coming from coredumpctl and return a Line object.
+
+ Example input:
+ Mon 2015-09-28 23:22:24 CEST 10606 1000 1000 11 /usr/bin/python3.4
+ """
+ fields = {
+ 'time': (0, 28, str),
+ 'pid': (29, 35, int),
+ 'uid': (36, 41, int),
+ 'gid': (42, 47, int),
+ 'sig': (48, 51, int),
+ 'present': (52, 53, _convert_present),
+ 'exe': (54, None, str),
+ }
+
+ data = {}
+ for name, (start, end, converter) in fields.items():
+ data[name] = converter(line[start:end])
+ return Line(**data)
+
+
+def get_info(pid):
+ """Get and parse "coredumpctl info" output for the given PID."""
+ data = {}
+ output = subprocess.run(['coredumpctl', 'info', str(pid)], check=True,
+ stdout=subprocess.PIPE).stdout
+ output = output.decode('utf-8')
+ for line in output.split('\n'):
+ if not line.strip():
+ continue
+ try:
+ key, value = line.split(':', maxsplit=1)
+ except ValueError:
+ # systemd stack output
+ continue
+ data[key.strip()] = value.strip()
+ return data
+
+
+def is_qutebrowser_dump(parsed):
+ """Check if the given Line is a qutebrowser dump."""
+ basename = os.path.basename(parsed.exe)
+ if basename == 'python' or basename.startswith('python3'):
+ info = get_info(parsed.pid)
+ try:
+ cmdline = info['Command Line']
+ except KeyError:
+ return True
+ else:
+ return '-m qutebrowser' in cmdline
+ else:
+ return basename == 'qutebrowser'
+
+
+def dump_infos_gdb(parsed):
+ """Dump all needed infos for the given crash using gdb."""
+ with tempfile.TemporaryDirectory() as tempdir:
+ coredump = os.path.join(tempdir, 'dump')
+ subprocess.run(['coredumpctl', 'dump', '-o', coredump,
+ str(parsed.pid)], check=True)
+ subprocess.run(['gdb', parsed.exe, coredump,
+ '-ex', 'info threads',
+ '-ex', 'thread apply all bt full',
+ '-ex', 'quit'], check=True)
+
+
+def dump_infos(parsed):
+ """Dump all possible infos for the given crash."""
+ if not parsed.present:
+ info = get_info(parsed.pid)
+ print("{}: Signal {} with no coredump: {}".format(
+ parsed.time, info.get('Signal', None),
+ info.get('Command Line', None)))
+ else:
+ print('\n\n\n')
+ utils.print_title('{} - {}'.format(parsed.time, parsed.pid))
+ sys.stdout.flush()
+ dump_infos_gdb(parsed)
+
+
+def check_prerequisites():
+ """Check if coredumpctl/gdb are installed."""
+ for binary in ['coredumpctl', 'gdb']:
+ try:
+ subprocess.run([binary, '--version'], check=True)
+ except FileNotFoundError:
+ print("{} is needed to run this script!".format(binary),
+ file=sys.stderr)
+ sys.exit(1)
+
+
+def main():
+ check_prerequisites()
+
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--all', help="Also list crashes without coredumps.",
+ action='store_true')
+ args = parser.parse_args()
+
+ coredumps = subprocess.run(['coredumpctl', 'list'], check=True,
+ stdout=subprocess.PIPE).stdout
+ lines = coredumps.decode('utf-8').split('\n')
+ for line in lines[1:]:
+ if not line.strip():
+ continue
+ parsed = parse_coredumpctl_line(line)
+ if not parsed.present and not args.all:
+ continue
+ if is_qutebrowser_dump(parsed):
+ dump_infos(parsed)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/.config/qutebrowser/scripts/dev/misc_checks.py b/.config/qutebrowser/scripts/dev/misc_checks.py
new file mode 100644
index 0000000..2992464
--- /dev/null
+++ b/.config/qutebrowser/scripts/dev/misc_checks.py
@@ -0,0 +1,163 @@
+#!/usr/bin/env python3
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>.
+
+"""Various small code checkers."""
+
+import os
+import os.path
+import re
+import sys
+import argparse
+import subprocess
+import tokenize
+import traceback
+import collections
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir,
+ os.pardir))
+
+from scripts import utils
+
+
+def _get_files(only_py=False):
+ """Iterate over all python files and yield filenames."""
+ for (dirpath, _dirnames, filenames) in os.walk('.'):
+ parts = dirpath.split(os.sep)
+ if len(parts) >= 2:
+ rootdir = parts[1]
+ if rootdir.startswith('.') or rootdir == 'htmlcov':
+ # ignore hidden dirs and htmlcov
+ continue
+
+ if only_py:
+ endings = {'.py'}
+ else:
+ endings = {'.py', '.asciidoc', '.js', '.feature'}
+ files = (e for e in filenames if os.path.splitext(e)[1] in endings)
+ for name in files:
+ yield os.path.join(dirpath, name)
+
+
+def check_git():
+ """Check for uncommitted git files.."""
+ if not os.path.isdir(".git"):
+ print("No .git dir, ignoring")
+ print()
+ return False
+ untracked = []
+ gitst = subprocess.run(['git', 'status', '--porcelain'], check=True,
+ stdout=subprocess.PIPE).stdout
+ gitst = gitst.decode('UTF-8').strip()
+ for line in gitst.splitlines():
+ s, name = line.split(maxsplit=1)
+ if s == '??' and name != '.venv/':
+ untracked.append(name)
+ status = True
+ if untracked:
+ status = False
+ utils.print_col("Untracked files:", 'red')
+ print('\n'.join(untracked))
+ print()
+ return status
+
+
+def check_spelling():
+ """Check commonly misspelled words."""
+ # Words which I often misspell
+ words = {'[Bb]ehaviour', '[Qq]uitted', 'Ll]ikelyhood', '[Ss]ucessfully',
+ '[Oo]ccur[^rs .]', '[Ss]eperator', '[Ee]xplicitely',
+ '[Aa]uxillary', '[Aa]ccidentaly', '[Aa]mbigious', '[Ll]oosly',
+ '[Ii]nitialis', '[Cc]onvienence', '[Ss]imiliar', '[Uu]ncommited',
+ '[Rr]eproducable', '[Aa]n [Uu]ser', '[Cc]onvienience',
+ '[Ww]ether', '[Pp]rogramatically', '[Ss]plitted', '[Ee]xitted',
+ '[Mm]ininum', '[Rr]esett?ed', '[Rr]ecieved', '[Rr]egularily',
+ '[Uu]nderlaying', '[Ii]nexistant', '[Ee]lipsis', 'commiting',
+ 'existant', '[Rr]esetted', '[Ss]imilarily', '[Ii]nformations'}
+
+ # Words which look better when splitted, but might need some fine tuning.
+ words |= {'[Ww]ebelements', '[Mm]ouseevent', '[Kk]eysequence',
+ '[Nn]ormalmode', '[Ee]ventloops', '[Ss]izehint',
+ '[Ss]tatemachine', '[Mm]etaobject', '[Ll]ogrecord',
+ '[Ff]iletype'}
+
+ # Files which should be ignored, e.g. because they come from another
+ # package
+ ignored = [
+ os.path.join('.', 'scripts', 'dev', 'misc_checks.py'),
+ os.path.join('.', 'qutebrowser', '3rdparty', 'pdfjs'),
+ os.path.join('.', 'tests', 'end2end', 'data', 'hints', 'ace',
+ 'ace.js'),
+ ]
+
+ seen = collections.defaultdict(list)
+ try:
+ ok = True
+ for fn in _get_files():
+ with tokenize.open(fn) as f:
+ if any(fn.startswith(i) for i in ignored):
+ continue
+ for line in f:
+ for w in words:
+ if (re.search(w, line) and
+ fn not in seen[w] and
+ '# pragma: no spellcheck' not in line):
+ print('Found "{}" in {}!'.format(w, fn))
+ seen[w].append(fn)
+ ok = False
+ print()
+ return ok
+ except Exception:
+ traceback.print_exc()
+ return None
+
+
+def check_vcs_conflict():
+ """Check VCS conflict markers."""
+ try:
+ ok = True
+ for fn in _get_files(only_py=True):
+ with tokenize.open(fn) as f:
+ for line in f:
+ if any(line.startswith(c * 7) for c in '<>=|'):
+ print("Found conflict marker in {}".format(fn))
+ ok = False
+ print()
+ return ok
+ except Exception:
+ traceback.print_exc()
+ return None
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument('checker', choices=('git', 'vcs', 'spelling'),
+ help="Which checker to run.")
+ args = parser.parse_args()
+ if args.checker == 'git':
+ ok = check_git()
+ elif args.checker == 'vcs':
+ ok = check_vcs_conflict()
+ elif args.checker == 'spelling':
+ ok = check_spelling()
+ return 0 if ok else 1
+
+
+if __name__ == '__main__':
+ sys.exit(main())
diff --git a/.config/qutebrowser/scripts/dev/pylint_checkers/qute_pylint/__init__.py b/.config/qutebrowser/scripts/dev/pylint_checkers/qute_pylint/__init__.py
new file mode 100644
index 0000000..1341a93
--- /dev/null
+++ b/.config/qutebrowser/scripts/dev/pylint_checkers/qute_pylint/__init__.py
@@ -0,0 +1 @@
+"""Custom pylint checkers."""
diff --git a/.config/qutebrowser/scripts/dev/pylint_checkers/qute_pylint/config.py b/.config/qutebrowser/scripts/dev/pylint_checkers/qute_pylint/config.py
new file mode 100644
index 0000000..5aa5250
--- /dev/null
+++ b/.config/qutebrowser/scripts/dev/pylint_checkers/qute_pylint/config.py
@@ -0,0 +1,84 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>.
+
+"""Custom astroid checker for config calls."""
+
+import sys
+import pathlib
+
+import yaml
+import astroid
+from pylint import interfaces, checkers
+from pylint.checkers import utils
+
+
+OPTIONS = None
+FAILED_LOAD = False
+
+
+class ConfigChecker(checkers.BaseChecker):
+
+ """Custom astroid checker for config calls."""
+
+ __implements__ = interfaces.IAstroidChecker
+ name = 'config'
+ msgs = {
+ 'E9998': ('%s is no valid config option.', # flake8: disable=S001
+ 'bad-config-option',
+ None),
+ }
+ priority = -1
+ printed_warning = False
+
+ @utils.check_messages('bad-config-option')
+ def visit_attribute(self, node):
+ """Visit a getattr node."""
+ # At the end of a config.val.foo.bar chain
+ if not isinstance(node.parent, astroid.Attribute):
+ # FIXME:conf do some proper check for this...
+ node_str = node.as_string()
+ prefix = 'config.val.'
+ if node_str.startswith(prefix):
+ self._check_config(node, node_str[len(prefix):])
+
+ def _check_config(self, node, name):
+ """Check that we're accessing proper config options."""
+ if FAILED_LOAD:
+ if not ConfigChecker.printed_warning:
+ print("[WARN] Could not find configdata.yml. Please run "
+ "pylint from qutebrowser root.", file=sys.stderr)
+ print("Skipping some checks...", file=sys.stderr)
+ ConfigChecker.printed_warning = True
+ return
+ if name not in OPTIONS:
+ self.add_message('bad-config-option', node=node, args=name)
+
+
+def register(linter):
+ """Register this checker."""
+ linter.register_checker(ConfigChecker(linter))
+ global OPTIONS
+ global FAILED_LOAD
+ yaml_file = pathlib.Path('qutebrowser') / 'config' / 'configdata.yml'
+ if not yaml_file.exists():
+ OPTIONS = None
+ FAILED_LOAD = True
+ return
+ with yaml_file.open(mode='r', encoding='utf-8') as f:
+ OPTIONS = list(yaml.load(f))
diff --git a/.config/qutebrowser/scripts/dev/pylint_checkers/qute_pylint/modeline.py b/.config/qutebrowser/scripts/dev/pylint_checkers/qute_pylint/modeline.py
new file mode 100644
index 0000000..429974c
--- /dev/null
+++ b/.config/qutebrowser/scripts/dev/pylint_checkers/qute_pylint/modeline.py
@@ -0,0 +1,63 @@
+# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>.
+
+"""Checker for vim modelines in files."""
+
+import os.path
+import contextlib
+
+from pylint import interfaces, checkers
+
+
+class ModelineChecker(checkers.BaseChecker):
+
+ """Check for vim modelines in files."""
+
+ __implements__ = interfaces.IRawChecker
+
+ name = 'modeline'
+ msgs = {'W9002': ('Does not have vim modeline', 'modeline-missing', None),
+ 'W9003': ('Modeline is invalid', 'invalid-modeline', None),
+ 'W9004': ('Modeline position is wrong', 'modeline-position', None)}
+ options = ()
+ priority = -1
+
+ def process_module(self, node):
+ """Process the module."""
+ if os.path.basename(os.path.splitext(node.file)[0]) == '__init__':
+ return
+ max_lineno = 1
+ with contextlib.closing(node.stream()) as stream:
+ for (lineno, line) in enumerate(stream):
+ if lineno == 1 and line.startswith(b'#!'):
+ max_lineno += 1
+ continue
+ elif line.startswith(b'# vim:'):
+ if lineno > max_lineno:
+ self.add_message('modeline-position', line=lineno)
+ if (line.rstrip() != b'# vim: ft=python '
+ b'fileencoding=utf-8 sts=4 sw=4 et:'):
+ self.add_message('invalid-modeline', line=lineno)
+ break
+ else:
+ self.add_message('modeline-missing', line=1)
+
+
+def register(linter):
+ """Register the checker."""
+ linter.register_checker(ModelineChecker(linter))
diff --git a/.config/qutebrowser/scripts/dev/pylint_checkers/qute_pylint/openencoding.py b/.config/qutebrowser/scripts/dev/pylint_checkers/qute_pylint/openencoding.py
new file mode 100644
index 0000000..f577011
--- /dev/null
+++ b/.config/qutebrowser/scripts/dev/pylint_checkers/qute_pylint/openencoding.py
@@ -0,0 +1,83 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>.
+
+"""Make sure open() has an encoding set."""
+
+import astroid
+from pylint import interfaces, checkers
+from pylint.checkers import utils
+
+
+class OpenEncodingChecker(checkers.BaseChecker):
+
+ """Checker to check open() has an encoding set."""
+
+ __implements__ = interfaces.IAstroidChecker
+ name = 'open-encoding'
+
+ msgs = {
+ 'W9400': ('open() called without encoding', 'open-without-encoding',
+ None),
+ }
+
+ @utils.check_messages('open-without-encoding')
+ def visit_call(self, node):
+ """Visit a Call node."""
+ if hasattr(node, 'func'):
+ infer = utils.safe_infer(node.func)
+ if infer and infer.root().name == '_io':
+ if getattr(node.func, 'name', None) in ['open', 'file']:
+ self._check_open_encoding(node)
+
+ def _check_open_encoding(self, node):
+ """Check that an open() call always has an encoding set."""
+ try:
+ mode_arg = utils.get_argument_from_call(node, position=1,
+ keyword='mode')
+ except utils.NoSuchArgumentError:
+ mode_arg = None
+ _encoding = None
+ try:
+ _encoding = utils.get_argument_from_call(node, position=2)
+ except utils.NoSuchArgumentError:
+ try:
+ _encoding = utils.get_argument_from_call(node,
+ keyword='encoding')
+ except utils.NoSuchArgumentError:
+ pass
+ if _encoding is None:
+ if mode_arg is None:
+ mode = None
+ else:
+ mode = utils.safe_infer(mode_arg)
+ if mode is not None and not isinstance(mode, astroid.Const):
+ # We can't say what mode is exactly.
+ return
+ if mode is None:
+ self.add_message('open-without-encoding', node=node)
+ elif 'b' in getattr(mode, 'value', ''):
+ # Files opened as binary don't need an encoding.
+ return
+ else:
+ self.add_message('open-without-encoding', node=node)
+
+
+def register(linter):
+ """Register this checker."""
+ linter.register_checker(OpenEncodingChecker(linter))
diff --git a/.config/qutebrowser/scripts/dev/pylint_checkers/qute_pylint/settrace.py b/.config/qutebrowser/scripts/dev/pylint_checkers/qute_pylint/settrace.py
new file mode 100644
index 0000000..c82d646
--- /dev/null
+++ b/.config/qutebrowser/scripts/dev/pylint_checkers/qute_pylint/settrace.py
@@ -0,0 +1,49 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>.
+
+"""Custom astroid checker for set_trace calls."""
+
+from pylint.interfaces import IAstroidChecker
+from pylint.checkers import BaseChecker, utils
+
+
+class SetTraceChecker(BaseChecker):
+
+ """Custom astroid checker for set_trace calls."""
+
+ __implements__ = IAstroidChecker
+ name = 'settrace'
+ msgs = {
+ 'E9101': ('set_trace call found', 'set-trace', None),
+ }
+ priority = -1
+
+ @utils.check_messages('set-trace')
+ def visit_call(self, node):
+ """Visit a Call node."""
+ if hasattr(node, 'func'):
+ infer = utils.safe_infer(node.func)
+ if infer:
+ if getattr(node.func, 'name', None) == 'set_trace':
+ self.add_message('set-trace', node=node)
+
+
+def register(linter):
+ """Register this checker."""
+ linter.register_checker(SetTraceChecker(linter))
diff --git a/.config/qutebrowser/scripts/dev/pylint_checkers/setup.py b/.config/qutebrowser/scripts/dev/pylint_checkers/setup.py
new file mode 100644
index 0000000..7833c7d
--- /dev/null
+++ b/.config/qutebrowser/scripts/dev/pylint_checkers/setup.py
@@ -0,0 +1,25 @@
+#!/usr/bin/env python3
+
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2016-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>.
+
+"""This is only here so we can install those plugins in tox.ini easily."""
+
+from setuptools import setup
+setup(name='qute_pylint', packages=['qute_pylint'])
diff --git a/.config/qutebrowser/scripts/dev/quit_segfault_test.sh b/.config/qutebrowser/scripts/dev/quit_segfault_test.sh
new file mode 100755
index 0000000..389f125
--- /dev/null
+++ b/.config/qutebrowser/scripts/dev/quit_segfault_test.sh
@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+[[ $PWD == */scripts ]] && cd ..
+
+echo > crash.log
+while :; do
+ exit=0
+ while (( exit == 0 )); do
+ duration=$(( RANDOM % 10000 ))
+ python3 -m qutebrowser --debug ":later $duration quit" http://www.heise.de/
+ exit=$?
+ done
+ echo "$(date) $exit $duration" >> crash.log
+done
diff --git a/.config/qutebrowser/scripts/dev/recompile_requirements.py b/.config/qutebrowser/scripts/dev/recompile_requirements.py
new file mode 100644
index 0000000..6e26145
--- /dev/null
+++ b/.config/qutebrowser/scripts/dev/recompile_requirements.py
@@ -0,0 +1,136 @@
+#!/usr/bin/env python3
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2016-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>.
+
+"""Script to regenerate requirements files in misc/requirements."""
+
+import re
+import sys
+import os.path
+import glob
+import subprocess
+import tempfile
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir,
+ os.pardir))
+
+from scripts import utils
+
+REPO_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)),
+ '..', '..') # /scripts/dev -> /scripts -> /
+REQ_DIR = os.path.join(REPO_DIR, 'misc', 'requirements')
+
+
+def convert_line(line, comments):
+ """Convert the given requirement line to place into the output."""
+ for pattern, repl in comments['replace'].items():
+ line = re.sub(pattern, repl, line)
+
+ pkgname = line.split('=')[0]
+
+ if pkgname in comments['ignore']:
+ line = '# ' + line
+
+ try:
+ line += ' # ' + comments['comment'][pkgname]
+ except KeyError:
+ pass
+
+ try:
+ line += ' # rq.filter: {}'.format(comments['filter'][pkgname])
+ except KeyError:
+ pass
+
+ return line
+
+
+def read_comments(fobj):
+ """Find special comments in the config.
+
+ Args:
+ fobj: A file object for the config.
+
+ Return:
+ A dict with the parsed comment data.
+ """
+ comments = {
+ 'filter': {},
+ 'comment': {},
+ 'ignore': [],
+ 'replace': {},
+ }
+ for line in fobj:
+ if line.startswith('#@'):
+ command, args = line[2:].split(':', maxsplit=1)
+ command = command.strip()
+ args = args.strip()
+ if command == 'filter':
+ pkg, filt = args.split(' ', maxsplit=1)
+ comments['filter'][pkg] = filt
+ elif command == 'comment':
+ pkg, comment = args.split(' ', maxsplit=1)
+ comments['comment'][pkg] = comment
+ elif command == 'ignore':
+ comments['ignore'] += args.split(', ')
+ elif command == 'replace':
+ pattern, replacement = args.split(' ', maxsplit=1)
+ comments['replace'][pattern] = replacement
+ return comments
+
+
+def get_all_names():
+ """Get all requirement names based on filenames."""
+ for filename in glob.glob(os.path.join(REQ_DIR, 'requirements-*.txt-raw')):
+ basename = os.path.basename(filename)
+ yield basename[len('requirements-'):-len('.txt-raw')]
+
+
+def main():
+ """Re-compile the given (or all) requirement files."""
+ names = sys.argv[1:] if len(sys.argv) > 1 else sorted(get_all_names())
+
+ for name in names:
+ utils.print_title(name)
+ filename = os.path.join(REQ_DIR,
+ 'requirements-{}.txt-raw'.format(name))
+ if name == 'qutebrowser':
+ outfile = os.path.join(REPO_DIR, 'requirements.txt')
+ else:
+ outfile = os.path.join(REQ_DIR, 'requirements-{}.txt'.format(name))
+
+ with tempfile.TemporaryDirectory() as tmpdir:
+ pip_bin = os.path.join(tmpdir, 'bin', 'pip')
+ subprocess.run(['virtualenv', tmpdir], check=True)
+ subprocess.run([pip_bin, 'install', '-r', filename], check=True)
+ proc = subprocess.run([pip_bin, 'freeze'], check=True,
+ stdout=subprocess.PIPE)
+ reqs = proc.stdout.decode('utf-8')
+
+ with open(filename, 'r', encoding='utf-8') as f:
+ comments = read_comments(f)
+
+ with open(outfile, 'w', encoding='utf-8') as f:
+ f.write("# This file is automatically generated by "
+ "scripts/dev/recompile_requirements.py\n\n")
+ for line in reqs.splitlines():
+ f.write(convert_line(line, comments) + '\n')
+
+
+if __name__ == '__main__':
+ main()
diff --git a/.config/qutebrowser/scripts/dev/run_profile.py b/.config/qutebrowser/scripts/dev/run_profile.py
new file mode 100755
index 0000000..93e0b61
--- /dev/null
+++ b/.config/qutebrowser/scripts/dev/run_profile.py
@@ -0,0 +1,93 @@
+#!/usr/bin/env python3
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>.
+
+"""Profile qutebrowser."""
+
+import sys
+import cProfile
+import os.path
+import os
+import tempfile
+import subprocess
+import shutil
+import argparse
+import shlex
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir,
+ os.pardir))
+
+import qutebrowser.qutebrowser
+
+
+def parse_args():
+ """Parse commandline arguments.
+
+ Return:
+ A (namespace, remaining_args) tuple from argparse.
+ """
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--profile-tool', metavar='TOOL',
+ action='store', choices=['kcachegrind', 'snakeviz',
+ 'gprof2dot', 'none'],
+ default='snakeviz',
+ help="The tool to use to view the profiling data")
+ parser.add_argument('--profile-file', metavar='FILE', action='store',
+ help="The filename to use with --profile-tool=none")
+ return parser.parse_known_args()
+
+
+def main():
+ args, remaining = parse_args()
+ tempdir = tempfile.mkdtemp()
+
+ if args.profile_tool == 'none':
+ profilefile = os.path.join(os.getcwd(), args.profile_file)
+ else:
+ profilefile = os.path.join(tempdir, 'profile')
+
+ sys.argv = [sys.argv[0]] + remaining
+
+ profiler = cProfile.Profile()
+ profiler.runcall(qutebrowser.qutebrowser.main)
+
+ # If we have an exception after here, we don't want the qutebrowser
+ # exception hook to take over.
+ sys.excepthook = sys.__excepthook__
+ profiler.dump_stats(profilefile)
+
+ if args.profile_tool == 'none':
+ pass
+ elif args.profile_tool == 'gprof2dot':
+ # yep, shell=True. I know what I'm doing.
+ subprocess.run(
+ 'gprof2dot -f pstats {} | dot -Tpng | feh -F -'.format(
+ shlex.quote(profilefile)), shell=True)
+ elif args.profile_tool == 'kcachegrind':
+ callgraphfile = os.path.join(tempdir, 'callgraph')
+ subprocess.run(['pyprof2calltree', '-k', '-i', profilefile,
+ '-o', callgraphfile])
+ elif args.profile_tool == 'snakeviz':
+ subprocess.run(['snakeviz', profilefile])
+
+ shutil.rmtree(tempdir)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/.config/qutebrowser/scripts/dev/run_pylint_on_tests.py b/.config/qutebrowser/scripts/dev/run_pylint_on_tests.py
new file mode 100644
index 0000000..7adf45f
--- /dev/null
+++ b/.config/qutebrowser/scripts/dev/run_pylint_on_tests.py
@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>.
+
+"""Run pylint on tests.
+
+This is needed because pylint can't check a folder which isn't a package:
+https://bitbucket.org/logilab/pylint/issue/512/
+"""
+
+import os
+import os.path
+import sys
+import subprocess
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir,
+ os.pardir))
+
+from scripts import utils
+
+
+def main():
+ """Main entry point.
+
+ Return:
+ The pylint exit status.
+ """
+ utils.change_cwd()
+ files = []
+ for dirpath, _dirnames, filenames in os.walk('tests'):
+ for fn in filenames:
+ if os.path.splitext(fn)[1] == '.py':
+ files.append(os.path.join(dirpath, fn))
+
+ disabled = [
+ # pytest fixtures
+ 'redefined-outer-name',
+ 'unused-argument',
+ # things which are okay in tests
+ 'missing-docstring',
+ 'protected-access',
+ 'len-as-condition',
+ # directories without __init__.py...
+ 'import-error',
+ ]
+
+ toxinidir = sys.argv[1]
+ pythonpath = os.environ.get('PYTHONPATH', '').split(os.pathsep) + [
+ toxinidir,
+ ]
+
+ args = (['--disable={}'.format(','.join(disabled)),
+ '--ignored-modules=helpers,pytest,PyQt5'] +
+ sys.argv[2:] + files)
+ env = os.environ.copy()
+ env['PYTHONPATH'] = os.pathsep.join(pythonpath)
+
+ ret = subprocess.run(['pylint'] + args, env=env).returncode
+ return ret
+
+
+if __name__ == '__main__':
+ sys.exit(main())
diff --git a/.config/qutebrowser/scripts/dev/run_vulture.py b/.config/qutebrowser/scripts/dev/run_vulture.py
new file mode 100755
index 0000000..cb19d62
--- /dev/null
+++ b/.config/qutebrowser/scripts/dev/run_vulture.py
@@ -0,0 +1,196 @@
+#!/usr/bin/env python
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2015-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>.
+
+"""Run vulture on the source files and filter out false-positives."""
+
+import sys
+import os
+import re
+import tempfile
+import inspect
+import argparse
+
+import vulture
+
+import qutebrowser.app # pylint: disable=unused-import
+from qutebrowser.commands import cmdutils
+from qutebrowser.utils import utils
+from qutebrowser.browser.webkit import rfc6266
+# To run the decorators from there
+# pylint: disable=unused-import
+from qutebrowser.browser.webkit.network import webkitqutescheme
+# pylint: enable=unused-import
+from qutebrowser.browser import qutescheme
+from qutebrowser.config import configtypes
+
+
+def whitelist_generator(): # noqa
+ """Generator which yields lines to add to a vulture whitelist."""
+ # qutebrowser commands
+ for cmd in cmdutils.cmd_dict.values():
+ yield utils.qualname(cmd.handler)
+
+ # pyPEG2 classes
+ for name, member in inspect.getmembers(rfc6266, inspect.isclass):
+ for attr in ['grammar', 'regex']:
+ if hasattr(member, attr):
+ yield 'qutebrowser.browser.webkit.rfc6266.{}.{}'.format(name,
+ attr)
+
+ # PyQt properties
+ yield 'qutebrowser.mainwindow.statusbar.bar.StatusBar.color_flags'
+ yield 'qutebrowser.mainwindow.statusbar.url.UrlText.urltype'
+
+ # Not used yet, but soon (or when debugging)
+ yield 'qutebrowser.utils.debug.log_events'
+ yield 'qutebrowser.utils.debug.log_signals'
+ yield 'qutebrowser.utils.debug.qflags_key'
+ yield 'qutebrowser.utils.qtutils.QtOSError.qt_errno'
+ yield 'scripts.utils.bg_colors'
+
+ # Qt attributes
+ yield 'PyQt5.QtWebKit.QWebPage.ErrorPageExtensionReturn().baseUrl'
+ yield 'PyQt5.QtWebKit.QWebPage.ErrorPageExtensionReturn().content'
+ yield 'PyQt5.QtWebKit.QWebPage.ErrorPageExtensionReturn().encoding'
+ yield 'PyQt5.QtWebKit.QWebPage.ErrorPageExtensionReturn().fileNames'
+ yield 'PyQt5.QtWidgets.QStyleOptionViewItem.backgroundColor'
+
+ ## qute://... handlers
+ for name in qutescheme._HANDLERS: # pylint: disable=protected-access
+ name = name.replace('-', '_')
+ yield 'qutebrowser.browser.qutescheme.qute_' + name
+
+ # Other false-positives
+ yield 'qutebrowser.completion.models.listcategory.ListCategory().lessThan'
+ yield 'qutebrowser.utils.jinja.Loader.get_source'
+ yield 'qutebrowser.utils.log.QtWarningFilter.filter'
+ yield 'qutebrowser.browser.pdfjs.is_available'
+ yield 'qutebrowser.misc.guiprocess.spawn_output'
+ yield 'QEvent.posted'
+ yield 'log_stack' # from message.py
+ yield 'propagate' # logging.getLogger('...).propagate = False
+ # vulture doesn't notice the hasattr() and thus thinks netrc_used is unused
+ # in NetworkManager.on_authentication_required
+ yield 'PyQt5.QtNetwork.QNetworkReply.netrc_used'
+ yield 'qutebrowser.browser.downloads.last_used_directory'
+ yield 'PaintContext.clip' # from completiondelegate.py
+ yield 'logging.LogRecord.log_color' # from logging.py
+ yield 'scripts.utils.use_color' # from asciidoc2html.py
+ for attr in ['pyeval_output', 'log_clipboard', 'fake_clipboard']:
+ yield 'qutebrowser.misc.utilcmds.' + attr
+
+ for attr in ['fileno', 'truncate', 'closed', 'readable']:
+ yield 'qutebrowser.utils.qtutils.PyQIODevice.' + attr
+
+ for attr in ['msgs', 'priority', 'visit_attribute']:
+ yield 'scripts.dev.pylint_checkers.config.' + attr
+ for attr in ['visit_call', 'process_module']:
+ yield 'scripts.dev.pylint_checkers.modeline.' + attr
+
+ for name, _member in inspect.getmembers(configtypes, inspect.isclass):
+ yield 'qutebrowser.config.configtypes.' + name
+ yield 'qutebrowser.config.configexc.ConfigErrorDesc.traceback'
+ yield 'qutebrowser.config.configfiles.ConfigAPI.load_autoconfig'
+ yield 'types.ModuleType.c' # configfiles:read_config_py
+ for name in ['configdir', 'datadir']:
+ yield 'qutebrowser.config.configfiles.ConfigAPI.' + name
+
+ yield 'include_aliases'
+
+ for attr in ['_get_default_metavar_for_optional',
+ '_get_default_metavar_for_positional', '_metavar_formatter']:
+ yield 'scripts.dev.src2asciidoc.UsageFormatter.' + attr
+
+ # attrs
+ yield 'qutebrowser.browser.webkit.network.networkmanager.ProxyId.hostname'
+ yield 'qutebrowser.command.command.ArgInfo._validate_exclusive'
+ yield 'scripts.get_coredumpctl_traces.Line.uid'
+ yield 'scripts.get_coredumpctl_traces.Line.gid'
+ yield 'scripts.importer.import_moz_places.places.row_factory'
+
+
+def filter_func(item):
+ """Check if a missing function should be filtered or not.
+
+ Return:
+ True if the missing function should be filtered/ignored, False
+ otherwise.
+ """
+ return bool(re.fullmatch(r'[a-z]+[A-Z][a-zA-Z]+', item.name))
+
+
+def report(items):
+ """Generate a report based on the given vulture.Item's.
+
+ Based on vulture.Vulture.report, but we can't use that as we can't set the
+ properties which get used for the items.
+ """
+ output = []
+ for item in sorted(items,
+ key=lambda e: (e.filename.lower(), e.first_lineno)):
+ output.append(item.get_report())
+ return output
+
+
+def run(files):
+ """Run vulture over the given files."""
+ with tempfile.NamedTemporaryFile(mode='w', delete=False) as whitelist_file:
+ for line in whitelist_generator():
+ whitelist_file.write(line + '\n')
+
+ whitelist_file.close()
+
+ vult = vulture.Vulture(verbose=False)
+ vult.scavenge(files + [whitelist_file.name])
+
+ os.remove(whitelist_file.name)
+
+ filters = {
+ 'unused_funcs': filter_func,
+ 'unused_props': lambda item: False,
+ 'unused_vars': lambda item: False,
+ 'unused_attrs': lambda item: False,
+ }
+
+ items = []
+
+ for attr, func in filters.items():
+ sub_items = getattr(vult, attr)
+ for item in sub_items:
+ filtered = func(item)
+ if not filtered:
+ items.append(item)
+
+ return report(items)
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument('files', nargs='*', default=['qutebrowser', 'scripts',
+ 'setup.py'])
+ args = parser.parse_args()
+ out = run(args.files)
+ for line in out:
+ print(line)
+ sys.exit(bool(out))
+
+
+if __name__ == '__main__':
+ main()
diff --git a/.config/qutebrowser/scripts/dev/segfault_test.py b/.config/qutebrowser/scripts/dev/segfault_test.py
new file mode 100755
index 0000000..aaf495f
--- /dev/null
+++ b/.config/qutebrowser/scripts/dev/segfault_test.py
@@ -0,0 +1,120 @@
+#!/usr/bin/env python3
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>.
+
+"""Tester for Qt segfaults with different harfbuzz engines."""
+
+import os
+import os.path
+import signal
+import sys
+import subprocess
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir,
+ os.pardir))
+
+from scripts import utils
+
+
+SCRIPT = """
+import sys
+
+from PyQt5.QtCore import QUrl
+from PyQt5.QtWidgets import QApplication
+from PyQt5.QtWebKitWidgets import QWebView
+
+def on_load_finished(ok):
+ if ok:
+ app.exit(0)
+ else:
+ app.exit(1)
+
+app = QApplication([])
+wv = QWebView()
+wv.loadFinished.connect(on_load_finished)
+wv.load(QUrl(sys.argv[1]))
+#wv.show()
+app.exec_()
+"""
+
+
+def print_ret(ret):
+ """Print information about an exit status."""
+ if ret == 0:
+ utils.print_col("success", 'green')
+ elif ret == -signal.SIGSEGV:
+ utils.print_col("segfault", 'red')
+ else:
+ utils.print_col("error {}".format(ret), 'yellow')
+ print()
+
+
+def main():
+ retvals = []
+ if len(sys.argv) < 2:
+ # pages which previously caused problems
+ pages = [
+ # ANGLE, https://bugreports.qt.io/browse/QTBUG-39723
+ ('http://www.binpress.com/', False),
+ ('http://david.li/flow/', False),
+ ('https://imzdl.com/', False),
+ # not reproducible
+ # https://bugreports.qt.io/browse/QTBUG-39847
+ ('http://www.20min.ch/', True),
+ # HarfBuzz, https://bugreports.qt.io/browse/QTBUG-39278
+ ('http://www.the-compiler.org/', True),
+ ('http://phoronix.com', True),
+ ('http://twitter.com', True),
+ # HarfBuzz #2, https://bugreports.qt.io/browse/QTBUG-36099
+ ('http://lenta.ru/', True),
+ # Unknown, https://bugreports.qt.io/browse/QTBUG-41360
+ ('http://salt.readthedocs.org/en/latest/topics/pillar/', True),
+ ]
+ else:
+ pages = [(e, True) for e in sys.argv[1:]]
+ for page, test_harfbuzz in pages:
+ utils.print_bold("==== {} ====".format(page))
+ if test_harfbuzz:
+ print("With system harfbuzz:")
+ ret = subprocess.run([sys.executable, '-c', SCRIPT, page]).returncode
+ print_ret(ret)
+ retvals.append(ret)
+ if test_harfbuzz:
+ print("With QT_HARFBUZZ=old:")
+ env = dict(os.environ)
+ env['QT_HARFBUZZ'] = 'old'
+ ret = subprocess.run([sys.executable, '-c', SCRIPT, page],
+ env=env).returncode
+ print_ret(ret)
+ retvals.append(ret)
+ print("With QT_HARFBUZZ=new:")
+ env = dict(os.environ)
+ env['QT_HARFBUZZ'] = 'new'
+ ret = subprocess.run([sys.executable, '-c', SCRIPT, page],
+ env=env).returncode
+ print_ret(ret)
+ retvals.append(ret)
+ if all(r == 0 for r in retvals):
+ sys.exit(0)
+ else:
+ sys.exit(1)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/.config/qutebrowser/scripts/dev/src2asciidoc.py b/.config/qutebrowser/scripts/dev/src2asciidoc.py
new file mode 100755
index 0000000..cc00c37
--- /dev/null
+++ b/.config/qutebrowser/scripts/dev/src2asciidoc.py
@@ -0,0 +1,561 @@
+#!/usr/bin/env python3
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>.
+
+"""Generate asciidoc source for qutebrowser based on docstrings."""
+
+import os
+import os.path
+import sys
+import shutil
+import inspect
+import subprocess
+import tempfile
+import argparse
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir,
+ os.pardir))
+
+# We import qutebrowser.app so all @cmdutils-register decorators are run.
+import qutebrowser.app
+from qutebrowser import qutebrowser, commands
+from qutebrowser.commands import cmdutils, argparser
+from qutebrowser.config import configdata, configtypes
+from qutebrowser.utils import docutils, usertypes
+from scripts import asciidoc2html, utils
+
+FILE_HEADER = """
+// DO NOT EDIT THIS FILE DIRECTLY!
+// It is autogenerated by running:
+// $ python3 scripts/dev/src2asciidoc.py
+// vim: readonly:
+
+""".lstrip()
+
+
+class UsageFormatter(argparse.HelpFormatter):
+
+ """Patched HelpFormatter to include some asciidoc markup in the usage.
+
+ This does some horrible things, but the alternative would be to reimplement
+ argparse.HelpFormatter while copying 99% of the code :-/
+ """
+
+ def _format_usage(self, usage, actions, groups, _prefix):
+ """Override _format_usage to not add the 'usage:' prefix."""
+ return super()._format_usage(usage, actions, groups, '')
+
+ def _get_default_metavar_for_optional(self, action):
+ """Do name transforming when getting metavar."""
+ return argparser.arg_name(action.dest.upper())
+
+ def _get_default_metavar_for_positional(self, action):
+ """Do name transforming when getting metavar."""
+ return argparser.arg_name(action.dest)
+
+ def _metavar_formatter(self, action, default_metavar):
+ """Override _metavar_formatter to add asciidoc markup to metavars.
+
+ Most code here is copied from Python 3.4's argparse.py.
+ """
+ if action.metavar is not None:
+ result = "'{}'".format(action.metavar)
+ elif action.choices is not None:
+ choice_strs = [str(choice) for choice in action.choices]
+ result = ('{' + ','.join('*{}*'.format(e) for e in choice_strs) +
+ '}')
+ else:
+ result = "'{}'".format(default_metavar)
+
+ def fmt(tuple_size):
+ """Format the result according to the tuple size."""
+ if isinstance(result, tuple):
+ return result
+ else:
+ return (result, ) * tuple_size
+ return fmt
+
+ def _format_actions_usage(self, actions, groups):
+ """Override _format_actions_usage to add asciidoc markup to flags.
+
+ Because argparse.py's _format_actions_usage is very complex, we first
+ monkey-patch the option strings to include the asciidoc markup, then
+ run the original method, then undo the patching.
+ """
+ old_option_strings = {}
+ for action in actions:
+ old_option_strings[action] = action.option_strings[:]
+ action.option_strings = ['*{}*'.format(s)
+ for s in action.option_strings]
+ ret = super()._format_actions_usage(actions, groups)
+ for action in actions:
+ action.option_strings = old_option_strings[action]
+ return ret
+
+
+def _open_file(name, mode='w'):
+ """Open a file with a preset newline/encoding mode."""
+ return open(name, mode, newline='\n', encoding='utf-8')
+
+
+def _get_cmd_syntax(_name, cmd):
+ """Get the command syntax for a command.
+
+ We monkey-patch the parser's formatter_class here to use our UsageFormatter
+ which adds some asciidoc markup.
+ """
+ old_fmt_class = cmd.parser.formatter_class
+ cmd.parser.formatter_class = UsageFormatter
+ usage = cmd.parser.format_usage().rstrip()
+ cmd.parser.formatter_class = old_fmt_class
+ return usage
+
+
+def _get_command_quickref(cmds):
+ """Generate the command quick reference."""
+ out = []
+ out.append('[options="header",width="75%",cols="25%,75%"]')
+ out.append('|==============')
+ out.append('|Command|Description')
+ for name, cmd in cmds:
+ desc = inspect.getdoc(cmd.handler).splitlines()[0]
+ out.append('|<<{},{}>>|{}'.format(name, name, desc))
+ out.append('|==============')
+ return '\n'.join(out)
+
+
+def _get_setting_quickref():
+ """Generate the settings quick reference."""
+ out = []
+ out.append('')
+ out.append('[options="header",width="75%",cols="25%,75%"]')
+ out.append('|==============')
+ out.append('|Setting|Description')
+ for opt in sorted(configdata.DATA.values()):
+ desc = opt.description.splitlines()[0]
+ out.append('|<<{},{}>>|{}'.format(opt.name, opt.name, desc))
+ out.append('|==============')
+ return '\n'.join(out)
+
+
+def _get_configtypes():
+ """Get configtypes classes to document."""
+ predicate = lambda e: (
+ inspect.isclass(e) and
+ # pylint: disable=protected-access
+ e not in [configtypes.BaseType, configtypes.MappingType,
+ configtypes._Numeric] and
+ # pylint: enable=protected-access
+ issubclass(e, configtypes.BaseType))
+ yield from inspect.getmembers(configtypes, predicate)
+
+
+def _get_setting_types_quickref():
+ """Generate the setting types quick reference."""
+ out = []
+ out.append('[[types]]')
+ out.append('[options="header",width="75%",cols="25%,75%"]')
+ out.append('|==============')
+ out.append('|Type|Description')
+
+ for name, typ in _get_configtypes():
+ parser = docutils.DocstringParser(typ)
+ desc = parser.short_desc
+ if parser.long_desc:
+ desc += '\n\n' + parser.long_desc
+ out.append('|{}|{}'.format(name, desc))
+
+ out.append('|==============')
+ return '\n'.join(out)
+
+
+def _get_command_doc(name, cmd):
+ """Generate the documentation for a command."""
+ output = ['[[{}]]'.format(name)]
+ output += ['=== {}'.format(name)]
+ syntax = _get_cmd_syntax(name, cmd)
+ if syntax != name:
+ output.append('Syntax: +:{}+'.format(syntax))
+ output.append("")
+ parser = docutils.DocstringParser(cmd.handler)
+ output.append(parser.short_desc)
+ if parser.long_desc:
+ output.append("")
+ output.append(parser.long_desc)
+
+ output += list(_get_command_doc_args(cmd, parser))
+ output += list(_get_command_doc_count(cmd, parser))
+ output += list(_get_command_doc_notes(cmd))
+
+ output.append("")
+ output.append("")
+ return '\n'.join(output)
+
+
+def _get_command_doc_args(cmd, parser):
+ """Get docs for the arguments of a command.
+
+ Args:
+ cmd: The Command to get the docs for.
+ parser: The DocstringParser to use.
+
+ Yield:
+ Strings which should be added to the docs.
+ """
+ if cmd.pos_args:
+ yield ""
+ yield "==== positional arguments"
+ for arg, name in cmd.pos_args:
+ try:
+ yield "* +'{}'+: {}".format(name, parser.arg_descs[arg])
+ except KeyError as e:
+ raise KeyError("No description for arg {} of command "
+ "'{}'!".format(e, cmd.name)) from e
+
+ if cmd.opt_args:
+ yield ""
+ yield "==== optional arguments"
+ for arg, (long_flag, short_flag) in cmd.opt_args.items():
+ try:
+ yield '* +*{}*+, +*{}*+: {}'.format(short_flag, long_flag,
+ parser.arg_descs[arg])
+ except KeyError as e:
+ raise KeyError("No description for arg {} of command "
+ "'{}'!".format(e, cmd.name)) from e
+
+
+def _get_command_doc_count(cmd, parser):
+ """Get docs for the count of a command.
+
+ Args:
+ cmd: The Command to get the docs for.
+ parser: The DocstringParser to use.
+
+ Yield:
+ Strings which should be added to the docs.
+ """
+ for param in inspect.signature(cmd.handler).parameters.values():
+ if cmd.get_arg_info(param).count:
+ yield ""
+ yield "==== count"
+ try:
+ yield parser.arg_descs[param.name]
+ except KeyError as e:
+ raise KeyError("No description for count arg {!r} of command "
+ "{!r}!".format(param.name, cmd.name)) from e
+
+
+def _get_command_doc_notes(cmd):
+ """Get docs for the notes of a command.
+
+ Args:
+ cmd: The Command to get the docs for.
+ parser: The DocstringParser to use.
+
+ Yield:
+ Strings which should be added to the docs.
+ """
+ if (cmd.maxsplit is not None or cmd.no_cmd_split or
+ cmd.no_replace_variables and cmd.name != "spawn"):
+ yield ""
+ yield "==== note"
+ if cmd.maxsplit is not None:
+ yield ("* This command does not split arguments after the last "
+ "argument and handles quotes literally.")
+ if cmd.no_cmd_split:
+ yield ("* With this command, +;;+ is interpreted literally "
+ "instead of splitting off a second command.")
+ if cmd.no_replace_variables and cmd.name != "spawn":
+ yield r"* This command does not replace variables like +\{url\}+."
+
+
+def _get_action_metavar(action, nargs=1):
+ """Get the metavar to display for an argparse action.
+
+ Args:
+ action: The argparse action to get the metavar for.
+ nargs: The nargs setting for the related argument.
+ """
+ if action.metavar is not None:
+ if isinstance(action.metavar, str):
+ elems = [action.metavar] * nargs
+ else:
+ elems = action.metavar
+ return ' '.join("'{}'".format(e) for e in elems)
+ elif action.choices is not None:
+ choices = ','.join(str(e) for e in action.choices)
+ return "'{{{}}}'".format(choices)
+ else:
+ return "'{}'".format(action.dest.upper())
+
+
+def _format_action_args(action):
+ """Get an argument string based on an argparse action."""
+ if action.nargs is None:
+ return _get_action_metavar(action)
+ elif action.nargs == '?':
+ return '[{}]'.format(_get_action_metavar(action))
+ elif action.nargs == '*':
+ return '[{mv} [{mv} ...]]'.format(mv=_get_action_metavar(action))
+ elif action.nargs == '+':
+ return '{mv} [{mv} ...]'.format(mv=_get_action_metavar(action))
+ elif action.nargs == '...':
+ return '...'
+ else:
+ return _get_action_metavar(action, nargs=action.nargs)
+
+
+def _format_action(action):
+ """Get an invocation string/help from an argparse action."""
+ if action.help == argparse.SUPPRESS:
+ return None
+ if not action.option_strings:
+ invocation = '*{}*::'.format(_get_action_metavar(action))
+ else:
+ parts = []
+ if action.nargs == 0:
+ # Doesn't take a value, so the syntax is -s, --long
+ parts += ['*{}*'.format(s) for s in action.option_strings]
+ else:
+ # Takes a value, so the syntax is -s ARGS or --long ARGS.
+ args_string = _format_action_args(action)
+ for opt in action.option_strings:
+ parts.append('*{}* {}'.format(opt, args_string))
+ invocation = ', '.join(parts) + '::'
+ return '{}\n {}\n'.format(invocation, action.help)
+
+
+def generate_commands(filename):
+ """Generate the complete commands section."""
+ with _open_file(filename) as f:
+ f.write(FILE_HEADER)
+ f.write("= Commands\n\n")
+ f.write(commands.__doc__)
+ normal_cmds = []
+ other_cmds = []
+ debug_cmds = []
+ for name, cmd in cmdutils.cmd_dict.items():
+ if cmd.deprecated:
+ continue
+ if usertypes.KeyMode.normal not in cmd.modes:
+ other_cmds.append((name, cmd))
+ elif cmd.debug:
+ debug_cmds.append((name, cmd))
+ else:
+ normal_cmds.append((name, cmd))
+ normal_cmds.sort()
+ other_cmds.sort()
+ debug_cmds.sort()
+ f.write("\n")
+ f.write("== Normal commands\n")
+ f.write(".Quick reference\n")
+ f.write(_get_command_quickref(normal_cmds) + '\n')
+ for name, cmd in normal_cmds:
+ f.write(_get_command_doc(name, cmd))
+ f.write("\n")
+ f.write("== Commands not usable in normal mode\n")
+ f.write(".Quick reference\n")
+ f.write(_get_command_quickref(other_cmds) + '\n')
+ for name, cmd in other_cmds:
+ f.write(_get_command_doc(name, cmd))
+ f.write("\n")
+ f.write("== Debugging commands\n")
+ f.write("These commands are mainly intended for debugging. They are "
+ "hidden if qutebrowser was started without the "
+ "`--debug`-flag.\n")
+ f.write("\n")
+ f.write(".Quick reference\n")
+ f.write(_get_command_quickref(debug_cmds) + '\n')
+ for name, cmd in debug_cmds:
+ f.write(_get_command_doc(name, cmd))
+
+
+def _generate_setting_backend_info(f, opt):
+ """Generate backend information for the given option."""
+ all_backends = [usertypes.Backend.QtWebKit, usertypes.Backend.QtWebEngine]
+ if opt.raw_backends is not None:
+ for name, conditional in sorted(opt.raw_backends.items()):
+ if conditional is True:
+ pass
+ elif conditional is False:
+ f.write("\nOn {}, this setting is unavailable.\n".format(name))
+ else:
+ f.write("\nOn {}, this setting requires {} or newer.\n"
+ .format(name, conditional))
+ elif opt.backends == all_backends:
+ pass
+ elif opt.backends == [usertypes.Backend.QtWebKit]:
+ f.write("\nThis setting is only available with the QtWebKit "
+ "backend.\n")
+ elif opt.backends == [usertypes.Backend.QtWebEngine]:
+ f.write("\nThis setting is only available with the QtWebEngine "
+ "backend.\n")
+ else:
+ raise ValueError("Invalid value {!r} for opt.backends"
+ .format(opt.backends))
+
+
+def _generate_setting_option(f, opt):
+ """Generate documentation for a single section."""
+ f.write("\n")
+ f.write('[[{}]]'.format(opt.name) + "\n")
+ f.write("=== {}".format(opt.name) + "\n")
+ f.write(opt.description + "\n")
+ if opt.restart:
+ f.write("This setting requires a restart.\n")
+ if opt.supports_pattern:
+ f.write("\nThis setting supports URL patterns.\n")
+ if opt.no_autoconfig:
+ f.write("\nThis setting can only be set in config.py.\n")
+ f.write("\n")
+ typ = opt.typ.get_name().replace(',', '&#44;')
+ f.write('Type: <<types,{typ}>>\n'.format(typ=typ))
+ f.write("\n")
+
+ valid_values = opt.typ.get_valid_values()
+ if valid_values is not None and valid_values.generate_docs:
+ f.write("Valid values:\n")
+ f.write("\n")
+ for val in valid_values:
+ try:
+ desc = valid_values.descriptions[val]
+ f.write(" * +{}+: {}".format(val, desc) + "\n")
+ except KeyError:
+ f.write(" * +{}+".format(val) + "\n")
+ f.write("\n")
+
+ f.write("Default: {}\n".format(opt.typ.to_doc(opt.default)))
+ _generate_setting_backend_info(f, opt)
+
+
+def generate_settings(filename):
+ """Generate the complete settings section."""
+ configdata.init()
+ with _open_file(filename) as f:
+ f.write(FILE_HEADER)
+ f.write("= Setting reference\n\n")
+ f.write("== All settings\n")
+ f.write(_get_setting_quickref() + "\n")
+ for opt in sorted(configdata.DATA.values()):
+ _generate_setting_option(f, opt)
+ f.write("\n== Setting types\n")
+ f.write(_get_setting_types_quickref() + "\n")
+
+
+def _format_block(filename, what, data):
+ """Format a block in a file.
+
+ The block is delimited by markers like these:
+ // QUTE_*_START
+ ...
+ // QUTE_*_END
+
+ The * part is the part which should be given as 'what'.
+
+ Args:
+ filename: The file to change.
+ what: What to change (authors, options, etc.)
+ data; A list of strings which is the new data.
+ """
+ what = what.upper()
+ oshandle, tmpname = tempfile.mkstemp()
+ try:
+ with _open_file(filename, mode='r') as infile, \
+ _open_file(oshandle, mode='w') as temp:
+ found_start = False
+ found_end = False
+ for line in infile:
+ if line.strip() == '// QUTE_{}_START'.format(what):
+ temp.write(line)
+ temp.write(''.join(data))
+ found_start = True
+ elif line.strip() == '// QUTE_{}_END'.format(what.upper()):
+ temp.write(line)
+ found_end = True
+ elif (not found_start) or found_end:
+ temp.write(line)
+ if not found_start:
+ raise Exception("Marker '// QUTE_{}_START' not found in "
+ "'{}'!".format(what, filename))
+ elif not found_end:
+ raise Exception("Marker '// QUTE_{}_END' not found in "
+ "'{}'!".format(what, filename))
+ except:
+ os.remove(tmpname)
+ raise
+ else:
+ os.remove(filename)
+ shutil.move(tmpname, filename)
+
+
+def regenerate_manpage(filename):
+ """Update manpage OPTIONS using an argparse parser."""
+ parser = qutebrowser.get_argparser()
+ groups = []
+ # positionals, optionals and user-defined groups
+ # pylint: disable=protected-access
+ for group in parser._action_groups:
+ groupdata = []
+ groupdata.append('=== {}'.format(group.title))
+ if group.description is not None:
+ groupdata.append(group.description)
+ for action in group._group_actions:
+ action_data = _format_action(action)
+ if action_data is not None:
+ groupdata.append(action_data)
+ groups.append('\n'.join(groupdata))
+ # pylint: enable=protected-access
+ options = '\n'.join(groups)
+ # epilog
+ if parser.epilog is not None:
+ options += parser.epilog
+ _format_block(filename, 'options', options)
+
+
+def regenerate_cheatsheet():
+ """Generate cheatsheet PNGs based on the SVG."""
+ files = [
+ ('doc/img/cheatsheet-small.png', 300, 185),
+ ('doc/img/cheatsheet-big.png', 3342, 2060),
+ ]
+
+ for filename, x, y in files:
+ subprocess.run(['inkscape', '-e', filename, '-b', 'white',
+ '-w', str(x), '-h', str(y),
+ 'misc/cheatsheet.svg'], check=True)
+
+
+def main():
+ """Regenerate all documentation."""
+ utils.change_cwd()
+ print("Generating manpage...")
+ regenerate_manpage('doc/qutebrowser.1.asciidoc')
+ print("Generating settings help...")
+ generate_settings('doc/help/settings.asciidoc')
+ print("Generating command help...")
+ generate_commands('doc/help/commands.asciidoc')
+ if '--cheatsheet' in sys.argv:
+ print("Regenerating cheatsheet .pngs")
+ regenerate_cheatsheet()
+ if '--html' in sys.argv:
+ asciidoc2html.main()
+
+
+if __name__ == '__main__':
+ main()
diff --git a/.config/qutebrowser/scripts/dev/standardpaths_tester.py b/.config/qutebrowser/scripts/dev/standardpaths_tester.py
new file mode 100644
index 0000000..27b8382
--- /dev/null
+++ b/.config/qutebrowser/scripts/dev/standardpaths_tester.py
@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2017-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>.
+
+"""Show various QStandardPath paths."""
+
+import os
+import sys
+
+from PyQt5.QtCore import (QT_VERSION_STR, PYQT_VERSION_STR, qVersion,
+ QStandardPaths, QCoreApplication)
+
+
+def print_header():
+ """Show system information."""
+ print("Python {}".format(sys.version))
+ print("os.name: {}".format(os.name))
+ print("sys.platform: {}".format(sys.platform))
+ print()
+
+ print("Qt {}, compiled {}".format(qVersion(), QT_VERSION_STR))
+ print("PyQt {}".format(PYQT_VERSION_STR))
+ print()
+
+
+def print_paths():
+ """Print all QStandardPaths.StandardLocation members."""
+ for name, obj in vars(QStandardPaths).items():
+ if isinstance(obj, QStandardPaths.StandardLocation):
+ location = QStandardPaths.writableLocation(obj)
+ print("{:25} {}".format(name, location))
+
+
+def main():
+ print_header()
+
+ print("No QApplication")
+ print("===============")
+ print()
+ print_paths()
+
+ app = QCoreApplication(sys.argv)
+ app.setApplicationName("qapp_name")
+
+ print()
+ print("With QApplication")
+ print("=================")
+ print()
+ print_paths()
+
+
+if __name__ == '__main__':
+ main()
diff --git a/.config/qutebrowser/scripts/dev/strip_whitespace.sh b/.config/qutebrowser/scripts/dev/strip_whitespace.sh
new file mode 100644
index 0000000..ee14278
--- /dev/null
+++ b/.config/qutebrowser/scripts/dev/strip_whitespace.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# Strip trailing whitespace from files in this repo
+
+find qutebrowser scripts tests \
+ -type f \( \
+ -name '*.py' -o \
+ -name '*.feature' -o \
+ -name '*.sh' \
+ \) -exec sed -i 's/ \+$//' {} +
diff --git a/.config/qutebrowser/scripts/dev/ua_fetch.py b/.config/qutebrowser/scripts/dev/ua_fetch.py
new file mode 100755
index 0000000..75ce4c2
--- /dev/null
+++ b/.config/qutebrowser/scripts/dev/ua_fetch.py
@@ -0,0 +1,122 @@
+#!/usr/bin/env python3
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2015-2018 lamarpavel
+# Copyright 2015-2018 Alexey Nabrodov (Averrin)
+# Copyright 2015-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>.
+
+
+"""Fetch list of popular user-agents.
+
+The script is based on a gist posted by github.com/averrin, the output of this
+script is formatted to be pasted into configdata.yml
+"""
+
+import requests
+from lxml import html # pylint: disable=import-error
+
+
+def fetch():
+ """Fetch list of popular user-agents.
+
+ Return:
+ List of relevant strings.
+ """
+ url = 'https://techblog.willshouse.com/2012/01/03/most-common-user-agents/'
+ page = requests.get(url)
+ page = html.fromstring(page.text)
+ path = '//*[@id="post-2229"]/div[2]/table/tbody'
+ return page.xpath(path)[0]
+
+
+def filter_list(complete_list, browsers):
+ """Filter the received list based on a look up table.
+
+ The LUT should be a dictionary of the format {browser: versions}, where
+ 'browser' is the name of the browser (eg. "Firefox") as string and
+ 'versions' is a set of different versions of this browser that should be
+ included when found (eg. {"Linux", "MacOSX"}). This function returns a
+ dictionary with the same keys as the LUT, but storing lists of tuples
+ (user_agent, browser_description) as values.
+ """
+ # pylint: disable=too-many-nested-blocks
+ table = {}
+ for entry in complete_list:
+ # Tuple of (user_agent, browser_description)
+ candidate = (entry[1].text_content(), entry[2].text_content())
+ for name in browsers:
+ found = False
+ if name.lower() in candidate[1].lower():
+ for version in browsers[name]:
+ if version.lower() in candidate[1].lower():
+ if table.get(name) is None:
+ table[name] = []
+ table[name].append(candidate)
+ browsers[name].remove(version)
+ found = True
+ break
+ if found:
+ break
+ return table
+
+
+def add_diversity(table):
+ """Insert a few additional entries for diversity into the dict.
+
+ (as returned by filter_list())
+ """
+ table["Obscure"] = [
+ ('Mozilla/5.0 (compatible; Googlebot/2.1; '
+ '+http://www.google.com/bot.html',
+ "Google Bot"),
+ ('Wget/1.16.1 (linux-gnu)',
+ "wget 1.16.1"),
+ ('curl/7.40.0',
+ "curl 7.40.0"),
+ ('Mozilla/5.0 (Linux; U; Android 7.1.2) AppleWebKit/534.30 '
+ '(KHTML, like Gecko) Version/4.0 Mobile Safari/534.30',
+ "Mobile Generic Android"),
+ ('Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like '
+ 'Gecko',
+ "IE 11.0 for Desktop Win7 64-bit"),
+ ]
+ return table
+
+
+def main():
+ """Generate user agent code."""
+ fetched = fetch()
+ lut = {
+ "Firefox": {"Win", "MacOSX", "Linux", "Android"},
+ "Chrome": {"Win", "MacOSX", "Linux"},
+ "Safari": {"MacOSX", "iOS"}
+ }
+ filtered = filter_list(fetched, lut)
+ filtered = add_diversity(filtered)
+
+ tab = " "
+ for browser in ["Firefox", "Safari", "Chrome", "Obscure"]:
+ for it in filtered[browser]:
+ print('{}- - "{}"'.format(3 * tab, it[0]))
+ desc = it[1].replace('\xa0', ' ').replace(' ', ' ')
+ print("{}- {}".format(4 * tab, desc))
+ print("")
+
+
+if __name__ == '__main__':
+ main()
diff --git a/.config/qutebrowser/scripts/dev/update_3rdparty.py b/.config/qutebrowser/scripts/dev/update_3rdparty.py
new file mode 100755
index 0000000..c40015d
--- /dev/null
+++ b/.config/qutebrowser/scripts/dev/update_3rdparty.py
@@ -0,0 +1,166 @@
+#!/usr/bin/env python3
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2015 Daniel Schadt
+# Copyright 2016-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>.
+
+"""Update all third-party-modules."""
+
+import argparse
+import urllib.request
+import urllib.error
+import shutil
+import json
+import os
+import sys
+
+sys.path.insert(
+ 0, os.path.join(os.path.dirname(__file__), os.pardir, os.pardir))
+from scripts import dictcli
+from qutebrowser.config import configdata
+
+
+def get_latest_pdfjs_url():
+ """Get the URL of the latest pdf.js prebuilt package.
+
+ Returns a (version, url)-tuple.
+ """
+ github_api = 'https://api.github.com'
+ endpoint = 'repos/mozilla/pdf.js/releases/latest'
+ request_url = '{}/{}'.format(github_api, endpoint)
+ with urllib.request.urlopen(request_url) as fp:
+ data = json.loads(fp.read().decode('utf-8'))
+
+ download_url = data['assets'][0]['browser_download_url']
+ version_name = data['name']
+ return (version_name, download_url)
+
+
+def update_pdfjs(target_version=None):
+ """Download and extract the latest pdf.js version.
+
+ If target_version is not None, download the given version instead.
+
+ Args:
+ target_version: None or version string ('x.y.z')
+ """
+ if target_version is None:
+ version, url = get_latest_pdfjs_url()
+ else:
+ # We need target_version as x.y.z, without the 'v' prefix, though the
+ # user might give it on the command line
+ if target_version.startswith('v'):
+ target_version = target_version[1:]
+ # version should have the prefix to be consistent with the return value
+ # of get_latest_pdfjs_url()
+ version = 'v' + target_version
+ url = ('https://github.com/mozilla/pdf.js/releases/download/'
+ 'v{0}/pdfjs-{0}-dist.zip').format(target_version)
+
+ os.chdir(os.path.join(os.path.dirname(os.path.abspath(__file__)),
+ '..', '..'))
+ target_path = os.path.join('qutebrowser', '3rdparty', 'pdfjs')
+ print("=> Downloading pdf.js {}".format(version))
+ try:
+ (archive_path, _headers) = urllib.request.urlretrieve(url)
+ except urllib.error.HTTPError as error:
+ print("Could not retrieve pdfjs {}: {}".format(version, error))
+ return
+ if os.path.isdir(target_path):
+ print("Removing old version in {}".format(target_path))
+ shutil.rmtree(target_path)
+ os.makedirs(target_path)
+ print("Extracting new version")
+ with open(archive_path, 'rb') as archive:
+ shutil.unpack_archive(archive, target_path, 'zip')
+ urllib.request.urlcleanup()
+
+
+def update_dmg_makefile():
+ """Update fancy-dmg Makefile.
+
+ See https://el-tramo.be/blog/fancy-dmg/
+ """
+ print("Updating fancy-dmg Makefile...")
+ url = 'https://raw.githubusercontent.com/remko/fancy-dmg/master/Makefile'
+ target_path = os.path.join('scripts', 'dev', 'Makefile-dmg')
+ urllib.request.urlretrieve(url, target_path)
+ urllib.request.urlcleanup()
+
+
+def update_ace():
+ """Update ACE.
+
+ See https://ace.c9.io/ and https://github.com/ajaxorg/ace-builds/
+ """
+ print("Updating ACE...")
+ url = 'https://raw.githubusercontent.com/ajaxorg/ace-builds/master/src/ace.js'
+ target_path = os.path.join('tests', 'end2end', 'data', 'hints', 'ace',
+ 'ace.js')
+ urllib.request.urlretrieve(url, target_path)
+ urllib.request.urlcleanup()
+
+
+def test_dicts():
+ """Test available dictionaries."""
+ configdata.init()
+ for lang in dictcli.available_languages():
+ print('Testing dictionary {}... '.format(lang.code), end='')
+ lang_url = urllib.parse.urljoin(dictcli.API_URL, lang.remote_path)
+ request = urllib.request.Request(lang_url, method='HEAD')
+ response = urllib.request.urlopen(request)
+ if response.status == 200:
+ print('OK')
+ else:
+ print('ERROR: {}'.format(response.status))
+
+
+def run(ace=False, pdfjs=True, fancy_dmg=False, pdfjs_version=None,
+ dicts=False):
+ """Update components based on the given arguments."""
+ if pdfjs:
+ update_pdfjs(pdfjs_version)
+ if ace:
+ update_ace()
+ if fancy_dmg:
+ update_dmg_makefile()
+ if dicts:
+ test_dicts()
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ '--pdfjs', '-p',
+ help='Specify pdfjs version. If not given, '
+ 'the latest version is used.',
+ required=False, metavar='VERSION')
+ parser.add_argument('--fancy-dmg', help="Update fancy-dmg Makefile",
+ action='store_true')
+ parser.add_argument(
+ '--dicts', '-d',
+ help='Test whether all available dictionaries '
+ 'can be reached at the remote repository.',
+ required=False, action='store_true')
+ args = parser.parse_args()
+ run(ace=True, pdfjs=True, fancy_dmg=args.fancy_dmg,
+ pdfjs_version=args.pdfjs, dicts=args.dicts)
+
+
+if __name__ == '__main__':
+ main()