diff options
Diffstat (limited to '.config/qutebrowser/scripts')
49 files changed, 5309 insertions, 0 deletions
diff --git a/.config/qutebrowser/scripts/__init__.py b/.config/qutebrowser/scripts/__init__.py new file mode 100755 index 0000000..90be1e0 --- /dev/null +++ b/.config/qutebrowser/scripts/__init__.py @@ -0,0 +1,3 @@ +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +"""Various utility scripts.""" diff --git a/.config/qutebrowser/scripts/asciidoc2html.py b/.config/qutebrowser/scripts/asciidoc2html.py new file mode 100755 index 0000000..c4af174 --- /dev/null +++ b/.config/qutebrowser/scripts/asciidoc2html.py @@ -0,0 +1,303 @@ +#!/usr/bin/env python3 +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org> + +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + +"""Generate the html documentation based on the asciidoc files.""" + +import re +import os +import os.path +import sys +import subprocess +import glob +import shutil +import tempfile +import argparse +import io + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir)) + +from scripts import utils + + +class AsciiDoc: + + """Abstraction of an asciidoc subprocess.""" + + FILES = ['faq', 'changelog', 'contributing', 'quickstart', 'userscripts'] + + def __init__(self, args): + self._cmd = None + self._args = args + self._homedir = None + self._themedir = None + self._tempdir = None + self._failed = False + + def prepare(self): + """Get the asciidoc command and create the homedir to use.""" + self._cmd = self._get_asciidoc_cmd() + self._homedir = tempfile.mkdtemp() + self._themedir = os.path.join( + self._homedir, '.asciidoc', 'themes', 'qute') + self._tempdir = os.path.join(self._homedir, 'tmp') + os.makedirs(self._tempdir) + os.makedirs(self._themedir) + + def cleanup(self): + """Clean up the temporary home directory for asciidoc.""" + if self._homedir is not None and not self._failed: + shutil.rmtree(self._homedir) + + def build(self): + """Build either the website or the docs.""" + if self._args.website: + self._build_website() + else: + self._build_docs() + self._copy_images() + + def _build_docs(self): + """Render .asciidoc files to .html sites.""" + files = [('doc/{}.asciidoc'.format(f), + 'qutebrowser/html/doc/{}.html'.format(f)) + for f in self.FILES] + for src in glob.glob('doc/help/*.asciidoc'): + name, _ext = os.path.splitext(os.path.basename(src)) + dst = 'qutebrowser/html/doc/{}.html'.format(name) + files.append((src, dst)) + + # patch image links to use local copy + replacements = [ + ("https://raw.githubusercontent.com/qutebrowser/qutebrowser/master/doc/img/cheatsheet-big.png", + "qute://help/img/cheatsheet-big.png"), + ("https://raw.githubusercontent.com/qutebrowser/qutebrowser/master/doc/img/cheatsheet-small.png", + "qute://help/img/cheatsheet-small.png") + ] + asciidoc_args = ['-a', 'source-highlighter=pygments'] + + for src, dst in files: + src_basename = os.path.basename(src) + modified_src = os.path.join(self._tempdir, src_basename) + with open(modified_src, 'w', encoding='utf-8') as modified_f, \ + open(src, 'r', encoding='utf-8') as f: + for line in f: + for orig, repl in replacements: + line = line.replace(orig, repl) + modified_f.write(line) + self.call(modified_src, dst, *asciidoc_args) + + def _copy_images(self): + """Copy image files to qutebrowser/html/doc.""" + print("Copying files...") + dst_path = os.path.join('qutebrowser', 'html', 'doc', 'img') + try: + os.mkdir(dst_path) + except FileExistsError: + pass + for filename in ['cheatsheet-big.png', 'cheatsheet-small.png']: + src = os.path.join('doc', 'img', filename) + dst = os.path.join(dst_path, filename) + shutil.copy(src, dst) + + def _build_website_file(self, root, filename): + """Build a single website file.""" + src = os.path.join(root, filename) + src_basename = os.path.basename(src) + parts = [self._args.website[0]] + dirname = os.path.dirname(src) + if dirname: + parts.append(os.path.relpath(os.path.dirname(src))) + parts.append( + os.extsep.join((os.path.splitext(src_basename)[0], + 'html'))) + dst = os.path.join(*parts) + os.makedirs(os.path.dirname(dst), exist_ok=True) + + modified_src = os.path.join(self._tempdir, src_basename) + shutil.copy('www/header.asciidoc', modified_src) + + outfp = io.StringIO() + + with open(modified_src, 'r', encoding='utf-8') as header_file: + header = header_file.read() + header += "\n\n" + + with open(src, 'r', encoding='utf-8') as infp: + outfp.write("\n\n") + hidden = False + found_title = False + title = "" + last_line = "" + + for line in infp: + line = line.rstrip() + if line == '// QUTE_WEB_HIDE': + assert not hidden + hidden = True + elif line == '// QUTE_WEB_HIDE_END': + assert hidden + hidden = False + elif line == "The Compiler <mail@qutebrowser.org>": + continue + elif re.fullmatch(r':\w+:.*', line): + # asciidoc field + continue + + if not found_title: + if re.fullmatch(r'=+', line): + line = line.replace('=', '-') + found_title = True + title = last_line + " | qutebrowser\n" + title += "=" * (len(title) - 1) + elif re.fullmatch(r'= .+', line): + line = '==' + line[1:] + found_title = True + title = last_line + " | qutebrowser\n" + title += "=" * (len(title) - 1) + + if not hidden: + outfp.write(line.replace(".asciidoc[", ".html[") + '\n') + last_line = line + + current_lines = outfp.getvalue() + outfp.close() + + with open(modified_src, 'w+', encoding='utf-8') as final_version: + final_version.write(title + "\n\n" + header + current_lines) + + asciidoc_args = ['--theme=qute', '-a toc', '-a toc-placement=manual', + '-a', 'source-highlighter=pygments'] + self.call(modified_src, dst, *asciidoc_args) + + def _build_website(self): + """Prepare and build the website.""" + theme_file = os.path.abspath(os.path.join('www', 'qute.css')) + shutil.copy(theme_file, self._themedir) + + outdir = self._args.website[0] + + for root, _dirs, files in os.walk(os.getcwd()): + for filename in files: + basename, ext = os.path.splitext(filename) + if (ext != '.asciidoc' or + basename in ['header', 'OpenSans-License']): + continue + self._build_website_file(root, filename) + + copy = {'icons': 'icons', 'doc/img': 'doc/img', 'www/media': 'media/'} + + for src, dest in copy.items(): + full_dest = os.path.join(outdir, dest) + try: + shutil.rmtree(full_dest) + except FileNotFoundError: + pass + shutil.copytree(src, full_dest) + + for dst, link_name in [ + ('README.html', 'index.html'), + (os.path.join('doc', 'quickstart.html'), 'quickstart.html')]: + try: + os.symlink(dst, os.path.join(outdir, link_name)) + except FileExistsError: + pass + + def _get_asciidoc_cmd(self): + """Try to find out what commandline to use to invoke asciidoc.""" + if self._args.asciidoc is not None: + return self._args.asciidoc + + try: + subprocess.run(['asciidoc'], stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) + except OSError: + pass + else: + return ['asciidoc'] + + try: + subprocess.run(['asciidoc.py'], stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) + except OSError: + pass + else: + return ['asciidoc.py'] + + raise FileNotFoundError + + def call(self, src, dst, *args): + """Call asciidoc for the given files. + + Args: + src: The source .asciidoc file. + dst: The destination .html file, or None to auto-guess. + *args: Additional arguments passed to asciidoc. + """ + print("Calling asciidoc for {}...".format(os.path.basename(src))) + cmdline = self._cmd[:] + if dst is not None: + cmdline += ['--out-file', dst] + cmdline += args + cmdline.append(src) + try: + env = os.environ.copy() + env['HOME'] = self._homedir + subprocess.run(cmdline, check=True, env=env) + except (subprocess.CalledProcessError, OSError) as e: + self._failed = True + utils.print_col(str(e), 'red') + print("Keeping modified sources in {}.".format(self._homedir)) + sys.exit(1) + + +def main(colors=False): + """Generate html files for the online documentation.""" + utils.change_cwd() + utils.use_color = colors + parser = argparse.ArgumentParser() + parser.add_argument('--website', help="Build website into a given " + "directory.", nargs=1) + parser.add_argument('--asciidoc', help="Full path to python and " + "asciidoc.py. If not given, it's searched in PATH.", + nargs=2, required=False, + metavar=('PYTHON', 'ASCIIDOC')) + args = parser.parse_args() + try: + os.mkdir('qutebrowser/html/doc') + except FileExistsError: + pass + + asciidoc = AsciiDoc(args) + try: + asciidoc.prepare() + except FileNotFoundError: + utils.print_col("Could not find asciidoc! Please install it, or use " + "the --asciidoc argument to point this script to the " + "correct python/asciidoc.py location!", 'red') + sys.exit(1) + + try: + asciidoc.build() + finally: + asciidoc.cleanup() + + +if __name__ == '__main__': + main(colors=True) diff --git a/.config/qutebrowser/scripts/cycle-inputs.js b/.config/qutebrowser/scripts/cycle-inputs.js new file mode 100644 index 0000000..bb667bd --- /dev/null +++ b/.config/qutebrowser/scripts/cycle-inputs.js @@ -0,0 +1,46 @@ +/* Cycle <input> text boxes. + * works with the types defined in 'types'. + * Note: Does not work for <textarea>. + * + * Example keybind: + * CYCLE_INPUTS = "jseval -q -f ~/.config/qutebrowser/cycle-inputs.js" + * config.bind('gi', CYCLE_INPUTS) + * + * By dive on freenode <dave@dawoodfall.net> + */ + +(function() { + "use strict"; + const inputs = document.getElementsByTagName("input"); + const types = /text|password|date|email|month|number|range|search|tel|time|url|week/; + const hidden = /hidden/; + let found = false; + + function ishidden(el) { + return hidden.test(el.attributes.value) || el.offsetParent === null; + } + + for (let i = 0; i < inputs.length; i++) { + if (inputs[i] === document.activeElement) { + for (let k = i + 1; k < inputs.length; k++) { + if (!ishidden(inputs[k]) && types.test(inputs[k].type)) { + inputs[k].focus(); + found = true; + break; + } + } + break; + } + } + + if (!found) { + for (let i = 0; i < inputs.length; i++) { + if (!ishidden(inputs[i]) && types.test(inputs[i].type)) { + inputs[i].focus(); + break; + } + } + } +})(); + +// vim: tw=0 expandtab tabstop=4 softtabstop=4 shiftwidth=4 diff --git a/.config/qutebrowser/scripts/dev/Makefile-dmg b/.config/qutebrowser/scripts/dev/Makefile-dmg new file mode 100644 index 0000000..1cf4cfb --- /dev/null +++ b/.config/qutebrowser/scripts/dev/Makefile-dmg @@ -0,0 +1,71 @@ +# +# Build file for creating DMG files. +# +# The DMG packager looks for a template.dmg.bz2 for using as its +# DMG template. If it doesn't find one, it generates a clean one. +# +# If you create a DMG template, you should make one containing all +# the files listed in $(SOURCE_FILES) below, and arrange everything to suit +# your style. The contents of the files themselves does not matter, so +# they can be empty (they will be overwritten later). +# +# Remko Tronçon +# https://el-tramo.be +# Licensed under the MIT License. See COPYING for details. + + +################################################################################ +# Customizable variables +################################################################################ + +NAME ?= qutebrowser + +SOURCE_DIR ?= . +SOURCE_FILES ?= dist/qutebrowser.app LICENSE + +TEMPLATE_DMG ?= template.dmg +TEMPLATE_SIZE ?= 300m + +################################################################################ +# DMG building. No editing should be needed beyond this point. +################################################################################ + +MASTER_DMG=$(NAME).dmg +WC_DMG=wc.dmg +WC_DIR=wc + +.PHONY: all +all: $(MASTER_DMG) + +$(TEMPLATE_DMG): + @echo + @echo --------------------- Generating empty template -------------------- + mkdir template + hdiutil create -fs HFSX -layout SPUD -size $(TEMPLATE_SIZE) "$(TEMPLATE_DMG)" -srcfolder template -format UDRW -volname "$(NAME)" -quiet + rmdir template + +$(WC_DMG): $(TEMPLATE_DMG) + cp $< $@ + +$(MASTER_DMG): $(WC_DMG) $(addprefix $(SOURCE_DIR)/,$(SOURCE_FILES)) + @echo + @echo --------------------- Creating Disk Image -------------------- + mkdir -p $(WC_DIR) + hdiutil attach "$(WC_DMG)" -noautoopen -quiet -mountpoint "$(WC_DIR)" + for i in $(SOURCE_FILES); do \ + rm -rf "$(WC_DIR)/$$i"; \ + ditto -rsrc "$(SOURCE_DIR)/$$i" "$(WC_DIR)/$${i##*/}"; \ + done + ln -s /Applications $(WC_DIR) + #rm -f "$@" + #hdiutil create -srcfolder "$(WC_DIR)" -format UDZO -imagekey zlib-level=9 "$@" -volname "$(NAME) $(VERSION)" -scrub -quiet + WC_DEV=`hdiutil info | grep "$(WC_DIR)" | grep "Apple_HFS" | awk '{print $$1}'` && \ + hdiutil detach $$WC_DEV -quiet -force + rm -f "$(MASTER_DMG)" + hdiutil convert "$(WC_DMG)" -quiet -format UDZO -imagekey zlib-level=9 -o "$@" + rm -rf $(WC_DIR) + @echo + +.PHONY: clean +clean: + -rm -rf $(TEMPLATE_DMG) $(MASTER_DMG) $(WC_DMG) diff --git a/.config/qutebrowser/scripts/dev/__init__.py b/.config/qutebrowser/scripts/dev/__init__.py new file mode 100644 index 0000000..7dc0433 --- /dev/null +++ b/.config/qutebrowser/scripts/dev/__init__.py @@ -0,0 +1,3 @@ +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +"""Various scripts used for developing qutebrowser.""" diff --git a/.config/qutebrowser/scripts/dev/build_release.py b/.config/qutebrowser/scripts/dev/build_release.py new file mode 100755 index 0000000..254132b --- /dev/null +++ b/.config/qutebrowser/scripts/dev/build_release.py @@ -0,0 +1,419 @@ +#!/usr/bin/env python3 +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org> +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + +"""Build a new release.""" + + +import os +import os.path +import sys +import time +import glob +import shutil +import plistlib +import subprocess +import argparse +import tarfile +import tempfile +import collections + +try: + import winreg +except ImportError: + pass + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir, + os.pardir)) + +import qutebrowser +from scripts import utils +# from scripts.dev import update_3rdparty + + +def call_script(name, *args, python=sys.executable): + """Call a given shell script. + + Args: + name: The script to call. + *args: The arguments to pass. + python: The python interpreter to use. + """ + path = os.path.join(os.path.dirname(__file__), os.pardir, name) + subprocess.run([python, path] + list(args), check=True) + + +def call_tox(toxenv, *args, python=sys.executable): + """Call tox. + + Args: + toxenv: Which tox environment to use + *args: The arguments to pass. + python: The python interpreter to use. + """ + env = os.environ.copy() + env['PYTHON'] = python + env['PATH'] = os.environ['PATH'] + os.pathsep + os.path.dirname(python) + subprocess.run( + [sys.executable, '-m', 'tox', '-vv', '-e', toxenv] + list(args), + env=env, check=True) + + +def run_asciidoc2html(args): + """Common buildsteps used for all OS'.""" + utils.print_title("Running asciidoc2html.py") + if args.asciidoc is not None: + a2h_args = ['--asciidoc'] + args.asciidoc + else: + a2h_args = [] + call_script('asciidoc2html.py', *a2h_args) + + +def _maybe_remove(path): + """Remove a path if it exists.""" + try: + shutil.rmtree(path) + except FileNotFoundError: + pass + + +def smoke_test(executable): + """Try starting the given qutebrowser executable.""" + subprocess.run([executable, '--no-err-windows', '--nowindow', + '--temp-basedir', 'about:blank', ':later 500 quit'], + check=True) + + +def patch_mac_app(): + """Patch .app to copy missing data and link some libs. + + See https://github.com/pyinstaller/pyinstaller/issues/2276 + """ + app_path = os.path.join('dist', 'qutebrowser.app') + qtwe_core_dir = os.path.join('.tox', 'pyinstaller', 'lib', 'python3.6', + 'site-packages', 'PyQt5', 'Qt', 'lib', + 'QtWebEngineCore.framework') + # Copy QtWebEngineProcess.app + proc_app = 'QtWebEngineProcess.app' + shutil.copytree(os.path.join(qtwe_core_dir, 'Helpers', proc_app), + os.path.join(app_path, 'Contents', 'MacOS', proc_app)) + # Copy resources + for f in glob.glob(os.path.join(qtwe_core_dir, 'Resources', '*')): + dest = os.path.join(app_path, 'Contents', 'Resources') + if os.path.isdir(f): + dir_dest = os.path.join(dest, os.path.basename(f)) + print("Copying directory {} to {}".format(f, dir_dest)) + shutil.copytree(f, dir_dest) + else: + print("Copying {} to {}".format(f, dest)) + shutil.copy(f, dest) + # Link dependencies + for lib in ['QtCore', 'QtWebEngineCore', 'QtQuick', 'QtQml', 'QtNetwork', + 'QtGui', 'QtWebChannel', 'QtPositioning']: + dest = os.path.join(app_path, lib + '.framework', 'Versions', '5') + os.makedirs(dest) + os.symlink(os.path.join(os.pardir, os.pardir, os.pardir, 'Contents', + 'MacOS', lib), + os.path.join(dest, lib)) + # Patch Info.plist - pyinstaller's options are too limiting + plist_path = os.path.join(app_path, 'Contents', 'Info.plist') + with open(plist_path, "rb") as f: + plist_data = plistlib.load(f) + plist_data.update(INFO_PLIST_UPDATES) + with open(plist_path, "wb") as f: + plistlib.dump(plist_data, f) + + +INFO_PLIST_UPDATES = { + 'CFBundleVersion': qutebrowser.__version__, + 'CFBundleShortVersionString': qutebrowser.__version__, + 'NSSupportsAutomaticGraphicsSwitching': True, + 'NSHighResolutionCapable': True, + 'CFBundleURLTypes': [{ + "CFBundleURLName": "http(s) URL", + "CFBundleURLSchemes": ["http", "https"] + }, { + "CFBundleURLName": "local file URL", + "CFBundleURLSchemes": ["file"] + }], + 'CFBundleDocumentTypes': [{ + "CFBundleTypeExtensions": ["html", "htm"], + "CFBundleTypeMIMETypes": ["text/html"], + "CFBundleTypeName": "HTML document", + "CFBundleTypeOSTypes": ["HTML"], + "CFBundleTypeRole": "Viewer", + }, { + "CFBundleTypeExtensions": ["xhtml"], + "CFBundleTypeMIMETypes": ["text/xhtml"], + "CFBundleTypeName": "XHTML document", + "CFBundleTypeRole": "Viewer", + }] +} + + +def build_mac(): + """Build macOS .dmg/.app.""" + utils.print_title("Cleaning up...") + for f in ['wc.dmg', 'template.dmg']: + try: + os.remove(f) + except FileNotFoundError: + pass + for d in ['dist', 'build']: + shutil.rmtree(d, ignore_errors=True) + utils.print_title("Updating 3rdparty content") + # Currently disabled because QtWebEngine has no pdfjs support + # update_3rdparty.run(ace=False, pdfjs=True, fancy_dmg=False) + utils.print_title("Building .app via pyinstaller") + call_tox('pyinstaller', '-r') + utils.print_title("Patching .app") + patch_mac_app() + utils.print_title("Building .dmg") + subprocess.run(['make', '-f', 'scripts/dev/Makefile-dmg'], check=True) + + dmg_name = 'qutebrowser-{}.dmg'.format(qutebrowser.__version__) + os.rename('qutebrowser.dmg', dmg_name) + + utils.print_title("Running smoke test") + + try: + with tempfile.TemporaryDirectory() as tmpdir: + subprocess.run(['hdiutil', 'attach', dmg_name, + '-mountpoint', tmpdir], check=True) + try: + binary = os.path.join(tmpdir, 'qutebrowser.app', 'Contents', + 'MacOS', 'qutebrowser') + smoke_test(binary) + finally: + time.sleep(5) + subprocess.run(['hdiutil', 'detach', tmpdir]) + except PermissionError as e: + print("Failed to remove tempdir: {}".format(e)) + + return [(dmg_name, 'application/x-apple-diskimage', 'macOS .dmg')] + + +def patch_windows(out_dir): + """Copy missing DLLs for windows into the given output.""" + dll_dir = os.path.join('.tox', 'pyinstaller', 'lib', 'site-packages', + 'PyQt5', 'Qt', 'bin') + dlls = ['libEGL.dll', 'libGLESv2.dll', 'libeay32.dll', 'ssleay32.dll'] + for dll in dlls: + shutil.copy(os.path.join(dll_dir, dll), out_dir) + + +def build_windows(): + """Build windows executables/setups.""" + utils.print_title("Updating 3rdparty content") + # Currently disabled because QtWebEngine has no pdfjs support + # update_3rdparty.run(ace=False, pdfjs=True, fancy_dmg=False) + + utils.print_title("Building Windows binaries") + parts = str(sys.version_info.major), str(sys.version_info.minor) + ver = ''.join(parts) + dot_ver = '.'.join(parts) + + # Get python path from registry if possible + try: + reg64_key = winreg.OpenKeyEx(winreg.HKEY_LOCAL_MACHINE, + r'SOFTWARE\Python\PythonCore' + r'\{}\InstallPath'.format(dot_ver)) + python_x64 = winreg.QueryValueEx(reg64_key, 'ExecutablePath')[0] + except FileNotFoundError: + python_x64 = r'C:\Python{}\python.exe'.format(ver) + + out_pyinstaller = os.path.join('dist', 'qutebrowser') + out_64 = os.path.join('dist', + 'qutebrowser-{}-x64'.format(qutebrowser.__version__)) + + artifacts = [] + + from scripts.dev import gen_versioninfo + utils.print_title("Updating VersionInfo file") + gen_versioninfo.main() + + utils.print_title("Running pyinstaller 64bit") + _maybe_remove(out_64) + call_tox('pyinstaller', '-r', python=python_x64) + shutil.move(out_pyinstaller, out_64) + patch_windows(out_64) + + utils.print_title("Building installers") + subprocess.run(['makensis.exe', + '/DX64', + '/DVERSION={}'.format(qutebrowser.__version__), + 'misc/qutebrowser.nsi'], check=True) + + name_64 = 'qutebrowser-{}-amd64.exe'.format(qutebrowser.__version__) + + artifacts += [ + (os.path.join('dist', name_64), + 'application/vnd.microsoft.portable-executable', + 'Windows 64bit installer'), + ] + + utils.print_title("Running 64bit smoke test") + smoke_test(os.path.join(out_64, 'qutebrowser.exe')) + + utils.print_title("Zipping 64bit standalone...") + name = 'qutebrowser-{}-windows-standalone-amd64'.format( + qutebrowser.__version__) + shutil.make_archive(name, 'zip', 'dist', os.path.basename(out_64)) + artifacts.append(('{}.zip'.format(name), + 'application/zip', + 'Windows 64bit standalone')) + + return artifacts + + +def build_sdist(): + """Build an sdist and list the contents.""" + utils.print_title("Building sdist") + + _maybe_remove('dist') + + subprocess.run([sys.executable, 'setup.py', 'sdist'], check=True) + dist_files = os.listdir(os.path.abspath('dist')) + assert len(dist_files) == 1 + + dist_file = os.path.join('dist', dist_files[0]) + subprocess.run(['gpg', '--detach-sign', '-a', dist_file], check=True) + + tar = tarfile.open(dist_file) + by_ext = collections.defaultdict(list) + + for tarinfo in tar.getmembers(): + if not tarinfo.isfile(): + continue + name = os.sep.join(tarinfo.name.split(os.sep)[1:]) + _base, ext = os.path.splitext(name) + by_ext[ext].append(name) + + assert '.pyc' not in by_ext + + utils.print_title("sdist contents") + + for ext, files in sorted(by_ext.items()): + utils.print_subtitle(ext) + print('\n'.join(files)) + + filename = 'qutebrowser-{}.tar.gz'.format(qutebrowser.__version__) + artifacts = [ + (os.path.join('dist', filename), 'application/gzip', 'Source release'), + (os.path.join('dist', filename + '.asc'), 'application/pgp-signature', + 'Source release - PGP signature'), + ] + + return artifacts + + +def test_makefile(): + """Make sure the Makefile works correctly.""" + utils.print_title("Testing makefile") + with tempfile.TemporaryDirectory() as tmpdir: + subprocess.run(['make', '-f', 'misc/Makefile', + 'DESTDIR={}'.format(tmpdir), 'install'], check=True) + + +def read_github_token(): + """Read the GitHub API token from disk.""" + token_file = os.path.join(os.path.expanduser('~'), '.gh_token') + with open(token_file, encoding='ascii') as f: + token = f.read().strip() + return token + + +def github_upload(artifacts, tag): + """Upload the given artifacts to GitHub. + + Args: + artifacts: A list of (filename, mimetype, description) tuples + tag: The name of the release tag + """ + import github3 + utils.print_title("Uploading to github...") + + token = read_github_token() + gh = github3.login(token=token) + repo = gh.repository('qutebrowser', 'qutebrowser') + + release = None # to satisfy pylint + for release in repo.releases(): + if release.tag_name == tag: + break + else: + raise Exception("No release found for {!r}!".format(tag)) + + for filename, mimetype, description in artifacts: + with open(filename, 'rb') as f: + basename = os.path.basename(filename) + asset = release.upload_asset(mimetype, basename, f) + asset.edit(basename, description) + + +def pypi_upload(artifacts): + """Upload the given artifacts to PyPI using twine.""" + filenames = [a[0] for a in artifacts] + subprocess.run(['twine', 'upload'] + filenames, check=True) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--asciidoc', help="Full path to python and " + "asciidoc.py. If not given, it's searched in PATH.", + nargs=2, required=False, + metavar=('PYTHON', 'ASCIIDOC')) + parser.add_argument('--upload', help="Tag to upload the release for", + nargs=1, required=False, metavar='TAG') + args = parser.parse_args() + utils.change_cwd() + + upload_to_pypi = False + + if args.upload is not None: + # Fail early when trying to upload without github3 installed + # or without API token + import github3 # pylint: disable=unused-variable + read_github_token() + + run_asciidoc2html(args) + if os.name == 'nt': + artifacts = build_windows() + elif sys.platform == 'darwin': + artifacts = build_mac() + else: + test_makefile() + artifacts = build_sdist() + upload_to_pypi = True + + if args.upload is not None: + utils.print_title("Press enter to release...") + input() + github_upload(artifacts, args.upload[0]) + if upload_to_pypi: + pypi_upload(artifacts) + else: + print() + utils.print_title("Artifacts") + for artifact in artifacts: + print(artifact) + + +if __name__ == '__main__': + main() diff --git a/.config/qutebrowser/scripts/dev/check_coverage.py b/.config/qutebrowser/scripts/dev/check_coverage.py new file mode 100644 index 0000000..32c5afc --- /dev/null +++ b/.config/qutebrowser/scripts/dev/check_coverage.py @@ -0,0 +1,348 @@ +#!/usr/bin/env python3 +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2015-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org> + +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + +"""Enforce perfect coverage on some files.""" + +import os +import os.path +import sys +import enum +import subprocess +from xml.etree import ElementTree + +import attr + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir, + os.pardir)) + +from scripts import utils as scriptutils +from qutebrowser.utils import utils + + +@attr.s +class Message: + + """A message shown by coverage.py.""" + + typ = attr.ib() + filename = attr.ib() + text = attr.ib() + + +MsgType = enum.Enum('MsgType', 'insufficent_coverage, perfect_file') + + +# A list of (test_file, tested_file) tuples. test_file can be None. +PERFECT_FILES = [ + (None, + 'commands/cmdexc.py'), + ('tests/unit/commands/test_cmdutils.py', + 'commands/cmdutils.py'), + ('tests/unit/commands/test_argparser.py', + 'commands/argparser.py'), + + ('tests/unit/browser/webkit/test_cache.py', + 'browser/webkit/cache.py'), + ('tests/unit/browser/webkit/test_cookies.py', + 'browser/webkit/cookies.py'), + ('tests/unit/browser/test_history.py', + 'browser/history.py'), + ('tests/unit/browser/webkit/http/test_http.py', + 'browser/webkit/http.py'), + ('tests/unit/browser/webkit/http/test_content_disposition.py', + 'browser/webkit/rfc6266.py'), + # ('tests/unit/browser/webkit/test_webkitelem.py', + # 'browser/webkit/webkitelem.py'), + # ('tests/unit/browser/webkit/test_webkitelem.py', + # 'browser/webelem.py'), + ('tests/unit/browser/webkit/network/test_filescheme.py', + 'browser/webkit/network/filescheme.py'), + ('tests/unit/browser/webkit/network/test_networkreply.py', + 'browser/webkit/network/networkreply.py'), + + ('tests/unit/browser/test_signalfilter.py', + 'browser/signalfilter.py'), + (None, + 'browser/webengine/certificateerror.py'), + # ('tests/unit/browser/test_tab.py', + # 'browser/tab.py'), + + ('tests/unit/keyinput/test_basekeyparser.py', + 'keyinput/basekeyparser.py'), + ('tests/unit/keyinput/test_keyutils.py', + 'keyinput/keyutils.py'), + + ('tests/unit/misc/test_autoupdate.py', + 'misc/autoupdate.py'), + ('tests/unit/misc/test_readline.py', + 'misc/readline.py'), + ('tests/unit/misc/test_split.py', + 'misc/split.py'), + ('tests/unit/misc/test_msgbox.py', + 'misc/msgbox.py'), + ('tests/unit/misc/test_checkpyver.py', + 'misc/checkpyver.py'), + ('tests/unit/misc/test_guiprocess.py', + 'misc/guiprocess.py'), + ('tests/unit/misc/test_editor.py', + 'misc/editor.py'), + ('tests/unit/misc/test_cmdhistory.py', + 'misc/cmdhistory.py'), + ('tests/unit/misc/test_ipc.py', + 'misc/ipc.py'), + ('tests/unit/misc/test_keyhints.py', + 'misc/keyhintwidget.py'), + ('tests/unit/misc/test_pastebin.py', + 'misc/pastebin.py'), + (None, + 'misc/objects.py'), + + (None, + 'mainwindow/statusbar/keystring.py'), + ('tests/unit/mainwindow/statusbar/test_percentage.py', + 'mainwindow/statusbar/percentage.py'), + ('tests/unit/mainwindow/statusbar/test_progress.py', + 'mainwindow/statusbar/progress.py'), + ('tests/unit/mainwindow/statusbar/test_tabindex.py', + 'mainwindow/statusbar/tabindex.py'), + ('tests/unit/mainwindow/statusbar/test_textbase.py', + 'mainwindow/statusbar/textbase.py'), + ('tests/unit/mainwindow/statusbar/test_url.py', + 'mainwindow/statusbar/url.py'), + ('tests/unit/mainwindow/statusbar/test_backforward.py', + 'mainwindow/statusbar/backforward.py'), + ('tests/unit/mainwindow/test_messageview.py', + 'mainwindow/messageview.py'), + + ('tests/unit/config/test_config.py', + 'config/config.py'), + ('tests/unit/config/test_configdata.py', + 'config/configdata.py'), + ('tests/unit/config/test_configexc.py', + 'config/configexc.py'), + ('tests/unit/config/test_configfiles.py', + 'config/configfiles.py'), + ('tests/unit/config/test_configtypes.py', + 'config/configtypes.py'), + ('tests/unit/config/test_configinit.py', + 'config/configinit.py'), + ('tests/unit/config/test_configcommands.py', + 'config/configcommands.py'), + ('tests/unit/config/test_configutils.py', + 'config/configutils.py'), + + ('tests/unit/utils/test_qtutils.py', + 'utils/qtutils.py'), + ('tests/unit/utils/test_standarddir.py', + 'utils/standarddir.py'), + ('tests/unit/utils/test_urlutils.py', + 'utils/urlutils.py'), + ('tests/unit/utils/usertypes', + 'utils/usertypes.py'), + ('tests/unit/utils/test_utils.py', + 'utils/utils.py'), + ('tests/unit/utils/test_version.py', + 'utils/version.py'), + ('tests/unit/utils/test_debug.py', + 'utils/debug.py'), + ('tests/unit/utils/test_jinja.py', + 'utils/jinja.py'), + ('tests/unit/utils/test_error.py', + 'utils/error.py'), + ('tests/unit/utils/test_javascript.py', + 'utils/javascript.py'), + ('tests/unit/utils/test_urlmatch.py', + 'utils/urlmatch.py'), + + (None, + 'completion/models/util.py'), + ('tests/unit/completion/test_models.py', + 'completion/models/urlmodel.py'), + ('tests/unit/completion/test_models.py', + 'completion/models/configmodel.py'), + ('tests/unit/completion/test_histcategory.py', + 'completion/models/histcategory.py'), + ('tests/unit/completion/test_listcategory.py', + 'completion/models/listcategory.py'), + + ('tests/unit/browser/webengine/test_spell.py', + 'browser/webengine/spell.py'), + +] + + +# 100% coverage because of end2end tests, but no perfect unit tests yet. +WHITELISTED_FILES = [ + 'browser/webkit/webkitinspector.py', + 'keyinput/macros.py', + 'browser/webkit/webkitelem.py', +] + + +class Skipped(Exception): + + """Exception raised when skipping coverage checks.""" + + def __init__(self, reason): + self.reason = reason + super().__init__("Skipping coverage checks " + reason) + + +def _get_filename(filename): + """Transform the absolute test filenames to relative ones.""" + if os.path.isabs(filename): + basedir = os.path.abspath( + os.path.join(os.path.dirname(__file__), '..', '..')) + common_path = os.path.commonprefix([basedir, filename]) + if common_path: + filename = filename[len(common_path):].lstrip('/') + if filename.startswith('qutebrowser/'): + filename = filename.split('/', maxsplit=1)[1] + + return filename + + +def check(fileobj, perfect_files): + """Main entry point which parses/checks coverage.xml if applicable.""" + if not utils.is_linux: + raise Skipped("on non-Linux system.") + elif '-k' in sys.argv[1:]: + raise Skipped("because -k is given.") + elif '-m' in sys.argv[1:]: + raise Skipped("because -m is given.") + elif '--lf' in sys.argv[1:]: + raise Skipped("because --lf is given.") + + perfect_src_files = [e[1] for e in perfect_files] + + filename_args = [arg for arg in sys.argv[1:] + if arg.startswith('tests' + os.sep)] + filtered_files = [tpl[1] for tpl in perfect_files if tpl[0] in + filename_args] + + if filename_args and not filtered_files: + raise Skipped("because there is nothing to check.") + + tree = ElementTree.parse(fileobj) + classes = tree.getroot().findall('./packages/package/classes/class') + + messages = [] + + for klass in classes: + filename = _get_filename(klass.attrib['filename']) + + line_cov = float(klass.attrib['line-rate']) * 100 + branch_cov = float(klass.attrib['branch-rate']) * 100 + + if filtered_files and filename not in filtered_files: + continue + + assert 0 <= line_cov <= 100, line_cov + assert 0 <= branch_cov <= 100, branch_cov + assert '\\' not in filename, filename + + is_bad = line_cov < 100 or branch_cov < 100 + + if filename in perfect_src_files and is_bad: + text = "{} has {:.2f}% line and {:.2f}% branch coverage!".format( + filename, line_cov, branch_cov) + messages.append(Message(MsgType.insufficent_coverage, filename, + text)) + elif (filename not in perfect_src_files and not is_bad and + filename not in WHITELISTED_FILES): + text = ("{} has 100% coverage but is not in " + "perfect_files!".format(filename)) + messages.append(Message(MsgType.perfect_file, filename, text)) + + return messages + + +def main_check(): + """Check coverage after a test run.""" + try: + with open('coverage.xml', encoding='utf-8') as f: + messages = check(f, PERFECT_FILES) + except Skipped as e: + print(e) + messages = [] + + if messages: + print() + print() + scriptutils.print_title("Coverage check failed") + for msg in messages: + print(msg.text) + print() + filters = ','.join('qutebrowser/' + msg.filename for msg in messages) + subprocess.run([sys.executable, '-m', 'coverage', 'report', + '--show-missing', '--include', filters], check=True) + print() + print("To debug this, run 'tox -e py36-pyqt59-cov' " + "(or py35-pyqt59-cov) locally and check htmlcov/index.html") + print("or check https://codecov.io/github/qutebrowser/qutebrowser") + print() + + if 'CI' in os.environ: + print("Keeping coverage.xml on CI.") + else: + os.remove('coverage.xml') + return 1 if messages else 0 + + +def main_check_all(): + """Check the coverage for all files individually. + + This makes sure the files have 100% coverage without running unrelated + tests. + + This runs pytest with the used executable, so check_coverage.py should be + called with something like ./.tox/py36/bin/python. + """ + for test_file, src_file in PERFECT_FILES: + if test_file is None: + continue + subprocess.run( + [sys.executable, '-m', 'pytest', '--cov', 'qutebrowser', + '--cov-report', 'xml', test_file], check=True) + with open('coverage.xml', encoding='utf-8') as f: + messages = check(f, [(test_file, src_file)]) + os.remove('coverage.xml') + + messages = [msg for msg in messages + if msg.typ == MsgType.insufficent_coverage] + if messages: + for msg in messages: + print(msg.text) + return 1 + else: + print("Check ok!") + return 0 + + +def main(): + scriptutils.change_cwd() + if '--check-all' in sys.argv: + return main_check_all() + else: + return main_check() + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/.config/qutebrowser/scripts/dev/check_doc_changes.py b/.config/qutebrowser/scripts/dev/check_doc_changes.py new file mode 100755 index 0000000..3d90bea --- /dev/null +++ b/.config/qutebrowser/scripts/dev/check_doc_changes.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2016-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org> + +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + +"""Check if docs changed and output an error if so.""" + +import sys +import subprocess +import os + +code = subprocess.run(['git', '--no-pager', 'diff', + '--exit-code', '--stat']).returncode + +if os.environ.get('TRAVIS_PULL_REQUEST', 'false') != 'false': + if code != 0: + print("Docs changed but ignoring change as we're building a PR") + sys.exit(0) + +if code != 0: + print() + print('The autogenerated docs changed, please run this to update them:') + print(' tox -e docs') + print(' git commit -am "Update docs"') + print() + print('(Or you have uncommitted changes, in which case you can ignore ' + 'this.)') + if 'TRAVIS' in os.environ: + print() + print("travis_fold:start:gitdiff") + subprocess.run(['git', '--no-pager', 'diff']) + print("travis_fold:end:gitdiff") +sys.exit(code) diff --git a/.config/qutebrowser/scripts/dev/ci/travis_backtrace.sh b/.config/qutebrowser/scripts/dev/ci/travis_backtrace.sh new file mode 100644 index 0000000..227dde8 --- /dev/null +++ b/.config/qutebrowser/scripts/dev/ci/travis_backtrace.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# +# Find all possible core files under current directory. Attempt +# to determine exe using file(1) and dump stack trace with gdb. +# + +case $TESTENV in + py3*-pyqt*) + exe=$(readlink -f ".tox/$TESTENV/bin/python") + full= + ;; + *) + echo "Skipping coredump analysis in testenv $TESTENV!" + exit 0 + ;; +esac + +find . \( -name "*.core" -o -name core \) -exec gdb --batch --quiet -ex "thread apply all bt $full" "$exe" {} \; diff --git a/.config/qutebrowser/scripts/dev/ci/travis_install.sh b/.config/qutebrowser/scripts/dev/ci/travis_install.sh new file mode 100644 index 0000000..18f5aa9 --- /dev/null +++ b/.config/qutebrowser/scripts/dev/ci/travis_install.sh @@ -0,0 +1,108 @@ +#!/bin/bash +# vim: ft=sh fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org> + +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + +# Stolen from https://github.com/travis-ci/travis-build/blob/master/lib/travis/build/templates/header.sh +# and adjusted to use ((...)) +travis_retry() { + local ANSI_RED='\033[31;1m' + local ANSI_RESET='\033[0m' + local result=0 + local count=1 + while (( count < 3 )); do + if (( result != 0 )); then + echo -e "\\n${ANSI_RED}The command \"$*\" failed. Retrying, $count of 3.${ANSI_RESET}\\n" >&2 + fi + "$@" + result=$? + (( result == 0 )) && break + count=$(( count + 1 )) + sleep 1 + done + + if (( count > 3 )); then + echo -e "\\n${ANSI_RED}The command \"$*\" failed 3 times.${ANSI_RESET}\\n" >&2 + fi + + return $result +} + +pip_install() { + travis_retry python3 -m pip install "$@" +} + +npm_install() { + # Make sure npm is up-to-date first + travis_retry npm install -g npm + travis_retry npm install -g "$@" +} + +check_pyqt() { + python3 <<EOF +import sys +from PyQt5.QtCore import QT_VERSION_STR, PYQT_VERSION_STR, qVersion +try: + from PyQt.sip import SIP_VERSION_STR +except ModuleNotFoundError: + from sip import SIP_VERSION_STR + +print("Python {}".format(sys.version)) +print("PyQt5 {}".format(PYQT_VERSION_STR)) +print("Qt5 {} (runtime {})".format(QT_VERSION_STR, qVersion())) +print("sip {}".format(SIP_VERSION_STR)) +EOF +} + +set -e + +if [[ $DOCKER ]]; then + exit 0 +elif [[ $TRAVIS_OS_NAME == osx ]]; then + # Disable App Nap + defaults write NSGlobalDomain NSAppSleepDisabled -bool YES + + curl -LO https://bootstrap.pypa.io/get-pip.py + sudo -H python get-pip.py + + brew --version + brew update + brew upgrade python libyaml + brew install qt5 pyqt5 + + pip_install -r misc/requirements/requirements-tox.txt + python3 -m pip --version + tox --version + check_pyqt + exit 0 +fi + +case $TESTENV in + eslint) + npm_install eslint + ;; + shellcheck) + ;; + *) + pip_install pip + pip_install -r misc/requirements/requirements-tox.txt + if [[ $TESTENV == *-cov ]]; then + pip_install -r misc/requirements/requirements-codecov.txt + fi + ;; +esac diff --git a/.config/qutebrowser/scripts/dev/ci/travis_run.sh b/.config/qutebrowser/scripts/dev/ci/travis_run.sh new file mode 100644 index 0000000..55ca7c1 --- /dev/null +++ b/.config/qutebrowser/scripts/dev/ci/travis_run.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +if [[ $DOCKER ]]; then + docker run \ + --privileged \ + -v "$PWD:/outside" \ + -e "QUTE_BDD_WEBENGINE=$QUTE_BDD_WEBENGINE" \ + -e "DOCKER=$DOCKER" \ + -e "CI=$CI" \ + -e "TRAVIS=$TRAVIS" \ + "qutebrowser/travis:$DOCKER" +elif [[ $TESTENV == eslint ]]; then + # Can't run this via tox as we can't easily install tox in the javascript + # travis env + cd qutebrowser/javascript || exit 1 + eslint --color --report-unused-disable-directives . +elif [[ $TESTENV == shellcheck ]]; then + SCRIPTS=$( mktemp ) + find scripts/dev/ -name '*.sh' >"$SCRIPTS" + find misc/userscripts/ -type f -exec grep -lE '[/ ][bd]ash$|[/ ]sh$|[/ ]ksh$' {} + >>"$SCRIPTS" + mapfile -t scripts <"$SCRIPTS" + rm -f "$SCRIPTS" + docker run \ + -v "$PWD:/outside" \ + -w /outside \ + koalaman/shellcheck:latest "${scripts[@]}" +else + args=() + [[ $TRAVIS_OS_NAME == osx ]] && args=('--qute-bdd-webengine' '--no-xvfb' 'tests/unit') + + tox -e "$TESTENV" -- "${args[@]}" +fi diff --git a/.config/qutebrowser/scripts/dev/cleanup.py b/.config/qutebrowser/scripts/dev/cleanup.py new file mode 100755 index 0000000..d1bb84a --- /dev/null +++ b/.config/qutebrowser/scripts/dev/cleanup.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org> +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + +"""Script to clean up the mess made by Python/setuptools/PyInstaller.""" + +import os +import os.path +import sys +import glob +import shutil +import fnmatch + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir, + os.pardir)) + +from scripts import utils + + +recursive_lint = ('__pycache__', '*.pyc') +lint = ('build', 'dist', 'pkg/pkg', 'pkg/qutebrowser-*.pkg.tar.xz', 'pkg/src', + 'pkg/qutebrowser', 'qutebrowser.egg-info', 'setuptools-*.egg', + 'setuptools-*.zip', 'doc/qutebrowser.asciidoc', 'doc/*.html', + 'doc/qutebrowser.1', 'README.html', 'qutebrowser/html/doc') + + +def remove(path): + """Remove either a file or directory unless --dry-run is given.""" + if os.path.isdir(path): + print("rm -r '{}'".format(path)) + if '--dry-run' not in sys.argv: + shutil.rmtree(path) + else: + print("rm '{}'".format(path)) + if '--dry-run' not in sys.argv: + os.remove(path) + + +def main(): + """Clean up lint in the current dir.""" + utils.change_cwd() + for elem in lint: + for f in glob.glob(elem): + remove(f) + + for root, _dirs, _files in os.walk(os.getcwd()): + path = os.path.basename(root) + if any(fnmatch.fnmatch(path, e) for e in recursive_lint): + remove(root) + + +if __name__ == '__main__': + main() diff --git a/.config/qutebrowser/scripts/dev/download_release.sh b/.config/qutebrowser/scripts/dev/download_release.sh new file mode 100644 index 0000000..207da21 --- /dev/null +++ b/.config/qutebrowser/scripts/dev/download_release.sh @@ -0,0 +1,34 @@ +#!/bin/bash +set -e + +# This script downloads the given release from GitHub so we can mirror it on +# qutebrowser.org. + +tmpdir=$(mktemp -d) +oldpwd=$PWD + +if [[ $# != 1 ]]; then + echo "Usage: $0 <version>" >&2 + exit 1 +fi + +cd "$tmpdir" +mkdir windows + +base="https://github.com/qutebrowser/qutebrowser/releases/download/v$1" + +wget "$base/qutebrowser-$1.tar.gz" +wget "$base/qutebrowser-$1.tar.gz.asc" +wget "$base/qutebrowser-$1.dmg" +wget "$base/qutebrowser_${1}-1_all.deb" + +cd windows +wget "$base/qutebrowser-${1}-amd64.msi" +wget "$base/qutebrowser-${1}-win32.msi" +wget "$base/qutebrowser-${1}-windows-standalone-amd64.zip" +wget "$base/qutebrowser-${1}-windows-standalone-win32.zip" + +dest="/srv/http/qutebrowser/releases/v$1" +cd "$oldpwd" +sudo mv "$tmpdir" "$dest" +sudo chown -R http:http "$dest" diff --git a/.config/qutebrowser/scripts/dev/gen_resources.py b/.config/qutebrowser/scripts/dev/gen_resources.py new file mode 100644 index 0000000..cbfc69b --- /dev/null +++ b/.config/qutebrowser/scripts/dev/gen_resources.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# copyright 2014 florian bruhin (the compiler) <mail@qutebrowser.org> + +# this file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the gnu general public license as published by +# the free software foundation, either version 3 of the license, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but without any warranty; without even the implied warranty of +# merchantability or fitness for a particular purpose. see the +# gnu general public license for more details. +# +# you should have received a copy of the gnu general public license +# along with qutebrowser. if not, see <http://www.gnu.org/licenses/>. + +"""Generate Qt resources based on source files.""" + +import subprocess + +with open('qutebrowser/resources.py', 'w', encoding='utf-8') as f: + subprocess.run(['pyrcc5', 'qutebrowser.rcc'], stdout=f, check=True) diff --git a/.config/qutebrowser/scripts/dev/gen_versioninfo.py b/.config/qutebrowser/scripts/dev/gen_versioninfo.py new file mode 100644 index 0000000..1aa4b64 --- /dev/null +++ b/.config/qutebrowser/scripts/dev/gen_versioninfo.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org> +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + +"""Generate file_version_info.txt for Pyinstaller use with Windows builds.""" + +import os.path +import sys + +# pylint: disable=import-error,no-member,useless-suppression +from PyInstaller.utils.win32 import versioninfo as vs +# pylint: enable=import-error,no-member,useless-suppression + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir, + os.pardir)) + +import qutebrowser +from scripts import utils + + +def main(): + utils.change_cwd() + out_filename = 'misc/file_version_info.txt' + + filevers = qutebrowser.__version_info__ + (0,) + prodvers = qutebrowser.__version_info__ + (0,) + str_filevers = qutebrowser.__version__ + str_prodvers = qutebrowser.__version__ + + comment_text = qutebrowser.__doc__ + copyright_text = qutebrowser.__copyright__ + trademark_text = ("qutebrowser is free software under the GNU General " + "Public License") + + # https://www.science.co.il/language/Locale-codes.php#definitions + # https://msdn.microsoft.com/en-us/library/windows/desktop/dd317756.aspx + en_us = 1033 # 0x0409 + utf_16 = 1200 # 0x04B0 + + ffi = vs.FixedFileInfo(filevers, prodvers) + + kids = [ + vs.StringFileInfo([ + # 0x0409: MAKELANGID(LANG_ENGLISH, SUBLANG_ENGLISH_US) + # 0x04B0: codepage 1200 (UTF-16LE) + vs.StringTable('040904B0', [ + vs.StringStruct('Comments', comment_text), + vs.StringStruct('CompanyName', "qutebrowser.org"), + vs.StringStruct('FileDescription', "qutebrowser"), + vs.StringStruct('FileVersion', str_filevers), + vs.StringStruct('InternalName', "qutebrowser"), + vs.StringStruct('LegalCopyright', copyright_text), + vs.StringStruct('LegalTrademarks', trademark_text), + vs.StringStruct('OriginalFilename', "qutebrowser.exe"), + vs.StringStruct('ProductName', "qutebrowser"), + vs.StringStruct('ProductVersion', str_prodvers) + ]), + ]), + vs.VarFileInfo([vs.VarStruct('Translation', [en_us, utf_16])]), + ] + + file_version_info = vs.VSVersionInfo(ffi, kids) + + with open(out_filename, 'w', encoding='utf-8') as f: + f.write(str(file_version_info)) + + +if __name__ == '__main__': + main() diff --git a/.config/qutebrowser/scripts/dev/get_coredumpctl_traces.py b/.config/qutebrowser/scripts/dev/get_coredumpctl_traces.py new file mode 100644 index 0000000..d286d38 --- /dev/null +++ b/.config/qutebrowser/scripts/dev/get_coredumpctl_traces.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2015-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org> + +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + +"""Get qutebrowser crash information and stacktraces from coredumpctl.""" + +import os +import os.path +import sys +import argparse +import subprocess +import tempfile + +import attr + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir, + os.pardir)) + +from scripts import utils + + +@attr.s +class Line: + + """A line in "coredumpctl list".""" + + time = attr.ib() + pid = attr.ib() + uid = attr.ib() + gid = attr.ib() + sig = attr.ib() + present = attr.ib() + exe = attr.ib() + + +def _convert_present(data): + """Convert " "/"*" to True/False for parse_coredumpctl_line.""" + if data == '*': + return True + elif data == ' ': + return False + else: + raise ValueError(data) + + +def parse_coredumpctl_line(line): + """Parse a given string coming from coredumpctl and return a Line object. + + Example input: + Mon 2015-09-28 23:22:24 CEST 10606 1000 1000 11 /usr/bin/python3.4 + """ + fields = { + 'time': (0, 28, str), + 'pid': (29, 35, int), + 'uid': (36, 41, int), + 'gid': (42, 47, int), + 'sig': (48, 51, int), + 'present': (52, 53, _convert_present), + 'exe': (54, None, str), + } + + data = {} + for name, (start, end, converter) in fields.items(): + data[name] = converter(line[start:end]) + return Line(**data) + + +def get_info(pid): + """Get and parse "coredumpctl info" output for the given PID.""" + data = {} + output = subprocess.run(['coredumpctl', 'info', str(pid)], check=True, + stdout=subprocess.PIPE).stdout + output = output.decode('utf-8') + for line in output.split('\n'): + if not line.strip(): + continue + try: + key, value = line.split(':', maxsplit=1) + except ValueError: + # systemd stack output + continue + data[key.strip()] = value.strip() + return data + + +def is_qutebrowser_dump(parsed): + """Check if the given Line is a qutebrowser dump.""" + basename = os.path.basename(parsed.exe) + if basename == 'python' or basename.startswith('python3'): + info = get_info(parsed.pid) + try: + cmdline = info['Command Line'] + except KeyError: + return True + else: + return '-m qutebrowser' in cmdline + else: + return basename == 'qutebrowser' + + +def dump_infos_gdb(parsed): + """Dump all needed infos for the given crash using gdb.""" + with tempfile.TemporaryDirectory() as tempdir: + coredump = os.path.join(tempdir, 'dump') + subprocess.run(['coredumpctl', 'dump', '-o', coredump, + str(parsed.pid)], check=True) + subprocess.run(['gdb', parsed.exe, coredump, + '-ex', 'info threads', + '-ex', 'thread apply all bt full', + '-ex', 'quit'], check=True) + + +def dump_infos(parsed): + """Dump all possible infos for the given crash.""" + if not parsed.present: + info = get_info(parsed.pid) + print("{}: Signal {} with no coredump: {}".format( + parsed.time, info.get('Signal', None), + info.get('Command Line', None))) + else: + print('\n\n\n') + utils.print_title('{} - {}'.format(parsed.time, parsed.pid)) + sys.stdout.flush() + dump_infos_gdb(parsed) + + +def check_prerequisites(): + """Check if coredumpctl/gdb are installed.""" + for binary in ['coredumpctl', 'gdb']: + try: + subprocess.run([binary, '--version'], check=True) + except FileNotFoundError: + print("{} is needed to run this script!".format(binary), + file=sys.stderr) + sys.exit(1) + + +def main(): + check_prerequisites() + + parser = argparse.ArgumentParser() + parser.add_argument('--all', help="Also list crashes without coredumps.", + action='store_true') + args = parser.parse_args() + + coredumps = subprocess.run(['coredumpctl', 'list'], check=True, + stdout=subprocess.PIPE).stdout + lines = coredumps.decode('utf-8').split('\n') + for line in lines[1:]: + if not line.strip(): + continue + parsed = parse_coredumpctl_line(line) + if not parsed.present and not args.all: + continue + if is_qutebrowser_dump(parsed): + dump_infos(parsed) + + +if __name__ == '__main__': + main() diff --git a/.config/qutebrowser/scripts/dev/misc_checks.py b/.config/qutebrowser/scripts/dev/misc_checks.py new file mode 100644 index 0000000..2992464 --- /dev/null +++ b/.config/qutebrowser/scripts/dev/misc_checks.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org> +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + +"""Various small code checkers.""" + +import os +import os.path +import re +import sys +import argparse +import subprocess +import tokenize +import traceback +import collections + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir, + os.pardir)) + +from scripts import utils + + +def _get_files(only_py=False): + """Iterate over all python files and yield filenames.""" + for (dirpath, _dirnames, filenames) in os.walk('.'): + parts = dirpath.split(os.sep) + if len(parts) >= 2: + rootdir = parts[1] + if rootdir.startswith('.') or rootdir == 'htmlcov': + # ignore hidden dirs and htmlcov + continue + + if only_py: + endings = {'.py'} + else: + endings = {'.py', '.asciidoc', '.js', '.feature'} + files = (e for e in filenames if os.path.splitext(e)[1] in endings) + for name in files: + yield os.path.join(dirpath, name) + + +def check_git(): + """Check for uncommitted git files..""" + if not os.path.isdir(".git"): + print("No .git dir, ignoring") + print() + return False + untracked = [] + gitst = subprocess.run(['git', 'status', '--porcelain'], check=True, + stdout=subprocess.PIPE).stdout + gitst = gitst.decode('UTF-8').strip() + for line in gitst.splitlines(): + s, name = line.split(maxsplit=1) + if s == '??' and name != '.venv/': + untracked.append(name) + status = True + if untracked: + status = False + utils.print_col("Untracked files:", 'red') + print('\n'.join(untracked)) + print() + return status + + +def check_spelling(): + """Check commonly misspelled words.""" + # Words which I often misspell + words = {'[Bb]ehaviour', '[Qq]uitted', 'Ll]ikelyhood', '[Ss]ucessfully', + '[Oo]ccur[^rs .]', '[Ss]eperator', '[Ee]xplicitely', + '[Aa]uxillary', '[Aa]ccidentaly', '[Aa]mbigious', '[Ll]oosly', + '[Ii]nitialis', '[Cc]onvienence', '[Ss]imiliar', '[Uu]ncommited', + '[Rr]eproducable', '[Aa]n [Uu]ser', '[Cc]onvienience', + '[Ww]ether', '[Pp]rogramatically', '[Ss]plitted', '[Ee]xitted', + '[Mm]ininum', '[Rr]esett?ed', '[Rr]ecieved', '[Rr]egularily', + '[Uu]nderlaying', '[Ii]nexistant', '[Ee]lipsis', 'commiting', + 'existant', '[Rr]esetted', '[Ss]imilarily', '[Ii]nformations'} + + # Words which look better when splitted, but might need some fine tuning. + words |= {'[Ww]ebelements', '[Mm]ouseevent', '[Kk]eysequence', + '[Nn]ormalmode', '[Ee]ventloops', '[Ss]izehint', + '[Ss]tatemachine', '[Mm]etaobject', '[Ll]ogrecord', + '[Ff]iletype'} + + # Files which should be ignored, e.g. because they come from another + # package + ignored = [ + os.path.join('.', 'scripts', 'dev', 'misc_checks.py'), + os.path.join('.', 'qutebrowser', '3rdparty', 'pdfjs'), + os.path.join('.', 'tests', 'end2end', 'data', 'hints', 'ace', + 'ace.js'), + ] + + seen = collections.defaultdict(list) + try: + ok = True + for fn in _get_files(): + with tokenize.open(fn) as f: + if any(fn.startswith(i) for i in ignored): + continue + for line in f: + for w in words: + if (re.search(w, line) and + fn not in seen[w] and + '# pragma: no spellcheck' not in line): + print('Found "{}" in {}!'.format(w, fn)) + seen[w].append(fn) + ok = False + print() + return ok + except Exception: + traceback.print_exc() + return None + + +def check_vcs_conflict(): + """Check VCS conflict markers.""" + try: + ok = True + for fn in _get_files(only_py=True): + with tokenize.open(fn) as f: + for line in f: + if any(line.startswith(c * 7) for c in '<>=|'): + print("Found conflict marker in {}".format(fn)) + ok = False + print() + return ok + except Exception: + traceback.print_exc() + return None + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('checker', choices=('git', 'vcs', 'spelling'), + help="Which checker to run.") + args = parser.parse_args() + if args.checker == 'git': + ok = check_git() + elif args.checker == 'vcs': + ok = check_vcs_conflict() + elif args.checker == 'spelling': + ok = check_spelling() + return 0 if ok else 1 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/.config/qutebrowser/scripts/dev/pylint_checkers/qute_pylint/__init__.py b/.config/qutebrowser/scripts/dev/pylint_checkers/qute_pylint/__init__.py new file mode 100644 index 0000000..1341a93 --- /dev/null +++ b/.config/qutebrowser/scripts/dev/pylint_checkers/qute_pylint/__init__.py @@ -0,0 +1 @@ +"""Custom pylint checkers.""" diff --git a/.config/qutebrowser/scripts/dev/pylint_checkers/qute_pylint/config.py b/.config/qutebrowser/scripts/dev/pylint_checkers/qute_pylint/config.py new file mode 100644 index 0000000..5aa5250 --- /dev/null +++ b/.config/qutebrowser/scripts/dev/pylint_checkers/qute_pylint/config.py @@ -0,0 +1,84 @@ +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org> +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + +"""Custom astroid checker for config calls.""" + +import sys +import pathlib + +import yaml +import astroid +from pylint import interfaces, checkers +from pylint.checkers import utils + + +OPTIONS = None +FAILED_LOAD = False + + +class ConfigChecker(checkers.BaseChecker): + + """Custom astroid checker for config calls.""" + + __implements__ = interfaces.IAstroidChecker + name = 'config' + msgs = { + 'E9998': ('%s is no valid config option.', # flake8: disable=S001 + 'bad-config-option', + None), + } + priority = -1 + printed_warning = False + + @utils.check_messages('bad-config-option') + def visit_attribute(self, node): + """Visit a getattr node.""" + # At the end of a config.val.foo.bar chain + if not isinstance(node.parent, astroid.Attribute): + # FIXME:conf do some proper check for this... + node_str = node.as_string() + prefix = 'config.val.' + if node_str.startswith(prefix): + self._check_config(node, node_str[len(prefix):]) + + def _check_config(self, node, name): + """Check that we're accessing proper config options.""" + if FAILED_LOAD: + if not ConfigChecker.printed_warning: + print("[WARN] Could not find configdata.yml. Please run " + "pylint from qutebrowser root.", file=sys.stderr) + print("Skipping some checks...", file=sys.stderr) + ConfigChecker.printed_warning = True + return + if name not in OPTIONS: + self.add_message('bad-config-option', node=node, args=name) + + +def register(linter): + """Register this checker.""" + linter.register_checker(ConfigChecker(linter)) + global OPTIONS + global FAILED_LOAD + yaml_file = pathlib.Path('qutebrowser') / 'config' / 'configdata.yml' + if not yaml_file.exists(): + OPTIONS = None + FAILED_LOAD = True + return + with yaml_file.open(mode='r', encoding='utf-8') as f: + OPTIONS = list(yaml.load(f)) diff --git a/.config/qutebrowser/scripts/dev/pylint_checkers/qute_pylint/modeline.py b/.config/qutebrowser/scripts/dev/pylint_checkers/qute_pylint/modeline.py new file mode 100644 index 0000000..429974c --- /dev/null +++ b/.config/qutebrowser/scripts/dev/pylint_checkers/qute_pylint/modeline.py @@ -0,0 +1,63 @@ +# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org> +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + +"""Checker for vim modelines in files.""" + +import os.path +import contextlib + +from pylint import interfaces, checkers + + +class ModelineChecker(checkers.BaseChecker): + + """Check for vim modelines in files.""" + + __implements__ = interfaces.IRawChecker + + name = 'modeline' + msgs = {'W9002': ('Does not have vim modeline', 'modeline-missing', None), + 'W9003': ('Modeline is invalid', 'invalid-modeline', None), + 'W9004': ('Modeline position is wrong', 'modeline-position', None)} + options = () + priority = -1 + + def process_module(self, node): + """Process the module.""" + if os.path.basename(os.path.splitext(node.file)[0]) == '__init__': + return + max_lineno = 1 + with contextlib.closing(node.stream()) as stream: + for (lineno, line) in enumerate(stream): + if lineno == 1 and line.startswith(b'#!'): + max_lineno += 1 + continue + elif line.startswith(b'# vim:'): + if lineno > max_lineno: + self.add_message('modeline-position', line=lineno) + if (line.rstrip() != b'# vim: ft=python ' + b'fileencoding=utf-8 sts=4 sw=4 et:'): + self.add_message('invalid-modeline', line=lineno) + break + else: + self.add_message('modeline-missing', line=1) + + +def register(linter): + """Register the checker.""" + linter.register_checker(ModelineChecker(linter)) diff --git a/.config/qutebrowser/scripts/dev/pylint_checkers/qute_pylint/openencoding.py b/.config/qutebrowser/scripts/dev/pylint_checkers/qute_pylint/openencoding.py new file mode 100644 index 0000000..f577011 --- /dev/null +++ b/.config/qutebrowser/scripts/dev/pylint_checkers/qute_pylint/openencoding.py @@ -0,0 +1,83 @@ +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org> + +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + +"""Make sure open() has an encoding set.""" + +import astroid +from pylint import interfaces, checkers +from pylint.checkers import utils + + +class OpenEncodingChecker(checkers.BaseChecker): + + """Checker to check open() has an encoding set.""" + + __implements__ = interfaces.IAstroidChecker + name = 'open-encoding' + + msgs = { + 'W9400': ('open() called without encoding', 'open-without-encoding', + None), + } + + @utils.check_messages('open-without-encoding') + def visit_call(self, node): + """Visit a Call node.""" + if hasattr(node, 'func'): + infer = utils.safe_infer(node.func) + if infer and infer.root().name == '_io': + if getattr(node.func, 'name', None) in ['open', 'file']: + self._check_open_encoding(node) + + def _check_open_encoding(self, node): + """Check that an open() call always has an encoding set.""" + try: + mode_arg = utils.get_argument_from_call(node, position=1, + keyword='mode') + except utils.NoSuchArgumentError: + mode_arg = None + _encoding = None + try: + _encoding = utils.get_argument_from_call(node, position=2) + except utils.NoSuchArgumentError: + try: + _encoding = utils.get_argument_from_call(node, + keyword='encoding') + except utils.NoSuchArgumentError: + pass + if _encoding is None: + if mode_arg is None: + mode = None + else: + mode = utils.safe_infer(mode_arg) + if mode is not None and not isinstance(mode, astroid.Const): + # We can't say what mode is exactly. + return + if mode is None: + self.add_message('open-without-encoding', node=node) + elif 'b' in getattr(mode, 'value', ''): + # Files opened as binary don't need an encoding. + return + else: + self.add_message('open-without-encoding', node=node) + + +def register(linter): + """Register this checker.""" + linter.register_checker(OpenEncodingChecker(linter)) diff --git a/.config/qutebrowser/scripts/dev/pylint_checkers/qute_pylint/settrace.py b/.config/qutebrowser/scripts/dev/pylint_checkers/qute_pylint/settrace.py new file mode 100644 index 0000000..c82d646 --- /dev/null +++ b/.config/qutebrowser/scripts/dev/pylint_checkers/qute_pylint/settrace.py @@ -0,0 +1,49 @@ +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org> +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + +"""Custom astroid checker for set_trace calls.""" + +from pylint.interfaces import IAstroidChecker +from pylint.checkers import BaseChecker, utils + + +class SetTraceChecker(BaseChecker): + + """Custom astroid checker for set_trace calls.""" + + __implements__ = IAstroidChecker + name = 'settrace' + msgs = { + 'E9101': ('set_trace call found', 'set-trace', None), + } + priority = -1 + + @utils.check_messages('set-trace') + def visit_call(self, node): + """Visit a Call node.""" + if hasattr(node, 'func'): + infer = utils.safe_infer(node.func) + if infer: + if getattr(node.func, 'name', None) == 'set_trace': + self.add_message('set-trace', node=node) + + +def register(linter): + """Register this checker.""" + linter.register_checker(SetTraceChecker(linter)) diff --git a/.config/qutebrowser/scripts/dev/pylint_checkers/setup.py b/.config/qutebrowser/scripts/dev/pylint_checkers/setup.py new file mode 100644 index 0000000..7833c7d --- /dev/null +++ b/.config/qutebrowser/scripts/dev/pylint_checkers/setup.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 + +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2016-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org> +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + +"""This is only here so we can install those plugins in tox.ini easily.""" + +from setuptools import setup +setup(name='qute_pylint', packages=['qute_pylint']) diff --git a/.config/qutebrowser/scripts/dev/quit_segfault_test.sh b/.config/qutebrowser/scripts/dev/quit_segfault_test.sh new file mode 100755 index 0000000..389f125 --- /dev/null +++ b/.config/qutebrowser/scripts/dev/quit_segfault_test.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +[[ $PWD == */scripts ]] && cd .. + +echo > crash.log +while :; do + exit=0 + while (( exit == 0 )); do + duration=$(( RANDOM % 10000 )) + python3 -m qutebrowser --debug ":later $duration quit" http://www.heise.de/ + exit=$? + done + echo "$(date) $exit $duration" >> crash.log +done diff --git a/.config/qutebrowser/scripts/dev/recompile_requirements.py b/.config/qutebrowser/scripts/dev/recompile_requirements.py new file mode 100644 index 0000000..6e26145 --- /dev/null +++ b/.config/qutebrowser/scripts/dev/recompile_requirements.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2016-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org> +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + +"""Script to regenerate requirements files in misc/requirements.""" + +import re +import sys +import os.path +import glob +import subprocess +import tempfile + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir, + os.pardir)) + +from scripts import utils + +REPO_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), + '..', '..') # /scripts/dev -> /scripts -> / +REQ_DIR = os.path.join(REPO_DIR, 'misc', 'requirements') + + +def convert_line(line, comments): + """Convert the given requirement line to place into the output.""" + for pattern, repl in comments['replace'].items(): + line = re.sub(pattern, repl, line) + + pkgname = line.split('=')[0] + + if pkgname in comments['ignore']: + line = '# ' + line + + try: + line += ' # ' + comments['comment'][pkgname] + except KeyError: + pass + + try: + line += ' # rq.filter: {}'.format(comments['filter'][pkgname]) + except KeyError: + pass + + return line + + +def read_comments(fobj): + """Find special comments in the config. + + Args: + fobj: A file object for the config. + + Return: + A dict with the parsed comment data. + """ + comments = { + 'filter': {}, + 'comment': {}, + 'ignore': [], + 'replace': {}, + } + for line in fobj: + if line.startswith('#@'): + command, args = line[2:].split(':', maxsplit=1) + command = command.strip() + args = args.strip() + if command == 'filter': + pkg, filt = args.split(' ', maxsplit=1) + comments['filter'][pkg] = filt + elif command == 'comment': + pkg, comment = args.split(' ', maxsplit=1) + comments['comment'][pkg] = comment + elif command == 'ignore': + comments['ignore'] += args.split(', ') + elif command == 'replace': + pattern, replacement = args.split(' ', maxsplit=1) + comments['replace'][pattern] = replacement + return comments + + +def get_all_names(): + """Get all requirement names based on filenames.""" + for filename in glob.glob(os.path.join(REQ_DIR, 'requirements-*.txt-raw')): + basename = os.path.basename(filename) + yield basename[len('requirements-'):-len('.txt-raw')] + + +def main(): + """Re-compile the given (or all) requirement files.""" + names = sys.argv[1:] if len(sys.argv) > 1 else sorted(get_all_names()) + + for name in names: + utils.print_title(name) + filename = os.path.join(REQ_DIR, + 'requirements-{}.txt-raw'.format(name)) + if name == 'qutebrowser': + outfile = os.path.join(REPO_DIR, 'requirements.txt') + else: + outfile = os.path.join(REQ_DIR, 'requirements-{}.txt'.format(name)) + + with tempfile.TemporaryDirectory() as tmpdir: + pip_bin = os.path.join(tmpdir, 'bin', 'pip') + subprocess.run(['virtualenv', tmpdir], check=True) + subprocess.run([pip_bin, 'install', '-r', filename], check=True) + proc = subprocess.run([pip_bin, 'freeze'], check=True, + stdout=subprocess.PIPE) + reqs = proc.stdout.decode('utf-8') + + with open(filename, 'r', encoding='utf-8') as f: + comments = read_comments(f) + + with open(outfile, 'w', encoding='utf-8') as f: + f.write("# This file is automatically generated by " + "scripts/dev/recompile_requirements.py\n\n") + for line in reqs.splitlines(): + f.write(convert_line(line, comments) + '\n') + + +if __name__ == '__main__': + main() diff --git a/.config/qutebrowser/scripts/dev/run_profile.py b/.config/qutebrowser/scripts/dev/run_profile.py new file mode 100755 index 0000000..93e0b61 --- /dev/null +++ b/.config/qutebrowser/scripts/dev/run_profile.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org> +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + +"""Profile qutebrowser.""" + +import sys +import cProfile +import os.path +import os +import tempfile +import subprocess +import shutil +import argparse +import shlex + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir, + os.pardir)) + +import qutebrowser.qutebrowser + + +def parse_args(): + """Parse commandline arguments. + + Return: + A (namespace, remaining_args) tuple from argparse. + """ + parser = argparse.ArgumentParser() + parser.add_argument('--profile-tool', metavar='TOOL', + action='store', choices=['kcachegrind', 'snakeviz', + 'gprof2dot', 'none'], + default='snakeviz', + help="The tool to use to view the profiling data") + parser.add_argument('--profile-file', metavar='FILE', action='store', + help="The filename to use with --profile-tool=none") + return parser.parse_known_args() + + +def main(): + args, remaining = parse_args() + tempdir = tempfile.mkdtemp() + + if args.profile_tool == 'none': + profilefile = os.path.join(os.getcwd(), args.profile_file) + else: + profilefile = os.path.join(tempdir, 'profile') + + sys.argv = [sys.argv[0]] + remaining + + profiler = cProfile.Profile() + profiler.runcall(qutebrowser.qutebrowser.main) + + # If we have an exception after here, we don't want the qutebrowser + # exception hook to take over. + sys.excepthook = sys.__excepthook__ + profiler.dump_stats(profilefile) + + if args.profile_tool == 'none': + pass + elif args.profile_tool == 'gprof2dot': + # yep, shell=True. I know what I'm doing. + subprocess.run( + 'gprof2dot -f pstats {} | dot -Tpng | feh -F -'.format( + shlex.quote(profilefile)), shell=True) + elif args.profile_tool == 'kcachegrind': + callgraphfile = os.path.join(tempdir, 'callgraph') + subprocess.run(['pyprof2calltree', '-k', '-i', profilefile, + '-o', callgraphfile]) + elif args.profile_tool == 'snakeviz': + subprocess.run(['snakeviz', profilefile]) + + shutil.rmtree(tempdir) + + +if __name__ == '__main__': + main() diff --git a/.config/qutebrowser/scripts/dev/run_pylint_on_tests.py b/.config/qutebrowser/scripts/dev/run_pylint_on_tests.py new file mode 100644 index 0000000..7adf45f --- /dev/null +++ b/.config/qutebrowser/scripts/dev/run_pylint_on_tests.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org> + +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + +"""Run pylint on tests. + +This is needed because pylint can't check a folder which isn't a package: +https://bitbucket.org/logilab/pylint/issue/512/ +""" + +import os +import os.path +import sys +import subprocess + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir, + os.pardir)) + +from scripts import utils + + +def main(): + """Main entry point. + + Return: + The pylint exit status. + """ + utils.change_cwd() + files = [] + for dirpath, _dirnames, filenames in os.walk('tests'): + for fn in filenames: + if os.path.splitext(fn)[1] == '.py': + files.append(os.path.join(dirpath, fn)) + + disabled = [ + # pytest fixtures + 'redefined-outer-name', + 'unused-argument', + # things which are okay in tests + 'missing-docstring', + 'protected-access', + 'len-as-condition', + # directories without __init__.py... + 'import-error', + ] + + toxinidir = sys.argv[1] + pythonpath = os.environ.get('PYTHONPATH', '').split(os.pathsep) + [ + toxinidir, + ] + + args = (['--disable={}'.format(','.join(disabled)), + '--ignored-modules=helpers,pytest,PyQt5'] + + sys.argv[2:] + files) + env = os.environ.copy() + env['PYTHONPATH'] = os.pathsep.join(pythonpath) + + ret = subprocess.run(['pylint'] + args, env=env).returncode + return ret + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/.config/qutebrowser/scripts/dev/run_vulture.py b/.config/qutebrowser/scripts/dev/run_vulture.py new file mode 100755 index 0000000..cb19d62 --- /dev/null +++ b/.config/qutebrowser/scripts/dev/run_vulture.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2015-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org> + +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + +"""Run vulture on the source files and filter out false-positives.""" + +import sys +import os +import re +import tempfile +import inspect +import argparse + +import vulture + +import qutebrowser.app # pylint: disable=unused-import +from qutebrowser.commands import cmdutils +from qutebrowser.utils import utils +from qutebrowser.browser.webkit import rfc6266 +# To run the decorators from there +# pylint: disable=unused-import +from qutebrowser.browser.webkit.network import webkitqutescheme +# pylint: enable=unused-import +from qutebrowser.browser import qutescheme +from qutebrowser.config import configtypes + + +def whitelist_generator(): # noqa + """Generator which yields lines to add to a vulture whitelist.""" + # qutebrowser commands + for cmd in cmdutils.cmd_dict.values(): + yield utils.qualname(cmd.handler) + + # pyPEG2 classes + for name, member in inspect.getmembers(rfc6266, inspect.isclass): + for attr in ['grammar', 'regex']: + if hasattr(member, attr): + yield 'qutebrowser.browser.webkit.rfc6266.{}.{}'.format(name, + attr) + + # PyQt properties + yield 'qutebrowser.mainwindow.statusbar.bar.StatusBar.color_flags' + yield 'qutebrowser.mainwindow.statusbar.url.UrlText.urltype' + + # Not used yet, but soon (or when debugging) + yield 'qutebrowser.utils.debug.log_events' + yield 'qutebrowser.utils.debug.log_signals' + yield 'qutebrowser.utils.debug.qflags_key' + yield 'qutebrowser.utils.qtutils.QtOSError.qt_errno' + yield 'scripts.utils.bg_colors' + + # Qt attributes + yield 'PyQt5.QtWebKit.QWebPage.ErrorPageExtensionReturn().baseUrl' + yield 'PyQt5.QtWebKit.QWebPage.ErrorPageExtensionReturn().content' + yield 'PyQt5.QtWebKit.QWebPage.ErrorPageExtensionReturn().encoding' + yield 'PyQt5.QtWebKit.QWebPage.ErrorPageExtensionReturn().fileNames' + yield 'PyQt5.QtWidgets.QStyleOptionViewItem.backgroundColor' + + ## qute://... handlers + for name in qutescheme._HANDLERS: # pylint: disable=protected-access + name = name.replace('-', '_') + yield 'qutebrowser.browser.qutescheme.qute_' + name + + # Other false-positives + yield 'qutebrowser.completion.models.listcategory.ListCategory().lessThan' + yield 'qutebrowser.utils.jinja.Loader.get_source' + yield 'qutebrowser.utils.log.QtWarningFilter.filter' + yield 'qutebrowser.browser.pdfjs.is_available' + yield 'qutebrowser.misc.guiprocess.spawn_output' + yield 'QEvent.posted' + yield 'log_stack' # from message.py + yield 'propagate' # logging.getLogger('...).propagate = False + # vulture doesn't notice the hasattr() and thus thinks netrc_used is unused + # in NetworkManager.on_authentication_required + yield 'PyQt5.QtNetwork.QNetworkReply.netrc_used' + yield 'qutebrowser.browser.downloads.last_used_directory' + yield 'PaintContext.clip' # from completiondelegate.py + yield 'logging.LogRecord.log_color' # from logging.py + yield 'scripts.utils.use_color' # from asciidoc2html.py + for attr in ['pyeval_output', 'log_clipboard', 'fake_clipboard']: + yield 'qutebrowser.misc.utilcmds.' + attr + + for attr in ['fileno', 'truncate', 'closed', 'readable']: + yield 'qutebrowser.utils.qtutils.PyQIODevice.' + attr + + for attr in ['msgs', 'priority', 'visit_attribute']: + yield 'scripts.dev.pylint_checkers.config.' + attr + for attr in ['visit_call', 'process_module']: + yield 'scripts.dev.pylint_checkers.modeline.' + attr + + for name, _member in inspect.getmembers(configtypes, inspect.isclass): + yield 'qutebrowser.config.configtypes.' + name + yield 'qutebrowser.config.configexc.ConfigErrorDesc.traceback' + yield 'qutebrowser.config.configfiles.ConfigAPI.load_autoconfig' + yield 'types.ModuleType.c' # configfiles:read_config_py + for name in ['configdir', 'datadir']: + yield 'qutebrowser.config.configfiles.ConfigAPI.' + name + + yield 'include_aliases' + + for attr in ['_get_default_metavar_for_optional', + '_get_default_metavar_for_positional', '_metavar_formatter']: + yield 'scripts.dev.src2asciidoc.UsageFormatter.' + attr + + # attrs + yield 'qutebrowser.browser.webkit.network.networkmanager.ProxyId.hostname' + yield 'qutebrowser.command.command.ArgInfo._validate_exclusive' + yield 'scripts.get_coredumpctl_traces.Line.uid' + yield 'scripts.get_coredumpctl_traces.Line.gid' + yield 'scripts.importer.import_moz_places.places.row_factory' + + +def filter_func(item): + """Check if a missing function should be filtered or not. + + Return: + True if the missing function should be filtered/ignored, False + otherwise. + """ + return bool(re.fullmatch(r'[a-z]+[A-Z][a-zA-Z]+', item.name)) + + +def report(items): + """Generate a report based on the given vulture.Item's. + + Based on vulture.Vulture.report, but we can't use that as we can't set the + properties which get used for the items. + """ + output = [] + for item in sorted(items, + key=lambda e: (e.filename.lower(), e.first_lineno)): + output.append(item.get_report()) + return output + + +def run(files): + """Run vulture over the given files.""" + with tempfile.NamedTemporaryFile(mode='w', delete=False) as whitelist_file: + for line in whitelist_generator(): + whitelist_file.write(line + '\n') + + whitelist_file.close() + + vult = vulture.Vulture(verbose=False) + vult.scavenge(files + [whitelist_file.name]) + + os.remove(whitelist_file.name) + + filters = { + 'unused_funcs': filter_func, + 'unused_props': lambda item: False, + 'unused_vars': lambda item: False, + 'unused_attrs': lambda item: False, + } + + items = [] + + for attr, func in filters.items(): + sub_items = getattr(vult, attr) + for item in sub_items: + filtered = func(item) + if not filtered: + items.append(item) + + return report(items) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('files', nargs='*', default=['qutebrowser', 'scripts', + 'setup.py']) + args = parser.parse_args() + out = run(args.files) + for line in out: + print(line) + sys.exit(bool(out)) + + +if __name__ == '__main__': + main() diff --git a/.config/qutebrowser/scripts/dev/segfault_test.py b/.config/qutebrowser/scripts/dev/segfault_test.py new file mode 100755 index 0000000..aaf495f --- /dev/null +++ b/.config/qutebrowser/scripts/dev/segfault_test.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org> +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + +"""Tester for Qt segfaults with different harfbuzz engines.""" + +import os +import os.path +import signal +import sys +import subprocess + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir, + os.pardir)) + +from scripts import utils + + +SCRIPT = """ +import sys + +from PyQt5.QtCore import QUrl +from PyQt5.QtWidgets import QApplication +from PyQt5.QtWebKitWidgets import QWebView + +def on_load_finished(ok): + if ok: + app.exit(0) + else: + app.exit(1) + +app = QApplication([]) +wv = QWebView() +wv.loadFinished.connect(on_load_finished) +wv.load(QUrl(sys.argv[1])) +#wv.show() +app.exec_() +""" + + +def print_ret(ret): + """Print information about an exit status.""" + if ret == 0: + utils.print_col("success", 'green') + elif ret == -signal.SIGSEGV: + utils.print_col("segfault", 'red') + else: + utils.print_col("error {}".format(ret), 'yellow') + print() + + +def main(): + retvals = [] + if len(sys.argv) < 2: + # pages which previously caused problems + pages = [ + # ANGLE, https://bugreports.qt.io/browse/QTBUG-39723 + ('http://www.binpress.com/', False), + ('http://david.li/flow/', False), + ('https://imzdl.com/', False), + # not reproducible + # https://bugreports.qt.io/browse/QTBUG-39847 + ('http://www.20min.ch/', True), + # HarfBuzz, https://bugreports.qt.io/browse/QTBUG-39278 + ('http://www.the-compiler.org/', True), + ('http://phoronix.com', True), + ('http://twitter.com', True), + # HarfBuzz #2, https://bugreports.qt.io/browse/QTBUG-36099 + ('http://lenta.ru/', True), + # Unknown, https://bugreports.qt.io/browse/QTBUG-41360 + ('http://salt.readthedocs.org/en/latest/topics/pillar/', True), + ] + else: + pages = [(e, True) for e in sys.argv[1:]] + for page, test_harfbuzz in pages: + utils.print_bold("==== {} ====".format(page)) + if test_harfbuzz: + print("With system harfbuzz:") + ret = subprocess.run([sys.executable, '-c', SCRIPT, page]).returncode + print_ret(ret) + retvals.append(ret) + if test_harfbuzz: + print("With QT_HARFBUZZ=old:") + env = dict(os.environ) + env['QT_HARFBUZZ'] = 'old' + ret = subprocess.run([sys.executable, '-c', SCRIPT, page], + env=env).returncode + print_ret(ret) + retvals.append(ret) + print("With QT_HARFBUZZ=new:") + env = dict(os.environ) + env['QT_HARFBUZZ'] = 'new' + ret = subprocess.run([sys.executable, '-c', SCRIPT, page], + env=env).returncode + print_ret(ret) + retvals.append(ret) + if all(r == 0 for r in retvals): + sys.exit(0) + else: + sys.exit(1) + + +if __name__ == '__main__': + main() diff --git a/.config/qutebrowser/scripts/dev/src2asciidoc.py b/.config/qutebrowser/scripts/dev/src2asciidoc.py new file mode 100755 index 0000000..cc00c37 --- /dev/null +++ b/.config/qutebrowser/scripts/dev/src2asciidoc.py @@ -0,0 +1,561 @@ +#!/usr/bin/env python3 +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org> + +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + +"""Generate asciidoc source for qutebrowser based on docstrings.""" + +import os +import os.path +import sys +import shutil +import inspect +import subprocess +import tempfile +import argparse + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir, + os.pardir)) + +# We import qutebrowser.app so all @cmdutils-register decorators are run. +import qutebrowser.app +from qutebrowser import qutebrowser, commands +from qutebrowser.commands import cmdutils, argparser +from qutebrowser.config import configdata, configtypes +from qutebrowser.utils import docutils, usertypes +from scripts import asciidoc2html, utils + +FILE_HEADER = """ +// DO NOT EDIT THIS FILE DIRECTLY! +// It is autogenerated by running: +// $ python3 scripts/dev/src2asciidoc.py +// vim: readonly: + +""".lstrip() + + +class UsageFormatter(argparse.HelpFormatter): + + """Patched HelpFormatter to include some asciidoc markup in the usage. + + This does some horrible things, but the alternative would be to reimplement + argparse.HelpFormatter while copying 99% of the code :-/ + """ + + def _format_usage(self, usage, actions, groups, _prefix): + """Override _format_usage to not add the 'usage:' prefix.""" + return super()._format_usage(usage, actions, groups, '') + + def _get_default_metavar_for_optional(self, action): + """Do name transforming when getting metavar.""" + return argparser.arg_name(action.dest.upper()) + + def _get_default_metavar_for_positional(self, action): + """Do name transforming when getting metavar.""" + return argparser.arg_name(action.dest) + + def _metavar_formatter(self, action, default_metavar): + """Override _metavar_formatter to add asciidoc markup to metavars. + + Most code here is copied from Python 3.4's argparse.py. + """ + if action.metavar is not None: + result = "'{}'".format(action.metavar) + elif action.choices is not None: + choice_strs = [str(choice) for choice in action.choices] + result = ('{' + ','.join('*{}*'.format(e) for e in choice_strs) + + '}') + else: + result = "'{}'".format(default_metavar) + + def fmt(tuple_size): + """Format the result according to the tuple size.""" + if isinstance(result, tuple): + return result + else: + return (result, ) * tuple_size + return fmt + + def _format_actions_usage(self, actions, groups): + """Override _format_actions_usage to add asciidoc markup to flags. + + Because argparse.py's _format_actions_usage is very complex, we first + monkey-patch the option strings to include the asciidoc markup, then + run the original method, then undo the patching. + """ + old_option_strings = {} + for action in actions: + old_option_strings[action] = action.option_strings[:] + action.option_strings = ['*{}*'.format(s) + for s in action.option_strings] + ret = super()._format_actions_usage(actions, groups) + for action in actions: + action.option_strings = old_option_strings[action] + return ret + + +def _open_file(name, mode='w'): + """Open a file with a preset newline/encoding mode.""" + return open(name, mode, newline='\n', encoding='utf-8') + + +def _get_cmd_syntax(_name, cmd): + """Get the command syntax for a command. + + We monkey-patch the parser's formatter_class here to use our UsageFormatter + which adds some asciidoc markup. + """ + old_fmt_class = cmd.parser.formatter_class + cmd.parser.formatter_class = UsageFormatter + usage = cmd.parser.format_usage().rstrip() + cmd.parser.formatter_class = old_fmt_class + return usage + + +def _get_command_quickref(cmds): + """Generate the command quick reference.""" + out = [] + out.append('[options="header",width="75%",cols="25%,75%"]') + out.append('|==============') + out.append('|Command|Description') + for name, cmd in cmds: + desc = inspect.getdoc(cmd.handler).splitlines()[0] + out.append('|<<{},{}>>|{}'.format(name, name, desc)) + out.append('|==============') + return '\n'.join(out) + + +def _get_setting_quickref(): + """Generate the settings quick reference.""" + out = [] + out.append('') + out.append('[options="header",width="75%",cols="25%,75%"]') + out.append('|==============') + out.append('|Setting|Description') + for opt in sorted(configdata.DATA.values()): + desc = opt.description.splitlines()[0] + out.append('|<<{},{}>>|{}'.format(opt.name, opt.name, desc)) + out.append('|==============') + return '\n'.join(out) + + +def _get_configtypes(): + """Get configtypes classes to document.""" + predicate = lambda e: ( + inspect.isclass(e) and + # pylint: disable=protected-access + e not in [configtypes.BaseType, configtypes.MappingType, + configtypes._Numeric] and + # pylint: enable=protected-access + issubclass(e, configtypes.BaseType)) + yield from inspect.getmembers(configtypes, predicate) + + +def _get_setting_types_quickref(): + """Generate the setting types quick reference.""" + out = [] + out.append('[[types]]') + out.append('[options="header",width="75%",cols="25%,75%"]') + out.append('|==============') + out.append('|Type|Description') + + for name, typ in _get_configtypes(): + parser = docutils.DocstringParser(typ) + desc = parser.short_desc + if parser.long_desc: + desc += '\n\n' + parser.long_desc + out.append('|{}|{}'.format(name, desc)) + + out.append('|==============') + return '\n'.join(out) + + +def _get_command_doc(name, cmd): + """Generate the documentation for a command.""" + output = ['[[{}]]'.format(name)] + output += ['=== {}'.format(name)] + syntax = _get_cmd_syntax(name, cmd) + if syntax != name: + output.append('Syntax: +:{}+'.format(syntax)) + output.append("") + parser = docutils.DocstringParser(cmd.handler) + output.append(parser.short_desc) + if parser.long_desc: + output.append("") + output.append(parser.long_desc) + + output += list(_get_command_doc_args(cmd, parser)) + output += list(_get_command_doc_count(cmd, parser)) + output += list(_get_command_doc_notes(cmd)) + + output.append("") + output.append("") + return '\n'.join(output) + + +def _get_command_doc_args(cmd, parser): + """Get docs for the arguments of a command. + + Args: + cmd: The Command to get the docs for. + parser: The DocstringParser to use. + + Yield: + Strings which should be added to the docs. + """ + if cmd.pos_args: + yield "" + yield "==== positional arguments" + for arg, name in cmd.pos_args: + try: + yield "* +'{}'+: {}".format(name, parser.arg_descs[arg]) + except KeyError as e: + raise KeyError("No description for arg {} of command " + "'{}'!".format(e, cmd.name)) from e + + if cmd.opt_args: + yield "" + yield "==== optional arguments" + for arg, (long_flag, short_flag) in cmd.opt_args.items(): + try: + yield '* +*{}*+, +*{}*+: {}'.format(short_flag, long_flag, + parser.arg_descs[arg]) + except KeyError as e: + raise KeyError("No description for arg {} of command " + "'{}'!".format(e, cmd.name)) from e + + +def _get_command_doc_count(cmd, parser): + """Get docs for the count of a command. + + Args: + cmd: The Command to get the docs for. + parser: The DocstringParser to use. + + Yield: + Strings which should be added to the docs. + """ + for param in inspect.signature(cmd.handler).parameters.values(): + if cmd.get_arg_info(param).count: + yield "" + yield "==== count" + try: + yield parser.arg_descs[param.name] + except KeyError as e: + raise KeyError("No description for count arg {!r} of command " + "{!r}!".format(param.name, cmd.name)) from e + + +def _get_command_doc_notes(cmd): + """Get docs for the notes of a command. + + Args: + cmd: The Command to get the docs for. + parser: The DocstringParser to use. + + Yield: + Strings which should be added to the docs. + """ + if (cmd.maxsplit is not None or cmd.no_cmd_split or + cmd.no_replace_variables and cmd.name != "spawn"): + yield "" + yield "==== note" + if cmd.maxsplit is not None: + yield ("* This command does not split arguments after the last " + "argument and handles quotes literally.") + if cmd.no_cmd_split: + yield ("* With this command, +;;+ is interpreted literally " + "instead of splitting off a second command.") + if cmd.no_replace_variables and cmd.name != "spawn": + yield r"* This command does not replace variables like +\{url\}+." + + +def _get_action_metavar(action, nargs=1): + """Get the metavar to display for an argparse action. + + Args: + action: The argparse action to get the metavar for. + nargs: The nargs setting for the related argument. + """ + if action.metavar is not None: + if isinstance(action.metavar, str): + elems = [action.metavar] * nargs + else: + elems = action.metavar + return ' '.join("'{}'".format(e) for e in elems) + elif action.choices is not None: + choices = ','.join(str(e) for e in action.choices) + return "'{{{}}}'".format(choices) + else: + return "'{}'".format(action.dest.upper()) + + +def _format_action_args(action): + """Get an argument string based on an argparse action.""" + if action.nargs is None: + return _get_action_metavar(action) + elif action.nargs == '?': + return '[{}]'.format(_get_action_metavar(action)) + elif action.nargs == '*': + return '[{mv} [{mv} ...]]'.format(mv=_get_action_metavar(action)) + elif action.nargs == '+': + return '{mv} [{mv} ...]'.format(mv=_get_action_metavar(action)) + elif action.nargs == '...': + return '...' + else: + return _get_action_metavar(action, nargs=action.nargs) + + +def _format_action(action): + """Get an invocation string/help from an argparse action.""" + if action.help == argparse.SUPPRESS: + return None + if not action.option_strings: + invocation = '*{}*::'.format(_get_action_metavar(action)) + else: + parts = [] + if action.nargs == 0: + # Doesn't take a value, so the syntax is -s, --long + parts += ['*{}*'.format(s) for s in action.option_strings] + else: + # Takes a value, so the syntax is -s ARGS or --long ARGS. + args_string = _format_action_args(action) + for opt in action.option_strings: + parts.append('*{}* {}'.format(opt, args_string)) + invocation = ', '.join(parts) + '::' + return '{}\n {}\n'.format(invocation, action.help) + + +def generate_commands(filename): + """Generate the complete commands section.""" + with _open_file(filename) as f: + f.write(FILE_HEADER) + f.write("= Commands\n\n") + f.write(commands.__doc__) + normal_cmds = [] + other_cmds = [] + debug_cmds = [] + for name, cmd in cmdutils.cmd_dict.items(): + if cmd.deprecated: + continue + if usertypes.KeyMode.normal not in cmd.modes: + other_cmds.append((name, cmd)) + elif cmd.debug: + debug_cmds.append((name, cmd)) + else: + normal_cmds.append((name, cmd)) + normal_cmds.sort() + other_cmds.sort() + debug_cmds.sort() + f.write("\n") + f.write("== Normal commands\n") + f.write(".Quick reference\n") + f.write(_get_command_quickref(normal_cmds) + '\n') + for name, cmd in normal_cmds: + f.write(_get_command_doc(name, cmd)) + f.write("\n") + f.write("== Commands not usable in normal mode\n") + f.write(".Quick reference\n") + f.write(_get_command_quickref(other_cmds) + '\n') + for name, cmd in other_cmds: + f.write(_get_command_doc(name, cmd)) + f.write("\n") + f.write("== Debugging commands\n") + f.write("These commands are mainly intended for debugging. They are " + "hidden if qutebrowser was started without the " + "`--debug`-flag.\n") + f.write("\n") + f.write(".Quick reference\n") + f.write(_get_command_quickref(debug_cmds) + '\n') + for name, cmd in debug_cmds: + f.write(_get_command_doc(name, cmd)) + + +def _generate_setting_backend_info(f, opt): + """Generate backend information for the given option.""" + all_backends = [usertypes.Backend.QtWebKit, usertypes.Backend.QtWebEngine] + if opt.raw_backends is not None: + for name, conditional in sorted(opt.raw_backends.items()): + if conditional is True: + pass + elif conditional is False: + f.write("\nOn {}, this setting is unavailable.\n".format(name)) + else: + f.write("\nOn {}, this setting requires {} or newer.\n" + .format(name, conditional)) + elif opt.backends == all_backends: + pass + elif opt.backends == [usertypes.Backend.QtWebKit]: + f.write("\nThis setting is only available with the QtWebKit " + "backend.\n") + elif opt.backends == [usertypes.Backend.QtWebEngine]: + f.write("\nThis setting is only available with the QtWebEngine " + "backend.\n") + else: + raise ValueError("Invalid value {!r} for opt.backends" + .format(opt.backends)) + + +def _generate_setting_option(f, opt): + """Generate documentation for a single section.""" + f.write("\n") + f.write('[[{}]]'.format(opt.name) + "\n") + f.write("=== {}".format(opt.name) + "\n") + f.write(opt.description + "\n") + if opt.restart: + f.write("This setting requires a restart.\n") + if opt.supports_pattern: + f.write("\nThis setting supports URL patterns.\n") + if opt.no_autoconfig: + f.write("\nThis setting can only be set in config.py.\n") + f.write("\n") + typ = opt.typ.get_name().replace(',', ',') + f.write('Type: <<types,{typ}>>\n'.format(typ=typ)) + f.write("\n") + + valid_values = opt.typ.get_valid_values() + if valid_values is not None and valid_values.generate_docs: + f.write("Valid values:\n") + f.write("\n") + for val in valid_values: + try: + desc = valid_values.descriptions[val] + f.write(" * +{}+: {}".format(val, desc) + "\n") + except KeyError: + f.write(" * +{}+".format(val) + "\n") + f.write("\n") + + f.write("Default: {}\n".format(opt.typ.to_doc(opt.default))) + _generate_setting_backend_info(f, opt) + + +def generate_settings(filename): + """Generate the complete settings section.""" + configdata.init() + with _open_file(filename) as f: + f.write(FILE_HEADER) + f.write("= Setting reference\n\n") + f.write("== All settings\n") + f.write(_get_setting_quickref() + "\n") + for opt in sorted(configdata.DATA.values()): + _generate_setting_option(f, opt) + f.write("\n== Setting types\n") + f.write(_get_setting_types_quickref() + "\n") + + +def _format_block(filename, what, data): + """Format a block in a file. + + The block is delimited by markers like these: + // QUTE_*_START + ... + // QUTE_*_END + + The * part is the part which should be given as 'what'. + + Args: + filename: The file to change. + what: What to change (authors, options, etc.) + data; A list of strings which is the new data. + """ + what = what.upper() + oshandle, tmpname = tempfile.mkstemp() + try: + with _open_file(filename, mode='r') as infile, \ + _open_file(oshandle, mode='w') as temp: + found_start = False + found_end = False + for line in infile: + if line.strip() == '// QUTE_{}_START'.format(what): + temp.write(line) + temp.write(''.join(data)) + found_start = True + elif line.strip() == '// QUTE_{}_END'.format(what.upper()): + temp.write(line) + found_end = True + elif (not found_start) or found_end: + temp.write(line) + if not found_start: + raise Exception("Marker '// QUTE_{}_START' not found in " + "'{}'!".format(what, filename)) + elif not found_end: + raise Exception("Marker '// QUTE_{}_END' not found in " + "'{}'!".format(what, filename)) + except: + os.remove(tmpname) + raise + else: + os.remove(filename) + shutil.move(tmpname, filename) + + +def regenerate_manpage(filename): + """Update manpage OPTIONS using an argparse parser.""" + parser = qutebrowser.get_argparser() + groups = [] + # positionals, optionals and user-defined groups + # pylint: disable=protected-access + for group in parser._action_groups: + groupdata = [] + groupdata.append('=== {}'.format(group.title)) + if group.description is not None: + groupdata.append(group.description) + for action in group._group_actions: + action_data = _format_action(action) + if action_data is not None: + groupdata.append(action_data) + groups.append('\n'.join(groupdata)) + # pylint: enable=protected-access + options = '\n'.join(groups) + # epilog + if parser.epilog is not None: + options += parser.epilog + _format_block(filename, 'options', options) + + +def regenerate_cheatsheet(): + """Generate cheatsheet PNGs based on the SVG.""" + files = [ + ('doc/img/cheatsheet-small.png', 300, 185), + ('doc/img/cheatsheet-big.png', 3342, 2060), + ] + + for filename, x, y in files: + subprocess.run(['inkscape', '-e', filename, '-b', 'white', + '-w', str(x), '-h', str(y), + 'misc/cheatsheet.svg'], check=True) + + +def main(): + """Regenerate all documentation.""" + utils.change_cwd() + print("Generating manpage...") + regenerate_manpage('doc/qutebrowser.1.asciidoc') + print("Generating settings help...") + generate_settings('doc/help/settings.asciidoc') + print("Generating command help...") + generate_commands('doc/help/commands.asciidoc') + if '--cheatsheet' in sys.argv: + print("Regenerating cheatsheet .pngs") + regenerate_cheatsheet() + if '--html' in sys.argv: + asciidoc2html.main() + + +if __name__ == '__main__': + main() diff --git a/.config/qutebrowser/scripts/dev/standardpaths_tester.py b/.config/qutebrowser/scripts/dev/standardpaths_tester.py new file mode 100644 index 0000000..27b8382 --- /dev/null +++ b/.config/qutebrowser/scripts/dev/standardpaths_tester.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2017-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org> +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + +"""Show various QStandardPath paths.""" + +import os +import sys + +from PyQt5.QtCore import (QT_VERSION_STR, PYQT_VERSION_STR, qVersion, + QStandardPaths, QCoreApplication) + + +def print_header(): + """Show system information.""" + print("Python {}".format(sys.version)) + print("os.name: {}".format(os.name)) + print("sys.platform: {}".format(sys.platform)) + print() + + print("Qt {}, compiled {}".format(qVersion(), QT_VERSION_STR)) + print("PyQt {}".format(PYQT_VERSION_STR)) + print() + + +def print_paths(): + """Print all QStandardPaths.StandardLocation members.""" + for name, obj in vars(QStandardPaths).items(): + if isinstance(obj, QStandardPaths.StandardLocation): + location = QStandardPaths.writableLocation(obj) + print("{:25} {}".format(name, location)) + + +def main(): + print_header() + + print("No QApplication") + print("===============") + print() + print_paths() + + app = QCoreApplication(sys.argv) + app.setApplicationName("qapp_name") + + print() + print("With QApplication") + print("=================") + print() + print_paths() + + +if __name__ == '__main__': + main() diff --git a/.config/qutebrowser/scripts/dev/strip_whitespace.sh b/.config/qutebrowser/scripts/dev/strip_whitespace.sh new file mode 100644 index 0000000..ee14278 --- /dev/null +++ b/.config/qutebrowser/scripts/dev/strip_whitespace.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +# Strip trailing whitespace from files in this repo + +find qutebrowser scripts tests \ + -type f \( \ + -name '*.py' -o \ + -name '*.feature' -o \ + -name '*.sh' \ + \) -exec sed -i 's/ \+$//' {} + diff --git a/.config/qutebrowser/scripts/dev/ua_fetch.py b/.config/qutebrowser/scripts/dev/ua_fetch.py new file mode 100755 index 0000000..75ce4c2 --- /dev/null +++ b/.config/qutebrowser/scripts/dev/ua_fetch.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2015-2018 lamarpavel +# Copyright 2015-2018 Alexey Nabrodov (Averrin) +# Copyright 2015-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org> +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + + +"""Fetch list of popular user-agents. + +The script is based on a gist posted by github.com/averrin, the output of this +script is formatted to be pasted into configdata.yml +""" + +import requests +from lxml import html # pylint: disable=import-error + + +def fetch(): + """Fetch list of popular user-agents. + + Return: + List of relevant strings. + """ + url = 'https://techblog.willshouse.com/2012/01/03/most-common-user-agents/' + page = requests.get(url) + page = html.fromstring(page.text) + path = '//*[@id="post-2229"]/div[2]/table/tbody' + return page.xpath(path)[0] + + +def filter_list(complete_list, browsers): + """Filter the received list based on a look up table. + + The LUT should be a dictionary of the format {browser: versions}, where + 'browser' is the name of the browser (eg. "Firefox") as string and + 'versions' is a set of different versions of this browser that should be + included when found (eg. {"Linux", "MacOSX"}). This function returns a + dictionary with the same keys as the LUT, but storing lists of tuples + (user_agent, browser_description) as values. + """ + # pylint: disable=too-many-nested-blocks + table = {} + for entry in complete_list: + # Tuple of (user_agent, browser_description) + candidate = (entry[1].text_content(), entry[2].text_content()) + for name in browsers: + found = False + if name.lower() in candidate[1].lower(): + for version in browsers[name]: + if version.lower() in candidate[1].lower(): + if table.get(name) is None: + table[name] = [] + table[name].append(candidate) + browsers[name].remove(version) + found = True + break + if found: + break + return table + + +def add_diversity(table): + """Insert a few additional entries for diversity into the dict. + + (as returned by filter_list()) + """ + table["Obscure"] = [ + ('Mozilla/5.0 (compatible; Googlebot/2.1; ' + '+http://www.google.com/bot.html', + "Google Bot"), + ('Wget/1.16.1 (linux-gnu)', + "wget 1.16.1"), + ('curl/7.40.0', + "curl 7.40.0"), + ('Mozilla/5.0 (Linux; U; Android 7.1.2) AppleWebKit/534.30 ' + '(KHTML, like Gecko) Version/4.0 Mobile Safari/534.30', + "Mobile Generic Android"), + ('Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like ' + 'Gecko', + "IE 11.0 for Desktop Win7 64-bit"), + ] + return table + + +def main(): + """Generate user agent code.""" + fetched = fetch() + lut = { + "Firefox": {"Win", "MacOSX", "Linux", "Android"}, + "Chrome": {"Win", "MacOSX", "Linux"}, + "Safari": {"MacOSX", "iOS"} + } + filtered = filter_list(fetched, lut) + filtered = add_diversity(filtered) + + tab = " " + for browser in ["Firefox", "Safari", "Chrome", "Obscure"]: + for it in filtered[browser]: + print('{}- - "{}"'.format(3 * tab, it[0])) + desc = it[1].replace('\xa0', ' ').replace(' ', ' ') + print("{}- {}".format(4 * tab, desc)) + print("") + + +if __name__ == '__main__': + main() diff --git a/.config/qutebrowser/scripts/dev/update_3rdparty.py b/.config/qutebrowser/scripts/dev/update_3rdparty.py new file mode 100755 index 0000000..c40015d --- /dev/null +++ b/.config/qutebrowser/scripts/dev/update_3rdparty.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2015 Daniel Schadt +# Copyright 2016-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org> +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + +"""Update all third-party-modules.""" + +import argparse +import urllib.request +import urllib.error +import shutil +import json +import os +import sys + +sys.path.insert( + 0, os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)) +from scripts import dictcli +from qutebrowser.config import configdata + + +def get_latest_pdfjs_url(): + """Get the URL of the latest pdf.js prebuilt package. + + Returns a (version, url)-tuple. + """ + github_api = 'https://api.github.com' + endpoint = 'repos/mozilla/pdf.js/releases/latest' + request_url = '{}/{}'.format(github_api, endpoint) + with urllib.request.urlopen(request_url) as fp: + data = json.loads(fp.read().decode('utf-8')) + + download_url = data['assets'][0]['browser_download_url'] + version_name = data['name'] + return (version_name, download_url) + + +def update_pdfjs(target_version=None): + """Download and extract the latest pdf.js version. + + If target_version is not None, download the given version instead. + + Args: + target_version: None or version string ('x.y.z') + """ + if target_version is None: + version, url = get_latest_pdfjs_url() + else: + # We need target_version as x.y.z, without the 'v' prefix, though the + # user might give it on the command line + if target_version.startswith('v'): + target_version = target_version[1:] + # version should have the prefix to be consistent with the return value + # of get_latest_pdfjs_url() + version = 'v' + target_version + url = ('https://github.com/mozilla/pdf.js/releases/download/' + 'v{0}/pdfjs-{0}-dist.zip').format(target_version) + + os.chdir(os.path.join(os.path.dirname(os.path.abspath(__file__)), + '..', '..')) + target_path = os.path.join('qutebrowser', '3rdparty', 'pdfjs') + print("=> Downloading pdf.js {}".format(version)) + try: + (archive_path, _headers) = urllib.request.urlretrieve(url) + except urllib.error.HTTPError as error: + print("Could not retrieve pdfjs {}: {}".format(version, error)) + return + if os.path.isdir(target_path): + print("Removing old version in {}".format(target_path)) + shutil.rmtree(target_path) + os.makedirs(target_path) + print("Extracting new version") + with open(archive_path, 'rb') as archive: + shutil.unpack_archive(archive, target_path, 'zip') + urllib.request.urlcleanup() + + +def update_dmg_makefile(): + """Update fancy-dmg Makefile. + + See https://el-tramo.be/blog/fancy-dmg/ + """ + print("Updating fancy-dmg Makefile...") + url = 'https://raw.githubusercontent.com/remko/fancy-dmg/master/Makefile' + target_path = os.path.join('scripts', 'dev', 'Makefile-dmg') + urllib.request.urlretrieve(url, target_path) + urllib.request.urlcleanup() + + +def update_ace(): + """Update ACE. + + See https://ace.c9.io/ and https://github.com/ajaxorg/ace-builds/ + """ + print("Updating ACE...") + url = 'https://raw.githubusercontent.com/ajaxorg/ace-builds/master/src/ace.js' + target_path = os.path.join('tests', 'end2end', 'data', 'hints', 'ace', + 'ace.js') + urllib.request.urlretrieve(url, target_path) + urllib.request.urlcleanup() + + +def test_dicts(): + """Test available dictionaries.""" + configdata.init() + for lang in dictcli.available_languages(): + print('Testing dictionary {}... '.format(lang.code), end='') + lang_url = urllib.parse.urljoin(dictcli.API_URL, lang.remote_path) + request = urllib.request.Request(lang_url, method='HEAD') + response = urllib.request.urlopen(request) + if response.status == 200: + print('OK') + else: + print('ERROR: {}'.format(response.status)) + + +def run(ace=False, pdfjs=True, fancy_dmg=False, pdfjs_version=None, + dicts=False): + """Update components based on the given arguments.""" + if pdfjs: + update_pdfjs(pdfjs_version) + if ace: + update_ace() + if fancy_dmg: + update_dmg_makefile() + if dicts: + test_dicts() + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + '--pdfjs', '-p', + help='Specify pdfjs version. If not given, ' + 'the latest version is used.', + required=False, metavar='VERSION') + parser.add_argument('--fancy-dmg', help="Update fancy-dmg Makefile", + action='store_true') + parser.add_argument( + '--dicts', '-d', + help='Test whether all available dictionaries ' + 'can be reached at the remote repository.', + required=False, action='store_true') + args = parser.parse_args() + run(ace=True, pdfjs=True, fancy_dmg=args.fancy_dmg, + pdfjs_version=args.pdfjs, dicts=args.dicts) + + +if __name__ == '__main__': + main() diff --git a/.config/qutebrowser/scripts/dictcli.py b/.config/qutebrowser/scripts/dictcli.py new file mode 100755 index 0000000..4017159 --- /dev/null +++ b/.config/qutebrowser/scripts/dictcli.py @@ -0,0 +1,283 @@ +#!/usr/bin/env python3 +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2017-2018 Michal Siedlaczek <michal.siedlaczek@gmail.com> + +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + +"""A script installing Hunspell dictionaries. + +Use: python -m scripts.dictcli [-h] {list,update,remove-old,install} ... +""" + +import argparse +import base64 +import json +import os +import sys +import re +import urllib.request + +import attr + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir)) +from qutebrowser.browser.webengine import spell +from qutebrowser.config import configdata +from qutebrowser.utils import standarddir + + +API_URL = 'https://chromium.googlesource.com/chromium/deps/hunspell_dictionaries.git/+/master/' + + +class InvalidLanguageError(Exception): + + """Raised when requesting invalid languages.""" + + def __init__(self, invalid_langs): + msg = 'invalid languages: {}'.format(', '.join(invalid_langs)) + super().__init__(msg) + + +@attr.s +class Language: + + """Dictionary language specs.""" + + code = attr.ib() + name = attr.ib() + remote_filename = attr.ib() + local_filename = attr.ib(default=None) + _file_extension = attr.ib('bdic', init=False) + + def __attrs_post_init__(self): + if self.local_filename is None: + self.local_filename = spell.local_filename(self.code) + + @property + def remote_path(self): + """Resolve the filename with extension the remote dictionary.""" + return '.'.join([self.remote_filename, self._file_extension]) + + @property + def local_path(self): + """Resolve the filename with extension the local dictionary.""" + if self.local_filename is None: + return None + return '.'.join([self.local_filename, self._file_extension]) + + @property + def remote_version(self): + """Resolve the version of the local dictionary.""" + return spell.version(self.remote_path) + + @property + def local_version(self): + """Resolve the version of the local dictionary.""" + local_path = self.local_path + if local_path is None: + return None + return spell.version(local_path) + + +def get_argparser(): + """Get the argparse parser.""" + desc = 'Install and manage Hunspell dictionaries for QtWebEngine.' + parser = argparse.ArgumentParser(prog='dictcli', + description=desc) + subparsers = parser.add_subparsers(help='Command', dest='cmd') + subparsers.required = True + subparsers.add_parser('list', + help='Display the list of available languages.') + subparsers.add_parser('update', + help='Update dictionaries') + subparsers.add_parser('remove-old', + help='Remove old versions of dictionaries.') + + install_parser = subparsers.add_parser('install', + help='Install dictionaries') + install_parser.add_argument('language', + nargs='*', + help="A list of languages to install.") + + return parser + + +def version_str(version): + return '.'.join(str(n) for n in version) + + +def print_list(languages): + """Print the list of available languages.""" + pat = '{:<7}{:<26}{:<8}{:<5}' + print(pat.format('Code', 'Name', 'Version', 'Installed')) + for lang in languages: + remote_version = version_str(lang.remote_version) + local_version = '-' + if lang.local_version is not None: + local_version = version_str(lang.local_version) + if lang.local_version < lang.remote_version: + local_version += ' - update available!' + print(pat.format(lang.code, lang.name, remote_version, local_version)) + + +def valid_languages(): + """Return a mapping from valid language codes to their names.""" + option = configdata.DATA['spellcheck.languages'] + return option.typ.valtype.valid_values.descriptions + + +def parse_entry(entry): + """Parse an entry from the remote API.""" + dict_re = re.compile(r""" + (?P<filename>(?P<code>[a-z]{2}(-[A-Z]{2})?).*)\.bdic + """, re.VERBOSE) + match = dict_re.fullmatch(entry['name']) + if match is not None: + return match.group('code'), match.group('filename') + else: + return None + + +def language_list_from_api(): + """Return a JSON with a list of available languages from Google API.""" + listurl = API_URL + '?format=JSON' + response = urllib.request.urlopen(listurl) + # A special 5-byte prefix must be stripped from the response content + # See: https://github.com/google/gitiles/issues/22 + # https://github.com/google/gitiles/issues/82 + json_content = response.read()[5:] + entries = json.loads(json_content.decode('utf-8'))['entries'] + parsed_entries = [parse_entry(entry) for entry in entries] + return [entry for entry in parsed_entries if entry is not None] + + +def latest_yet(code2file, code, filename): + """Determine whether the latest version so far.""" + if code not in code2file: + return True + return spell.version(code2file[code]) < spell.version(filename) + + +def available_languages(): + """Return a list of Language objects of all available languages.""" + lang_map = valid_languages() + api_list = language_list_from_api() + code2file = {} + for code, filename in api_list: + if latest_yet(code2file, code, filename): + code2file[code] = filename + return [ + Language(code, name, code2file[code]) + for code, name in lang_map.items() + if code in code2file + ] + + +def download_dictionary(url, dest): + """Download a decoded dictionary file.""" + response = urllib.request.urlopen(url) + decoded = base64.decodebytes(response.read()) + with open(dest, 'bw') as dict_file: + dict_file.write(decoded) + + +def filter_languages(languages, selected): + """Filter a list of languages based on an inclusion list. + + Args: + languages: a list of languages to filter + selected: a list of keys to select + """ + filtered_languages = [] + for language in languages: + if language.code in selected: + filtered_languages.append(language) + selected.remove(language.code) + if selected: + raise InvalidLanguageError(selected) + return filtered_languages + + +def install_lang(lang): + """Install a single lang given by the argument.""" + lang_url = API_URL + lang.remote_path + '?format=TEXT' + if not os.path.isdir(spell.dictionary_dir()): + msg = '{} does not exist, creating the directory' + print(msg.format(spell.dictionary_dir())) + os.makedirs(spell.dictionary_dir()) + print('Downloading {}'.format(lang_url)) + dest = os.path.join(spell.dictionary_dir(), lang.remote_path) + download_dictionary(lang_url, dest) + print('Done.') + + +def install(languages): + """Install languages.""" + for lang in languages: + try: + print('Installing {}: {}'.format(lang.code, lang.name)) + install_lang(lang) + except PermissionError as e: + sys.exit(str(e)) + + +def update(languages): + """Update the given languages.""" + installed = [lang for lang in languages if lang.local_version is not None] + for lang in installed: + if lang.local_version < lang.remote_version: + print('Upgrading {} from {} to {}'.format( + lang.code, + version_str(lang.local_version), + version_str(lang.remote_version))) + install_lang(lang) + + +def remove_old(languages): + """Remove old versions of languages.""" + installed = [lang for lang in languages if lang.local_version is not None] + for lang in installed: + local_files = spell.local_files(lang.code) + for old_file in local_files[1:]: + os.remove(os.path.join(spell.dictionary_dir(), old_file)) + + +def main(): + if configdata.DATA is None: + configdata.init() + standarddir.init(None) + + parser = get_argparser() + argv = sys.argv[1:] + args = parser.parse_args(argv) + languages = available_languages() + if args.cmd == 'list': + print_list(languages) + elif args.cmd == 'update': + update(languages) + elif args.cmd == 'remove-old': + remove_old(languages) + elif not args.language: + sys.exit('You must provide a list of languages to install.') + else: + try: + install(filter_languages(languages, args.language)) + except InvalidLanguageError as e: + print(e) + + +if __name__ == '__main__': + main() diff --git a/.config/qutebrowser/scripts/hist_importer.py b/.config/qutebrowser/scripts/hist_importer.py new file mode 100755 index 0000000..914701a --- /dev/null +++ b/.config/qutebrowser/scripts/hist_importer.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2017-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org> +# Copyright 2017-2018 Josefson Souza <josefson.br@gmail.com> + +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + + +"""Tool to import browser history from other browsers.""" + + +import argparse +import sqlite3 +import sys +import os + + +class Error(Exception): + + """Exception for errors in this module.""" + + pass + + +def parse(): + """Parse command line arguments.""" + description = ("This program is meant to extract browser history from your" + " previous browser and import them into qutebrowser.") + epilog = ("Databases:\n\n\tqutebrowser: Is named 'history.sqlite' and can " + "be found at your --basedir. In order to find where your " + "basedir is you can run ':open qute:version' inside qutebrowser." + "\n\n\tFirefox: Is named 'places.sqlite', and can be found at " + "your system's profile folder. Check this link for where it is " + "located: http://kb.mozillazine.org/Profile_folder" + "\n\n\tChrome: Is named 'History', and can be found at the " + "respective User Data Directory. Check this link for where it is" + "located: https://chromium.googlesource.com/chromium/src/+/" + "master/docs/user_data_dir.md\n\n" + "Example: hist_importer.py -b firefox -s /Firefox/Profile/" + "places.sqlite -d /qutebrowser/data/history.sqlite") + parser = argparse.ArgumentParser( + description=description, epilog=epilog, + formatter_class=argparse.RawTextHelpFormatter + ) + parser.add_argument('-b', '--browser', dest='browser', required=True, + type=str, help='Browsers: {firefox, chrome}') + parser.add_argument('-s', '--source', dest='source', required=True, + type=str, help='Source: Full path to the sqlite data' + 'base file from the source browser.') + parser.add_argument('-d', '--dest', dest='dest', required=True, type=str, + help='\nDestination: Full path to the qutebrowser ' + 'sqlite database') + return parser.parse_args() + + +def open_db(data_base): + """Open connection with database.""" + if os.path.isfile(data_base): + return sqlite3.connect(data_base) + raise Error('The file {} does not exist.'.format(data_base)) + + +def extract(source, query): + """Get records from source database. + + Args: + source: File path to the source database where we want to extract the + data from. + query: The query string to be executed in order to retrieve relevant + attributes as (datetime, url, time) from the source database according + to the browser chosen. + """ + try: + conn = open_db(source) + cursor = conn.cursor() + cursor.execute(query) + history = cursor.fetchall() + conn.close() + return history + except sqlite3.OperationalError as op_e: + raise Error('Could not perform queries on the source database: ' + '{}'.format(op_e)) + + +def clean(history): + """Clean up records from source database. + + Receives a list of record and sanityze them in order for them to be + properly imported to qutebrowser. Sanitation requires adding a 4th + attribute 'redirect' which is filled with '0's, and also purging all + records that have a NULL/None datetime attribute. + + Args: + history: List of records (datetime, url, title) from source database. + """ + # replace missing titles with an empty string + for index, record in enumerate(history): + if record[1] is None: + cleaned = list(record) + cleaned[1] = '' + history[index] = tuple(cleaned) + + nulls = [record for record in history if None in record] + for null_record in nulls: + history.remove(null_record) + history = [list(record) for record in history] + for record in history: + record.append('0') + return history + + +def insert_qb(history, dest): + """Insert history into dest database. + + Args: + history: List of records. + dest: File path to the destination database, where history will be + inserted. + """ + conn = open_db(dest) + cursor = conn.cursor() + cursor.executemany( + 'INSERT INTO History (url,title,atime,redirect) VALUES (?,?,?,?)', + history + ) + cursor.execute('DROP TABLE CompletionHistory') + conn.commit() + conn.close() + + +def run(): + """Main control flux of the script.""" + args = parse() + browser = args.browser.lower() + source, dest = args.source, args.dest + query = { + 'firefox': 'select url,title,last_visit_date/1000000 as date ' + 'from moz_places where url like "http%" or url ' + 'like "ftp%" or url like "file://%"', + 'chrome': 'select url,title,last_visit_time/10000000 as date ' + 'from urls', + } + if browser not in query: + raise Error('Sorry, the selected browser: "{}" is not ' + 'supported.'.format(browser)) + else: + history = extract(source, query[browser]) + history = clean(history) + insert_qb(history, dest) + + +def main(): + try: + run() + except Error as e: + sys.exit(str(e)) + + +if __name__ == "__main__": + main() diff --git a/.config/qutebrowser/scripts/hostblock_blame.py b/.config/qutebrowser/scripts/hostblock_blame.py new file mode 100755 index 0000000..2f68d29 --- /dev/null +++ b/.config/qutebrowser/scripts/hostblock_blame.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org> +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + +"""Check by which hostblock list a host was blocked.""" + +import sys +import io +import os +import os.path +import urllib.request + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir)) +from qutebrowser.browser import adblock +from qutebrowser.config import configdata + + +def main(): + """Check by which hostblock list a host was blocked.""" + if len(sys.argv) != 2: + print("Usage: {} <host>".format(sys.argv[0]), file=sys.stderr) + sys.exit(1) + + configdata.init() + + for url in configdata.DATA['content.host_blocking.lists'].default: + print("checking {}...".format(url)) + raw_file = urllib.request.urlopen(url) + byte_io = io.BytesIO(raw_file.read()) + f = adblock.get_fileobj(byte_io) + for line in f: + line = line.decode('utf-8') + if sys.argv[1] in line: + print("FOUND {} in {}:".format(sys.argv[1], url)) + print(" " + line.rstrip()) + + +if __name__ == '__main__': + main() diff --git a/.config/qutebrowser/scripts/importer.py b/.config/qutebrowser/scripts/importer.py new file mode 100755 index 0000000..eb808a6 --- /dev/null +++ b/.config/qutebrowser/scripts/importer.py @@ -0,0 +1,349 @@ +#!/usr/bin/env python3 +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2014-2018 Claude (longneck) <longneck@scratchbook.ch> +# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org> + +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + + +"""Tool to import data from other browsers. + +Currently importing bookmarks from Netscape Bookmark files and Mozilla +profiles is supported. +""" + + +import argparse +import sqlite3 +import os +import urllib.parse +import json +import string + +browser_default_input_format = { + 'chromium': 'chrome', + 'chrome': 'chrome', + 'ie': 'netscape', + 'firefox': 'mozilla', + 'seamonkey': 'mozilla', + 'palemoon': 'mozilla', +} + + +def main(): + args = get_args() + bookmark_types = [] + output_format = None + input_format = args.input_format + if args.search_output: + bookmark_types = ['search'] + if args.oldconfig: + output_format = 'oldsearch' + else: + output_format = 'search' + else: + if args.bookmark_output: + output_format = 'bookmark' + elif args.quickmark_output: + output_format = 'quickmark' + if args.import_bookmarks: + bookmark_types.append('bookmark') + if args.import_keywords: + bookmark_types.append('keyword') + if not bookmark_types: + bookmark_types = ['bookmark', 'keyword'] + if not output_format: + output_format = 'quickmark' + if not input_format: + if args.browser: + input_format = browser_default_input_format[args.browser] + else: + #default to netscape + input_format = 'netscape' + + import_function = { + 'netscape': import_netscape_bookmarks, + 'mozilla': import_moz_places, + 'chrome': import_chrome, + } + import_function[input_format](args.bookmarks, bookmark_types, + output_format) + + +def get_args(): + """Get the argparse parser.""" + parser = argparse.ArgumentParser( + epilog="To import bookmarks from Chromium, Firefox or IE, " + "export them to HTML in your browsers bookmark manager. ") + parser.add_argument( + 'browser', + help="Which browser? {%(choices)s}", + choices=browser_default_input_format.keys(), + nargs='?', + metavar='browser') + parser.add_argument( + '-i', + '--input-format', + help='Which input format? (overrides browser default; "netscape" if ' + 'neither given)', + choices=set(browser_default_input_format.values()), + required=False) + parser.add_argument( + '-b', + '--bookmark-output', + help="Output in bookmark format.", + action='store_true', + default=False, + required=False) + parser.add_argument( + '-q', + '--quickmark-output', + help="Output in quickmark format (default).", + action='store_true', + default=False, + required=False) + parser.add_argument( + '-s', + '--search-output', + help="Output config.py search engine format (negates -B and -K)", + action='store_true', + default=False, + required=False) + parser.add_argument( + '--oldconfig', + help="Output search engine format for old qutebrowser.conf format", + default=False, + action='store_true', + required=False) + parser.add_argument( + '-B', + '--import-bookmarks', + help="Import plain bookmarks (can be combiend with -K)", + action='store_true', + default=False, + required=False) + parser.add_argument( + '-K', + '--import-keywords', + help="Import keywords (can be combined with -B)", + action='store_true', + default=False, + required=False) + parser.add_argument( + 'bookmarks', + help="Bookmarks file (html format) or " + "profile folder (Mozilla format)") + args = parser.parse_args() + return args + + +def search_escape(url): + """Escape URLs such that preexisting { and } are handled properly. + + Will obviously trash a properly-formatted qutebrowser URL. + """ + return url.replace('{', '{{').replace('}', '}}') + + +def opensearch_convert(url): + """Convert a basic OpenSearch URL into something qutebrowser can use. + + Exceptions: + KeyError: + An unknown and required parameter is present in the URL. This + usually means there's browser/addon specific functionality needed + to build the URL (I'm looking at you and your browser, Google) that + obviously won't be present here. + """ + subst = { + 'searchTerms': '%s', # for proper escaping later + 'language': '*', + 'inputEncoding': 'UTF-8', + 'outputEncoding': 'UTF-8' + } + + # remove optional parameters (even those we don't support) + for param in string.Formatter().parse(url): + if param[1]: + if param[1].endswith('?'): + url = url.replace('{' + param[1] + '}', '') + elif param[2] and param[2].endswith('?'): + url = url.replace('{' + param[1] + ':' + param[2] + '}', '') + return search_escape(url.format(**subst)).replace('%s', '{}') + + +def import_netscape_bookmarks(bookmarks_file, bookmark_types, output_format): + """Import bookmarks from a NETSCAPE-Bookmark-file v1. + + Generated by Chromium, Firefox, IE and possibly more browsers. Not all + export all possible bookmark types: + - Firefox mostly works with everything + - Chrome doesn't support keywords at all; searches are a separate + database + """ + import bs4 + with open(bookmarks_file, encoding='utf-8') as f: + soup = bs4.BeautifulSoup(f, 'html.parser') + bookmark_query = { + 'search': lambda tag: ( + (tag.name == 'a') and + ('shortcuturl' in tag.attrs) and + ('%s' in tag['href'])), + 'keyword': lambda tag: ( + (tag.name == 'a') and + ('shortcuturl' in tag.attrs) and + ('%s' not in tag['href'])), + 'bookmark': lambda tag: ( + (tag.name == 'a') and + ('shortcuturl' not in tag.attrs) and + (tag.string)), + } + output_template = { + 'search': { + 'search': + "c.url.searchengines['{tag[shortcuturl]}'] = " + "'{tag[href]}' #{tag.string}" + }, + 'oldsearch': { + 'search': '{tag[shortcuturl]} = {tag[href]} #{tag.string}', + }, + 'bookmark': { + 'bookmark': '{tag[href]} {tag.string}', + 'keyword': '{tag[href]} {tag.string}' + }, + 'quickmark': { + 'bookmark': '{tag.string} {tag[href]}', + 'keyword': '{tag[shortcuturl]} {tag[href]}' + } + } + bookmarks = [] + for typ in bookmark_types: + tags = soup.findAll(bookmark_query[typ]) + for tag in tags: + if typ == 'search': + tag['href'] = search_escape(tag['href']).replace('%s', '{}') + if tag['href'] not in bookmarks: + bookmarks.append( + output_template[output_format][typ].format(tag=tag)) + for bookmark in bookmarks: + print(bookmark) + + +def import_moz_places(profile, bookmark_types, output_format): + """Import bookmarks from a Mozilla profile's places.sqlite database.""" + place_query = { + 'bookmark': ( + "SELECT DISTINCT moz_bookmarks.title,moz_places.url " + "FROM moz_bookmarks,moz_places " + "WHERE moz_places.id=moz_bookmarks.fk " + "AND moz_places.id NOT IN (SELECT place_id FROM moz_keywords) " + "AND moz_places.url NOT LIKE 'place:%';" + ), # Bookmarks with no keywords assigned + 'keyword': ( + "SELECT moz_keywords.keyword,moz_places.url " + "FROM moz_keywords,moz_places,moz_bookmarks " + "WHERE moz_places.id=moz_bookmarks.fk " + "AND moz_places.id=moz_keywords.place_id " + "AND moz_places.url NOT LIKE '%!%s%' ESCAPE '!';" + ), # Bookmarks with keywords assigned but no %s substitution + 'search': ( + "SELECT moz_keywords.keyword, " + " moz_bookmarks.title, " + " search_conv(moz_places.url) AS url " + "FROM moz_keywords,moz_places,moz_bookmarks " + "WHERE moz_places.id=moz_bookmarks.fk " + "AND moz_places.id=moz_keywords.place_id " + "AND moz_places.url LIKE '%!%s%' ESCAPE '!';" + ) # bookmarks with keyword and %s substitution + } + out_template = { + 'bookmark': { + 'bookmark': '{url} {title}', + 'keyword': '{url} {keyword}' + }, + 'quickmark': { + 'bookmark': '{title} {url}', + 'keyword': '{keyword} {url}' + }, + 'oldsearch': { + 'search': '{keyword} {url} #{title}' + }, + 'search': { + 'search': "c.url.searchengines['{keyword}'] = '{url}' #{title}" + } + } + + def search_conv(url): + return search_escape(url).replace('%s', '{}') + + places = sqlite3.connect(os.path.join(profile, "places.sqlite")) + places.create_function('search_conv', 1, search_conv) + places.row_factory = sqlite3.Row + c = places.cursor() + for typ in bookmark_types: + c.execute(place_query[typ]) + for row in c: + print(out_template[output_format][typ].format(**row)) + + +def import_chrome(profile, bookmark_types, output_format): + """Import bookmarks and search keywords from Chrome-type profiles. + + On Chrome, keywords and search engines are the same thing and handled in + their own database table; bookmarks cannot have associated keywords. This + is why the dictionary lookups here are much simpler. + """ + out_template = { + 'bookmark': '{url} {name}', + 'quickmark': '{name} {url}', + 'search': "c.url.searchengines['{keyword}'] = '{url}'", + 'oldsearch': '{keyword} {url}' + } + + if 'search' in bookmark_types: + webdata = sqlite3.connect(os.path.join(profile, 'Web Data')) + c = webdata.cursor() + c.execute('SELECT keyword,url FROM keywords;') + for keyword, url in c: + try: + url = opensearch_convert(url) + print(out_template[output_format].format( + keyword=keyword, url=url)) + except KeyError: + print('# Unsupported parameter in url for {}; skipping....'. + format(keyword)) + + else: + with open(os.path.join(profile, 'Bookmarks'), encoding='utf-8') as f: + bookmarks = json.load(f) + + def bm_tree_walk(bm, template): + """Recursive function to walk through bookmarks.""" + assert 'type' in bm, bm + if bm['type'] == 'url': + if urllib.parse.urlparse(bm['url']).scheme != 'chrome': + print(template.format(**bm)) + elif bm['type'] == 'folder': + for child in bm['children']: + bm_tree_walk(child, template) + + for root in bookmarks['roots'].values(): + bm_tree_walk(root, out_template[output_format]) + + +if __name__ == '__main__': + main() diff --git a/.config/qutebrowser/scripts/keytester.py b/.config/qutebrowser/scripts/keytester.py new file mode 100755 index 0000000..ee5eb34 --- /dev/null +++ b/.config/qutebrowser/scripts/keytester.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org> + +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + +"""Small test script to show key presses. + +Use python3 -m scripts.keytester to launch it. +""" + +from PyQt5.QtWidgets import QApplication, QWidget, QLabel, QHBoxLayout + +from qutebrowser.keyinput import keyutils + + +class KeyWidget(QWidget): + + """Widget displaying key presses.""" + + def __init__(self, parent=None): + super().__init__(parent) + self._layout = QHBoxLayout(self) + self._label = QLabel(text="Waiting for keypress...") + self._layout.addWidget(self._label) + + def keyPressEvent(self, e): + """Show pressed keys.""" + lines = [ + str(keyutils.KeyInfo.from_event(e)), + '', + 'key: 0x{:x}'.format(int(e.key())), + 'modifiers: 0x{:x}'.format(int(e.modifiers())), + 'text: {!r}'.format(e.text()), + ] + self._label.setText('\n'.join(lines)) + + +app = QApplication([]) +w = KeyWidget() +w.show() +app.exec_() diff --git a/.config/qutebrowser/scripts/link_pyqt.py b/.config/qutebrowser/scripts/link_pyqt.py new file mode 100755 index 0000000..ae7eaa6 --- /dev/null +++ b/.config/qutebrowser/scripts/link_pyqt.py @@ -0,0 +1,233 @@ +#!/usr/bin/env python3 +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org> + +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + +"""Symlink PyQt into a given virtualenv.""" + +import os +import os.path +import argparse +import shutil +import sys +import subprocess +import tempfile +import filecmp + + +class Error(Exception): + + """Exception raised when linking fails.""" + + pass + + +def run_py(executable, *code): + """Run the given python code with the given executable.""" + if os.name == 'nt' and len(code) > 1: + # Windows can't do newlines in arguments... + oshandle, filename = tempfile.mkstemp() + with os.fdopen(oshandle, 'w') as f: + f.write('\n'.join(code)) + cmd = [executable, filename] + try: + ret = subprocess.run(cmd, universal_newlines=True, check=True, + stdout=subprocess.PIPE).stdout + finally: + os.remove(filename) + else: + cmd = [executable, '-c', '\n'.join(code)] + ret = subprocess.run(cmd, universal_newlines=True, check=True, + stdout=subprocess.PIPE).stdout + return ret.rstrip() + + +def verbose_copy(src, dst, *, follow_symlinks=True): + """Copy function for shutil.copytree which prints copied files.""" + if '-v' in sys.argv: + print('{} -> {}'.format(src, dst)) + shutil.copy(src, dst, follow_symlinks=follow_symlinks) + + +def get_ignored_files(directory, files): + """Get the files which should be ignored for link_pyqt() on Windows.""" + needed_exts = ('.py', '.dll', '.pyd', '.so') + ignored_dirs = ('examples', 'qml', 'uic', 'doc') + filtered = [] + for f in files: + ext = os.path.splitext(f)[1] + full_path = os.path.join(directory, f) + if os.path.isdir(full_path) and f in ignored_dirs: + filtered.append(f) + elif (ext not in needed_exts) and os.path.isfile(full_path): + filtered.append(f) + return filtered + + +def needs_update(source, dest): + """Check if a file to be linked/copied needs to be updated.""" + if os.path.islink(dest): + # No need to delete a link and relink -> skip this + return False + elif os.path.isdir(dest): + diffs = filecmp.dircmp(source, dest) + ignored = get_ignored_files(source, diffs.left_only) + has_new_files = set(ignored) != set(diffs.left_only) + return (has_new_files or diffs.right_only or diffs.common_funny or + diffs.diff_files or diffs.funny_files) + else: + return not filecmp.cmp(source, dest) + + +def get_lib_path(executable, name, required=True): + """Get the path of a python library. + + Args: + executable: The Python executable to use. + name: The name of the library to get the path for. + required: Whether Error should be raised if the lib was not found. + """ + code = [ + 'try:', + ' import {}'.format(name), + 'except ImportError as e:', + ' print("ImportError: " + str(e))', + 'else:', + ' print("path: " + {}.__file__)'.format(name) + ] + output = run_py(executable, *code) + + try: + prefix, data = output.split(': ') + except ValueError: + raise ValueError("Unexpected output: {!r}".format(output)) + + if prefix == 'path': + return data + elif prefix == 'ImportError': + if required: + raise Error("Could not import {} with {}: {}!".format( + name, executable, data)) + else: + return None + else: + raise ValueError("Unexpected output: {!r}".format(output)) + + +def link_pyqt(executable, venv_path): + """Symlink the systemwide PyQt/sip into the venv. + + Args: + executable: The python executable where the source files are present. + venv_path: The path to the virtualenv site-packages. + """ + try: + get_lib_path(executable, 'PyQt5.sip') + except Error: + # There is no PyQt5.sip, so we need to copy the toplevel sip. + sip_file = get_lib_path(executable, 'sip') + else: + # There is a PyQt5.sip, it'll get copied with the PyQt5 dir. + sip_file = None + + sipconfig_file = get_lib_path(executable, 'sipconfig', required=False) + pyqt_dir = os.path.dirname(get_lib_path(executable, 'PyQt5.QtCore')) + + for path in [sip_file, sipconfig_file, pyqt_dir]: + if path is None: + continue + + fn = os.path.basename(path) + dest = os.path.join(venv_path, fn) + + if os.path.exists(dest): + if needs_update(path, dest): + remove(dest) + else: + continue + + copy_or_link(path, dest) + + +def copy_or_link(source, dest): + """Copy or symlink source to dest.""" + if os.name == 'nt': + if os.path.isdir(source): + print('{} -> {}'.format(source, dest)) + shutil.copytree(source, dest, ignore=get_ignored_files, + copy_function=verbose_copy) + else: + print('{} -> {}'.format(source, dest)) + shutil.copy(source, dest) + else: + print('{} -> {}'.format(source, dest)) + os.symlink(source, dest) + + +def remove(filename): + """Remove a given filename, regardless of whether it's a file or dir.""" + if os.path.isdir(filename): + shutil.rmtree(filename) + else: + os.unlink(filename) + + +def get_venv_lib_path(path): + """Get the library path of a virtualenv.""" + subdir = 'Scripts' if os.name == 'nt' else 'bin' + executable = os.path.join(path, subdir, 'python') + return run_py(executable, + 'from distutils.sysconfig import get_python_lib', + 'print(get_python_lib())') + + +def get_tox_syspython(tox_path): + """Get the system python based on a virtualenv created by tox.""" + path = os.path.join(tox_path, '.tox-config1') + with open(path, encoding='ascii') as f: + line = f.readline() + _md5, sys_python = line.rstrip().split(' ', 1) + return sys_python + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('path', help="Base path to the venv.") + parser.add_argument('--tox', help="Add when called via tox.", + action='store_true') + args = parser.parse_args() + + if args.tox: + # Workaround for the lack of negative factors in tox.ini + if 'LINK_PYQT_SKIP' in os.environ: + print('LINK_PYQT_SKIP set, exiting...') + sys.exit(0) + executable = get_tox_syspython(args.path) + else: + executable = sys.executable + + venv_path = get_venv_lib_path(args.path) + link_pyqt(executable, venv_path) + + +if __name__ == '__main__': + try: + main() + except Error as e: + print(str(e), file=sys.stderr) + sys.exit(1) diff --git a/.config/qutebrowser/scripts/open_url_in_instance.sh b/.config/qutebrowser/scripts/open_url_in_instance.sh new file mode 100755 index 0000000..a6ce0ed --- /dev/null +++ b/.config/qutebrowser/scripts/open_url_in_instance.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# initial idea: Florian Bruhin (The-Compiler) +# author: Thore Bödecker (foxxx0) + +_url="$1" +_qb_version='1.0.4' +_proto_version=1 +_ipc_socket="${XDG_RUNTIME_DIR}/qutebrowser/ipc-$(echo -n "$USER" | md5sum | cut -d' ' -f1)" +_qute_bin="/usr/bin/qutebrowser" + +printf '{"args": ["%s"], "target_arg": null, "version": "%s", "protocol_version": %d, "cwd": "%s"}\n' \ + "${_url}" \ + "${_qb_version}" \ + "${_proto_version}" \ + "${PWD}" | socat - UNIX-CONNECT:"${_ipc_socket}" 2>/dev/null || "$_qute_bin" "$@" & diff --git a/.config/qutebrowser/scripts/setupcommon.py b/.config/qutebrowser/scripts/setupcommon.py new file mode 100755 index 0000000..50eabac --- /dev/null +++ b/.config/qutebrowser/scripts/setupcommon.py @@ -0,0 +1,74 @@ +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org> + +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + + +"""Data used by setup.py and the PyInstaller qutebrowser.spec.""" + +import sys +import os +import os.path +import subprocess +sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir)) + + +if sys.hexversion >= 0x03000000: + open_file = open +else: + import codecs + open_file = codecs.open + + +BASEDIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), + os.path.pardir) + + +def _git_str(): + """Try to find out git version. + + Return: + string containing the git commit ID and timestamp. + None if there was an error or we're not in a git repo. + """ + if BASEDIR is None: + return None + if not os.path.isdir(os.path.join(BASEDIR, ".git")): + return None + try: + # https://stackoverflow.com/questions/21017300/21017394#21017394 + commit_hash = subprocess.run( + ['git', 'describe', '--match=NeVeRmAtCh', '--always', '--dirty'], + cwd=BASEDIR, check=True, + stdout=subprocess.PIPE).stdout.decode('UTF-8').strip() + date = subprocess.run( + ['git', 'show', '-s', '--format=%ci', 'HEAD'], + cwd=BASEDIR, check=True, + stdout=subprocess.PIPE).stdout.decode('UTF-8').strip() + return '{} ({})'.format(commit_hash, date) + except (subprocess.CalledProcessError, OSError): + return None + + +def write_git_file(): + """Write the git-commit-id file with the current commit.""" + gitstr = _git_str() + if gitstr is None: + gitstr = '' + path = os.path.join(BASEDIR, 'qutebrowser', 'git-commit-id') + with open_file(path, 'w', encoding='ascii') as f: + f.write(gitstr) diff --git a/.config/qutebrowser/scripts/testbrowser/cpp/webengine/main.cpp b/.config/qutebrowser/scripts/testbrowser/cpp/webengine/main.cpp new file mode 100644 index 0000000..311432e --- /dev/null +++ b/.config/qutebrowser/scripts/testbrowser/cpp/webengine/main.cpp @@ -0,0 +1,13 @@ +#include <QApplication> +#include <QWebEngineView> +#include <QUrl> + + +int main(int argc, char *argv[]) +{ + QApplication app(argc, argv); + QWebEngineView view; + view.load(QUrl(argv[1])); + view.show(); + return app.exec(); +} diff --git a/.config/qutebrowser/scripts/testbrowser/cpp/webengine/testbrowser.pro b/.config/qutebrowser/scripts/testbrowser/cpp/webengine/testbrowser.pro new file mode 100644 index 0000000..12a1cf7 --- /dev/null +++ b/.config/qutebrowser/scripts/testbrowser/cpp/webengine/testbrowser.pro @@ -0,0 +1,6 @@ +QT += core widgets webenginewidgets + +TARGET = testbrowser +TEMPLATE = app + +SOURCES += main.cpp diff --git a/.config/qutebrowser/scripts/testbrowser/cpp/webkit/main.cpp b/.config/qutebrowser/scripts/testbrowser/cpp/webkit/main.cpp new file mode 100644 index 0000000..06c3d1a --- /dev/null +++ b/.config/qutebrowser/scripts/testbrowser/cpp/webkit/main.cpp @@ -0,0 +1,13 @@ +#include <QApplication> +#include <QWebView> +#include <QUrl> + + +int main(int argc, char *argv[]) +{ + QApplication app(argc, argv); + QWebView view; + view.load(QUrl(argv[1])); + view.show(); + return app.exec(); +} diff --git a/.config/qutebrowser/scripts/testbrowser/cpp/webkit/testbrowser.pro b/.config/qutebrowser/scripts/testbrowser/cpp/webkit/testbrowser.pro new file mode 100644 index 0000000..59f55dd --- /dev/null +++ b/.config/qutebrowser/scripts/testbrowser/cpp/webkit/testbrowser.pro @@ -0,0 +1,6 @@ +QT += core widgets webkit webkitwidgets + +TARGET = testbrowser +TEMPLATE = app + +SOURCES += main.cpp diff --git a/.config/qutebrowser/scripts/testbrowser/testbrowser_webengine.py b/.config/qutebrowser/scripts/testbrowser/testbrowser_webengine.py new file mode 100755 index 0000000..fdf6728 --- /dev/null +++ b/.config/qutebrowser/scripts/testbrowser/testbrowser_webengine.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org> +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + +"""Very simple browser for testing purposes.""" + +import sys +import argparse + +from PyQt5.QtCore import QUrl +from PyQt5.QtWidgets import QApplication +from PyQt5.QtWebEngineWidgets import QWebEngineView + + +def parse_args(): + """Parse commandline arguments.""" + parser = argparse.ArgumentParser() + parser.add_argument('url', help='The URL to open') + return parser.parse_known_args()[0] + + +if __name__ == '__main__': + args = parse_args() + app = QApplication(sys.argv) + wv = QWebEngineView() + + wv.loadStarted.connect(lambda: print("Loading started")) + wv.loadProgress.connect(lambda p: print("Loading progress: {}%".format(p))) + wv.loadFinished.connect(lambda: print("Loading finished")) + + wv.load(QUrl.fromUserInput(args.url)) + wv.show() + + app.exec_() diff --git a/.config/qutebrowser/scripts/testbrowser/testbrowser_webkit.py b/.config/qutebrowser/scripts/testbrowser/testbrowser_webkit.py new file mode 100755 index 0000000..73cae08 --- /dev/null +++ b/.config/qutebrowser/scripts/testbrowser/testbrowser_webkit.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org> +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + +"""Very simple browser for testing purposes.""" + +import sys +import argparse + +from PyQt5.QtCore import QUrl +from PyQt5.QtWidgets import QApplication +from PyQt5.QtWebKit import QWebSettings +from PyQt5.QtWebKitWidgets import QWebView + + +def parse_args(): + """Parse commandline arguments.""" + parser = argparse.ArgumentParser() + parser.add_argument('url', help='The URL to open') + parser.add_argument('--plugins', '-p', help='Enable plugins', + default=False, action='store_true') + return parser.parse_known_args()[0] + + +if __name__ == '__main__': + args = parse_args() + app = QApplication(sys.argv) + wv = QWebView() + + wv.loadStarted.connect(lambda: print("Loading started")) + wv.loadProgress.connect(lambda p: print("Loading progress: {}%".format(p))) + wv.loadFinished.connect(lambda: print("Loading finished")) + + if args.plugins: + wv.settings().setAttribute(QWebSettings.PluginsEnabled, True) + + wv.load(QUrl.fromUserInput(args.url)) + wv.show() + + app.exec_() diff --git a/.config/qutebrowser/scripts/utils.py b/.config/qutebrowser/scripts/utils.py new file mode 100755 index 0000000..9a1a751 --- /dev/null +++ b/.config/qutebrowser/scripts/utils.py @@ -0,0 +1,103 @@ +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org> + +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. + +"""Utility functions for scripts.""" + +import os +import os.path + + +# Import side-effects are an evil thing, but here it's okay so scripts using +# colors work on Windows as well. +try: + import colorama +except ImportError: + colorama = None +else: + colorama.init() + + +use_color = os.name != 'nt' or colorama + + +fg_colors = { + 'black': 30, + 'red': 31, + 'green': 32, + 'yellow': 33, + 'blue': 34, + 'magenta': 35, + 'cyan': 36, + 'white': 37, + 'reset': 39, +} + + +bg_colors = {name: col + 10 for name, col in fg_colors.items()} + + +term_attributes = { + 'bright': 1, + 'dim': 2, + 'normal': 22, + 'reset': 0, +} + + +def _esc(code): + """Get an ANSI color code based on a color number.""" + return '\033[{}m'.format(code) + + +def print_col(text, color): + """Print a colorized text.""" + if use_color: + fg = _esc(fg_colors[color.lower()]) + reset = _esc(fg_colors['reset']) + print(''.join([fg, text, reset])) + else: + print(text) + + +def print_title(text): + """Print a title.""" + print_col("==================== {} ====================".format(text), + 'yellow') + + +def print_subtitle(text): + """Print a subtitle.""" + print_col("------ {} ------".format(text), 'cyan') + + +def print_bold(text): + """Print a bold text.""" + if use_color: + bold = _esc(term_attributes['bright']) + reset = _esc(term_attributes['reset']) + print(''.join([bold, text, reset])) + else: + print(text) + + +def change_cwd(): + """Change the scripts cwd if it was started inside the script folder.""" + cwd = os.getcwd() + if os.path.split(cwd)[1] == 'scripts': + os.chdir(os.path.join(cwd, os.pardir)) |