diff --git a/.codecov.yml b/.codecov.yml deleted file mode 100644 index 6866413f..00000000 --- a/.codecov.yml +++ /dev/null @@ -1,15 +0,0 @@ -comment: off -coverage: - range: 75..95 - precision: 0 - status: - patch: - default: - target: 90 - project: - default: - target: auto - threshold: 5 - # Fix for https://github.com/codecov/codecov-python/issues/136 - fixes: - - "__init__.py::pdoc/__init__.py" diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml new file mode 100644 index 00000000..b609154e --- /dev/null +++ b/.github/actions/setup/action.yml @@ -0,0 +1,21 @@ +name: 'Default Checkout' +description: 'checkout & setup' +inputs: + python-version: + description: 'Python version' + required: true + default: '>=3' +runs: + using: "composite" + steps: + - name: Set up Python ${{ inputs.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python-version }} + - uses: actions/cache@v4 + with: + path: | + ~/.cache/pip + ~\AppData\Local\pip\Cache + ~/Library/Caches/pip + key: ${{ runner.os }}-py${{ inputs.python-version }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 75b394f6..3c72bc53 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,72 +2,61 @@ name: CI on: push: { branches: [master] } pull_request: { branches: [master] } - schedule: [ cron: '12 2 * * 6' ] # Every Saturday, 02:12 + schedule: [ cron: '12 2 6 * *' ] jobs: - build: - name: Build - runs-on: ubuntu-18.04 - + test-matrix: + runs-on: ubuntu-latest strategy: matrix: - python-version: [3.6, 3.7] - include: - - python-version: 3.8 - test-type: lint - - python-version: 3.8 - test-type: docs - + python-version: ['3.9', '>=3'] steps: - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + - uses: actions/checkout@v4 + - uses: ./.github/actions/setup with: python-version: ${{ matrix.python-version }} - - uses: actions/cache@v2 - name: Set up caches - with: - path: ~/.cache/pip - key: ${{ runner.os }}-py${{ matrix.python-version }} + - run: pip install -U pip setuptools wheel && pip install -U . + - run: time python -m unittest -v pdoc.test - - name: Checkout repo - uses: actions/checkout@v2 + lint-test-coverage: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/setup with: - fetch-depth: 3 - - name: Fetch tags - run: git fetch --depth=1 origin +refs/tags/*:refs/tags/* + python-version: 3.11 - - name: Install dependencies - run: | - pip install -U pip setuptools wheel - pip install -U . + - run: pip install -U pip setuptools wheel && pip install -U . - name: Install lint dependencies - if: matrix.test-type == 'lint' run: | pip install flake8 coverage mypy types-Markdown sudo apt update && sudo apt-get install \ texlive-xetex lmodern texlive-fonts-recommended # test_pdf_pandoc - wget -O/tmp/pandoc.deb https://github.com/jgm/pandoc/releases/download/2.10/pandoc-2.10-1-amd64.deb && sudo dpkg -i /tmp/pandoc.deb - - - name: Install docs dependencies - if: matrix.test-type == 'docs' - run: pip install -e . + wget -O/tmp/pandoc.deb https://github.com/jgm/pandoc/releases/download/3.1.12.2/pandoc-3.1.12.2-1-amd64.deb && sudo dpkg -i /tmp/pandoc.deb + + - run: find -name '*.md' | xargs .github/lint-markdown.sh + - run: flake8 pdoc setup.py + - run: mypy -p pdoc + - run: time coverage run -m unittest -v pdoc.test + - run: coverage report + - run: PDOC_TEST_PANDOC=1 time python -m unittest -v pdoc.test.CliTest.test_pdf_pandoc + - uses: actions/upload-artifact@v4 + with: + name: Pdoc Documentation.pdf + path: /tmp/pdoc.pdf - - name: Test w/ Coverage, Lint - if: matrix.test-type == 'lint' - run: | - find -name '*.md' | xargs .github/lint-markdown.sh - flake8 - mypy -p pdoc - time coverage run -m unittest -v pdoc.test - PDOC_TEST_PANDOC=1 time catchsegv python -m unittest -v pdoc.test.CliTest.test_pdf_pandoc - bash <(curl -s https://codecov.io/bash) + docs: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/setup + with: + python-version: 3.11 - - name: Test - if: '! matrix.test-type' - run: time python -m unittest -v pdoc.test + - name: Fetch tags + run: git fetch --depth=1 origin +refs/tags/*:refs/tags/* - - name: Test docs - if: matrix.test-type == 'docs' - run: time doc/build.sh + - run: pip install -U pip setuptools wheel && pip install -e . + - run: time doc/build.sh \ No newline at end of file diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml index 117896c5..caf74c3e 100644 --- a/.github/workflows/deploy-docs.yml +++ b/.github/workflows/deploy-docs.yml @@ -9,17 +9,6 @@ jobs: runs-on: ubuntu-latest steps: - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - - uses: actions/cache@v2 - name: Set up caches - with: - path: ~/.cache/pip - key: ${{ runner.os }} - - name: Checkout repo uses: actions/checkout@v2 with: @@ -27,6 +16,10 @@ jobs: - name: Fetch tags run: git fetch --depth=1 origin +refs/tags/*:refs/tags/* + - uses: ./.github/actions/setup + with: + python-version: 3.11 + - name: Install dependencies run: | pip install -U pip setuptools wheel diff --git a/CHANGELOG b/CHANGELOG index c30ffb17..befdc51c 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,44 @@ +0.11.5 (2024-12-13) +====== + - A few default HTML template improvements + +0.11.4 (2024-12-13) +====== + - Fix Lunr.js prebuilt index introduced in v0.11.0 + - Fix showing Git link (git_link_template) for property, cached_property, + namedtuple, and member_descriptor types + +0.11.3 (2024-11-26) +====== + - Format Optional as `X | None` (#395) + - Support configurable Python-Markdown extensions (#440) + +0.11.2 (2024-11-25) +====== + - Improve formatting of Optional, Union and `collection.abc` types (#395) + - In HTML, format long function params on multiple lines + - Fix issue with `--skip-errors` (#421) + +0.11.1 (2024-06-26) +====== + - Handle union type expressions (|) for Google style docstrings (#443) + - Fix bug with Lunr.js search when `node` in not available (#446) + +0.11.0 (2024-06-22) +====== + - Handle import of distutils on Python 3.12 + - Discern properties from regular variables (#277) + - Prebuild Lunr.js search index if Node is available. + - Templates: Update CDN resource links + - Support MathJax inline $dollar-pattern$ + - Fix documenting classes that contain `unittest.mock.Mock` (#352) + - Strengthen signature detection for pybind-generated modules + - Bump pandoc integration to v3 + - Support Google Analytics 4 + - Fix deprecation warnings for PEP224 docstrings of class variables (#437) + - Skip `__editable__` paths during `iter_modules` (#408) + - Various bug fixes and tweaks + 0.10.0 (2021-08-03) ====== - Python 3.6+ required. diff --git a/README.md b/README.md index 8afc1953..55eca269 100644 --- a/README.md +++ b/README.md @@ -2,14 +2,13 @@ pdoc ==== - -[![Build Status](https://img.shields.io/github/workflow/status/pdoc3/pdoc/CI?style=for-the-badge)](https://github.com/pdoc3/pdoc/actions) +[![Build Status](https://img.shields.io/github/actions/workflow/status/pdoc3/pdoc/ci.yml?branch=master&style=for-the-badge)](https://github.com/pdoc3/pdoc/actions) [![Code Coverage](https://img.shields.io/codecov/c/gh/pdoc3/pdoc.svg?style=for-the-badge)](https://codecov.io/gh/pdoc3/pdoc) [![pdoc3 on PyPI](https://img.shields.io/pypi/v/pdoc3.svg?color=blue&style=for-the-badge)](https://pypi.org/project/pdoc3) [![package downloads](https://img.shields.io/pypi/dm/pdoc3.svg?color=skyblue&style=for-the-badge)](https://pypi.org/project/pdoc3) [![GitHub Sponsors](https://img.shields.io/github/sponsors/kernc?color=pink&style=for-the-badge)](https://github.com/sponsors/kernc) -Auto-generate API documentation for Python projects. +Auto-generate API documentation for Python 3+ projects. [**Project website**](https://pdoc3.github.io/pdoc/) diff --git a/doc/build.sh b/doc/build.sh index 044101c5..e53552cf 100755 --- a/doc/build.sh +++ b/doc/build.sh @@ -29,9 +29,9 @@ popd >/dev/null if [ "$IS_RELEASE" ]; then echo -e '\nAdding GAnalytics code\n' - ANALYTICS="" + ANALYTICS="" find "$BUILDROOT" -name '*.html' -print0 | - xargs -0 -- sed -i "s##$ANALYTICS#i" + xargs -0 -- sed -i "s##$ANALYTICS#i" ANALYTICS='' find "$BUILDROOT" -name '*.html' -print0 | xargs -0 -- sed -i "s##$ANALYTICS#i" @@ -41,6 +41,9 @@ fi echo echo 'Testing for broken links' echo +problematic_urls=' +https://www.gnu.org/licenses/agpl-3.0.html +' pushd "$BUILDROOT" >/dev/null grep -PR '/dev/null 2>&1 || - die "broken link in $file: $url" + curl --silent --fail --retry 3 --retry-delay 1 --connect-timeout 10 \ + --user-agent 'Mozilla/5.0 Firefox 125' "$url" >/dev/null 2>&1 || + grep -qF "$url" <(echo "$problematic_urls") || + die "broken link in $file: $url" done done popd >/dev/null diff --git a/pdoc/__init__.py b/pdoc/__init__.py index 0f05b2c0..bb21faf4 100644 --- a/pdoc/__init__.py +++ b/pdoc/__init__.py @@ -19,13 +19,14 @@ import typing from contextlib import contextmanager from copy import copy -from functools import lru_cache, reduce, partial, wraps +from functools import cached_property, lru_cache, reduce, partial, wraps from itertools import tee, groupby -from types import ModuleType +from types import FunctionType, ModuleType from typing import ( # noqa: F401 - cast, Any, Callable, Dict, Generator, Iterable, List, Mapping, NewType, + cast, Any, Callable, Dict, Generator, Iterable, List, Literal, Mapping, NewType, Optional, Set, Tuple, Type, TypeVar, Union, ) +from unittest.mock import Mock from warnings import warn from mako.lookup import TemplateLookup @@ -170,7 +171,7 @@ def html(module_name, docfilter=None, reload=False, skip_errors=False, **kwargs) that takes a single argument (a documentation object) and returns `True` or `False`. If `False`, that object will not be documented. """ - mod = Module(import_module(module_name, reload=reload), + mod = Module(import_module(module_name, reload=reload, skip_errors=False), docfilter=docfilter, skip_errors=skip_errors) link_inheritance() return mod.html(**kwargs) @@ -187,14 +188,18 @@ def text(module_name, docfilter=None, reload=False, skip_errors=False, **kwargs) that takes a single argument (a documentation object) and returns `True` or `False`. If `False`, that object will not be documented. """ - mod = Module(import_module(module_name, reload=reload), + mod = Module(import_module(module_name, reload=reload, skip_errors=False), docfilter=docfilter, skip_errors=skip_errors) link_inheritance() return mod.text(**kwargs) -def import_module(module: Union[str, ModuleType], - *, reload: bool = False) -> ModuleType: +def import_module( + module: Union[str, ModuleType], + *, + reload: bool = False, + skip_errors: bool = False, +) -> ModuleType: """ Return module object matching `module` specification (either a python module path or a filesystem path to file/directory). @@ -221,7 +226,11 @@ def _module_path(module): try: module = importlib.import_module(module_path) except Exception as e: - raise ImportError(f'Error importing {module!r}: {e.__class__.__name__}: {e}') + msg = f'Error importing {module!r}: {e.__class__.__name__}: {e}' + if not skip_errors: + raise ImportError(msg) + warn(msg, category=Module.ImportWarning, stacklevel=2) + module = ModuleType(module_path) assert inspect.ismodule(module) # If this is pdoc itself, return without reloading. Otherwise later @@ -267,8 +276,8 @@ def _pep224_docstrings(doc_obj: Union['Module', 'Class'], *, # Maybe raise exceptions with appropriate message # before using cleaned doc_obj.source _ = inspect.findsource(doc_obj.obj) - tree = ast.parse(doc_obj.source) # type: ignore - except (OSError, TypeError, SyntaxError) as exc: + tree = ast.parse(doc_obj.source) + except (OSError, TypeError, SyntaxError, UnicodeDecodeError) as exc: # Don't emit a warning for builtins that don't have source available is_builtin = getattr(doc_obj.obj, '__module__', None) == 'builtins' if not is_builtin: @@ -316,14 +325,14 @@ def get_name(assign_node): for assign_node, str_node in _pairwise(ast.iter_child_nodes(tree)): if not (isinstance(assign_node, (ast.Assign, ast.AnnAssign)) and isinstance(str_node, ast.Expr) and - isinstance(str_node.value, ast.Str)): + isinstance(str_node.value, ast.Constant)): continue name = get_name(assign_node) if not name: continue - docstring = inspect.cleandoc(str_node.value.s).strip() + docstring = inspect.cleandoc(str_node.value.value).strip() if not docstring: continue @@ -345,7 +354,7 @@ def get_name(assign_node): def get_indent(line): return len(line) - len(line.lstrip()) - source_lines = doc_obj.source.splitlines() # type: ignore + source_lines = doc_obj.source.splitlines() assign_line = source_lines[assign_node.lineno - 1] assign_indent = get_indent(assign_line) comment_lines = [] @@ -410,7 +419,7 @@ def _is_public(ident_name): def _is_function(obj): - return inspect.isroutine(obj) and callable(obj) + return inspect.isroutine(obj) and callable(obj) and not isinstance(obj, Mock) # Mock: GH-350 def _is_descriptor(obj): @@ -420,6 +429,25 @@ def _is_descriptor(obj): inspect.ismemberdescriptor(obj)) +def _unwrap_descriptor(dobj): + obj = dobj.obj + if isinstance(obj, property): + return (getattr(obj, 'fget', False) or + getattr(obj, 'fset', False) or + getattr(obj, 'fdel', obj)) + if isinstance(obj, cached_property): + return obj.func + if isinstance(obj, FunctionType): + return obj + if (inspect.ismemberdescriptor(obj) or + getattr(getattr(obj, '__class__', 0), '__name__', 0) == '_tuplegetter'): + class_name = dobj.qualname.rsplit('.', 1)[0] + obj = getattr(dobj.module.obj, class_name) + return obj + # XXX: Follow descriptor protocol? Already proved buggy in conditions above + return getattr(obj, '__get__', obj) + + def _filter_type(type: Type[T], values: Union[Iterable['Doc'], Mapping[str, 'Doc']]) -> List[T]: """ @@ -451,7 +479,7 @@ def _toposort(graph: Mapping[T, Set[T]]) -> Generator[T, None, None]: assert not graph, f"A cyclic dependency exists amongst {graph!r}" -def link_inheritance(context: Context = None): +def link_inheritance(context: Optional[Context] = None): """ Link inheritance relationsships between `pdoc.Class` objects (and between their members) of all `pdoc.Module` objects that @@ -491,7 +519,7 @@ class Doc: """ __slots__ = ('module', 'name', 'obj', 'docstring', 'inherits') - def __init__(self, name: str, module, obj, docstring: str = None): + def __init__(self, name: str, module, obj, docstring: str = ''): """ Initializes a documentation object, where `name` is the public identifier name, `module` is a `pdoc.Module` object where raw @@ -533,7 +561,7 @@ def __init__(self, name: str, module, obj, docstring: str = None): def __repr__(self): return f'<{self.__class__.__name__} {self.refname!r}>' - @property # type: ignore + @property @lru_cache() def source(self) -> str: """ @@ -541,7 +569,7 @@ def source(self) -> str: available, an empty string. """ try: - lines, _ = inspect.getsourcelines(self.obj) + lines, _ = inspect.getsourcelines(_unwrap_descriptor(self)) except (ValueError, TypeError, OSError): return '' return inspect.cleandoc(''.join(['\n'] + lines)) @@ -566,7 +594,7 @@ def qualname(self) -> str: return getattr(self.obj, '__qualname__', self.name) @lru_cache() - def url(self, relative_to: 'Module' = None, *, link_prefix: str = '', + def url(self, relative_to: Optional['Module'] = None, *, link_prefix: str = '', top_ancestor: bool = False) -> str: """ Canonical relative URL (including page fragment) for this @@ -624,8 +652,10 @@ class Module(Doc): __slots__ = ('supermodule', 'doc', '_context', '_is_inheritance_linked', '_skipped_submodules') - def __init__(self, module: Union[ModuleType, str], *, docfilter: Callable[[Doc], bool] = None, - supermodule: 'Module' = None, context: Context = None, + def __init__(self, module: Union[ModuleType, str], *, + docfilter: Optional[Callable[[Doc], bool]] = None, + supermodule: Optional['Module'] = None, + context: Optional[Context] = None, skip_errors: bool = False): """ Creates a `Module` documentation object given the actual @@ -645,7 +675,7 @@ def __init__(self, module: Union[ModuleType, str], *, docfilter: Callable[[Doc], of raising an exception. """ if isinstance(module, str): - module = import_module(module) + module = import_module(module, skip_errors=skip_errors) super().__init__(module.__name__, self, module) if self.name.endswith('.__init__') and not self.is_package: @@ -683,9 +713,10 @@ def __init__(self, module: Union[ModuleType, str], *, docfilter: Callable[[Doc], except AttributeError: warn(f"Module {self.module!r} doesn't contain identifier `{name}` " "exported in `__all__`") - if not _is_blacklisted(name, self): - obj = inspect.unwrap(obj) - public_objs.append((name, obj)) + else: + if not _is_blacklisted(name, self): + obj = inspect.unwrap(obj) + public_objs.append((name, obj)) else: def is_from_this_module(obj): mod = inspect.getmodule(inspect.unwrap(obj)) @@ -727,6 +758,9 @@ def iter_modules(paths): """ from os.path import isdir, join for pth in paths: + if pth.startswith("__editable__."): + # See https://github.com/pypa/pip/issues/11380 + continue for file in os.listdir(pth): if file.startswith(('.', '__pycache__', '__init__.py')): continue @@ -750,15 +784,9 @@ def iter_modules(paths): assert self.refname == self.name fullname = f"{self.name}.{root}" - try: - m = Module(import_module(fullname), - docfilter=docfilter, supermodule=self, - context=self._context, skip_errors=skip_errors) - except Exception as ex: - if skip_errors: - warn(str(ex), Module.ImportWarning) - continue - raise + m = Module(import_module(fullname, skip_errors=skip_errors), + docfilter=docfilter, supermodule=self, + context=self._context, skip_errors=skip_errors) self.doc[root] = m # Skip empty namespace packages because they may @@ -881,6 +909,8 @@ def html(self, minify=True, **kwargs) -> str: if minify: from pdoc.html_helpers import minify_html html = minify_html(html) + if not html.endswith('\n'): + html = html + '\n' return html @property @@ -1007,7 +1037,7 @@ class Class(Doc): """ __slots__ = ('doc', '_super_members') - def __init__(self, name: str, module: Module, obj, *, docstring: str = None): + def __init__(self, name: str, module: Module, obj, *, docstring: Optional[str] = None): assert inspect.isclass(obj) if docstring is None: @@ -1070,7 +1100,8 @@ def definition_order_index( var_docstrings.get(name) or (inspect.isclass(obj) or _is_descriptor(obj)) and inspect.getdoc(obj)), cls=self, - obj=getattr(obj, 'fget', getattr(obj, '__get__', None)), + kind="prop" if isinstance(obj, property) else "var", + obj=_is_descriptor(obj) and obj or None, instance_var=(_is_descriptor(obj) or name in getattr(self.obj, '__slots__', ()))) @@ -1272,9 +1303,19 @@ def _formatannotation(annot): `typing.Optional`, `nptyping.NDArray` and other types. >>> _formatannotation(NewType('MyType', str)) - 'MyType' + 'pdoc.MyType' >>> _formatannotation(Optional[Tuple[Optional[int], None]]) - 'Optional[Tuple[Optional[int], None]]' + 'Tuple[int | None, None] | None' + >>> _formatannotation(Optional[Union[int, float, None]]) + 'int | float | None' + >>> _formatannotation(Union[int, float]) + 'int | float' + >>> from typing import Callable + >>> _formatannotation(Callable[[Optional[int]], float]) + 'Callable[[int | None], float]' + >>> from collections.abc import Callable + >>> _formatannotation(Callable[[Optional[int]], float]) + 'Callable[[int | None], float]' """ class force_repr(str): __repr__ = str.__str__ @@ -1284,12 +1325,16 @@ def maybe_replace_reprs(a): if a is type(None): # noqa: E721 return force_repr('None') # Union[T, None] -> Optional[T] - if (getattr(a, '__origin__', None) is typing.Union and - len(a.__args__) == 2 and - type(None) in a.__args__): - t = inspect.formatannotation( - maybe_replace_reprs(next(filter(None, a.__args__)))) - return force_repr(f'Optional[{t}]') + if getattr(a, '__origin__', None) is typing.Union: + union_args = a.__args__ + is_optional = type(None) in union_args + if is_optional: + union_args = (x for x in union_args if x is not type(None)) + t = ' | '.join(inspect.formatannotation(maybe_replace_reprs(x)) + for x in union_args) + if is_optional: + t += ' | None' + return force_repr(t) # typing.NewType('T', foo) -> T module = getattr(a, '__module__', '') if module == 'typing' and getattr(a, '__qualname__', '').startswith('NewType.'): @@ -1298,11 +1343,25 @@ def maybe_replace_reprs(a): if module.startswith('nptyping.'): return force_repr(repr(a)) # Recurse into typing.Callable/etc. args - if hasattr(a, 'copy_with') and hasattr(a, '__args__'): - if a is typing.Callable: - # Bug on Python < 3.9, https://bugs.python.org/issue42195 - return a - a = a.copy_with(tuple([maybe_replace_reprs(arg) for arg in a.__args__])) + if hasattr(a, '__args__'): + if hasattr(a, 'copy_with'): + if a is typing.Callable: + # Bug on Python < 3.9, https://bugs.python.org/issue42195 + return a + a = a.copy_with(tuple([maybe_replace_reprs(arg) for arg in a.__args__])) + elif hasattr(a, '__origin__'): + args = tuple(map(maybe_replace_reprs, a.__args__)) + try: + a = a.__origin__[args] + except TypeError: + # collections.abc.Callable takes "([in], out)" + a = a.__origin__[(args[:-1], args[-1])] + # Recurse into lists + if isinstance(a, (list, tuple)): + return type(a)(map(maybe_replace_reprs, a)) + # Shorten standard collections: collections.abc.Callable -> Callable + if module == 'collections.abc': + return force_repr(repr(a).removeprefix('collections.abc.')) return a return str(inspect.formatannotation(maybe_replace_reprs(annot))) @@ -1314,7 +1373,7 @@ class Function(Doc): """ __slots__ = ('cls',) - def __init__(self, name: str, module: Module, obj, *, cls: Class = None): + def __init__(self, name: str, module: Module, obj, *, cls: Optional[Class] = None): """ Same as `pdoc.Doc`, except `obj` must be a Python function object. The docstring is gathered automatically. @@ -1383,7 +1442,8 @@ def return_annotation(self, *, link=None) -> str: lambda: _get_type_hints(cast(Class, self.cls).obj)[self.name], # global variables lambda: _get_type_hints(not self.cls and self.module.obj)[self.name], - lambda: inspect.signature(self.obj).return_annotation, + # properties + lambda: inspect.signature(_unwrap_descriptor(self)).return_annotation, # Use raw annotation strings in unmatched forward declarations lambda: cast(Class, self.cls).obj.__annotations__[self.name], # Extract annotation from the docstring for C builtin function @@ -1416,7 +1476,8 @@ def return_annotation(self, *, link=None) -> str: s = re.sub(r'[\w\.]+', partial(_linkify, link=link, module=self.module), s) return s - def params(self, *, annotate: bool = False, link: Callable[[Doc], str] = None) -> List[str]: + def params(self, *, annotate: bool = False, + link: Optional[Callable[[Doc], str]] = None) -> List[str]: """ Returns a list where each element is a nicely formatted parameter of this function. This includes argument lists, @@ -1564,7 +1625,7 @@ def _signature_from_string(self): try: exec(f'def {string}: pass', _globals, _locals) - except SyntaxError: + except Exception: continue signature = inspect.signature(_locals[self.name]) if cleanup_docstring and len(strings) == 1: @@ -1583,10 +1644,11 @@ class Variable(Doc): Representation of a variable's documentation. This includes module, class, and instance variables. """ - __slots__ = ('cls', 'instance_var') + __slots__ = ('cls', 'instance_var', 'kind') def __init__(self, name: str, module: Module, docstring, *, - obj=None, cls: Class = None, instance_var: bool = False): + obj=None, cls: Optional[Class] = None, instance_var: bool = False, + kind: Literal["prop", "var"] = 'var'): """ Same as `pdoc.Doc`, except `cls` should be provided as a `pdoc.Class` object when this is a class or instance @@ -1606,6 +1668,12 @@ def __init__(self, name: str, module: Module, docstring, *, opposed to class variable). """ + self.kind = kind + """ + `prop` if variable is a dynamic property (has getter/setter or deleter), + or `var` otherwise. + """ + @property def qualname(self) -> str: if self.cls: diff --git a/pdoc/build-index.js b/pdoc/build-index.js new file mode 100644 index 00000000..f5786b15 --- /dev/null +++ b/pdoc/build-index.js @@ -0,0 +1,94 @@ +import vm from 'vm'; + +const LUNR_SCRIPT = 'https://cdnjs.cloudflare.com/ajax/libs/lunr.js/2.3.9/lunr.min.js', + stdin = process.stdin, + stdout = process.stdout, + buffer = []; + +async function loadScript(url) { + const response = await fetch(url); + return await response.text(); +} +async function executeScript(script) { + const sandbox = { window: {}, self: {} }; + vm.runInContext(script, vm.createContext(sandbox)); + return sandbox; +} + +function compact(index) { + /* https://john-millikin.com/compacting-lunr-search-indices */ + function compactInvIndex(index) { + const fields = index["fields"]; + const fieldVectorIdxs = new Map(index["fieldVectors"].map((v, idx) => [v[0], idx])); + const items = new Map(index["invertedIndex"].map(item => { + const token = item[0]; + const props = item[1]; + const newItem = [token]; + fields.forEach(field => { + const fProps = props[field]; + const matches = []; + Object.keys(fProps).forEach(docRef => { + const fieldVectorIdx = fieldVectorIdxs.get(`${field}/${docRef}`); + if (fieldVectorIdx === undefined) { + throw new Error(); + } + matches.push(fieldVectorIdx); + matches.push(fProps[docRef]); + }); + newItem.push(matches); + }); + return [props["_index"], newItem]; + })); + const indexes = Array.from(items.keys()).sort((a, b) => a - b); + const compacted = Array.from(indexes, k => items.get(k)); + return compacted; + } + function compactVectors(index) { + return index["fieldVectors"].map(item => { + const id = item[0]; + const vectors = item[1]; + let prev = null; + const compacted = vectors.map((v, ii) => { + if (ii % 2 === 0) { + if (prev !== null && v === prev + 1) { + prev += 1; + return null; + } + prev = v; + } + return v; + }); + return [id, compacted]; + }); + } + index.invertedIndex = compactInvIndex(index); + index.fieldVectors = compactVectors(index); +} + +let lunr = (await executeScript(await loadScript(LUNR_SCRIPT)))['lunr']; + +stdin.resume(); +stdin.setEncoding('utf8'); + +stdin.on('data', function (data) {buffer.push(data)}); + +stdin.on('end', function () { + const documents = JSON.parse(buffer.join('')); + let idx = lunr(function () { + this.ref('i'); + this.field('name', {boost: 10}); + this.field('ref', {boost: 5}); + this.field('doc'); + this.metadataWhitelist = ['position']; + documents.forEach(function (doc, i) { + const parts = doc.ref.split('.'); + doc['name'] = parts[parts.length - 1]; + doc['i'] = i; + this.add(doc); + }, this) + }) + + let out = idx.toJSON(); + compact(out); + stdout.write(JSON.stringify([out, documents])); +}) diff --git a/pdoc/cli.py b/pdoc/cli.py index e1895819..26a056e6 100755 --- a/pdoc/cli.py +++ b/pdoc/cli.py @@ -9,11 +9,13 @@ import os.path as path import json import re +import subprocess import sys import warnings from contextlib import contextmanager from functools import lru_cache from http.server import BaseHTTPRequestHandler, HTTPServer +from pathlib import Path from typing import Dict, List, Sequence from warnings import warn @@ -397,6 +399,7 @@ def recursive_add_to_index(dobj): info['doc'] = trim_docstring(dobj.docstring) if isinstance(dobj, pdoc.Function): info['func'] = 1 + nonlocal index index.append(info) for member_dobj in getattr(dobj, 'doc', {}).values(): recursive_add_to_index(member_dobj) @@ -414,12 +417,30 @@ def to_url_id(module): recursive_add_to_index(top_module) urls = sorted(url_cache.keys(), key=url_cache.__getitem__) + cmd = ['node', str(Path(__file__).with_name('build-index.js'))] + proc = None + try: + proc = subprocess.Popen(cmd, text=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + stdout, stderr = proc.communicate(json.dumps(index)) + assert proc.poll() == 0, proc.poll() + except Exception as ex: + stderr = proc and proc.stderr and proc.stderr.read() # type: ignore + warn(f'Prebuilding Lunr index with command `{" ".join(cmd)}` failed with ' + f'"{ex.__class__.__name__}: {ex}". Stderr: {stderr or ""!r}. ' + f'Note, the search feature will still work, ' + f'but may be slower (with the index rebuilt just before use). ' + f'To prebuild an index in advance, ensure `node` is executable in the ' + f'pdoc environment.', category=RuntimeWarning) + stdout = ('URLS=' + json.dumps(urls, indent=0, separators=(',', ':')) + + ';\nINDEX=' + json.dumps(index, indent=0, separators=(',', ':'))) + else: + stdout = (f'let [INDEX, DOCS] = {stdout}; ' + f'let URLS={json.dumps(urls, indent=0, separators=(",", ":"))}') main_path = args.output_dir - with _open_write_file(path.join(main_path, 'index.js')) as f: - f.write("URLS=") - json.dump(urls, f, indent=0, separators=(',', ':')) - f.write(";\nINDEX=") - json.dump(index, f, indent=0, separators=(',', ':')) + index_path = Path(main_path).joinpath('index.js') + index_path.write_text(stdout) + print(str(index_path)) # Generate search.html with _open_write_file(path.join(main_path, 'doc-search.html')) as f: @@ -482,13 +503,13 @@ def main(_args=None): # Virtual environment handling for pdoc script run from system site try: - venv_dir = os.environ['VIRTUAL_ENV'] + os.environ['VIRTUAL_ENV'] except KeyError: pass # pdoc was not invoked while in a virtual environment else: from glob import glob - from distutils.sysconfig import get_python_lib - libdir = get_python_lib(prefix=venv_dir) + from sysconfig import get_path + libdir = get_path("platlib") sys.path.append(libdir) # Resolve egg-links from `setup.py develop` or `pip install -e` # XXX: Welcome a more canonical approach @@ -524,18 +545,31 @@ def main(_args=None): httpd.server_close() sys.exit(0) - docfilter = None if args.filter and args.filter.strip(): def docfilter(obj, _filters=args.filter.strip().split(',')): return any(f in obj.refname or isinstance(obj, pdoc.Class) and f in obj.doc for f in _filters) + else: + docfilter = None modules = [pdoc.Module(module, docfilter=docfilter, skip_errors=args.skip_errors) for module in args.modules] pdoc.link_inheritance() + # Loading is done. Output stage ... + config = pdoc._get_config(**template_config) + + # Load configured global markdown extensions + # XXX: This is hereby enabled only for CLI usage as for + # API use I couldn't figure out where reliably to put it. + if config.get('md_extensions'): + from .html_helpers import _md + _kwargs = {'extensions': [], 'configs': {}} + _kwargs.update(config.get('md_extensions', {})) + _md.registerExtensions(**_kwargs) + if args.pdf: _print_pdf(modules, **template_config) import textwrap @@ -582,7 +616,7 @@ def docfilter(obj, _filters=args.filter.strip().split(',')): sys.stdout.write(os.linesep * (1 + 2 * int(module != modules[-1]))) if args.html: - lunr_config = pdoc._get_config(**template_config).get('lunr_search') + lunr_config = config.get('lunr_search') if lunr_config is not None: _generate_lunr_search( modules, lunr_config.get("index_docstrings", True), template_config) @@ -592,7 +626,7 @@ def docfilter(obj, _filters=args.filter.strip().split(',')): pandoc --metadata=title:"MyProject Documentation" \\ --from=markdown+abbreviations+tex_math_single_backslash \\ --pdf-engine=xelatex --variable=mainfont:"DejaVu Sans" \\ - --toc --toc-depth=4 --output=pdf.pdf pdf.md\ + --toc --toc-depth=4 --output=/tmp/pdoc.pdf pdf.md ''' diff --git a/pdoc/documentation.md b/pdoc/documentation.md index 57e1a002..6d089827 100644 --- a/pdoc/documentation.md +++ b/pdoc/documentation.md @@ -81,19 +81,21 @@ by introducing syntax for docstrings for variables. following the normal class inheritance patterns. Consider the following code example: - >>> class A: - ... def test(self): - ... """Docstring for A.""" - ... pass - ... - >>> class B(A): - ... def test(self): - ... pass - ... - >>> A.test.__doc__ - 'Docstring for A.' - >>> B.test.__doc__ - None +```python-repl +>>> class A: +... def test(self): +... """Docstring for A.""" +... pass +... +>>> class B(A): +... def test(self): +... pass +... +>>> A.test.__doc__ +'Docstring for A.' +>>> B.test.__doc__ +None +``` In Python, the docstring for `B.test` doesn't exist, even though a docstring was defined for `A.test`. @@ -115,18 +117,20 @@ For example: [PEP-224]: http://www.python.org/dev/peps/pep-0224 - module_variable = 1 - """PEP 224 docstring for module_variable.""" +```python +module_variable = 1 +"""PEP 224 docstring for module_variable.""" - class C: - #: Documentation comment for class_variable - #: spanning over three lines. - class_variable = 2 #: Assignment line is included. +class C: + #: Documentation comment for class_variable + #: spanning over three lines. + class_variable = 2 #: Assignment line is included. - def __init__(self): - #: Instance variable's doc-comment - self.variable = 3 - """But note, PEP 224 docstrings take precedence.""" + def __init__(self): + #: Instance variable's doc-comment + self.variable = 3 + """But note, PEP 224 docstrings take precedence.""" +``` While the resulting variables have no `__doc__` attribute, `pdoc` compensates by reading the source code (when available) @@ -162,12 +166,14 @@ This feature is useful when there's no feasible way of attaching a docstring to something. A good example is a [namedtuple](https://docs.python.org/3/library/collections.html#collections.namedtuple): - __pdoc__ = {} +```python +__pdoc__ = {} - Table = namedtuple('Table', ['types', 'names', 'rows']) - __pdoc__['Table.types'] = 'Types for each column in the table.' - __pdoc__['Table.names'] = 'The names of each column in the table.' - __pdoc__['Table.rows'] = 'Lists corresponding to each row in the table.' +Table = namedtuple('Table', ['types', 'names', 'rows']) +__pdoc__['Table.types'] = 'Types for each column in the table.' +__pdoc__['Table.names'] = 'The names of each column in the table.' +__pdoc__['Table.rows'] = 'Lists corresponding to each row in the table.' +``` `pdoc` will then show `Table` as a class with documentation for the `types`, `names` and `rows` members. @@ -234,7 +240,7 @@ the identifier name must be fully qualified, for example `pdoc.Doc.docstring`) while \`Doc.docstring\` only works within `pdoc` module. -[backticks]: https://en.wikipedia.org/wiki/Grave_accent#Use_in_programming +[backticks]: https://en.wikipedia.org/wiki/Backtick Command-line interface @@ -246,11 +252,15 @@ For example, to produce HTML documentation of your whole package in subdirectory 'build' of the current directory, using the default HTML template, run: - $ pdoc --html --output-dir build my_package +```shell +$ pdoc --html --output-dir build my_package +``` If you want to omit the source code preview, run: - $ pdoc --html --config show_source_code=False my_package +```shell +$ pdoc --html --config show_source_code=False my_package +``` Find additional template configuration tunables in [custom templates] section below. @@ -258,20 +268,26 @@ section below. To run a local HTTP server while developing your package or writing docstrings for it, run: - $ pdoc --http : my_package +```shell +$ pdoc --http : my_package +``` To re-build documentation as part of your continuous integration (CI) best practice, i.e. ensuring all reference links are correct and up-to-date, make warnings error loudly by settings the environment variable [`PYTHONWARNINGS`][PYTHONWARNINGS] before running pdoc: - $ export PYTHONWARNINGS='error::UserWarning' +```shell +$ export PYTHONWARNINGS='error::UserWarning' +``` [PYTHONWARNINGS]: https://docs.python.org/3/using/cmdline.html#envvar-PYTHONWARNINGS For brief usage instructions, type: - $ pdoc --help +```shell +$ pdoc --help +``` Even more usage examples can be found in the [FAQ]. @@ -292,23 +308,25 @@ Afterwards, you can use `pdoc.Module.html` and `pdoc.Module.text` methods to output documentation in the desired format. For example: - import pdoc +```python +import pdoc - modules = ['a', 'b'] # Public submodules are auto-imported - context = pdoc.Context() +modules = ['a', 'b'] # Public submodules are auto-imported +context = pdoc.Context() - modules = [pdoc.Module(mod, context=context) - for mod in modules] - pdoc.link_inheritance(context) +modules = [pdoc.Module(mod, context=context) + for mod in modules] +pdoc.link_inheritance(context) - def recursive_htmls(mod): - yield mod.name, mod.html() - for submod in mod.submodules(): - yield from recursive_htmls(submod) +def recursive_htmls(mod): + yield mod.name, mod.html() + for submod in mod.submodules(): + yield from recursive_htmls(submod) - for mod in modules: - for module_name, html in recursive_htmls(mod): - ... # Process +for mod in modules: + for module_name, html in recursive_htmls(mod): + ... # Process +``` When documenting a single module, you might find functions `pdoc.html` and `pdoc.text` handy. @@ -353,7 +371,7 @@ modified templates into the `directories` list of the Compatibility ------------- -`pdoc` requires Python 3.6+. +`pdoc` requires Python 3.7+. The last version to support Python 2.x is [pdoc3 0.3.x]. [pdoc3 0.3.x]: https://pypi.org/project/pdoc3/0.3.13/ diff --git a/pdoc/html_helpers.py b/pdoc/html_helpers.py index 8ba63db6..deac0064 100644 --- a/pdoc/html_helpers.py +++ b/pdoc/html_helpers.py @@ -9,7 +9,7 @@ import traceback from contextlib import contextmanager from functools import partial, lru_cache -from typing import Callable, Match +from typing import Callable, Match, Optional from warnings import warn import xml.etree.ElementTree as etree @@ -73,12 +73,12 @@ def glimpse(text: str, max_length=153, *, paragraph=True, output_format='html5', # type: ignore[arg-type] extensions=[ "markdown.extensions.abbr", + "markdown.extensions.admonition", "markdown.extensions.attr_list", "markdown.extensions.def_list", "markdown.extensions.fenced_code", "markdown.extensions.footnotes", "markdown.extensions.tables", - "markdown.extensions.admonition", "markdown.extensions.smarty", "markdown.extensions.toc", ], @@ -248,14 +248,14 @@ def googledoc_sections(match): section = section.title() if section in ('Args', 'Attributes'): body = re.compile( - r'^([\w*]+)(?: \(([\w.,=\[\] -]+)\))?: ' + r'^([\w*]+)(?: \(([\w.,=|\[\] -]+)\))?: ' r'((?:.*)(?:\n(?: {2,}.*|$))*)', re.MULTILINE).sub( lambda m: _ToMarkdown._deflist(*_ToMarkdown._fix_indent(*m.groups())), inspect.cleandoc(f'\n{body}') ) elif section in ('Returns', 'Yields', 'Raises', 'Warns'): body = re.compile( - r'^()([\w.,\[\] ]+): ' + r'^()([\w.,|\[\] ]+): ' r'((?:.*)(?:\n(?: {2,}.*|$))*)', re.MULTILINE).sub( lambda m: _ToMarkdown._deflist(*_ToMarkdown._fix_indent(*m.groups())), inspect.cleandoc(f'\n{body}') @@ -275,6 +275,9 @@ def _admonition(match, module=None, limit_types=None): if limit_types and type not in limit_types: return match.group(0) + if text is None: + text = "" + if type == 'include' and module: try: return _ToMarkdown._include_file(indent, value, @@ -286,7 +289,7 @@ def _admonition(match, module=None, limit_types=None): '\n': ' ', '[': '\\[', ']': '\\]'})).strip() - return f'{indent}![{alt_text}]({value})\n' + return f'{indent}![{alt_text}]({value}){{: loading=lazy}}\n' if type == 'math': return _ToMarkdown.indent(indent, f'\\[ {text.strip()} \\]', @@ -323,7 +326,8 @@ def admonitions(text, module, limit_types=None): See: https://python-markdown.github.io/extensions/admonition/ """ substitute = partial(re.compile(r'^(?P *)\.\. ?(\w+)::(?: *(.*))?' - r'((?:\n(?:(?P=indent) +.*| *$))*)', re.MULTILINE).sub, + r'((?:\n(?:(?P=indent) +.*| *$))*[^\r\n])*', + re.MULTILINE).sub, partial(_ToMarkdown._admonition, module=module, limit_types=limit_types)) # Apply twice for nested (e.g. image inside warning) @@ -404,8 +408,9 @@ def handleMatch(self, m, data): def to_html(text: str, *, - docformat: str = None, - module: pdoc.Module = None, link: Callable[..., str] = None, + docformat: Optional[str] = None, + module: Optional[pdoc.Module] = None, + link: Optional[Callable[..., str]] = None, latex_math: bool = False): """ Returns HTML of `text` interpreted as `docformat`. `__docformat__` is respected @@ -429,8 +434,9 @@ def to_html(text: str, *, def to_markdown(text: str, *, - docformat: str = None, - module: pdoc.Module = None, link: Callable[..., str] = None): + docformat: Optional[str] = None, + module: Optional[pdoc.Module] = None, + link: Optional[Callable[..., str]] = None): """ Returns `text`, assumed to be a docstring in `docformat`, converted to markdown. `__docformat__` is respected @@ -558,20 +564,21 @@ def format_git_link(template: str, dobj: pdoc.Doc): try: if 'commit' in _str_template_fields(template): commit = _git_head_commit() - abs_path = inspect.getfile(inspect.unwrap(dobj.obj)) + obj = pdoc._unwrap_descriptor(dobj) + abs_path = inspect.getfile(inspect.unwrap(obj)) path = _project_relative_path(abs_path) # Urls should always use / instead of \\ if os.name == 'nt': path = path.replace('\\', '/') - lines, start_line = inspect.getsourcelines(dobj.obj) + lines, start_line = inspect.getsourcelines(obj) start_line = start_line or 1 # GH-296 end_line = start_line + len(lines) - 1 url = template.format(**locals()) return url except Exception: - warn(f'format_git_link for {dobj.obj} failed:\n{traceback.format_exc()}') + warn(f'format_git_link for {obj} failed:\n{traceback.format_exc()}') return None @@ -618,9 +625,9 @@ def _project_relative_path(absolute_path): Assumes the project's path is either the current working directory or Python library installation. """ - from distutils.sysconfig import get_python_lib - for prefix_path in (_git_project_root() or os.getcwd(), - get_python_lib()): + from sysconfig import get_path + libdir = get_path("platlib") + for prefix_path in (_git_project_root() or os.getcwd(), libdir): common_path = os.path.commonpath([prefix_path, absolute_path]) if os.path.samefile(common_path, prefix_path): # absolute_path is a descendant of prefix_path diff --git a/pdoc/package.json b/pdoc/package.json new file mode 100644 index 00000000..6990891f --- /dev/null +++ b/pdoc/package.json @@ -0,0 +1 @@ +{"type": "module"} diff --git a/pdoc/templates/_lunr_search.inc.mako b/pdoc/templates/_lunr_search.inc.mako index 19dd4323..d7b4f663 100644 --- a/pdoc/templates/_lunr_search.inc.mako +++ b/pdoc/templates/_lunr_search.inc.mako @@ -2,8 +2,8 @@ - - + + % if google_analytics: + + window.dataLayer = window.dataLayer || []; + function gtag(){dataLayer.push(arguments);} + gtag('js', new Date()); + gtag('config', '${google_analytics}'); + % endif % if google_search_query: @@ -408,12 +415,27 @@ % endif % if latex_math: + % endif % if syntax_highlighting: - - + + % endif <%include file="head.mako"/> diff --git a/pdoc/templates/search.mako b/pdoc/templates/search.mako index 090a7f66..6d8296a7 100644 --- a/pdoc/templates/search.mako +++ b/pdoc/templates/search.mako @@ -4,8 +4,8 @@ Search - - + +