'''
Python packaging operations, including PEP-517 support, for use by a
``setup.py`` script.

Overview:

    The intention is to take care of as many packaging details as possible so
    that setup.py contains only project-specific information, while also giving
    as much flexibility as possible.

    For example we provide a function `pipcl.build_extension()` that can be used
    to build a SWIG extension, but we also give access to the located
    compiler/linker so that a `setup.py` script can take over the details
    itself.

Doctests:
    Doctest strings are provided in some comments.

    Test in the usual way with::
    
        python -m doctest pipcl.py

    Test specific functions/classes with::
    
        python pipcl.py --doctest run_if ...

    If no functions or classes are specified, this tests everything.

Graal:
    For Graal we require that PIPCL_GRAAL_PYTHON is set to non-graal Python (we
    build for non-graal except with Graal Python's include paths and library
    directory).
'''

import atexit
import base64
import calendar
import codecs
import difflib
import glob
import hashlib
import inspect
import io
import os
import pickle
import platform
import re
import selectors
import shlex
import shutil
import subprocess
import sys
import sysconfig
import tarfile
import textwrap
import time
import zipfile

try:
    import wdev
except ImportError:
    from . import wdev


class Package:
    '''
    Represents a Python package.
    
    The constructor takes a definition of a Python package similar to that
    passed to `distutils.core.setup()` or `setuptools.setup()` (name, version,
    summary etc) plus callbacks for building, getting a list of sdist
    filenames, and cleaning.

    Methods are provided that can be used to implement a Python package's
    `setup.py` supporting PEP-517.

    Basic command line handling is also supported for
    use with a legacy (pre-PEP-517) pip, as implemented
    by legacy distutils/setuptools and described in:
    https://pip.pypa.io/en/stable/reference/build-system/setup-py/

    The file pyproject.toml must exist; this is checked if/when `fn_build()` is
    called.

    Here is a `doctest` example of using pipcl to create a SWIG extension
    module called 'foo', with a console-script command line tool called
    `cli`. Requires `swig`.

    Create an empty test directory:

        >>> import os
        >>> import shutil
        >>> shutil.rmtree('pipcl_test', ignore_errors=1)
        >>> os.mkdir('pipcl_test')

    Create a `setup.py` which uses `pipcl` to define an extension module.

        >>> import textwrap
        >>> with open('pipcl_test/setup.py', 'w') as f:
        ...     _ = f.write(textwrap.dedent("""
        ...             import sys
        ...             import pipcl
        ...
        ...             def build():
        ...                 so_leaf = pipcl.build_extension(
        ...                         name = 'foo',
        ...                         path_i = 'foo.i',
        ...                         outdir = 'build',
        ...                         source_extra = 'wibble.c',
        ...                         )
        ...                 return [
        ...                         ('build/foo.py', 'foo/__init__.py'),
        ...                         ('cli.py', 'foo/__main__.py'),
        ...                         (f'build/{so_leaf}', f'foo/'),
        ...                         (b'Hello world', 'foo/hw.txt'),
        ...                         ]
        ...
        ...             def sdist():
        ...                 return [
        ...                         'foo.i',
        ...                         'bar.i',
        ...                         'wibble.c',
        ...                         'setup.py',
        ...                         'pipcl.py',
        ...                         'pyproject.toml',
        ...                         'wdev.py',
        ...                         'README',
        ...                         ]
        ...
        ...             p = pipcl.Package(
        ...                     name = 'foo',
        ...                     version = '1.2.3',
        ...                     description = 'README',
        ...                     fn_build = build,
        ...                     fn_sdist = sdist,
        ...                     entry_points = (
        ...                         { 'console_scripts': [
        ...                             'foo_cli = foo.__main__:main',
        ...                             ],
        ...                         }),
        ...                     )
        ...
        ...             build_wheel = p.build_wheel
        ...             build_sdist = p.build_sdist
        ...
        ...             # Handle old-style setup.py command-line usage:
        ...             if __name__ == '__main__':
        ...                 p.handle_argv(sys.argv)
        ...             """))

    Create the files required by the above `setup.py` - pyproject.toml, the
    SWIG `.i` input file, the README file, and copies of `pipcl.py` and
    `wdev.py`.

        >>> with open('pipcl_test/pyproject.toml', 'w') as f:
        ...     _ = f.write(textwrap.dedent("""
        ...             [build-system]
        ...             requires = []
        ...
        ...             # See pep-517.
        ...             #
        ...             build-backend = "setup"
        ...             backend-path = ["."]
        ...             """))

        >>> with open('pipcl_test/foo.i', 'w') as f:
        ...     _ = f.write(textwrap.dedent("""
        ...             %include bar.i
        ...             %{
        ...             #include <stdio.h>
        ...             #include <string.h>
        ...             int bar(const char* text)
        ...             {
        ...                 printf("bar(): text: %s\\\\n", text);
        ...                 int len = (int) strlen(text);
        ...                 printf("bar(): len=%i\\\\n", len);
        ...                 fflush(stdout);
        ...                 return len;
        ...             }
        ...             %}
        ...             int bar(const char* text);
        ...             """))

        >>> with open('pipcl_test/bar.i', 'w') as f:
        ...     _ = f.write( '\\n')

        >>> with open('pipcl_test/wibble.c', 'w') as f:
        ...     _ = f.write( '\\n')

        >>> with open('pipcl_test/README', 'w') as f:
        ...     _ = f.write(textwrap.dedent("""
        ...             This is Foo.
        ...             """))

        >>> with open('pipcl_test/cli.py', 'w') as f:
        ...     _ = f.write(textwrap.dedent("""
        ...             def main():
        ...                 print('pipcl_test:main().')
        ...             if __name__ == '__main__':
        ...                 main()
        ...             """))

        >>> root = os.path.dirname(__file__)
        >>> _ = shutil.copy2(f'{root}/pipcl.py', 'pipcl_test/pipcl.py')
        >>> _ = shutil.copy2(f'{root}/wdev.py', 'pipcl_test/wdev.py')

    Use `setup.py`'s command-line interface to build and install the extension
    module into root `pipcl_test/install`.

        >>> _ = subprocess.run(
        ...         f'cd pipcl_test && {sys.executable} setup.py --root install install',
        ...         shell=1, check=1)

    The actual install directory depends on `sysconfig.get_path('platlib')`:

        >>> if windows():
        ...     install_dir = 'pipcl_test/install'
        ... else:
        ...     install_dir = f'pipcl_test/install/{sysconfig.get_path("platlib").lstrip(os.sep)}'
        >>> assert os.path.isfile( f'{install_dir}/foo/__init__.py')

    Create a test script which asserts that Python function call `foo.bar(s)`
    returns the length of `s`, and run it with `PYTHONPATH` set to the install
    directory:

        >>> with open('pipcl_test/test.py', 'w') as f:
        ...     _ = f.write(textwrap.dedent("""
        ...             import sys
        ...             import foo
        ...             text = 'hello'
        ...             print(f'test.py: calling foo.bar() with text={text!r}')
        ...             sys.stdout.flush()
        ...             l = foo.bar(text)
        ...             print(f'test.py: foo.bar() returned: {l}')
        ...             assert l == len(text)
        ...             """))
        >>> r = subprocess.run(
        ...         f'{sys.executable} pipcl_test/test.py',
        ...         shell=1, check=1, text=1,
        ...         stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
        ...         env=os.environ | dict(PYTHONPATH=install_dir),
        ...         )
        >>> print(r.stdout)
        test.py: calling foo.bar() with text='hello'
        bar(): text: hello
        bar(): len=5
        test.py: foo.bar() returned: 5
        <BLANKLINE>

    Check that building sdist and wheel succeeds. For now we don't attempt to
    check that the sdist and wheel actually work.

        >>> _ = subprocess.run(
        ...         f'cd pipcl_test && {sys.executable} setup.py sdist',
        ...         shell=1, check=1)

        >>> _ = subprocess.run(
        ...         f'cd pipcl_test && {sys.executable} setup.py bdist_wheel',
        ...         shell=1, check=1)

    Check that rebuild does nothing.

        >>> t0 = os.path.getmtime('pipcl_test/build/foo.py')
        >>> _ = subprocess.run(
        ...         f'cd pipcl_test && {sys.executable} setup.py bdist_wheel',
        ...         shell=1, check=1)
        >>> t = os.path.getmtime('pipcl_test/build/foo.py')
        >>> assert t == t0

    Check that touching bar.i forces rebuild.

        >>> os.utime('pipcl_test/bar.i')
        >>> _ = subprocess.run(
        ...         f'cd pipcl_test && {sys.executable} setup.py bdist_wheel',
        ...         shell=1, check=1)
        >>> t = os.path.getmtime('pipcl_test/build/foo.py')
        >>> assert t > t0

    Check that touching foo.i.cpp does not run swig, but does recompile/link.

        >>> so_suffix = 'pyd' if windows() else 'so'
        >>> t0 = time.time()
        >>> os.utime('pipcl_test/build/foo.i.cpp')
        >>> _ = subprocess.run(
        ...         f'cd pipcl_test && {sys.executable} setup.py bdist_wheel',
        ...         shell=1, check=1)
        >>> assert os.path.getmtime('pipcl_test/build/foo.py') <= t0
        >>> so = glob.glob(f'pipcl_test/build/*.{so_suffix}')
        >>> assert len(so) == 1, f'{so_suffix=} {so=} {os.path.abspath(".")=}'
        >>> so = so[0]
        >>> assert os.path.getmtime(so) > t0

    Check that touching wibbble.c does not run swig, but does recompile/link.

        >>> t0 = time.time()
        >>> os.utime('pipcl_test/wibble.c')
        >>> _ = subprocess.run(
        ...         f'cd pipcl_test && {sys.executable} setup.py bdist_wheel',
        ...         shell=1, check=1)
        >>> assert os.path.getmtime('pipcl_test/build/foo.py') <= t0
        >>> so = glob.glob(f'pipcl_test/build/*.{so_suffix}')
        >>> assert len(so) == 1
        >>> so = so[0]
        >>> assert os.path.getmtime(so) > t0

    Check `entry_points` causes creation of command `foo_cli` when we install
    from our wheel using pip. [As of 2024-02-24 using pipcl's CLI interface
    directly with `setup.py install` does not support entry points.]

        >>> print('Creating venv.', file=sys.stderr)
        >>> _ = subprocess.run(
        ...         f'cd pipcl_test && {sys.executable} -m venv pylocal',
        ...         shell=1, check=1)

        >>> print('Installing from wheel into venv using pip.', file=sys.stderr)
        >>> whl = glob.glob(f'pipcl_test/dist/*.whl')[0]
        >>> _ = subprocess.run(f'{venv_enter_cmd("pipcl_test/pylocal")} && pip install {whl}',
        ...         shell=1, check=1)

        >>> print('Running foo_cli.', file=sys.stderr)
        >>> _ = subprocess.run(
        ...         f'{venv_enter_cmd("pipcl_test/pylocal")} && foo_cli',
        ...         shell=1, check=1)
    
    Check that wheel contents' unix permissions are readable by all.
    
        >>> wheel_path = glob.glob('pipcl_test/dist/*.whl')[0]
        >>> with zipfile.ZipFile(wheel_path) as z:
        ...     for zi in z.infolist():
        ...         external_attr_lo = zi.external_attr & 15
        ...         external_attr_hi = zi.external_attr >> 16
        ...         assert external_attr_lo + (external_attr_hi << 16) == zi.external_attr
        ...         assert external_attr_hi & 0o444 == 0o444, (
        ...                 f'info.external_attr not readable by all:'
        ...                 f' {zi.external_attr=:#x}'
        ...                 f' {external_attr_hi=:#o}'
        ...                 f' {external_attr_lo=:#o}'
        ...                 f' {zi.filename=}'
        ...                 )

    Wheels and sdists

        Wheels:
            We generate wheels according to:
            https://packaging.python.org/specifications/binary-distribution-format/

            * `{name}-{version}.dist-info/RECORD` uses sha256 hashes.
            * Other `RECORD*` files such as `RECORD.jws` or `RECORD.p7s` are
              not generated.
            * `{name}-{version}.dist-info/WHEEL` has:

              * `Wheel-Version: 1.0`
              * `Root-Is-Purelib: false`
            * No support for signed wheels.

        Sdists:
            We generate sdist's according to:
            https://packaging.python.org/specifications/source-distribution-format/
    '''
    def __init__(self,
            name,
            version,
            *,
            author = None,
            author_email = None,
            classifier = None,
            description = None,
            description_content_type = None,
            download_url = None,
            home_page = None,
            keywords = None,
            license = None, # pylint: disable=redefined-builtin
            maintainer = None,
            maintainer_email = None,
            platform = None,    # pylint: disable=redefined-outer-name
            project_url = None,
            provides_extra = None,
            requires_dist = None,
            requires_external = None,
            requires_python = None,
            summary = None,
            supported_platform = None,
            
            entry_points = None,
            
            fn_build = None,
            fn_clean = None,
            fn_sdist = None,
            
            pure=False,
            py_limited_api = None,
            
            tag_python = None,
            tag_abi = None,
            tag_platform = None,
            
            root = None,
            
            wheel_compression = zipfile.ZIP_DEFLATED,
            wheel_compresslevel = None,
            ):
        '''
        The initial args before `entry_points` define the
        package metadata and closely follow the definitions in:
        https://packaging.python.org/specifications/core-metadata/

        Args:

            name:
                Used for metadata `Name`.
                A string, the name of the Python package.
            
            version:
                Used for metadata `Version`.
                A string, the version of the Python package. Also see PEP-440
                `Version Identification and Dependency Specification`.
            
            author:
                Used for metadata `Author`.
                Author name.
            
            author_email:
                Used for metadata `Author-email`.
                Author email.
            
            classifier:
                Used for metadata `Classifier`.
                A string or list of strings. Also see:

                * https://pypi.org/pypi?%3Aaction=list_classifiers
                * https://pypi.org/classifiers/
            
            description:
                Used for metadata `Description`.
                A string. If contains newlines, a detailed description of the
                package. Otherwise the path of a file containing the detailed
                description of the package.
            
            description_content_type:
                Used for metadata `Description-Content-Type`.
                A string describing markup of `description` arg. For example
                `text/markdown; variant=GFM`.
                If not specified we try to infer a value from <description>
                file suffix.
            
            download_url:
                Used for metadata `Download-URL`.
                Where this version can be downloaded from.
                Deprecated since metadata-1.2 - use project_url.
            
            home_page:
                Used for metadata `Home-page`.
                URL of home page.
                Deprecated since metadata-1.2 - use project_url.
            
            keywords:
                Used for metadata `Keywords`.
                A string containing comma-separated keywords.
            
            license:
                Used for metadata `License`.
                A string containing the license text. Written into metadata
                file `COPYING`. Is also written into metadata itself if not
                multi-line.
            
            maintainer:
                Used for metadata `Maintainer`.
                Maintainer.
            
            maintainer_email:
                Used for metadata `Maintainer-email`.
                Maintainer email.
            
            platform:
                Used for metadata `Platform`.
                A string or list of strings.
            
            project_url:
                Used for metadata `Project-URL`.
                A string or list of strings, each of the form: `{name}, {url}`.
                Well-known names include: homepage, source, documentation,
                issues; for more details, see:
                https://packaging.python.org/en/latest/specifications/well-known-project-urls/#well-known-project-urls
            
            provides_extra:
                Used for metadata `Provides-Extra`.
                A string or list of strings.
            
            requires_dist:
                Used for metadata `Requires-Dist`.
                A string or list of strings, Python packages required
                at runtime. None items are ignored.
            
            requires_external:
                Used for metadata `Requires-External`.
                A string or list of strings.
            
            requires_python:
                Used for metadata `Requires-Python`.
                A string or list of strings.
            
            summary:
                Used for metadata `Summary`.
                A string, short description of the package.
            
            supported_platform:
                Used for metadata `Supported-Platform`.
                A string or list of strings.

            entry_points:
                A string specifying *.dist-info/entry_points.txt, for
                example::

                    [console_scripts]
                    foo_cli = foo.__main__:main

                See: https://packaging.python.org/en/latest/specifications/entry-points/
                
                Alternatively one can specify a dict similar to that supported
                by setuptools (see
                https://setuptools.pypa.io/en/latest/userguide/entry_point.html)::

                    {
                        'console_scripts':
                        [
                            'foo_cli = foo.__main__:main',
                        ],
                    }

            fn_build:
                A function taking no args, or a single `config_settings` dict
                arg (as described in PEP-517). This should build all required
                files and return a list of items; each item should be a tuple
                `(from_, to_)`, or a single string `path` which is treated as
                the tuple `(path, path)`.

                `from_` can be a string or a `bytes`. If a string it should
                be the path to a file; a relative path is treated as relative
                to `root`. If a `bytes` it is the contents of the file to be
                added.

                `to_` identifies what the file should be called within a wheel
                or when installing. If `to_` is empty or `/` we set it to the
                leaf of `from_` (`from_` must not be a `bytes`) - i.e. we place
                the file in the root directory of the wheel; otherwise if
                `to_` ends with `/` the leaf of `from_` is appended to it (and
                `from_` must not be a `bytes`).

                Initial `$dist-info/` in `_to` is replaced by
                `{name}-{version}.dist-info/`; this is useful for license files
                etc.

                Initial `$data/` in `_to` is replaced by
                `{name}-{version}.data/`. We do not enforce particular
                subdirectories, instead it is up to `fn_build()` to specify
                specific subdirectories such as `purelib`, `headers`,
                `scripts`, `data` etc.

                If we are building a wheel (e.g. `python setup.py bdist_wheel`,
                or PEP-517 pip calls `self.build_wheel()`), we add file `from_`
                to the wheel archive with name `to_`.

                If we are installing (e.g. `install` command in
                the argv passed to `self.handle_argv()`), then
                we copy `from_` to `{sitepackages}/{to_}`, where
                `sitepackages` is the installation directory, the
                default being `sysconfig.get_path('platlib')` e.g.
                `myvenv/lib/python3.9/site-packages/`.

                When calling this function, we assert that the file
                pyproject.toml exists in the current directory. (We do this
                here rather than in pipcl.Package's constructor, as otherwise
                importing setup.py from non-package-related code could fail.)

            fn_clean:
                A function taking a single arg `all_` that cleans generated
                files. `all_` is true iff `--all` is in argv.

                For safety and convenience, can also returns a list of
                files/directory paths to be deleted. Relative paths are
                interpreted as relative to `root`. All paths are asserted to be
                within `root`.

            fn_sdist:
                A function taking no args, or a single `config_settings` dict
                arg (as described in PEP517), that returns a list of items to
                be copied into the sdist. The list should be in the same format
                as returned by `fn_build`.

                It can be convenient to use `pipcl.git_items()`.

                The specification for sdists requires that the list contains
                `pyproject.toml`; we enforce this with a Python assert.

            pure:
                If true we mark wheels as pure python - for example default
                wheel tag will be `py3-none-any`.

            py_limited_api:
                If true we build extension modules that use the Python
                Limited API. We use the version of `sys.executable` to define
                `Py_LIMITED_API` when compiling extensions, and use ABI tag
                `abi3` in the wheel name if argument `tag_abi` is None.
            
            tag_python:
                First element of wheel tag defined in PEP-425. If None we use
                `cp{version}`.

            tag_abi:
                Second element of wheel tag defined in PEP-425. If None we use
                `none`.

            tag_platform:
                Third element of wheel tag defined in PEP-425. Default
                is `os.environ('AUDITWHEEL_PLAT')` if set, otherwise
                derived from `sysconfig.get_platform()` (was
                `setuptools.distutils.util.get_platform()`, before that
                `distutils.util.get_platform()` as specified in the PEP), e.g.
                `openbsd_7_0_amd64`.

            root:
                Root of package, defaults to current directory.
            
            wheel_compression:
                Used as `zipfile.ZipFile()`'s `compression` parameter when
                creating wheels.

        Occurrences of `None` in lists are ignored.
        '''
        assert name
        assert version

        def assert_str( v):
            if v is not None:
                assert isinstance( v, str), f'Not a string: {v!r}'
        def assert_str_or_multi( v):
            if v is not None:
                assert isinstance( v, (str, tuple, list)), f'Not a string, tuple or list: {v!r}'

        assert_str( name)
        assert_str( version)
        assert_str_or_multi( platform)
        assert_str_or_multi( supported_platform)
        assert_str( summary)
        assert_str( description)
        assert_str( description_content_type)
        assert_str( keywords)
        assert_str( home_page)
        assert_str( download_url)
        assert_str( author)
        assert_str( author_email)
        assert_str( maintainer)
        assert_str( maintainer_email)
        assert_str( license)
        assert_str_or_multi( classifier)
        assert_str_or_multi( requires_dist)
        assert_str_or_multi( requires_python)
        assert_str_or_multi( requires_external)
        assert_str_or_multi( project_url)
        assert_str_or_multi( provides_extra)
        
        assert re.match('^([A-Z0-9]|[A-Z0-9][A-Z0-9._-]*[A-Z0-9])\\Z', name, re.IGNORECASE), (
                f'Invalid package name'
                f' (https://packaging.python.org/en/latest/specifications/name-normalization/)'
                f': {name!r}'
                )

        # https://packaging.python.org/en/latest/specifications/core-metadata/.
        assert re.match('([A-Z0-9]|[A-Z0-9][A-Z0-9._-]*[A-Z0-9])$', name, re.IGNORECASE), \
                f'Bad name: {name!r}'

        _assert_version_pep_440(version)

        # https://packaging.python.org/en/latest/specifications/binary-distribution-format/
        if tag_python:
            assert '-' not in tag_python
        if tag_abi:
            assert '-' not in tag_abi
        if tag_platform:
            assert '-' not in tag_platform

        self.name = name
        self.version = version
        self.pure = pure
        self.platform = platform
        self.supported_platform = supported_platform
        self.summary = summary
        self.description = description
        self.description_content_type = description_content_type
        self.keywords = keywords
        self.home_page = home_page
        self.download_url = download_url
        self.author = author
        self.author_email = author_email
        self.maintainer = maintainer
        self.maintainer_email = maintainer_email
        self.license = license
        self.classifier = classifier
        self.requires_dist = requires_dist
        self.requires_python = requires_python
        self.requires_external = requires_external
        self.project_url = project_url
        self.provides_extra = provides_extra
        self.entry_points = entry_points

        self.root = os.path.abspath(root if root else os.getcwd())
        self.fn_build = fn_build
        self.fn_clean = fn_clean
        self.fn_sdist = fn_sdist
        self.tag_python_ = tag_python
        self.tag_abi_ = tag_abi
        self.tag_platform_ = tag_platform
        self.py_limited_api = py_limited_api

        self.wheel_compression = wheel_compression
        self.wheel_compresslevel = wheel_compresslevel
        
        # If true and we are building for graal, we set PIPCL_PYTHON_CONFIG to
        # a command that will print includes/libs from graal_py's sysconfig.
        #
        self.graal_legacy_python_config = True

    def build_wheel(self,
            wheel_directory,
            config_settings=None,
            metadata_directory=None,
            ):
        '''
        A PEP-517 `build_wheel()` function.

        Also called by `handle_argv()` to handle the `bdist_wheel` command.

        Returns leafname of generated wheel within `wheel_directory`.
        '''
        log2(
                f' wheel_directory={wheel_directory!r}'
                f' config_settings={config_settings!r}'
                f' metadata_directory={metadata_directory!r}'
                )

        if os.environ.get('CIBUILDWHEEL') == '1':
            # Don't special-case graal builds when running under cibuildwheel.
            pass
        elif sys.implementation.name == 'graalpy':
            # We build for Graal by building a native Python wheel with Graal
            # Python's include paths and library directory. We then rename the
            # wheel to contain graal's tag etc.
            #
            log0(f'### Graal build: deferring to cpython.')
            python_native = os.environ.get('PIPCL_GRAAL_PYTHON')
            assert python_native, f'Graal build requires that PIPCL_GRAAL_PYTHON is set.'
            env_extra = dict(
                    PIPCL_SYSCONFIG_PATH_include = sysconfig.get_path('include'),
                    PIPCL_SYSCONFIG_PATH_platinclude = sysconfig.get_path('platinclude'),
                    PIPCL_SYSCONFIG_CONFIG_VAR_LIBDIR = sysconfig.get_config_var('LIBDIR'),
                    )
            # Tell native build to run pipcl.py itself to get python-config
            # information about include paths etc.
            if self.graal_legacy_python_config:
                env_extra['PIPCL_PYTHON_CONFIG'] = f'{python_native} {os.path.abspath(__file__)} --graal-legacy-python-config'
            
            # Create venv.
            venv_name = os.environ.get('PIPCL_GRAAL_NATIVE_VENV')
            if venv_name:
                log1(f'Graal using pre-existing {venv_name=}')
            else:
                venv_name = 'venv-pipcl-graal-native'
                run(f'{shlex.quote(python_native)} -m venv {venv_name}')
                log1(f'Graal using {venv_name=}')
            
            newfiles = NewFiles(f'{wheel_directory}/*.whl')
            run(
                    f'. {venv_enter_cmd(venv_name)} && python setup.py --dist-dir {shlex.quote(wheel_directory)} bdist_wheel',
                    env_extra = env_extra,
                    prefix = f'pipcl.py graal {python_native}: ',
                    )
            wheel = newfiles.get_one()
            wheel_leaf = os.path.basename(wheel)
            python_major_minor = run(f'{shlex.quote(python_native)} -c "import platform; import sys; sys.stdout.write(str().join(platform.python_version_tuple()[:2]))"', capture=1)
            cpabi = f'cp{python_major_minor}-abi3'
            assert cpabi in wheel_leaf, f'Expected wheel to be for {cpabi=}, but {wheel=}.'
            graalpy_ext_suffix = sysconfig.get_config_var('EXT_SUFFIX')
            log1(f'{graalpy_ext_suffix=}')
            m = re.match(r'\.graalpy(\d+[^\-]*)-(\d+)', graalpy_ext_suffix)
            gpver = m[1]
            cpver = m[2]
            graalpy_wheel_tag = f'graalpy{cpver}-graalpy{gpver}_{cpver}_native'
            name = wheel_leaf.replace(cpabi, graalpy_wheel_tag)
            destination = f'{wheel_directory}/{name}'
            log0(f'### Graal build: copying {wheel=} to {destination=}')
            # Copying results in two wheels which appears to confuse pip, showing:
            #   Found multiple .whl files; unspecified behaviour. Will call build_wheel.
            os.rename(wheel, destination)
            log1(f'Returning {name=}.')
            return name

        wheel_name = self.wheel_name()
        path = f'{wheel_directory}/{wheel_name}'

        # Do a build and get list of files to copy into the wheel.
        #
        items = list()
        if self.fn_build:
            items = self._call_fn_build(config_settings)

        log2(f'Creating wheel: {path}')
        os.makedirs(wheel_directory, exist_ok=True)
        from_maxlen = 0
        for item in items:
            from_, _ = self._fromto(item)
            if isinstance(from_, str):
                from_ = relpath(from_)
                from_maxlen = max(from_maxlen, len(from_))
        record = _Record()
        with zipfile.ZipFile(path, 'w', self.wheel_compression, self.wheel_compresslevel) as z:

            def add(from_, to_):
                if isinstance(from_, str):
                    z.write(from_, to_)
                    record.add_file(from_, to_, from_maxlen=from_maxlen)
                elif isinstance(from_, bytes):
                    zipfile_writestr(z, to_, from_, 0o644)
                    record.add_content(from_, to_)
                else:
                    assert 0

            def add_str(content, to_):
                add(content.encode('utf8'), to_)

            dist_info_dir = self._dist_info_dir()

            # Add the files returned by fn_build().
            #
            for item in items:
                from_, (_to_abs, to_rel) = self._fromto(item)
                add(from_, to_rel)

            # Add <name>-<version>.dist-info/WHEEL.
            #
            add_str(
                    f'Wheel-Version: 1.0\n'
                    f'Generator: pipcl\n'
                    f'Root-Is-Purelib: {"true" if self.pure else "false"}\n'
                    f'Tag: {self.wheel_tag_string()}\n'
                    ,
                    f'{dist_info_dir}/WHEEL',
                    )
            # Add <name>-<version>.dist-info/METADATA.
            #
            add_str(self._metainfo(), f'{dist_info_dir}/METADATA')

            # Add <name>-<version>.dist-info/COPYING.
            if self.license:
                add_str(self.license, f'{dist_info_dir}/COPYING')

            # Add <name>-<version>.dist-info/entry_points.txt.
            entry_points_text = self._entry_points_text()
            if entry_points_text:
                add_str(entry_points_text, f'{dist_info_dir}/entry_points.txt')

            # Update <name>-<version>.dist-info/RECORD. This must be last.
            #
            zipfile_writestr(
                    z,
                    f'{dist_info_dir}/RECORD',
                    record.get(f'{dist_info_dir}/RECORD'),
                    0o644,
                    )

        st = os.stat(path)
        log1( f'Have created wheel size={st.st_size:,}: {path}')
        if g_verbose >= 2:
            with zipfile.ZipFile(path, compression=self.wheel_compression) as z:
                log2(f'Contents are:')
                for zi in sorted(z.infolist(), key=lambda z: z.filename):
                    log2(f'    {zi.file_size: 10,d} {zi.filename}')

        return os.path.basename(path)

    def build_sdist(self,
            sdist_directory,
            formats,
            config_settings=None,
            ):
        '''
        A PEP-517 `build_sdist()` function.

        Also called by `handle_argv()` to handle the `sdist` command.

        Returns leafname of generated archive within `sdist_directory`.
        '''
        assert self.fn_sdist, f'fn_sdist() not provided.'
        log2(
                f' sdist_directory={sdist_directory!r}'
                f' formats={formats!r}'
                f' config_settings={config_settings!r}'
                )
        if formats and formats != 'gztar':
            raise Exception( f'Unsupported: formats={formats}')
        items = list()
        if inspect.signature(self.fn_sdist).parameters:
            items = self.fn_sdist(config_settings)
        else:
            items = self.fn_sdist()

        prefix = f'{_normalise2(self.name)}-{self.version}'
        os.makedirs(sdist_directory, exist_ok=True)
        tarpath = f'{sdist_directory}/{prefix}.tar.gz'
        log2(f'Creating sdist: {tarpath}')

        with tarfile.open(tarpath, 'w:gz') as tar:

            names_in_tar = list()
            def check_name(name):
                if name in names_in_tar:
                    raise Exception(f'Name specified twice: {name}')
                names_in_tar.append(name)

            def add(from_, name):
                check_name(name)
                if isinstance(from_, str):
                    log2( f'Adding file: {os.path.relpath(from_)} => {name}')
                    tar.add( from_, f'{prefix}/{name}', recursive=False)
                elif isinstance(from_, bytes):
                    log2( f'Adding: {name}')
                    ti = tarfile.TarInfo(f'{prefix}/{name}')
                    ti.size = len(from_)
                    ti.mtime = time.time()
                    tar.addfile(ti, io.BytesIO(from_))
                else:
                    assert 0

            def add_string(text, name):
                textb = text.encode('utf8')
                return add(textb, name)

            found_pyproject_toml = False
            for item in items:
                from_, (_to_abs, to_rel) = self._fromto(item)
                if isinstance(from_, bytes):
                    add(from_, to_rel)
                else:
                    if from_.startswith(f'{os.path.abspath(sdist_directory)}/'):
                        # Source files should not be inside <sdist_directory>.
                        assert 0, f'Path is inside sdist_directory={sdist_directory}: {from_!r}'
                    assert os.path.exists(from_), f'Path does not exist: {from_!r}'
                    assert os.path.isfile(from_), f'Path is not a file: {from_!r}'
                    add(from_, to_rel)
                if to_rel == 'pyproject.toml':
                    found_pyproject_toml = True

            assert found_pyproject_toml, f'Cannot create sdist because file not specified: pyproject.toml'

            # Always add a PKG-INFO file.
            add_string(self._metainfo(), 'PKG-INFO')

            if self.license:
                if 'COPYING' in names_in_tar:
                    log2(f'Not writing .license because file already in sdist: COPYING')
                else:
                    add_string(self.license, 'COPYING')

        log1( f'Have created sdist: {tarpath}')
        return os.path.basename(tarpath)

    def wheel_tag_string(self):
        '''
        Returns <tag_python>-<tag_abi>-<tag_platform>.
        '''
        return f'{self.tag_python()}-{self.tag_abi()}-{self.tag_platform()}'

    def tag_python(self):
        '''
        Get two-digit python version, e.g. 'cp3.8' for python-3.8.6.
        '''
        if self.tag_python_:
            ret = self.tag_python_
        elif self.pure:
            return 'py3'
        else:
            ret = 'cp' + ''.join(platform.python_version().split('.')[:2])
        assert '-' not in ret
        return ret

    def tag_abi(self):
        '''
        ABI tag.
        '''
        Py_GIL_DISABLED = sysconfig.get_config_var('Py_GIL_DISABLED')
        if self.tag_abi_:
            return self.tag_abi_
        elif self.pure:
            return 'none'
        elif self.py_limited_api:
            assert Py_GIL_DISABLED != 1, \
                    f'{self.py_limited_api=} and {Py_GIL_DISABLED=} are not supported together as of 2026-02-20, e.g. see PEP 803 and PEP 809.'
            return 'abi3'
        elif Py_GIL_DISABLED == 1:
            ret = ''
            ret += 'cp'
            ret += ''.join(platform.python_version().split('.')[:2])
            ret += 't'
            return ret
        else:
            return 'none'

    def tag_platform(self):
        '''
        Find platform tag used in wheel filename.
        '''
        ret = self.tag_platform_
        log0(f'From self.tag_platform_: {ret=}.')
        
        if not ret and self.pure:
            return 'any'
        
        if not ret:
            # Prefer this to PEP-425. Appears to be undocumented,
            # but set in manylinux docker images and appears
            # to be used by cibuildwheel and auditwheel, e.g.
            # https://github.com/rapidsai/shared-action-workflows/issues/80
            ret = os.environ.get( 'AUDITWHEEL_PLAT')
            log0(f'From AUDITWHEEL_PLAT: {ret=}.')

        if not ret:
            # Notes:
            #
            # PEP-425. On Linux gives `linux_x86_64` which is rejected by
            # pypi.org.
            #
            # On local MacOS/arm64 mac-mini have seen sysconfig.get_platform()
            # unhelpfully return `macosx-10.9-universal2` if `python3` is the
            # system Python /usr/bin/python3; this happens if we source `.
            # /etc/profile`.
            #
            ret = sysconfig.get_platform()
            ret = ret.replace('-', '_').replace('.', '_').lower()
            log0(f'From sysconfig.get_platform(): {ret=}.')

            ret = _macos_fixup_platform_tag(ret)

        log0( f'tag_platform(): returning {ret=}.')
        assert '-' not in ret
        return ret

    def wheel_name(self):
        ret = f'{_normalise2(self.name)}-{self.version}-{self.tag_python()}-{self.tag_abi()}-{self.tag_platform()}.whl'
        assert ret.count('-') == 4, f'Expected 4 dash characters in {ret=}.'
        return ret

    def wheel_name_match(self, wheel):
        '''
        Returns true if `wheel` matches our wheel. We basically require the
        name to be the same, except that we accept platform tags that contain
        extra items (see pep-0600/), for example we return true with:

            self:   foo-cp38-none-manylinux2014_x86_64.whl
            wheel:  foo-cp38-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl
        '''
        log2(f'{wheel=}')
        assert wheel.endswith('.whl')
        wheel2 = wheel[:-len('.whl')]
        name, version, tag_python, tag_abi, tag_platform = wheel2.split('-')
        
        py_limited_api_compatible = False
        if self.py_limited_api and tag_abi == 'abi3':
            # Allow lower tag_python number.
            m = re.match('cp([0-9]+)', tag_python)
            tag_python_int = int(m.group(1))
            m = re.match('cp([0-9]+)', self.tag_python())
            tag_python_int_self = int(m.group(1))
            if tag_python_int <= tag_python_int_self:
                # This wheel uses Python stable ABI same or older than ours, so
                # we can use it.
                log2(f'py_limited_api; {tag_python=} compatible with {self.tag_python()=}.')
                py_limited_api_compatible = True
            
        log2(f'{_normalise2(self.name) == name=}')
        log2(f'{self.version == version=}')
        log2(f'{self.tag_python() == tag_python=} {self.tag_python()=} {tag_python=}')
        log2(f'{py_limited_api_compatible=}')
        log2(f'{self.tag_abi() == tag_abi=}')
        log2(f'{self.tag_platform() in tag_platform.split(".")=}')
        log2(f'{self.tag_platform()=}')
        log2(f'{tag_platform.split(".")=}')
        ret = (1
                and _normalise2(self.name) == name
                and self.version == version
                and (self.tag_python() == tag_python or py_limited_api_compatible)
                and self.tag_abi() == tag_abi
                and self.tag_platform() in tag_platform.split('.')
                )
        log2(f'Returning {ret=}.')
        return ret
    
    def _entry_points_text(self):
        if self.entry_points:
            if isinstance(self.entry_points, str):
                return self.entry_points
            ret = ''
            for key, values in self.entry_points.items():
                ret += f'[{key}]\n'
                for value in values:
                    ret += f'{value}\n'
            return ret

    def _call_fn_build( self, config_settings=None):
        assert self.fn_build
        assert os.path.isfile('pyproject.toml'), (
                f'Cannot create package because file does not exist: {os.path.abspath("pyproject.toml")}'
                )
        log2(f'calling self.fn_build={self.fn_build}')
        if inspect.signature(self.fn_build).parameters:
            ret = self.fn_build(config_settings)
        else:
            ret = self.fn_build()
        assert isinstance( ret, (list, tuple)), \
                f'Expected list/tuple from {self.fn_build} but got: {ret!r}'

        # Check that any extensions that we have built, have same
        # py_limited_api value. If package is marked with py_limited_api=True
        # then non-py_limited_api extensions seem to fail at runtime on
        # Windows.
        #
        # (We could possibly allow package py_limited_api=False and extensions
        # py_limited_api=True, but haven't tested this, and it seems simpler to
        # be strict.)
        for item in ret:
            from_, (_to_abs, _to_rel) = self._fromto(item)
            from_abs = os.path.abspath(from_)
            is_py_limited_api = _extensions_to_py_limited_api.get(from_abs)
            if is_py_limited_api is not None:
                assert bool(self.py_limited_api) == bool(is_py_limited_api), (
                        f'Extension was built with'
                        f' py_limited_api={is_py_limited_api} but pipcl.Package'
                        f' name={self.name!r} has'
                        f' py_limited_api={self.py_limited_api}:'
                        f' {from_abs!r}'
                        )
        
        return ret

    def _argv_clean(self, all_):
        '''
        Called by `handle_argv()`.
        '''
        if not self.fn_clean:
            return
        paths = self.fn_clean(all_)
        if paths:
            if isinstance(paths, str):
                paths = paths,
            for path in paths:
                if not os.path.isabs(path):
                    path = os.path.join(self.root, path)
                path = os.path.abspath(path)
                assert path.startswith(self.root+os.sep), \
                        f'path={path!r} does not start with root={self.root+os.sep!r}'
                log(f'Removing: {path}')
                fs_remove(path)

    def install(self, record_path=None, root=None):
        '''
        Called by `handle_argv()` to handle `install` command..
        '''
        log2( f'{record_path=} {root=}')

        # Do a build and get list of files to install.
        #
        items = list()
        if self.fn_build:
            items = self._call_fn_build( dict())

        root2 = install_dir(root)
        log2( f'{root2=}')

        log1( f'Installing into: {root2!r}')
        dist_info_dir = self._dist_info_dir()

        if not record_path:
            record_path = f'{root2}/{dist_info_dir}/RECORD'
        record = _Record()

        def add_file(from_, to_abs, to_rel):
            os.makedirs( os.path.dirname( to_abs), exist_ok=True)
            if isinstance(from_, bytes):
                log2(f'Copying content into {to_abs}.')
                with open(to_abs, 'wb') as f:
                    f.write(from_)
                record.add_content(from_, to_rel)
            else:
                log0(f'{from_=}')
                log2(f'Copying from {os.path.relpath(from_, self.root)} to {to_abs}')
                shutil.copy2( from_, to_abs)
                record.add_file(from_, to_rel)

        def add_str(content, to_abs, to_rel):
            log2( f'Writing to: {to_abs}')
            os.makedirs( os.path.dirname( to_abs), exist_ok=True)
            with open( to_abs, 'w') as f:
                f.write( content)
            record.add_content(content, to_rel)

        for item in items:
            from_, (to_abs, to_rel) = self._fromto(item)
            log0(f'{from_=} {to_abs=} {to_rel=}')
            to_abs2 = f'{root2}/{to_rel}'
            add_file( from_, to_abs2, to_rel)

        add_str( self._metainfo(), f'{root2}/{dist_info_dir}/METADATA', f'{dist_info_dir}/METADATA')

        if self.license:
            add_str( self.license, f'{root2}/{dist_info_dir}/COPYING', f'{dist_info_dir}/COPYING')

        entry_points_text = self._entry_points_text()
        if entry_points_text:
            add_str(
                    entry_points_text,
                    f'{root2}/{dist_info_dir}/entry_points.txt',
                    f'{dist_info_dir}/entry_points.txt',
                    )

        log2( f'Writing to: {record_path}')
        with open(record_path, 'w') as f:
            f.write(record.get())

        log2(f'Finished.')

    def _argv_dist_info(self, root):
        '''
        Called by `handle_argv()`. There doesn't seem to be any documentation
        for `setup.py dist_info`, but it appears to be like `egg_info` except
        it writes to a slightly different directory.
        '''
        if root is None:
            root = f'{_normalise2(self.name)}-{self.version}.dist-info'
        self._write_info(f'{root}/METADATA')
        if self.license:
            with open( f'{root}/COPYING', 'w') as f:
                f.write( self.license)

    def _argv_egg_info(self, egg_base):
        '''
        Called by `handle_argv()`.
        '''
        if egg_base is None:
            egg_base = '.'
        self._write_info(f'{egg_base}/.egg-info')

    def _write_info(self, dirpath=None):
        '''
        Writes egg/dist info to files in directory `dirpath` or `self.root` if
        `None`.
        '''
        if dirpath is None:
            dirpath = self.root
        log2(f'Creating files in directory {dirpath}')
        os.makedirs(dirpath, exist_ok=True)
        with open(os.path.join(dirpath, 'PKG-INFO'), 'w') as f:
            f.write(self._metainfo())

        # These don't seem to be required?
        #
        #with open(os.path.join(dirpath, 'SOURCES.txt', 'w') as f:
        #    pass
        #with open(os.path.join(dirpath, 'dependency_links.txt', 'w') as f:
        #    pass
        #with open(os.path.join(dirpath, 'top_level.txt', 'w') as f:
        #    f.write(f'{self.name}\n')
        #with open(os.path.join(dirpath, 'METADATA', 'w') as f:
        #    f.write(self._metainfo())

    def handle_argv(self, argv):
        '''
        Attempt to handle old-style (pre PEP-517) command line passed by
        old releases of pip to a `setup.py` script, and manual running of
        `setup.py`.

        This is partial support at best.
        '''
        global g_verbose
        #log2(f'argv: {argv}')

        class ArgsRaise:
            pass

        class Args:
            '''
            Iterates over argv items.
            '''
            def __init__( self, argv):
                self.items = iter( argv)
            def next( self, eof=ArgsRaise):
                '''
                Returns next arg. If no more args, we return <eof> or raise an
                exception if <eof> is ArgsRaise.
                '''
                try:
                    return next( self.items)
                except StopIteration as e:
                    if eof is ArgsRaise:
                        raise Exception('Not enough args') from e
                    return eof

        command = None
        opt_all = None
        opt_dist_dir = 'dist'
        opt_egg_base = None
        opt_formats = None
        opt_install_headers = None  # pylint: disable=unused-variable
        opt_record = None
        opt_root = None

        args = Args(argv[1:])

        while 1:
            # pylint: disable=multiple-statements,consider-using-in
            arg = args.next(None)
            if arg is None:
                break
            if arg in ('-h', '--help', '--help-commands'):
                # codespell:ignore
                log0(textwrap.dedent('''
                        Usage:
                            [<options>...] <command> [<options>...]
                        Commands:
                            bdist_wheel
                                Creates a wheel called
                                <dist-dir>/<name>-<version>-<details>.whl, where
                                <dist-dir> is "dist" or as specified by --dist-dir,
                                and <details> encodes ABI and platform etc.
                            clean
                                Cleans build files.
                            dist_info
                                Creates files in <name>-<version>.dist-info/ or
                                directory specified by --egg-base.
                            egg_info
                                Creates files in .egg-info/ or directory
                                directory specified by --egg-base.
                            install
                                Builds and installs. Writes installation
                                information to <record> if --record was
                                specified.
                            sdist
                                Make a source distribution:
                                    <dist-dir>/<name>-<version>.tar.gz
                        Options:
                            --all
                                Used by "clean".
                            --compile
                                Ignored.
                            --dist-dir | -d <dist-dir>
                                Default is "dist".
                            --egg-base <egg-base>
                                Used by "egg_info".
                            --formats <formats>
                                Used by "sdist".
                            --install-headers <directory>
                                Ignored.
                            --python-tag <python-tag>
                                Ignored.
                            --record <record>
                                Used by "install".
                            --root <path>
                                Used by "install".
                            --single-version-externally-managed
                                Ignored.
                            --verbose -v
                                Extra diagnostics.
                        Other:
                            windows-vs [-y <year>] [-v <version>] [-g <grade] [--verbose]
                                Windows only; looks for matching Visual Studio.
                            windows-python [-v <version>] [--verbose]
                                Windows only; looks for matching Python.
                        '''))
                return

            elif arg in ('bdist_wheel', 'clean', 'dist_info', 'egg_info', 'install', 'sdist'):
                assert command is None, 'Two commands specified: {command} and {arg}.'
                command = arg

            elif arg in ('windows-vs', 'windows-python', 'show-sysconfig'):
                assert command is None, 'Two commands specified: {command} and {arg}.'
                command = arg

            elif arg == '--all':                                opt_all = True
            elif arg == '--compile':                            pass
            elif arg == '--dist-dir' or arg == '-d':            opt_dist_dir = args.next()
            elif arg == '--egg-base':                           opt_egg_base = args.next()
            elif arg == '--formats':                            opt_formats = args.next()
            elif arg == '--install-headers':                    opt_install_headers = args.next()   # noqa: F841
            elif arg == '--python-tag':                         pass
            elif arg == '--record':                             opt_record = args.next()
            elif arg == '--root':                               opt_root = args.next()
            elif arg == '--single-version-externally-managed':  pass
            elif arg == '--verbose' or arg == '-v':             g_verbose += 1

            else:
                raise Exception(f'Unrecognised arg: {arg}')

        assert command, 'No command specified'

        log1(f'Handling command={command}')
        # pylint: disable=multiple-statements
        if 0:   pass
        elif command == 'bdist_wheel':  self.build_wheel(opt_dist_dir)
        elif command == 'clean':        self._argv_clean(opt_all)
        elif command == 'dist_info':    self._argv_dist_info(opt_egg_base)
        elif command == 'egg_info':     self._argv_egg_info(opt_egg_base)
        elif command == 'install':      self.install(opt_record, opt_root)
        elif command == 'sdist':        self.build_sdist(opt_dist_dir, opt_formats)

        elif command == 'windows-python':
            version = None
            while 1:
                arg = args.next(None)
                if arg is None:
                    break
                elif arg == '-v':
                    version = args.next()
                elif arg == '--verbose':
                    g_verbose += 1
                else:
                    assert 0, f'Unrecognised {arg=}'
            python = wdev.WindowsPython(version=version)
            print(f'Python is:\n{python.description_ml("    ")}')

        elif command == 'windows-vs':
            grade = None
            version = None
            year = None
            while 1:
                arg = args.next(None)
                if arg is None:
                    break
                elif arg == '-g':
                    grade = args.next()
                elif arg == '-v':
                    version = args.next()
                elif arg == '-y':
                    year = args.next()
                elif arg == '--verbose':
                    g_verbose += 1
                else:
                    assert 0, f'Unrecognised {arg=}'
            vs = wdev.WindowsVS(year=year, grade=grade, version=version)
            print(f'Visual Studio is:\n{vs.description_ml("    ")}')

        # pylint: disable=too-many-nested-blocks
        elif command == 'show-sysconfig':
            show_sysconfig()
            for mod in platform, sys:
                log0(f'{mod.__name__}:')
                for n in dir(mod):
                    if n.startswith('_'):
                        continue
                    log0(f'{mod.__name__}.{n}')
                    if mod is platform and n == 'uname':
                        continue
                    if mod is platform and n == 'pdb':
                        continue
                    if mod is sys and n in ('breakpointhook', 'exit'):
                        # We don't want to call these.
                        continue
                    v = getattr(mod, n)
                    if callable(v):
                        try:
                            v = v()
                        except Exception:
                            pass
                        else:
                            #print(f'{n=}', flush=1)
                            try:
                                print(f'    {mod.__name__}.{n}()={v!r}')
                            except Exception:
                                print(f'    Failed to print value of {mod.__name__}.{n}().')
                    else:
                        try:
                            print(f'    {mod.__name__}.{n}={v!r}')
                        except Exception:
                            print(f'    Failed to print value of {mod.__name__}.{n}.')
        
        else:
            assert 0, f'Unrecognised command: {command}'

        log2(f'Finished handling command: {command}')

    def __str__(self):
        return ('{'
            f'name={self.name!r}'
            f' version={self.version!r}'
            f' platform={self.platform!r}'
            f' supported_platform={self.supported_platform!r}'
            f' summary={self.summary!r}'
            f' description={self.description!r}'
            f' description_content_type={self.description_content_type!r}'
            f' keywords={self.keywords!r}'
            f' home_page={self.home_page!r}'
            f' download_url={self.download_url!r}'
            f' author={self.author!r}'
            f' author_email={self.author_email!r}'
            f' maintainer={self.maintainer!r}'
            f' maintainer_email={self.maintainer_email!r}'
            f' license={self.license!r}'
            f' classifier={self.classifier!r}'
            f' requires_dist={self.requires_dist!r}'
            f' requires_python={self.requires_python!r}'
            f' requires_external={self.requires_external!r}'
            f' project_url={self.project_url!r}'
            f' provides_extra={self.provides_extra!r}'

            f' root={self.root!r}'
            f' fn_build={self.fn_build!r}'
            f' fn_sdist={self.fn_sdist!r}'
            f' fn_clean={self.fn_clean!r}'
            f' tag_python={self.tag_python_!r}'
            f' tag_abi={self.tag_abi_!r}'
            f' tag_platform={self.tag_platform_!r}'
            '}'
            )

    def _dist_info_dir( self):
        return f'{_normalise2(self.name)}-{self.version}.dist-info'

    def _metainfo(self):
        '''
        Returns text for `.egg-info/PKG-INFO` file, or `PKG-INFO` in an sdist
        `.tar.gz` file, or `...dist-info/METADATA` in a wheel.
        '''
        # 2021-04-30: Have been unable to get multiline content working on
        # test.pypi.org so we currently put the description as the body after
        # all the other headers.
        #
        ret = ['']
        def add(key, value):
            if value is None and key == 'Description-Content-Type':
                # Try to figure out from description suffix.
                if self.description and '\n' not in self.description:
                    if self.description.endswith('.md'):
                        value = 'text/markdown'
                    elif self.description.endswith('.rst'):
                        value = 'text/x-rst'
                    if value:
                        log(f'## Warning: description_content_type unspecified, inferring {value!r} from suffix of description={self.description!r}.')
            if value is None:
                return
            if isinstance( value, (tuple, list)):
                for v in value:
                    if v is not None:
                        add( key, v)
                return
            if key == 'License' and '\n' in value:
                # This is ok because we write `self.license` into
                # *.dist-info/COPYING.
                #
                log1( f'Omitting license because contains newline(s).')
                return
            assert '\n' not in value, f'key={key} value contains newline: {value!r}'
            if key == 'Project-URL':
                assert value.count(',') == 1, f'For {key=}, should have one comma in {value!r}.'
            ret[0] += f'{key}: {value}\n'
        #add('Description', self.description)
        add('Metadata-Version', '2.1')

        # These names are from:
        # https://packaging.python.org/specifications/core-metadata/
        #
        for name in (
                'Name',
                'Version',
                'Platform',
                'Supported-Platform',
                'Summary',
                'Description-Content-Type',
                'Keywords',
                'Home-page',
                'Download-URL',
                'Author',
                'Author-email',
                'Maintainer',
                'Maintainer-email',
                'License',
                'Classifier',
                'Requires-Dist',
                'Requires-Python',
                'Requires-External',
                'Project-URL',
                'Provides-Extra',
                ):
            identifier = name.lower().replace( '-', '_')
            add( name, getattr( self, identifier))

        ret = ret[0]

        # Append description as the body
        if self.description:
            if '\n' in self.description:
                description_text = self.description.strip()
            else:
                with open(self.description, encoding='utf8') as f:
                    description_text = f.read()
            ret += '\n' # Empty line separates headers from body.
            ret += description_text
            ret += '\n'
        return ret

    def _path_relative_to_root(self, path, assert_within_root=True):
        '''
        Returns `(path_abs, path_rel)`, where `path_abs` is absolute path and
        `path_rel` is relative to `self.root`.

        Interprets `path` as relative to `self.root` if not absolute.

        We use `os.path.realpath()` to resolve any links.

        if `assert_within_root` is true, assert-fails if `path` is not within
        `self.root`.
        '''
        if os.path.isabs(path):
            p = path
        else:
            p = os.path.join(self.root, path)
        p = os.path.realpath(os.path.abspath(p))
        if assert_within_root:
            assert p.startswith(self.root+os.sep) or p == self.root, \
                    f'Path not within root={self.root+os.sep!r}: {path=} {p=}'
        p_rel = os.path.relpath(p, self.root)
        return p, p_rel

    def _fromto(self, p):
        '''
        Returns `(from_, (to_abs, to_rel))`.

        If `p` is a string we convert to `(p, p)`. Otherwise we assert that
        `p` is a tuple `(from_, to_)` where `from_` is str/bytes and `to_` is
        str. If `from_` is a bytes it is contents of file to add, otherwise the
        path of an existing file; non-absolute paths are assumed to be relative
        to `self.root`.

        If `to_` is empty or `/` we set it to the leaf of `from_` (which must
        be a str) - i.e. we place the file in the root directory of the wheel;
        otherwise if `to_` ends with `/` we append the leaf of `from_` (which
        must be a str).

        If `to_` starts with `$dist-info/`, we replace this with
        `self._dist_info_dir()`.

        If `to_` starts with `$data/`, we replace this with
        `{self.name}-{self.version}.data/`.

        We assert that `to_abs` is `within self.root`.

        `to_rel` is derived from the `to_abs` and is relative to self.root`.
        '''
        if isinstance(p, str):
            p = p, p
        assert isinstance(p, tuple) and len(p) == 2

        from_, to_ = p
        assert isinstance(from_, (str, bytes))
        assert isinstance(to_, str)
        if to_ == '/' or to_ == '':
            to_ = os.path.basename(from_)
        elif to_.endswith('/'):
            to_ += os.path.basename(from_)
        prefix = '$dist-info/'
        if to_.startswith( prefix):
            to_ = f'{self._dist_info_dir()}/{to_[ len(prefix):]}'
        prefix = '$data/'
        if to_.startswith( prefix):
            to_ = f'{_normalise2(self.name)}-{self.version}.data/{to_[ len(prefix):]}'
        if isinstance(from_, str):
            from_, _ = self._path_relative_to_root( from_, assert_within_root=False)
        to_ = self._path_relative_to_root(to_)
        assert isinstance(from_, (str, bytes))
        log2(f'returning {from_=} {to_=}')
        return from_, to_


_extensions_to_py_limited_api = dict()


def build_extension(
        name,
        path_i,
        outdir,
        *,
        builddir=None,
        includes=None,
        defines=None,
        libpaths=None,
        libs=None,
        optimise=True,
        debug=False,
        compiler_extra='',
        linker_extra='',
        swig=None,
        cpp=True,
        source_extra=None,
        prerequisites_swig=None,
        prerequisites_compile=None,
        prerequisites_link=None,
        infer_swig_includes=True,
        py_limited_api=False,
        nogil=False,
        ):
    '''
    Builds a Python extension module using SWIG. Works on Windows, Linux, MacOS
    and OpenBSD.

    On Unix, sets rpath when linking shared libraries.

    Args:
        name:
            Name of generated extension module.
        path_i:
            Path of input SWIG `.i` file. Internally we use swig to generate a
            corresponding `.c` or `.cpp` file.
        outdir:
            Output directory for generated files:

                * `{outdir}/{name}.py`
                * `{outdir}/_{name}.so`     # Unix
                * `{outdir}/_{name}.*.pyd`  # Windows
            We return the leafname of the `.so` or `.pyd` file.
        builddir:
            Where to put intermediate files, for example the .cpp file
            generated by swig and `.d` dependency files. Default is `outdir`.
        includes:
            A string, or a sequence of extra include directories to be prefixed
            with `-I`.
        defines:
            A string, or a sequence of extra preprocessor defines to be
            prefixed with `-D`.
        libpaths
            A string, or a sequence of library paths to be prefixed with
            `/LIBPATH:` on Windows or `-L` on Unix.
        libs
            A string, or a sequence of library names. Each item is prefixed
            with `-l` on non-Windows.
        optimise:
            Whether to use compiler optimisations and define NDEBUG.
        debug:
            Whether to build with debug symbols.
        compiler_extra:
            Extra compiler flags. Can be None.
        linker_extra:
            Extra linker flags. Can be None.
        swig:
            Swig command; if false we use 'swig'.
        cpp:
            If true we tell SWIG to generate C++ code instead of C.
        source_extra:
            Extra source files to build into the shared library,
        prerequisites_swig:
        prerequisites_compile:
        prerequisites_link:

            [These are mainly for use on Windows. On other systems we
            automatically generate dynamic dependencies using swig/compile/link
            commands' `-MD` and `-MF` args.]

            Sequences of extra input files/directories that should force
            running of swig, compile or link commands if they are newer than
            any existing generated SWIG `.i` file, compiled object file or
            shared library file.

            If present, the first occurrence of `True` or `False` forces re-run
            or no re-run. Any occurrence of None is ignored. If an item is a
            directory path we look for newest file within the directory tree.

            If not a sequence, we convert into a single-item list.

            prerequisites_swig

                We use swig's -MD and -MF args to generate dynamic dependencies
                automatically, so this is not usually required.

            prerequisites_compile
            prerequisites_link

                On non-Windows we use cc's -MF and -MF args to generate dynamic
                dependencies so this is not usually required.
        infer_swig_includes:
            If true, we extract `-I<path>` and `-I <path>` args from
            `compile_extra` (also `/I` on windows) and use them with swig so
            that it can see the same header files as C/C++. This is useful
            when using enviromment variables such as `CC` and `CXX` to set
            `compile_extra`.
        py_limited_api:
            If true we build for current Python's limited API / stable ABI.

            Note that we will assert false if this extension is added to a
            pipcl.Package that has a different <py_limited_api>, because
            on Windows importing a non-py_limited_api extension inside a
            py_limited=True package fails.
        nogil:
            If true we use swig's `-nogil` flag which marks the extension as
            being thread safe.

    Returns the leafname of the generated library file within `outdir`, e.g.
    `_{name}.so` on Unix or `_{name}.cp311-win_amd64.pyd` on Windows.
    '''
    
    if builddir is None:
        builddir = outdir
    if not swig:
        swig = 'swig'
        
    if source_extra is None:
        source_extra = list()
    if isinstance(source_extra, str):
        source_extra = [source_extra]
    
    path_cpp = f'{builddir}/{os.path.basename(path_i)}'
    path_cpp += '.cpp' if cpp else '.c'
    os.makedirs(outdir, exist_ok=True)

    # Run SWIG.
    #
    if infer_swig_includes:
        # Extract include flags from `compiler_extra`.
        swig_includes_extra = ''
        compiler_extra_items = shlex.split(compiler_extra)
        i = 0
        while i < len(compiler_extra_items):
            item = compiler_extra_items[i]
            # Swig doesn't seem to like a space after `I`.
            if item == '-I' or (windows() and item == '/I'):
                swig_includes_extra += f' -I{compiler_extra_items[i+1]}'
                i += 1
            elif item.startswith('-I') or (windows() and item.startswith('/I')):
                swig_includes_extra += f' -I{compiler_extra_items[i][2:]}'
            i += 1
        swig_includes_extra = swig_includes_extra.strip()
    deps_path = f'{path_cpp}.d'
    prerequisites_swig2 = _get_prerequisites( deps_path)
    run_if(f'''
            {swig}
                -Wall
                {'-c++' if cpp else ''}
                -python
                -module {name}
                -outdir {outdir}
                -o {path_cpp}
                -MD -MF {deps_path}
                {'-nogil' if nogil else ''}
                {_flags( includes, '-I')}
                {swig_includes_extra}
                {path_i}
            '''
            ,
            path_cpp,
            path_i,
            prerequisites_swig,
            prerequisites_swig2,
            )

    if pyodide():
        so_suffix = '.so'
        log0(f'pyodide: PEP-3149 suffix untested, so omitting. {_so_suffix()=}.')
    else:
        so_suffix = _so_suffix(use_so_versioning = not py_limited_api)
    path_so_leaf = f'_{name}{so_suffix}'
    path_so = f'{outdir}/{path_so_leaf}'

    # setuptools on Linux seems to use slightly different compile flags:
    #
    # -fwrapv -O3 -Wall -O2 -g0 -DPY_CALL_TRAMPOLINE
    #

    # Fun fact - on Linux, if the -L and -l options are before '{path_cpp}'
    # they seem to be ignored...
    #
    
    # Compile source files to object files.
    #
    path_os = list()
    for path_source in [path_cpp] + source_extra:
        path_o = f'{path_source}.obj' if windows() else f'{path_source}.o'
        path_os.append(path_o)

        prerequisites_path = f'{path_o}.d'
        
        command = compiler_command(
                includes=includes,
                defines=defines,
                optimise=optimise,
                debug=debug,
                compiler_extra=compiler_extra,
                cpp=cpp,
                python=True,
                py_limited_api=py_limited_api,
                source_paths=path_source,
                path_o=path_o,
                )
        if not windows():
            command += '    -fPIC\n'
        run_if(
                command,
                path_o,
                path_source,
                [path_source] + _get_prerequisites(prerequisites_path),
                prerequisites_compile,
                )

    # Link object files to shared library.
    #
    prerequisites_path = f'{path_so}.d'
    
    command = linker_command(
            libpaths=libpaths,
            libs=libs,
            optimise=optimise,
            debug=debug,
            linker_extra=linker_extra,
            cpp=cpp,
            python=True,
            py_limited_api=py_limited_api,
            rpath=True,
            path_os=path_os,
            path_out=path_so,
            )
    
    if windows():
        base, _ = os.path.splitext(path_so_leaf)
        command += f'    /DLL\n'
        command += f'    /EXPORT:PyInit__{name}\n'
        command += f'    /IMPLIB:{base}.lib      # Overrides the default import library name.\n'
    else:
        command += f'    -MD -MF {prerequisites_path}\n'
        command += f'    -shared\n'
    
    link_was_run = run_if(
            command,
            path_so,
            path_cpp,
            *path_os,
            *_get_prerequisites(f'{path_so}.d'),
            prerequisites_link,
            )

    if link_was_run and darwin():
        macos_patch2(path_so, libpaths, libs)

    _extensions_to_py_limited_api[os.path.abspath(path_so)] = py_limited_api

    return path_so_leaf


# Functions that might be useful.
#


def base_compiler(vs=None, pythonflags=None, cpp=False, use_env=True):
    '''
    Returns basic compiler command and PythonFlags.

    Args:
        vs:
            Windows only. A `wdev.WindowsVS` instance or None to use default
            `wdev.WindowsVS` instance.
        pythonflags:
            A `pipcl.PythonFlags` instance or None to use default
            `pipcl.PythonFlags` instance.
        cpp:
            If true we return C++ compiler command instead of C. On Windows
            this has no effect - we always return `cl.exe`.
        use_env:
            If true we return '$CC' or '$CXX' if the corresponding
            environmental variable is set (without evaluating with `os.getenv()`
            or `os.environ`).

    Returns `(cc, pythonflags)`:
        cc:
            C or C++ command. On Windows this is of the form
            `{vs.vcvars}&&{vs.cl}`; otherwise it is typically `cc` or `c++`.
        pythonflags:
            The `pythonflags` arg or a new `pipcl.PythonFlags` instance.
    '''
    if not pythonflags:
        pythonflags = PythonFlags()
    cc = None
    if use_env:
        if cpp:
            if os.environ.get( 'CXX'):
                cc = '$CXX'
        else:
            if os.environ.get( 'CC'):
                cc = '$CC'
    if cc:
        pass
    elif windows():
        if not vs:
            vs = wdev.WindowsVS()
        cc = f'"{vs.vcvars}"&&"{vs.cl}"'
    elif wasm():
        cc = 'em++' if cpp else 'emcc'
    else:
        cc = 'c++' if cpp else 'cc'
    cc = macos_add_cross_flags( cc)
    return cc, pythonflags


def base_linker(vs=None, pythonflags=None, cpp=False, use_env=True):
    '''
    Returns basic linker command.

    Args:
        vs:
            Windows only. A `wdev.WindowsVS` instance or None to use default
            `wdev.WindowsVS` instance.
        pythonflags:
            A `pipcl.PythonFlags` instance or None to use default
            `pipcl.PythonFlags` instance.
        cpp:
            If true we return C++ linker command instead of C. On Windows this
            has no effect - we always return `link.exe`.
        use_env:
            If true we use `os.environ['LD']` if set.

    Returns `(linker, pythonflags)`:
        linker:
            Linker command. On Windows this is of the form
            `{vs.vcvars}&&{vs.link}`; otherwise it is typically `cc` or `c++`.
        pythonflags:
            The `pythonflags` arg or a new `pipcl.PythonFlags` instance.
    '''
    if not pythonflags:
        pythonflags = PythonFlags()
    linker = None
    if use_env:
        if os.environ.get( 'LD'):
            linker = '$LD'
    if linker:
        pass
    elif windows():
        if not vs:
            vs = wdev.WindowsVS()
        linker = f'"{vs.vcvars}"&&"{vs.link}"'
    elif wasm():
        linker = 'em++' if cpp else 'emcc'
    else:
        linker = 'c++' if cpp else 'cc'
    linker = macos_add_cross_flags( linker)
    return linker, pythonflags


def compiler_command(
        *,
        includes=None,
        defines=None,
        optimise=True,
        debug=False,
        compiler_extra='',
        cpp=True,
        python=False,
        py_limited_api=False,
        rpath=False,
        source_paths=None,
        path_o=None,
        ):
    '''
    Returns generic compiler and linker commands. Used by `build_extension()`.
    
    Args:
        includes:
            A string, or a sequence of extra include directories to be prefixed
            with `-I`.
        defines:
            A string, or a sequence of extra preprocessor defines to be
            prefixed with `-D`.
        optimise:
            Whether to use compiler optimisations and define NDEBUG.
        debug:
            Whether to build with debug symbols.
        compiler_extra:
            Extra compiler flags. Can be None.
        cpp:
            If true we tell SWIG to generate C++ code instead of C.
        python:
            If true we include flags for using Python headers and libraries.
        py_limited_api:
            If true we build for current Python's limited API / stable ABI.

            Note that we will assert false if this extension is added to a
            pipcl.Package that has a different <py_limited_api>, because
            on Windows importing a non-py_limited_api extension inside a
            py_limited=True package fails.
    '''
    if compiler_extra is None:
        compiler_extra = ''
    if isinstance(source_paths, str):
        source_paths = source_paths,
        
    includes_text = _flags( includes, '-I')
    defines_text = _flags( defines, '-D')

    py_limited_api2 = current_py_limited_api() if py_limited_api else None

    compiler_command, pythonflags = base_compiler(cpp=cpp)
    # setuptools on Linux seems to use slightly different compile flags:
    #
    # -fwrapv -O3 -Wall -O2 -g0 -DPY_CALL_TRAMPOLINE
    #

    general_flags = ''

    if darwin():
        general_flags += ' -undefined dynamic_lookup'
    
    if windows():
    
        permissive = '/permissive-'
        EHsc = '/EHsc'
        T = '/Tp' if cpp else '/Tc'
        optimise2 = '/DNDEBUG /O2' if optimise else '/D_DEBUG'
        debug2 = '/Zi' if debug else ''
        py_limited_api3 = f'/DPy_LIMITED_API={py_limited_api2}' if py_limited_api2 else ''
        
        compiler_command = f'''
                {compiler_command}
                    # General:
                    /c                          # Compiles without linking.
                    {EHsc}                      # Enable "Standard C++ exception handling".

                    #/MD                         # Creates a multithreaded DLL using MSVCRT.lib.
                    {'/MDd' if debug else '/MD'}
                    /Fo{path_o}                 # Output file. codespell:ignore

                    # Include paths:
                    {includes_text}
                    {pythonflags.includes if python else ''}      # Include path for Python headers.

                    # Code generation:
                    {optimise2}
                    {debug2}
                    {permissive}                # Set standard-conformance mode.

                    # Diagnostics:
                    #/FC                         # Display full path of source code files passed to cl.exe in diagnostic text.
                    /W3                         # Sets which warning level to output. /W3 is IDE default.
                    /diagnostics:caret          # Controls the format of diagnostic messages.
                    /nologo                     #

                    {defines_text}
                    {compiler_extra}

                    {py_limited_api3}
                '''
        for source_path in source_paths:
            compiler_command += f'    {T}{source_path}\n'
    
    else:
        if debug:
            general_flags += ' -g'
        if optimise:
            general_flags += ' -O2 -DNDEBUG'
        py_limited_api3 = f'-DPy_LIMITED_API={py_limited_api2}' if py_limited_api2 else ''
        
        compiler_command = f'''
                {compiler_command}
                    {general_flags.strip()}
                    {pythonflags.includes if python else ''}
                    {includes_text}
                    {defines_text}
                    {compiler_extra}
                    {py_limited_api3}
                    -c
                    -o {path_o}
                '''
        for source_path in source_paths:
            compiler_command += f'    {source_path}\n'
    
    return compiler_command


def linker_command(
        *,
        libpaths=None,
        libs=None,
        optimise=True,
        debug=False,
        linker_extra='',
        cpp=True,
        python=False,
        py_limited_api=False,
        rpath=False,
        path_os=None,
        path_out=None,
        ):
    '''
    Returns generic compiler and linker commands. Used by `build_extension()`.
    
    Args:
        libpaths
            A string, or a sequence of library paths to be prefixed with
            `/LIBPATH:` on Windows or `-L` on Unix.
        libs
            A string, or a sequence of library names. Each item is prefixed
            with `-l` on non-Windows.
        optimise:
            Whether to use compiler optimisations and define NDEBUG.
        debug:
            Whether to build with debug symbols.
        linker_extra:
            Extra linker flags. Can be None.
        cpp:
            .
        python:
            If true we include flags for using Python headers and libraries.
        py_limited_api:
            If true we build for current Python's limited API / stable ABI.

            Note that we will assert false if this extension is added to a
            pipcl.Package that has a different <py_limited_api>, because
            on Windows importing a non-py_limited_api extension inside a
            py_limited=True package fails.
        rpath:
            If true we include an rpath arg to linker command, and patch up
            libraries on MacOS.
    '''
    if linker_extra is None:
        linker_extra = ''
        
    libpaths_text = _flags( libpaths, '/LIBPATH:', '"') if windows() else _flags( libpaths, '-L')
    libs_text = _flags( libs, '' if windows() else '-l')

    py_limited_api2 = current_py_limited_api() if py_limited_api else None

    general_flags = ''

    if rpath:
        if windows():
            pass
        elif darwin():
            # MacOS's linker does not like `-z origin`.
            rpath_flag = "-Wl,-rpath,@loader_path/"
            # Avoid `Undefined symbols for ... "_PyArg_UnpackTuple" ...'.
        elif pyodide():
            # Setting `-Wl,-rpath,'$ORIGIN',-z,origin` gives:
            #   emcc: warning: ignoring unsupported linker flag: `-rpath` [-Wlinkflags]
            #   wasm-ld: error: unknown -z value: origin
            #
            rpath_flag = "-Wl,-rpath,'$ORIGIN'"
        else:
            rpath_flag = "-Wl,-rpath,'$ORIGIN',-z,origin"
    
    if darwin():
        general_flags += ' -undefined dynamic_lookup'
    
    linker_command, pythonflags = base_linker(cpp=cpp)
    
    if windows():
        debug2 = '/Zi' if debug else ''
        py_limited_api3 = f'/DPy_LIMITED_API={py_limited_api2}' if py_limited_api2 else ''
        debug2 = '/DEBUG' if debug else ''
        linker_command = textwrap.dedent(f'''
                {linker_command}
                    {libpaths_text}
                    {pythonflags.ldflags if python  else ''}
                    {debug2}
                    /nologo
                    {libs_text}
                    {linker_extra}
                    {'/OUT:'+path_out if path_out else ''}
                ''')
        for path_o in path_os:
            linker_command += f'    {path_o}\n'
    else:
        if debug:
            general_flags += ' -g'
        if optimise:
            general_flags += ' -O2 -DNDEBUG'
        py_limited_api3 = f'-DPy_LIMITED_API={py_limited_api2}' if py_limited_api2 else ''
        
        # Fun fact - on Linux, if the -L and -l options are before '{path_cpp}'
        # they seem to be ignored...
        #

        for path_o in path_os:
            linker_command += f'    {path_o}\n'
        if path_out:
            linker_command += f'    -o {path_out}\n'
        t = textwrap.dedent(f'''
                {general_flags.strip()}
                {libpaths_text}
                {libs_text}
                {linker_extra}
                {pythonflags.ldflags if python else ''}
                {rpath_flag}
                {py_limited_api3}
                ''')
        linker_command += textwrap.indent(t, '    ')
    return linker_command


def macos_patch2(path_so, libpaths, libs):
    assert darwin()
    # We need to patch up references to shared libraries in `libs`.
    sublibraries = list()
    for lib in () if libs is None else libs:
        for libpath in libpaths:
            found = list()
            for suffix in '.so', '.dylib':
                path = f'{libpath}/lib{os.path.basename(lib)}{suffix}'
                if os.path.exists( path):
                    found.append( path)
            if found:
                assert len(found) == 1, f'More than one file matches lib={lib!r}: {found}'
                sublibraries.append( found[0])
                break
        else:
            log2(f'Warning: can not find path of lib={lib!r} in libpaths={libpaths}')
    macos_patch( path_so, *sublibraries)

    #run(f'ls -l {path_so}', check=0)
    #run(f'file {path_so}', check=0)
    
        
def fs_ensure_empty_dir(path):
    '''
    Ensures <path> is an empty directory.
    '''
    os.makedirs(path, exist_ok=True)
    fs_remove_dir_contents( path)


def fs_ensure_dir(path):
    '''
    Ensures directory <path> exists.
    '''
    os.makedirs(path, exist_ok=True)


def fs_ensure_parent_dir(path):
    '''
    Ensures parent directory of <path> exists.
    '''
    parent = os.path.dirname(path)
    if parent:
        fs_ensure_dir(parent)


def fs_filesize( filename, default=0):
    '''
    Returns size of <filename> or <default> if error (e.g. doesn't exist).
    '''
    try:
        return os.path.getsize( filename)
    except OSError:
        return default


def fs_read(path, binary=False):
    '''
    Returns string containing contents of file <path>. Returns bytes if
    <binary> is true.
    '''
    with open(path, 'rb' if binary else 'r') as f:
        return f.read()


def fs_remove(path):
    '''
    Removes file or directory, without raising exception if it doesn't exist.

    We assert-fail if the path still exists when we return, in case of
    permission problems etc.
    '''
    try:
        os.remove(path)
    except Exception:
        pass
    shutil.rmtree(path, ignore_errors=1)
    assert not os.path.exists(path), f'Path still exists after deletion: {path!r}.'


def fs_remove_dir_contents(path):
    '''
    Removes all items in directory `path`; does not remove `path` itself.
    '''
    for leaf in os.listdir(path):
        path2 = os.path.join(path, leaf)
        fs_remove(path2)


def fs_write(path, data, binary=False):
    '''
    Writes <data> to file <path>.
    '''
    with open(path, 'wb' if binary else 'w') as f:
        return f.write( data)


def git_info( directory):
    '''
    Returns `(sha, comment, diff, branch)`, all items are str or None if not
    available. <comment> is the first line of the commit message.

    directory:
        Root of git checkout.
    '''
    sha, comment, diff, branch = None, None, None, None
    
    def run_git(command):
        return run(command, capture=1, check=0, verbose=0, env_extra=dict(PAGER=''))
    
    # We are careful to run git with `--no-pager` to avoid problems on Windows where
    # Defender appears to cause problems. See:
    #   https://www.reddit.com/r/sysadmin/comments/1msxc9o/git_commands_freezing_for_210_seconds_on_windows/
    #
    
    e, out = run_git(f'cd {directory} && git --no-pager log -1 --pretty=oneline')
    if not e:
        line, _ = out.split('\n', 1)
        sha, comment = line.split(' ', 1)
    
    if 0 and platform.system() == 'Windows':    # pylint: disable=condition-evals-to-constant
        # Have seen `git diff` sometimes hang when using a rsync'd
        # checkout. But it seems to work ok if we first diff a single file.
        #log(f'Doing dummy `git diff` of a single file, to avoid potential hang in later `git diff` on Windows.')
        e, text = run_git(f'cd {directory} && git ls-files')
        assert not e, f'git ls-files failed: {e=}'
        path0 = text.split('\n', 1)[0].strip()
        run_git(f'cd {directory} && git diff {path0}')
        
    e, out = run_git(f'cd {directory} && git --no-pager diff')
    if not e:
        diff = out
    
    e, out = run_git(f'cd {directory} && git --no-pager rev-parse --abbrev-ref HEAD')
    if not e:
        branch = out.strip()
    
    #log(f'git_info(): {directory=} {branch=} {sha=} {comment=}')
    return sha, comment, diff, branch


def get_time_iso_8601(text):
    '''
    Converts ISO-8601 text to Unix time.
    '''
    t = time.strptime(text, '%Y-%m-%dT%H:%M:%S%z')
    t = calendar.timegm(t)
    return t


def git_info_author_date(directory):
    '''
    Returns author date (as a Unix time) of HEAD.
    '''
    text = run(f'cd {directory} && git show -s --format=%aI HEAD', capture=1)
    return get_time_iso_8601(text.strip())


def git_info_committer_date(directory):
    '''
    Returns committer date (as a Unix time) of HEAD.
    '''
    text = run(f'cd {directory} && git show -s --format=%cI HEAD', capture=1)
    return get_time_iso_8601(text.strip())


def git_items( directory, submodules=False):
    '''
    Returns list of paths for all files known to git within a `directory`.

    Args:
        directory:
            Must be somewhere within a git checkout.
        submodules:
            If true we also include git submodules.

    Returns:
        A list of paths for all files known to git within `directory`. Each
        path is relative to `directory`. `directory` must be somewhere within a
        git checkout.

    We run a `git ls-files` command internally.

    This function can be useful for the `fn_sdist()` callback.
    '''
    command = 'cd ' + directory + ' && git ls-files'
    if submodules:
        command += ' --recurse-submodules'
    log1(f'Running {command=}')
    text = subprocess.check_output( command, shell=True)
    ret = []
    for path in text.decode('utf8').strip().split( '\n'):
        path2 = os.path.join(directory, path)
        # Sometimes git ls-files seems to list empty/non-existent directories
        # within submodules.
        #
        if not os.path.exists(path2):
            log2(f'Ignoring git ls-files item that does not exist: {path2}')
        elif os.path.isdir(path2):
            log2(f'Ignoring git ls-files item that is actually a directory: {path2}')
        else:
            ret.append(path)
    return ret


def fs_write_key(path, data):
    '''
    Writes <data> to <path>, ensuring that <path> is created with appropriate
    permissions.
    '''
    fs_remove(path)
    if platform.system() == 'Windows':
        # For unknown reasons, the code below for non-Windows does not work
        # on Windows.
        #
        # Also for unknown reasons, using backslashes in
        # path doesn't work - we write the file but it
        # doesn't appear in the filesystem.
        path = path.replace('\\', '/')
        with open(path, 'wb') as f:
            f.write(data.encode('utf8').replace(b'\r', b''))
    else:
        # Need to create file as read/write for current user only, so we have
        # to use `os.open()` instead of `open()`.
        #
        fd = os.open(path, os.O_WRONLY|os.O_CREAT|os.O_TRUNC|os.O_EXCL, 0o600)
        try:
            os.write(fd, data.encode('utf8'))
        finally:
            os.close(fd)


def git_get(
        local,
        *,
        remote=None,
        branch=None,
        tag=None,
        sha=None,
        text=None,
        depth=1,
        env_extra=None,
        update=True,
        submodules=True,
        devel=1,
        key=None,
        keyfile=None,
        clone_extra=None,
        ):
    '''
    Creates/updates local checkout <local> of remote repository and returns
    absolute path of <local>.

    If <text> is set but does not start with 'git:', it is assumed to be an up
    to date local checkout, and we return absolute path of <text> without doing
    any git operations.
    
    Args:
        local:
            Local directory. Created and/or updated using `git clone` and `git
            fetch` etc.
        remote:
            Remote git repostitory, for example
            'https://github.com/ArtifexSoftware/mupdf.git'. Can be overridden
            by <text>.
        branch:
            Branch to use; can be overridden by <text>.
        tag:
            Tag to use; can be overridden by <text>.
        sha:
            Use this sha; must be the full length sha,
            e.g. `5ea4449c3edba8840d5c27e15095536cbfc7c1f`
        text:
            If None or empty:
                Ignored.
            
            If starts with 'git:':
                The remaining text should be a command-line
                style string containing some or all of these args:
                    -b|--branch <branch>
                    -t|-tag <tag>
                    --depth <depth>
                    <remote>
                These overrides <branch>, <tag>, <remote> and <depth>.
            Otherwise:
                <text> is assumed to be a local directory, and we simply return
                it as an absolute path without doing any git operations.
            
            For example these all clone/update/branch master of https://foo.bar/qwerty.git to local
            checkout 'foo-local':
            
                git_get('foo-local', remote='https://foo.bar/qwerty.git', branch='master')
                git_get('foo-local', text='git:--branch master https://foo.bar/qwerty.git')
                git_get('foo-local', text='git:--branch master', remote='https://foo.bar/qwerty.git')
                git_get('foo-local', text='git:', branch='master', remote='https://foo.bar/qwerty.git')
        depth:
            Depth of local checkout when cloning and fetching, or None.
        env_extra:
            Dict of extra name=value environment variables to use whenever we
            run git.
        update:
            If false we do not update existing repository. Might be useful if
            testing without network access.
        submodules:
            If true, we clone with `--recursive --shallow-submodules` and run
            `git submodule update --init --recursive` before returning.
        key:
            Ssh key to use.
        keyfile:
            Ssh key file to use. Only used if <key> is None.
        clone_extra:
            If true, is added to any `git clone` command. For example
            `clone_extra='--config core.autocrlf=input'` to avoid converting
            text files to Windows line endings.
    '''
    log(f'{os.getcwd()=}')
    log(f'{remote=}')
    log(f'{local=}')
    log(f'{branch=}')
    log(f'{tag=}')
    log(f'{text=}')
    log(f'{env_extra=}')
    log(f'{update=}')
    log(f'{submodules=}')
    log(f'{devel=}')
    log(f'{bool(key)=}')
    log(f'{keyfile=}')
    
    if text:
        if text.startswith('git:'):
            args = iter(shlex.split(text[len('git:'):]))
            while 1:
                try:
                    arg = next(args)
                except StopIteration:
                    break
                if arg in ('-b', '--branch'):
                    branch = next(args)
                    tag = None
                    sha = None
                elif arg in ('-t', '--tag'):
                    tag = next(args)
                    branch = None
                    sha = None
                elif arg in ('-s', '--sha'):
                    sha = next(args)
                    branch = None
                    tag = None
                elif arg == '--depth':
                    depth = int(next(args))
                elif arg.startswith('-'):
                    assert 0, f'Unrecognised {arg=} in {text=}.'
                else:
                    remote = arg
            assert remote, f'<remote> unset and no remote specified in {text=}.'
            assert branch or tag or sha, f'<branch> and <tag> unset and no branch/tag specified in {text=}.'
        else:
            log0(f'Using local directory {text!r}.')
            return os.path.abspath(text)
        
    assert bool(branch) ^ bool(tag) ^ bool(sha), \
            f'Must specify exactly one of <branch>, <tag> or <tag>; {branch=} {tag=} {sha=}.'
    
    depth_arg = f' --depth {depth}' if depth else ''
    
    paths_to_delete = list()
    if key:
        keyfile = os.path.abspath('pipcl-tmp-git-key')
        paths_to_delete.append(keyfile)
        fs_write_key(keyfile, key)
    if keyfile:
        GIT_SSH_COMMAND = f'ssh -i {os.path.abspath(keyfile)} -o StrictHostKeyChecking=no'
        GIT_SSH_COMMAND = GIT_SSH_COMMAND.replace('\\', '/')    # Required on windows.
        log(f'{keyfile=}')
        log(f'{GIT_SSH_COMMAND=}')
        env_extra = (env_extra or dict()) | dict(GIT_SSH_COMMAND=GIT_SSH_COMMAND)
    
    if ':' not in remote:
        # Allow clone/fetch from local directory.
        remote = os.path.abspath(remote)
    
    try:
        # Convert local remotes to absolute path so it still works after we've
        # cd-ed into local directory.
        #
        if ':' not in remote:
            remote_0 = remote
            remote = os.path.abspath(remote)
            log(f'Changed local remote {remote_0!r} to {remote=}')
        
        def do_update():
            # This seems to pull in the entire repository.
            log0(f'do_update(): attempting to update {local=}.')
            # Remove any local changes.
            run(
                    f'cd {local} && git reset --hard',
                    env_extra=env_extra,
                    prefix='git reset: ',
                    )

            if tag:
                # `-u` avoids `fatal: Refusing to fetch into current branch`.
                # Using '+' and `revs/tags/` prefix seems to avoid errors like:
                #   error: cannot update ref 'refs/heads/v3.16.44':
                #   trying to write non-commit object
                #   06c4ae5fe39a03b37a25a8b95214d9f8f8a867b8 to branch
                #   'refs/heads/v3.16.44'
                #
                run(
                        f'cd {local} && git fetch -fuv{depth_arg} {remote} +refs/tags/{tag}:refs/tags/{tag}',
                        env_extra=env_extra,
                        prefix='git fetch: ',
                        )
                run(
                        f'cd {local} && git checkout -f {tag}',
                        env_extra=env_extra,
                        prefix='git checkout: ',
                        )
            if branch:
                # `-u` avoids `fatal: Refusing to fetch into current branch`.
                run(
                        f'cd {local} && git fetch -fuv{depth_arg} {remote} {branch}:{branch}',
                        env_extra=env_extra,
                        prefix='git fetch: ',
                        )
                run(
                        f'cd {local} && git checkout -f {branch}',
                        env_extra=env_extra,
                        prefix='git checkout: ',
                        )
            if sha:
                # `-u` avoids `fatal: Refusing to fetch into current branch`.
                run(
                        f'cd {local} && git fetch -fuv{depth_arg} {remote} {sha}',
                        env_extra=env_extra,
                        prefix='git fetch: ',
                        )
                run(
                        f'cd {local} && git checkout -f {sha}',
                        env_extra=env_extra,
                        prefix='git checkout: ',
                        )

        do_clone = True
        if os.path.isdir(f'{local}/.git'):
            if update:
                # Try to update existing checkout.
                try:
                    do_update()
                    do_clone = False
                except Exception as e:
                    log0(f'Failed to update existing checkout {local}: {e}')
            else:
                do_clone = False

        if do_clone:
            # No existing git checkout, so do a fresh clone.
            #_fs_remove(local)
            log0(f'{os.getcwd()=}')
            log0(f'Cloning to: {os.path.abspath(local)}')
            command = f'git clone --config core.longpaths=true{depth_arg}'
            if clone_extra:
                command += f' {clone_extra}'
            if submodules:
                command += f' --recursive --shallow-submodules'
            if branch:
                command += f' -b {branch}'
            #if tag:
            #    command += f' -t {tag}'
            command += f' {remote} {local}'
            run(
                    command,
                    env_extra=env_extra,
                    prefix='git clone: ',
                    )
            do_update()

        if submodules:
            run(
                    f'cd {local} && git submodule update --init --recursive',
                    env_extra=env_extra,
                    prefix='git submodule update: ',
                    )

        # Show sha of checkout.
        run(
                f'cd {local} && git --no-pager log -1 --pretty=oneline',
                check=False,
                prefix='git show: ',
                )
        if 0:
            # Some extra checking that what we're doing is correct.
            if tag:
                diff = run(f'cd {local} && git --no-pager diff {tag}', capture=1)
            if branch:
                diff = run(f'cd {local} && git --no-pager diff {branch}', capture=1)
            if sha:
                diff = run(f'cd {local} && git --no-pager diff {sha}', capture=1)
            log(f'{diff=}')
            diff = diff.strip()
            assert not diff, f'Diff is not empty:\n{textwrap.indent(diff, "    ")}'

        return os.path.abspath(local)
    
    finally:
        for path in paths_to_delete:
            fs_remove(path)
    

def run(
        command,
        *,
        capture=False,
        check=1,
        verbose=1,
        env=None,
        env_extra=None,
        timeout=None,
        caller=1,
        prefix=None,
        encoding=None,  # System default.
        errors='backslashreplace',
        ticker=0,
        out=None,
        tee=None,
        log=None,
        ):
    '''
    Runs a command using `subprocess.run()`.

    Args:
        command:
            A string command to run or list of string args to run.

            If a string:
                Multiple lines in `command` are treated as a single command.

                * If a line starts with `#` it is discarded.
                * If a line contains ` #`, the trailing text is discarded.

                When running the command on Windows, newlines are replaced by
                spaces; otherwise each line is terminated by a backslash character.
        capture:
            If true, we include the command's output in our return value.
        check:
            If true we raise an exception on error; otherwise we include the
            command's returncode in our return value.
        verbose:
            If true we show the command.
        env:
            None or dict to use instead of <os.environ>.
        env_extra:
            None or dict to add to <os.environ> or <env>.
        timeout:
            If not None, timeout in seconds; passed directly to
            `subprocess.run()`. Note that on MacOS `subprocess.run()` seems to
            leave processes running if timeout expires.
        prefix:
            String prefix for each line of output.

            If true:
            * We run command with stdout=subprocess.PIPE and
              stderr=subprocess.STDOUT, repetaedly reading the command's output
              and writing it with <prefix>.
            * We do not support <timeout>, which must be None.
        encoding:
        errors:
            Encoding of child process output.
        ticker:
            If non-zero, we show a rotating bar when waiting for child process
            output, with <ticker> seconds between each rotation.
        out:
            Where to write output. Single item or list/tuple of items. Each item
            can be a callable or an object with .write() and .flush() members.
            Special value 'log' writes to `pipcl.log()`.
        tee:
            Name of file to also write to. We refuse to overwrite existing file.
        log:
            If true we write to `pipcl.log()`. This is the default.
    
    On Windows:
        <ticker> is not supported and ignored.
        <prefix> and <timeout> both being true is not supported, and will raise
        an exception.
        These restrictions are because the Python module `selectors` does not
        handle pipes.
    
    Returns:
        check capture   Return
        --------------------------
          false false   returncode
          false  true   (returncode, output)
          true  false   None or raise exception
          true   true   output or raise exception
    '''
    if env is None:
        env = os.environ
        if env_extra:
            env = env.copy()
    if env_extra:
        env.update(env_extra)
    lines = _command_lines( command)
    if verbose:
        text = f'Running:'
        nl = '\n    '
        text += f' {nl.join(lines)}'
        if env_extra:
            text += f'\nwith:\n'
            for k in sorted(env_extra.keys()):
                text += f'    {k}={shlex.quote(env_extra[k])}\n'
        log1(text, caller=caller+1)
    if isinstance(command, str):
        sep = ' ' if windows() else ' \\\n'
        command2 = sep.join( lines)
    else:
        command2 = command
    
    if platform.system() == 'Windows':
        # The `selectors` module does not support pipes.
        ticker = False
    
    cleanup = list()
    try:
        if out:
            if isinstance(out, tuple):
                out = list(out)
            elif not isinstance(out, list):
                out = [out]
            else:
                out = out.copy()
        else:
            out = list()
        if log:
            out.append('log')
        if not out and not capture:
            out = ['log']

        if tee:
            assert not os.path.exists(tee), f'Tee file already exists: {tee}'
            tee_f = open(tee, 'w')  # pylint: disable=consider-using-with
            cleanup.append(tee_f.close)
            out.append(tee_f)

        # Convert `out` into list of (write, flush) callables.
        for i, o in enumerate(out):
            if callable(o):
                out[i] = (o, lambda: None)
            elif o == 'log':
                write2 = lambda text: _log(text, level=0, caller=caller+3, raw=True)
                out[i] = (write2, None)
                def log_cleanup():
                    if not _log_text_line_start:
                        write2('\n')
                cleanup.append(log_cleanup)
            else:
                out[i] = (o.write, o.flush)

        def write(text):
            for write2, _flush2 in out:
                write2(text)
        def flush():
            for _write2, flush2 in out:
                if flush2:
                    flush2()    # pylint: disable=not-callable

        # pylint: disable=too-many-nested-blocks
        if prefix or ticker or out!=[sys.stdout]:
            # Use explicit read loop from child process.
            if platform.system() == 'Windows':
                # The `selectors` module does not support pipes.
                assert not timeout, 'timeout with (prefix or out) not supported on Windows'
            
            child = subprocess.Popen(   # pylint: disable=consider-using-with
                    command2,
                    shell=isinstance(command2, str),
                    stdout=subprocess.PIPE,
                    stderr=subprocess.STDOUT,
                    encoding=encoding,
                    errors=errors,
                    env=env,
                    )

            if capture:
                capture_text = ''
            decoder = codecs.getincrementaldecoder(child.stdout.encoding)(errors)
            line_start = True
            timed_out = False
            
            waiter = _Wait(child.stdout.fileno(), ticker, timeout)
            cleanup.append(waiter.close)
            is_windows = windows()
            while 1:
                if waiter.wait():
                    child.terminate()
                    timed_out = True
                    break

                raw = os.read( child.stdout.fileno(), 10000)
                text = decoder.decode(raw, final=not raw)
                if is_windows:
                    # Presence of \r bytes on windows causes them to
                    # proliferate in nested runs, leading to logs being
                    # difficult to read. So we remove them explicitly. (Our
                    # subprocess.Popen instance will probably have text mode
                    # stdout/stderr, but we read from child.stdout.fileno()
                    # which will always be binary.)
                    #
                    text = text.replace('\r\n', '\n')
                if text:
                    waiter.cancel_ticker()
                    if capture:
                        capture_text += text
                    lines = text.split('\n')
                    for i, line in enumerate(lines):
                        if line_start:
                            if prefix:
                                write(prefix)
                            line_start = False
                        write(line)
                        if i < len(lines) - 1:
                            write('\n')
                            line_start = True
                        #else:
                        #   line_start = not line
                    flush()
                if not raw:
                    break
            waiter.cancel_ticker()
            if not line_start:
                write('\n')
            e = child.wait()
            if timed_out and check:
                raise subprocess.TimeoutExpired(e, command2, capture_text if capture else None)
            if check and e:
                raise subprocess.CalledProcessError(e, command2, capture_text if capture else None)
            if check:
                return capture_text if capture else None
            else:
                return (e, capture_text) if capture else e
        else:
            # Simple case - we can simply call subprocess.run().
            cp = subprocess.run(
                    command2,
                    shell=True,
                    stdout=subprocess.PIPE if capture else None,
                    stderr=subprocess.STDOUT if capture else None,
                    check=check,
                    encoding=encoding,
                    errors=errors,
                    env=env,
                    timeout=timeout,
                    )
        if check:
            return cp.stdout if capture else None
        else:
            return (cp.returncode, cp.stdout) if capture else cp.returncode
    finally:
        for fn in cleanup:
            fn()


def darwin():
    '''
    Returns true of we are running on MacOS.
    '''
    return sys.platform.startswith( 'darwin')

def windows():
    '''
    Returns true of we are running on Windows.
    '''
    return platform.system() == 'Windows'

def wasm():
    '''
    Returns true of we are running on Wasm.
    '''
    return os.environ.get( 'OS') in ('wasm', 'wasm-mt')

def pyodide():
    '''
    Returns true of we are running on Pyodide.
    '''
    return os.environ.get( 'PYODIDE') == '1'

def linux():
    '''
    Returns true of we are running on Linux.
    '''
    return platform.system() == 'Linux'

def openbsd():
    '''
    Returns true of we are running on OpenBSD.
    '''
    return platform.system() == 'OpenBSD'


def show_system():
    '''
    Show useful information about the system plus argv and environ.
    
    Omits os.environ if $PIPCL_SHOW_ENV is '0'.
    '''
    def log(text):
        log0(text, caller=3)
    
    #log(f'{__file__=}')
    #log(f'{__name__=}')
    log(f'{os.getcwd()=}')
    log(f'{platform.machine()=}')
    log(f'{platform.platform()=}')
    log(f'{platform.python_implementation()=}')
    log(f'{platform.python_version()=}')
    log(f'{platform.system()=}')
    if sys.implementation.name != 'graalpy':
        log(f'{platform.uname()=}')
    log(f'{sys.executable=}')
    log(f'{sys.version=}')
    log(f'{sys.version_info=}')
    log(f'{list(sys.version_info)=}')
    log(f'{sys.prefix=}')
    log(f'{sys.base_prefix=}')
    
    log(f'{sysconfig.get_config_var("Py_GIL_DISABLED")=}')
    try:
        log(f'{sys._is_gil_enabled()=}')    # # pylint: disable=protected-access
    except AttributeError as e:
        log(f'sys._is_gil_enabled() => exception {e}')
    
    log(f'CPU bits: {cpu_bits()}')
    
    log(f'sys.argv ({len(sys.argv)}):')
    for i, arg in enumerate(sys.argv):
        log(f'    {i}: {arg!r}')
    
    PIPCL_SHOW_ENV = os.environ.get('PIPCL_SHOW_ENV')
    if PIPCL_SHOW_ENV == '0':
        log(f'[Not showing os.environ because {PIPCL_SHOW_ENV=}.]')
    else:
        log(f'os.environ ({len(os.environ)}):')
        for k in sorted( os.environ.keys()):
            v = os.environ[ k]
            if 'BEGIN OPENSSH PRIVATE KEY' in v:
                # Don't show private keys.
                log(f'    {k} ****')
            else:
                log( f'    {k}: {v!r}')


class PythonFlags:
    '''
    Compile/link flags for the current python, for example the include path
    needed to get `Python.h`.

    The 'PIPCL_PYTHON_CONFIG' environment variable allows to override
    the location of the python-config executable.

    Members:
        .includes:
            String containing compiler flags for include paths.
        .ldflags:
            String containing linker flags for library paths.
    '''
    def __init__(self):
        
        # Experimental detection of python flags from sysconfig.*() instead of
        # python-config command.
        includes_, ldflags_ = sysconfig_python_flags()
      
        if pyodide():
            _include_dir = os.environ[ 'PYO3_CROSS_INCLUDE_DIR']
            _lib_dir = os.environ[ 'PYO3_CROSS_LIB_DIR']
            self.includes = f'-I {_include_dir}'
            self.ldflags = f'-L {_lib_dir}'

        elif 0:

            self.includes = includes_
            self.ldflags = ldflags_
        
        elif windows():
            wp = wdev.WindowsPython()
            self.includes = f'/I"{wp.include}"'
            self.ldflags = f'/LIBPATH:"{wp.libs}"'

        elif pyodide():
            _include_dir = os.environ[ 'PYO3_CROSS_INCLUDE_DIR']
            _lib_dir = os.environ[ 'PYO3_CROSS_LIB_DIR']
            self.includes = f'-I {_include_dir}'
            self.ldflags = f'-L {_lib_dir}'

        else:
            python_config = os.environ.get("PIPCL_PYTHON_CONFIG")
            if not python_config:
                # We use python-config which appears to work better than pkg-config
                # because it copes with multiple installed python's, e.g.
                # manylinux_2014's /opt/python/cp*-cp*/bin/python*.
                #
                # But... on non-macos it seems that we should not attempt to specify
                # libpython on the link command. The manylinux docker containers
                # don't actually contain libpython.so, and it seems that this
                # deliberate. And the link command runs ok.
                #
                python_exe = os.path.realpath( sys.executable)
                if darwin():
                    # Basic install of dev tools with `xcode-select --install` doesn't
                    # seem to provide a `python3-config` or similar, but there is a
                    # `python-config.py` accessible via sysconfig.
                    #
                    # We try different possibilities and use the last one that
                    # works.
                    #
                    python_config = None
                    for pc in (
                            f'python3-config',
                            f'{sys.executable} {sysconfig.get_config_var("srcdir")}/python-config.py',
                            f'{python_exe}-config',
                            ):
                        e = subprocess.run(
                                f'{pc} --includes',
                                shell=1,
                                stdout=subprocess.DEVNULL,
                                stderr=subprocess.DEVNULL,
                                check=0,
                                ).returncode
                        log2(f'{e=} from {pc!r}.')
                        if e == 0:
                            python_config = pc
                    assert python_config, f'Cannot find python-config'
                else:
                    python_config = f'{python_exe}-config'
            log2(f'Using {python_config=}.')
            try:
                self.includes = run( f'{python_config} --includes', capture=1, verbose=0).strip()
            except Exception as e:
                raise Exception('We require python development tools to be installed.') from e
            self.ldflags = run( f'{python_config} --ldflags', capture=1, verbose=0).strip()
            if linux():
                # It seems that with python-3.10 on Linux, we can get an
                # incorrect -lcrypt flag that on some systems (e.g. WSL)
                # causes:
                #
                #   ImportError: libcrypt.so.2: cannot open shared object file: No such file or directory
                #
                ldflags2 = self.ldflags.replace(' -lcrypt ', ' ')
                if ldflags2 != self.ldflags:
                    log2(f'### Have removed `-lcrypt` from ldflags: {self.ldflags!r} -> {ldflags2!r}')
                    self.ldflags = ldflags2

        if 0:
            log1(f'{self.includes=}')
            log1(f'    {includes_=}')
            log1(f'{self.ldflags=}')
            log1(f'    {ldflags_=}')


def macos_add_cross_flags(command):
    '''
    If running on MacOS and environment variables ARCHFLAGS is set
    (indicating we are cross-building, e.g. for arm64), returns
    `command` with extra flags appended. Otherwise returns unchanged
    `command`.
    '''
    if darwin():
        archflags = os.environ.get( 'ARCHFLAGS')
        if archflags:
            command = f'{command} {archflags}'
            log2(f'Appending ARCHFLAGS to command: {command}')
            return command
    return command


def macos_patch( library, *sublibraries):
    '''
    If running on MacOS, patches `library` so that all references to items in
    `sublibraries` are changed to `@rpath/{leafname}`. Does nothing on other
    platforms.

    library:
        Path of shared library.
    sublibraries:
        List of paths of shared libraries; these have typically been
        specified with `-l` when `library` was created.
    '''
    log2( f'macos_patch(): library={library}  sublibraries={sublibraries}')
    if not darwin():
        return
    if not sublibraries:
        return
    subprocess.run( f'otool -L {library}', shell=1, check=1)
    command = 'install_name_tool'
    for sublibrary in sublibraries:
        name = subprocess.run(
                f'otool -D {sublibrary}',
                shell=1,
                check=1,
                capture_output=1,
                encoding='utf8',
                ).stdout.strip()
        name = name.split('\n')
        assert len(name) == 2 and name[0] == f'{sublibrary}:', f'{name=}'
        name = name[1]
        # strip trailing so_name.
        leaf = os.path.basename(name)
        m = re.match('^(.+[.]((so)|(dylib)))[0-9.]*$', leaf)
        assert m
        log2(f'Changing {leaf=} to {m.group(1)}')
        leaf = m.group(1)
        command += f' -change {name} @rpath/{leaf}'
    command += f' {library}'
    log2( f'Running: {command}')
    subprocess.run( command, shell=1, check=1)
    subprocess.run( f'otool -L {library}', shell=1, check=1)


def _macos_fixup_platform_tag(tag):
    '''
    Patch up platform tag on MacOS.

    E.g. `foo-1.2.3-cp311-none-macosx_13_x86_64.whl` causes `pip` to fail with:
    `not a supported wheel on this platform`. We seem to need to add `_0` to
    the OS version. (This is documented at
    https://packaging.python.org/en/latest/specifications/platform-compatibility-tags/#macos).

    And with graal we need to replace trailing `universal2` with x86_64
    or arm64. On non-graal this causes problems because non-universal
    platform tags seem more restricted than platform tags from
    `sysconfig.get_platform()`. For example:
    
        pip install ...-macosx_10_13_arm64.whl
            ERROR: ...-macosx_10_13_arm64.whl is not a supported wheel on this platform.
        pip install ...-macosx_10_13_universal2.whl
            Ok.
    '''
    m = re.match( '^macosx_([0-9_]+)_([^0-9].+)$', tag)
    if not m:
        return tag
    a = m.group(1)
    if '_' not in a:
        a += '_0'
    b = m.group(2)
    if sys.implementation.name == 'graalpy' and b == 'universal2':
        # Replace 'universal2' with x86_64 or arm64.
        b = platform.machine()
    ret = f'macosx_{a}_{b}'
    #log0(f'Changing from {tag=} to {ret=}.')
    return ret
    

# Internal helpers.
#

class _Wait:
    def __init__(self, fd, ticker, timeout):
        '''
        Helper when reading output from child process.
        fd: child's output.
        ticker: ticker period, or false for no ticker.
        timeout: .
        '''
        self.fd = fd
        self.ticker = ticker
        if ticker:
            self.ticker_state = _TickerState()
        self.endtime = time.time() + timeout if timeout else None
        self.ticker_start = None
        self.selector = selectors.DefaultSelector()
        self.selector.register(fd, selectors.EVENT_READ)
    
    def wait(self):
        '''
        Waits for output or timeout or ticker-related timeout. Updates ticker
        as required.

        Returns true if original timeout exceeded, else None.

        After we return, the caller should call .cancel_ticker() if it reads
        non-empty output.
        '''
        if not self.endtime and not self.ticker:
            return
        while 1:
            # We need to use a timeout.
            t = time.time()
            if self.ticker:
                if self.ticker_start is None:
                    self.ticker_start = t
                assert t >= self.ticker_start
                endtime_ticker = self.ticker_start + ((t - self.ticker_start) // self.ticker + 1) * self.ticker
                assert endtime_ticker > self.ticker_start
                assert endtime_ticker > t
                assert endtime_ticker <= t + self.ticker

            if self.ticker and self.endtime:
                endtime2 = min(self.endtime, endtime_ticker) # pylint: disable=possibly-used-before-assignment
            elif self.ticker:
                endtime2 = endtime_ticker
            else:
                endtime2 = self.endtime

            assert endtime2 > t, f'{t=} {endtime2=}'
            events = self.selector.select(endtime2 - t)

            if events:
                # Data is available.
                return
            else:
                # timeout
                t2 = time.time()
                if self.endtime and t2 >= self.endtime:
                    return True
                if self.ticker and t2 >= endtime_ticker:
                    self.ticker_state.increment(self.ticker_start, endtime_ticker)
                continue
    
    def cancel_ticker(self):
        self.ticker_start = None
        if self.ticker:
            self.ticker_state.cancel()
    
    def close(self):
        self.cancel_ticker()
        self.selector.close()
    

class _TickerState:
    '''
    Helper for ticker in `pipcl.run()`.
    '''
    def __init__(self):
        self.current_length = 0
        self.rotate_count = 0
        self.time_t0 = None
    
    def increment(self, t1, t2):
        '''
        Update the ticker display.
        t1: Start time; only used if we were not already showing ticker.
        t2: Current time to show.
        '''
        self.rotate_count += 1
        rotate_c = '\\|/-'[self.rotate_count % 4]
        if self.time_t0 is None:
            self.time_t0 = t1
        dt = _duration(t2 - self.time_t0)
        dt = f'[{rotate_c} {dt}]'
        tail_len = max(0, self.current_length - len(dt))
        tail = ' ' * tail_len + '\b' * tail_len
        sys.stdout.write('\b' * self.current_length + dt + tail)
        self.current_length = len(dt)
        sys.stdout.flush()
    
    def cancel(self):
        if self.rotate_count:
            sys.stdout.write('\b \b' * self.current_length)
            sys.stdout.flush()
            self.current_length = 0
            self.rotate_count = 0
            self.time_t0 = None


def _command_lines( command):
    '''
    Process multiline command by running through `textwrap.dedent()`, removes
    comments (lines starting with `#` or ` #` until end of line), removes
    entirely blank lines.

    Returns list of lines.
    '''
    if isinstance(command, str):
        command = textwrap.dedent( command)
        lines = []
        for line in command.split( '\n'):
            if line.startswith( '#'):
                h = 0
            else:
                h = line.find( ' #')
            if h >= 0:
                line = line[:h]
            if line.strip():
                lines.append(line.rstrip())
    else:
        return command
    return lines


def cpu_bits():
    '''
    Returns CPU wordsize in bits.
    '''
    return int.bit_length(sys.maxsize+1)


def _cpu_name():
    '''
    Returns `x32` or `x64` depending on Python build.
    '''
    #log(f'sys.maxsize={hex(sys.maxsize)}')
    return f'x{32 if sys.maxsize == 2**31 - 1 else 64}'


def run_if( command, out, *prerequisites, caller=1):
    '''
    Runs a command only if the output file is not up to date.

    Args:
        command:
            The command to run. We write this and a hash of argv[0] into a file
            <out>.cmd so that we know to run a command if the command itself
            has changed.
        out:
            Path of the output file.

        prerequisites:
            List of prerequisite paths or true/false/None items. If an item
            is None it is ignored, otherwise if an item is not a string we
            immediately return it cast to a bool. We recurse into directories,
            effectively using the newest file in the directory.

    Returns:
        True if we ran the command, otherwise None.


    If the output file does not exist, the command is run:

        >>> del _log_f[:]
        >>> _log_f.append(sys.stdout)
        >>> log_prefix('%f(): ')
        >>> out = 'tests/run_if_test_out'
        >>> fs_remove(out)
        >>> fs_remove( f'{out}.cmd')
        >>> run_if( f'touch {out}', out, caller=0)
        run_if(): Running command because: File does not exist: 'tests/run_if_test_out'
        run_if(): Running: touch tests/run_if_test_out
        True

    If we repeat, the output file will be up to date so the command is not run:

        >>> run_if( f'touch {out}', out, caller=0)
        run_if(): Not running command because up to date: 'tests/run_if_test_out'

    If we change the command, the command is run:

        >>> run_if( f'touch {out};', out, caller=0)
        run_if(): Running command because: Command has changed:
        run_if():     @@ -1,2 +1,2 @@
        run_if():      touch
        run_if():     -tests/run_if_test_out
        run_if():     +tests/run_if_test_out;
        run_if(): Running: touch tests/run_if_test_out;
        True

    If we add a prerequisite that is newer than the output, the command is run:

        >>> time.sleep(1)
        >>> prerequisite = 'tests/run_if_test_prerequisite'
        >>> run( f'touch {prerequisite}', caller=0)
        run(): Running: touch tests/run_if_test_prerequisite
        >>> run_if( f'touch  {out}', out, prerequisite, caller=0)
        run_if(): Running command because: Command has changed:
        run_if():     @@ -1,2 +1,2 @@
        run_if():      touch
        run_if():     -tests/run_if_test_out;
        run_if():     +tests/run_if_test_out
        run_if(): Running: touch  tests/run_if_test_out
        True

    If we repeat, the output will be newer than the prerequisite, so the
    command is not run:

        >>> run_if( f'touch  {out}', out, prerequisite, caller=0)
        run_if(): Not running command because up to date: 'tests/run_if_test_out'
    
    Also see OS-specific doctest examples for
    `_run_if_test_scripting_windows()` and `_run_if_test_scripting_unix()`.
    '''
    doit = False
    
    # Path of file containing pickle data for command and hash of command's
    # first arg.
    cmd_path = f'{out}.cmd'
    
    def hash_get(path):
        try:
            with open(path, 'rb') as f:
                return hashlib.md5(f.read()).hexdigest()
        except Exception:
            #log(f'Failed to get hash of {path=}: {e}')
            return None
    
    lines = _command_lines( command)
    command2 = ' '.join(lines)
    command_args = shlex.split(command2 or '')
    command_arg0_path = fs_find_in_paths(command_args[0])
    command_arg0_hash = hash_get(command_arg0_path)
    
    cmd_args, cmd_arg0_hash = (None, None)
    if os.path.isfile(cmd_path):
        with open(cmd_path, 'rb') as f:
            try:
                cmd_args, cmd_arg0_hash = pickle.load(f)
            except Exception:
                pass

    if not doit:
        # Set doit if outfile does not exist.
        out_mtime = _fs_mtime( out)
        if out_mtime == 0:
            doit = f'File does not exist: {out!r}'

    if not doit:
        # Set doit if command has changed.
        if command_args != cmd_args:
            if cmd_args is None:
                doit = 'No previous command stored'
            else:
                doit = f'Command has changed'
                # Show diff based on commands split into pseudo lines by
                # shlex.split().
                doit += ':\n'
                lines = difflib.unified_diff(
                        cmd_args,
                        command_args,
                        lineterm='',
                        )
                # Skip initial lines.
                assert next(lines) == '--- '
                assert next(lines) == '+++ '
                for line in lines:
                    doit += f'    {line}\n'

    if not doit:
        # Set doit if argv[0] hash has changed.
        #print(f'{cmd_arg0_hash=} {command_arg0_hash=}', file=sys.stderr)
        if command_arg0_hash != cmd_arg0_hash:
            doit = f'arg0 hash has changed.'
            #doit = f'arg0 hash has changed from {cmd_arg0_hash=} to {command_arg0_hash=}..'
    
    if not doit:
        # See whether any prerequisites are newer than target.
        def _make_prerequisites(p):
            if isinstance( p, (list, tuple)):
                return list(p)
            else:
                return [p]
        prerequisites_all = list()
        for p in prerequisites:
            prerequisites_all += _make_prerequisites( p)
        if 0:
            log2( 'prerequisites_all:', caller=caller+1)
            for i in  prerequisites_all:
                log2( f'    {i!r}', caller=caller+1)
        pre_mtime = 0
        pre_path = None
        for prerequisite in prerequisites_all:
            if isinstance( prerequisite, str):
                mtime = _fs_mtime_newest( prerequisite)
                if mtime >= pre_mtime:
                    pre_mtime = mtime
                    pre_path = prerequisite
            elif prerequisite is None:
                pass
            elif prerequisite:
                doit = str(prerequisite)
                break
        if not doit:
            if pre_mtime > out_mtime:
                doit = f'Prerequisite is new: {os.path.abspath(pre_path)!r}'

    if doit:
        # Remove `cmd_path` before we run the command, so any failure
        # will force rerun next time.
        #
        fs_remove(cmd_path)
        log1( f'Running command because: {doit}', caller=caller+1)

        run( command, caller=caller+1)

        # Write the command we ran, into `cmd_path`.
        with open(cmd_path, 'wb') as f:
            pickle.dump((command_args, command_arg0_hash), f)
        return True
    else:
        log1( f'Not running command because up to date: {out!r}', caller=caller+1)


if windows():
    def _run_if_test_scripting_windows():
        '''
        Dummy function to allow Windows-only doctests for pipcl.run_if().
        
        We detect changes to the contents of argv[0]:

        Create a shell script and run it:

        >>> del _log_f[:]
        >>> _log_f.append(sys.stdout)
        >>> log_prefix('%f(): ')
        
        >>> for path in glob.glob('tests/run_if_test_argv0.*'):
        ...     fs_remove(path)
        >>> with open('tests/run_if_test_argv0.bat', 'w') as f:
        ...     print('@echo off', file=f)
        ...     print('echo hello world > tests/run_if_test_argv0.out', file=f)
        >>> import stat
        >>> os.chmod('tests/run_if_test_argv0.bat', os.stat('tests/run_if_test_argv0.bat').st_mode | stat.S_IEXEC)

        >>> run_if('"tests/run_if_test_argv0.bat"', f'tests/run_if_test_argv0.out', caller=0)
        run_if(): Running command because: File does not exist: 'tests/run_if_test_argv0.out'
        run_if(): Running: "tests/run_if_test_argv0.bat"
        True

        Running it a second time does nothing:

        >>> run_if(f'"tests/run_if_test_argv0.bat"', f'tests/run_if_test_argv0.out', caller=0)
        run_if(): Not running command because up to date: 'tests/run_if_test_argv0.out'

        Modify the script.

        >>> with open('tests/run_if_test_argv0.bat', 'a') as f:
        ...     print('echo hello >> tests/run_if_test_argv0.out', file=f)

        And now it is run because the hash of argv[0] has changed:

        >>> run_if(f'"tests/run_if_test_argv0.bat"', f'tests/run_if_test_argv0.out', caller=0)
        run_if(): Running command because: arg0 hash has changed.
        run_if(): Running: "tests/run_if_test_argv0.bat"
        True
        '''
        pass
else:
    def _run_if_test_scripting_unix():
        '''
        Dummy function to allow non-Windows doctests for pipcl.run_if().
        
        We detect changes to the contents of argv[0]:

        Create a shell script and run it:

        >>> del _log_f[:]
        >>> _log_f.append(sys.stdout)
        >>> log_prefix('%f(): ')
        
        >>> for path in glob.glob('tests/run_if_test_argv0.*'):
        ...     fs_remove(path)
        >>> with open('tests/run_if_test_argv0.sh', 'w') as f:
        ...     print('#! /bin/sh', file=f)
        ...     print('echo hello world > tests/run_if_test_argv0.out', file=f)
        >>> import stat
        >>> os.chmod('tests/run_if_test_argv0.sh', os.stat('tests/run_if_test_argv0.sh').st_mode | stat.S_IEXEC)

        
        >>> run_if('tests/run_if_test_argv0.sh', f'tests/run_if_test_argv0.out', caller=0)
        run_if(): Running command because: File does not exist: 'tests/run_if_test_argv0.out'
        run_if(): Running: tests/run_if_test_argv0.sh
        True

        Running it a second time does nothing:

        >>> run_if( f'tests/run_if_test_argv0.sh', f'tests/run_if_test_argv0.out', caller=0)
        run_if(): Not running command because up to date: 'tests/run_if_test_argv0.out'

        Modify the script.

        >>> with open('tests/run_if_test_argv0.sh', 'a') as f:
        ...     print('echo hello >> tests/run_if_test_argv0.out', file=f)

        And now it is run because the hash of argv[0] has changed:

        >>> run_if( f'tests/run_if_test_argv0.sh', f'tests/run_if_test_argv0.out', caller=0)
        run_if(): Running command because: arg0 hash has changed.
        run_if(): Running: tests/run_if_test_argv0.sh
        True
        '''
        pass


def fs_find_in_paths( name, paths=None, verbose=False):
    '''
    Looks for `name` in paths and returns complete path. `paths` is list/tuple
    or `os.pathsep`-separated string; if `None` we use `$PATH`. If `name`
    contains `/`, we return `name` itself if it is a file or None, regardless
    of <paths>.
    '''
    if '/' in name:
        return name if os.path.isfile( name) else None
    if paths is None:
        paths = os.environ.get( 'PATH', '')
        if verbose:
            log('From os.environ["PATH"]: {paths=}')
    if isinstance( paths, str):
        paths = paths.split( os.pathsep)
        if verbose:
            log('After split: {paths=}')
    for path in paths:
        p = os.path.join( path, name)
        if verbose:
            log('Checking {p=}')
        if os.path.isfile( p):
            if verbose:
                log('Returning because is file: {p!r}')
            return p
    if verbose:
        log('Returning None because not found: {name!r}')


def fs_symlink(from_, to_):
    '''
    Makes symbolic link from <from_> to <to_>.
    '''
    fs_remove(from_)
    os.symlink(to_, from_)


def _get_prerequisites(path):
    '''
    Returns list of prerequisites from Makefile-style dependency file, e.g.
    created by `cc -MD -MF <path>`.
    '''
    ret = list()
    if os.path.isfile(path):
        with open(path) as f:
            for line in f:
                for item in line.split():
                    if item.endswith( (':', '\\')):
                        continue
                    ret.append( item)
    return ret


def _fs_mtime_newest( path):
    '''
    path:
        If a file, returns mtime of the file. If a directory, returns mtime of
        newest file anywhere within directory tree. Otherwise returns 0.
    '''
    ret = 0
    if os.path.isdir( path):
        for dirpath, _dirnames, filenames in os.walk( path):
            for filename in filenames:
                path = os.path.join( dirpath, filename)
                ret = max( ret, _fs_mtime( path))
    else:
        ret = _fs_mtime( path)
    return ret


def _flags( items, prefix='', quote=''):
    '''
    Turns sequence into string, prefixing/quoting each item.
    '''
    if not items:
        return ''
    if isinstance( items, str):
        items = items,
    ret = ''
    for item in items:
        if ret:
            ret += ' '
        ret += f'{prefix}{quote}{item}{quote}'
    return ret.strip()


def _fs_mtime( filename, default=0):
    '''
    Returns mtime of file, or `default` if error - e.g. doesn't exist.
    '''
    try:
        return os.path.getmtime( filename)
    except OSError:
        return default


def _normalise(name):
    '''
    https://packaging.python.org/en/latest/specifications/name-normalization/#name-normalization
    '''
    return re.sub(r"[-_.]+", "-", name).lower()


def _normalise2(name):
    '''
    shttps://packaging.python.org/en/latest/specifications/binary-distribution-format/
    '''
    return _normalise(name).replace('-', '_')


def _assert_version_pep_440(version):
    '''
    Asserts that <version> cconforms with PEP 440.
    '''
    assert re.match(
                r'^([1-9][0-9]*!)?(0|[1-9][0-9]*)(\.(0|[1-9][0-9]*))*((a|b|rc)(0|[1-9][0-9]*))?(\.post(0|[1-9][0-9]*))?(\.dev(0|[1-9][0-9]*))?$',
                version,
            ), \
            f'Bad version: {version!r}.'


g_verbose = int(os.environ.get('PIPCL_VERBOSE', '1'))

def verbose(level=None):
    '''
    Sets verbose level if `level` is not None.
    Returns verbose level.
    '''
    global g_verbose
    if level is not None:
        g_verbose = level
    return g_verbose

def log(text='', caller=1):
    '''
    Writes <text> to log. Caller identifies the frame to use if file:line is
    included in prefix; default 1 identifies the caller.
    '''
    _log(text, 0, caller+1)

def log0(text='', caller=1):
    _log(text, 0, caller+1)

def log1(text='', caller=1):
    _log(text, 1, caller+1)

def log2(text='', caller=1):
    _log(text, 2, caller+1)


def _duration(s):
    '''
    Return representation of elapsed time +[[[DAYSd]HOURSh]MINSm]Ss, for
    example 1h03m45.3s.
    '''
    m, s = divmod(s, 60)
    h, m = divmod(m, 60)
    d, h = divmod(h, 24)
    m = int(m)
    h = int(h)
    d = int(d)
    if d:
        return f'{d}d{h:02}h{m:02}m{s:04.1f}s'
    elif h:
        return f'{h:}h{m:02}m{s:04.1f}s'
    elif m:
        return f'{m}m{s:04.1f}s'
    else:
        return f'{s:.1f}s'


# Log prefix pattern.
# %d - elapsed time +[[[DAYSd]HOURSh]MINSm]Ss, for example 1h03m45.3s.
# %f - function name
# %l - line number
# %p - path.
# %t - current time YYYY-mm-dd-HH-MM-SS
g_log_format = '[+%d] %p:%l: %f(): '
g_log_t0 = time.time()

def log_prefix(pattern):
    '''
    Sets log prefix pattern.
    '''
    global g_log_format
    g_log_format = pattern

_log_prefix_stack = list()

class LogPrefix:
    '''
    A context manager to add a temporary prefix to log output.
    '''
    def __init__(self, prefix):
        self.prefix = prefix
    
    def __enter__(self):
        _log_prefix_stack.append(self.prefix)
    
    def __exit__(self, *args):
        del _log_prefix_stack[-1]


def _log_prefix(format_, caller):
    '''
    Evaluates log format for <caller>.
    '''
    ret = ''
    
    _get_frame_cache = [None]
    def get_frame():
        if _get_frame_cache[0] is None:
            frame = inspect.stack(context=0)[caller+1]
            path = relpath(frame.filename)
            _get_frame_cache[0] = frame, path
        return _get_frame_cache[0]
    # pylint: disable=unpacking-non-sequence
    pos = 0
    while pos < len(format_):
        pos0 = pos
        if 0:
            pass
        elif format_[pos:].startswith('%d'):
            # Elapsed time.
            pos += 2
            s = time.time() - g_log_t0
            ret += _duration(s)
        elif format_[pos:].startswith('%f'):
            # Function name.
            pos += 2
            frame, path = get_frame()
            ret += frame.function
        elif format_[pos:].startswith('%l'):
            # Line number.
            pos += 2
            frame, path = get_frame()
            ret += str(frame.lineno)
        elif format_[pos:].startswith('%p'):
            # Path.
            pos += 2
            frame, path = get_frame()
            ret += path
        elif format_[pos:].startswith('%t'):
            # Current date and time.
            pos += 2
            ret += time.strftime(f'%F-%T')
        else:
            ret += format_[pos]
            pos += 1
        assert pos > pos0
    ret += ''.join(_log_prefix_stack)
    return ret

_log_text_line_start = True

_log_f = [sys.stdout]

_log_tee_f = None
_log_tee_exit_fn = None

def log_tee(path, path_simple=None):
    '''
    Copies log output to <path>. If path_simple is specified, on
    exit we make a convenience symlink from <path_simple> to <path>.
    '''
    global _log_tee_f
    global _log_tee_exit_fn
    
    # Remove any current tee setting.
    if _log_tee_f:
        _log_f.remove(_log_tee_f)
        _log_tee_f = None
    if _log_tee_exit_fn:
        atexit.unregister(_log_tee_exit_fn)
        _log_tee_exit_fn = None
    
    if path:
        _log_tee_f = open(path, 'w') # pylint: disable=consider-using-with
        _log_f.append(_log_tee_f)
        if path_simple:
            def final():
                fs_symlink(path_simple, path)
            _log_tee_exit_fn = final
            atexit.register(final)

def _log2(text):
    #print(f'####### {_log_f=}')
    #sys.exit()
    if _log_f is None:
        f = [sys.stdout]
    elif hasattr(_log_f, '__len__'):
        f = _log_f
    else:
        f = [_log_f]
    for ff in f:
        ff.write(text)
        ff.flush()

def _log(text, level, caller, *, raw=False, nl=True, format_=None):
    '''
    Logs lines with prefix, if <level> is lower or equal to <g_verbose>.
    '''
    if level > g_verbose:
        return
    if format_ is None:
        format_ = g_log_format
    prefix = None
    global _log_text_line_start
    pos = 0
    while 1:
        if pos == len(text):
            break
        if not raw or _log_text_line_start:
            if prefix is None:
                prefix = _log_prefix(format_, caller+1)
            _log2(prefix)
        nlp = text.find('\n', pos)
        if nlp == -1:
            _log2(text[pos:])
            if not raw and nl:
                _log2('\n')
            pos = len(text)
        else:
            _log2(text[pos:nlp+1])
            pos = nlp+1
        if raw:
            _log_text_line_start = (nlp >= 0)


def relpath(path, start=None, allow_up=True):
    '''
    A safe alternative to `os.path.relpath()`, avoiding an exception on Windows
    if the drive needs to change - in this case we use `os.path.abspath()`.
    
    Args:
        path:
            Path to be processed.
        start:
            Start directory or current directory if None.
        allow_up:
            If false we return absolute path is <path> is not within <start>.
    '''
    if windows():
        try:
            ret = os.path.relpath(path, start)
        except ValueError:
            # os.path.relpath() fails if trying to change drives.
            ret = os.path.abspath(path)
    else:
        ret = os.path.relpath(path, start)
    if not allow_up and ret.startswith('../') or ret.startswith('..\\'):
        ret = os.path.abspath(path)
    return ret


def _so_suffix(use_so_versioning=True):
    '''
    Filename suffix for shared libraries is defined in pep-3149.  The
    pep claims to only address posix systems, but the recommended
    sysconfig.get_config_var('EXT_SUFFIX') also seems to give the
    right string on Windows.
    
    If use_so_versioning is false, we return only the last component of
    the suffix, which removes any version number, for example changing
    `.cp312-win_amd64.pyd` to `.pyd`.
    '''
    # Example values:
    #   linux:      .cpython-311-x86_64-linux-gnu.so
    #   macos:      .cpython-311-darwin.so
    #   openbsd:    .cpython-310.so
    #   windows     .cp311-win_amd64.pyd
    #
    # Only Linux and Windows seem to identify the cpu. For example shared
    # libraries in numpy-1.25.2-cp311-cp311-macosx_11_0_arm64.whl are called
    # things like `numpy/core/_simd.cpython-311-darwin.so`.
    #
    ret = sysconfig.get_config_var('EXT_SUFFIX')
    if not use_so_versioning:
        # Use last component only.
        ret = os.path.splitext(ret)[1]
    return ret


def get_soname(path):
    '''
    If we are on Linux and `path` is softlink and points to a shared library
    for which `objdump -p` contains 'SONAME', return the pointee. Otherwise
    return `path`. Useful if Linux shared libraries have been created with
    `-Wl,-soname,...`, where we need to embed the versioned library.
    '''
    if linux() and os.path.islink(path):
        path2 = os.path.realpath(path)
        if subprocess.run(f'objdump -p {path2}|grep SONAME', shell=1, check=0).returncode == 0:
            return path2
    elif openbsd():
        # Return newest .so with version suffix.
        sos = glob.glob(f'{path}.*')
        log1(f'{sos=}')
        sos2 = list()
        for so in sos:
            suffix = so[len(path):]
            if not suffix or re.match('^[.][0-9.]*[0-9]$', suffix):
                sos2.append(so)
        sos2.sort(key=lambda p: os.path.getmtime(p))
        log1(f'{sos2=}')
        return sos2[-1]
    return path


def current_py_limited_api():
    '''
    Returns value of PyLIMITED_API to build for current Python.
    '''
    a, b = map(int, platform.python_version().split('.')[:2])
    return f'0x{a:02x}{b:02x}0000'
    

def install_dir(root=None):
    '''
    Returns install directory used by `install()`.

    This will be `sysconfig.get_path('platlib')`, modified by `root` if not
    None.
    '''
    # todo: for pure-python we should use sysconfig.get_path('purelib') ?
    root2 = sysconfig.get_path('platlib')
    if root:
        if windows():
            # If we are in a venv, `sysconfig.get_path('platlib')`
            # can be absolute, e.g.
            # `C:\\...\\venv-pypackage-3.11.1-64\\Lib\\site-packages`, so it's
            # not clear how to append it to `root`. So we just use `root`.
            return root
        else:
            # E.g. if `root` is `install' and `sysconfig.get_path('platlib')`
            # is `/usr/local/lib/python3.9/site-packages`, we set `root2` to
            # `install/usr/local/lib/python3.9/site-packages`.
            #
            return os.path.join( root, root2.lstrip( os.sep))
    else:
        return root2


def zipfile_writestr(zf, name, contents, unix_permissions=0o600):
    '''
    Alternative to `zipfile.ZipFile.writestr()`, which allows control over
    permissions. Our default 0o600 matches `zipfile.ZipFile.writestr()`.
    '''
    # There isn't a good way of setting permissions, zipfile seems very badly
    # designed here.
    zipinfo = zipfile.ZipInfo(
            filename = name,
            date_time = time.localtime(time.time())[:6],
            )
    zipinfo.compress_type = zf.compression
    if python_version_tuple() >= (3, 13):
        zipinfo.compress_level = zf.compresslevel
    else:
        zipinfo._compresslevel = zf.compresslevel
    zipinfo.external_attr = unix_permissions << 16
    zf.writestr(zipinfo, contents)


class _Record:
    '''
    Internal - builds up text suitable for writing to a RECORD item, e.g.
    within a wheel.
    '''
    def __init__(self):
        self.lines = list()

    def add_content(self, content, to_, verbose=True):
        if isinstance(content, str):
            content = content.encode('utf8')

        # Specification for the line we write is supposed to be in
        # https://packaging.python.org/en/latest/specifications/binary-distribution-format
        # but it's not very clear.
        #
        h = hashlib.sha256(content)
        digest = h.digest()
        digest = base64.urlsafe_b64encode(digest)
        digest = digest.rstrip(b'=')
        digest = digest.decode('utf8')

        self.lines.append(f'{to_.replace(os.sep, "/")},sha256={digest},{len(content)}\n')
        if verbose:
            log2(f'Adding {to_}')

    def add_file(self, from_, to_, from_maxlen=0):
        log1(f'Adding file: {os.path.relpath(from_).ljust(from_maxlen)} => {to_}')
        with open(from_, 'rb') as f:
            content = f.read()
        self.add_content(content, to_, verbose=False)

    def get(self, record_path=None):
        '''
        Returns contents of the RECORD file. If `record_path` is
        specified we append a final line `<record_path>,,`; this can be
        used to include the RECORD file itself in the contents, with
        empty hash and size fields.
        '''
        self.lines.sort()
        ret = ''.join(self.lines)
        if record_path:
            ret += f'{record_path},,\n'
        log(f'{record_path=}')
        log(f'ret is:\n{textwrap.indent(ret, "    ")}')
        return ret


class NewFiles:
    '''
    Detects new/modified/updated files matching a glob pattern. Useful for
    detecting wheels created by pip or cubuildwheel etc.
    '''
    def __init__(self, glob_pattern, recursive=None):
        '''
        Sets things up to look for new files matching <glob_pattern>.
        <recursive> is passed to `glob.glob()`, so for example set to true if
        using `**`.
        '''
        # Find current matches of <glob_pattern>.
        self.glob_pattern = glob_pattern
        self.recursive = recursive
        self.items0 = self._items()
    
    def get(self):
        '''
        Returns list of new matches of <glob_pattern> - paths of files that
        were not present previously, or have different mtimes or have different
        contents.
        '''
        ret = list()
        items = self._items()
        for path, id_ in items.items():
            id0 = self.items0.get(path)
            if id0 != id_:
                ret.append(path)
        return ret
    
    def get_n(self, n):
        '''
        Returns new files matching <glob_pattern>, asserting that there are
        exactly <n>.
        '''
        ret = self.get()
        sep = '\n    '
        assert len(ret) == n, f'{len(ret)=} != {n=}:{sep}{sep.join(ret)}. {self.glob_pattern=}.'
        return ret
    
    def get_one(self):
        '''
        Returns new match of <glob_pattern>, asserting that there is exactly
        one.
        '''
        return self.get_n(1)[0]
    
    def _file_id(self, path):
        mtime = os.stat(path).st_mtime
        with open(path, 'rb') as f:
            content = f.read()
            hash_ = hashlib.md5(content).digest()
            # With python >= 3.11 we can do:
            #hash_ = hashlib.file_digest(f, hashlib.md5).digest()
        return mtime, hash_
    
    def _items(self):
        ret = dict()
        for path in glob.glob(self.glob_pattern, recursive=self.recursive):
            if os.path.isfile(path):
                ret[path] = self._file_id(path)
        return ret


def swig_prepare_build(swig_env_extra):
    '''
    Builds required tools for building swig (not Windows).
    
    Adds to swig_env_extra['PATH'].
    '''
    if darwin():
        run(f'brew install automake')
        run(f'brew install pcre2')
        run(f'brew install bison')
        # Default bison doesn't work, and Brew's bison is not added to $PATH.
        #
        # > bison is keg-only, which means it was not symlinked into /opt/homebrew,
        # > because macOS already provides this software and installing another version in
        # > parallel can cause all kinds of trouble.
        # >
        # > If you need to have bison first in your PATH, run:
        # >   echo 'export PATH="/opt/homebrew/opt/bison/bin:$PATH"' >> ~/.zshrc
        #
        macos_add_brew_path('bison', swig_env_extra)
        run(f'which bison')
        run(f'which bison', env_extra=swig_env_extra)

    # Building swig requires bison>=3.5.
    bison_ok = 0
    e, text = run(f'bison --version', capture=1, check=0, env_extra=swig_env_extra)
    if not e:
        log(textwrap.indent(text, '    '))
        m = re.search('bison (GNU Bison) ([0-9]+)[.]([0-9]+)', text)
        if m:
            assert m, f'Unexpected output from `bison --version`: {text!r}'
            version_tuple = int(m.group(1)), int(m.group2())
            if version_tuple >= (3, 5):
                bison_ok = 1
    if not bison_ok:
        if 0:
            # Use git checkout. Fails to find scan-code.c. Presumably
            # something wrong with ./bootstrap?
            log(f'Cloning/fetching/build/installing bison.')
            bison_git = git_get(
                    'pipcl-bison-git',
                    remote='https://git.savannah.gnu.org/git/bison.git',
                    #branch='master',
                    tag='v3.5.4',
                    submodules=0, # recursive update fails.
                    )
            run(f'cd {bison_git} && git submodule update --init', prefix='bison git submodule update --init: ')
            run(f'cd {bison_git} && ./bootstrap', prefix='bison bootstrap: ')
            run(f'cd {bison_git} && ./configure', prefix='bison configure: ')
            run(f'cd {bison_git} && make', prefix='bison make: ')
            run(f'cd {bison_git} && sudo make install', prefix='bison make install: ')
        else:
            bison_version = 'bison-3.5.4'
            if not os.path.exists(f'{bison_version}.tar.gz'):
                run(
                        f'wget -O {bison_version}.tar.gz-0 http://www.mirrorservice.org/sites/ftp.gnu.org/gnu/bison/{bison_version}.tar.gz',
                        prefix='bison wget: ',
                        )
                os.rename(f'{bison_version}.tar.gz-0', f'{bison_version}.tar.gz')
            if not os.path.exists(f'{bison_version}'):
                run(f'tar -xzf {bison_version}.tar.gz', prefix='bison extract: ')
            # 2026-02-23: builds have started failing, perhaps recent compilers have extra detection?
            run(f'cd {bison_version} && ./configure', prefix='bison configure: ', env_extra=dict(CFLAGS='-Wno-incompatible-function-pointer-types'))
            run(f'cd {bison_version} && make', prefix='bison make: ')
            run(f'cd {bison_version} && sudo make install', prefix='bison make install: ')


def swig_get(swig, quick, swig_local='pipcl-swig-git'):
    '''
    Returns <swig> or a new swig binary.
    
    If <swig> is true and starts with 'git:' (not Windows), the remaining text
    is passed to `git_get()` and we clone/update/build swig, and return the built
    binary. We default to the main swig repository, branch master, so for
    example 'git:' will return the latest swig from branch master.
    
    Otherwise we simply return <swig>.
    
    Args:
        swig:
            If starts with 'git:', passed as <text> arg to `git_get()`, and we
            return the absolute path of the build binary. (Not windows.)
            
            If starts with 'pip:' we do: `pip install swig<swig[4:]>` and
            return 'swig'. E.g. 'pip:==4.3.1'.
            
            Otherwise we simply return <swig>.
        quick:
            If true, we do not update/build local checkout if the binary is
            already present.
        swig_local:
            path to use for checkout.
    '''
    #log(f'{swig=}')
    if swig and swig.startswith('git:'):
        assert platform.system() != 'Windows', f'Cannot build swig on Windows.'
        # Note that {swig_local}/install/bin/swig doesn't work on MacOS because
        # {swig_local}/INSTALL is a file and the fs is case-insensitive.
        swig_binary = os.path.abspath(f'{swig_local}/install-dir/bin/swig')
        if quick and os.path.isfile(swig_binary):
            log1(f'{quick=} and {swig_binary=} already exists, so not downloading/building.')
        else:
            # Clone swig.
            swig_env_extra = None
            swig_local = git_get(
                    swig_local,
                    text=swig,
                    remote='https://github.com/swig/swig.git',
                    branch='master',
                    )
            
            swig_prepare_build(swig_env_extra)
                
            # Build swig.
            run(f'cd {swig_local} && ./autogen.sh',
                    out=log,
                    env_extra=swig_env_extra,
                    prefix='build swig autogen.sh: ',
                    )
            run(f'cd {swig_local} && ./configure --prefix={swig_local}/install-dir',
                    out=log,
                    env_extra=swig_env_extra,
                    prefix='build swig configure: ',
                    )
            run(f'cd {swig_local} && make',
                    out=log,
                    env_extra=swig_env_extra,
                    prefix='build swig make: ',
                    )
            run(f'cd {swig_local} && make install',
                    out=log,
                    env_extra=swig_env_extra,
                    prefix='build swig install: ',
                    )
        assert os.path.isfile(swig_binary)
        return swig_binary
    
    elif swig and swig.startswith('pip:'):
        run(f'pip uninstall -y swig', check=0)
        run(f'pip install swig{swig[4:]}')
        return 'swig'
    
    else:
        return swig


def macos_add_brew_path(package, env=None, gnubin=True):
    '''
    Adds path(s) for Brew <package>'s binaries to env['PATH'].
    
    We assert-fail if the relevant directory does no exist.
    
    Args:
        package:
            Name of package. We get <package_root> of installed package by
            running `brew --prefix <package>`.
        env:
            The environment dict to modify. If None we use os.environ. If PATH
            is not in <env>, we first copy os.environ['PATH'] into <env>.
        gnubin:
            If true, we also add path to gnu binaries if it exists,
            <package_root>/libexe/gnubin.
    '''
    if not darwin():
        return
    if env is None:
        env = os.environ
    if 'PATH' not in env:
        env['PATH'] = os.environ['PATH']
    package_root = run(f'brew --prefix {package}', capture=1).strip()
    log(f'{package=} {package_root=}')
    def add(path):
        log(f'{path=}')
        if os.path.isdir(path):
            log(f'Prepending to $PATH: {path}')
            PATH = env['PATH']
            env['PATH'] = f'{path}:{PATH}'
            return 1
        else:
            log(f'Not a directory: {path=}')
            return 0
    n = 0
    n += add(f'{package_root}/bin')
    if gnubin:
        n += add(f'{package_root}/libexec/gnubin')
    assert n, f'Failed to add to $PATH, {package=} {gnubin=}.'


def _show_dict(d):
    ret = ''
    for n in sorted(d.keys()):
        v = d[n]
        ret += f'    {n}: {v!r}\n'
    return ret

def show_sysconfig():
    '''
    Shows contents of `sysconfig.get_paths()` and `sysconfig.get_config_vars()` dicts.
    '''
    log0(f'show_sysconfig().')
    log0(f'sysconfig.get_paths():\n{_show_dict(sysconfig.get_paths())}')
    log0(f'sysconfig.get_config_vars():\n{_show_dict(sysconfig.get_config_vars())}')


def sysconfig_python_flags():
    '''
    Returns include paths and library directory for Python.
    
    Uses sysconfig.*(), overridden by environment variables
    PIPCL_SYSCONFIG_PATH_include, PIPCL_SYSCONFIG_PATH_platinclude and
    PIPCL_SYSCONFIG_CONFIG_VAR_LIBDIR if set.
    '''
    include1_ = os.environ.get('PIPCL_SYSCONFIG_PATH_include') or sysconfig.get_path('include')
    include2_ = os.environ.get('PIPCL_SYSCONFIG_PATH_platinclude') or sysconfig.get_path('platinclude')
    ldflags_ = os.environ.get('PIPCL_SYSCONFIG_CONFIG_VAR_LIBDIR') or sysconfig.get_config_var('LIBDIR')

    includes_ = [include1_]
    if include2_ != include1_:
        includes_.append(include2_)
    if windows():
        includes_ = [f'/I"{i}"' for i in includes_]
        ldflags_ = f'/LIBPATH:"{ldflags_}"'
    else:
        includes_ = [f'-I {i}' for i in includes_]
        ldflags_ = f'-L {ldflags_}'
    includes_ = ' '.join(includes_)
    return includes_, ldflags_


def _commands_similar(a, b):
    '''
    Experimental debug fn for comparing command lines.
    '''
    def canonalisee(command):
        args = shlex.split(command)
        i = 0
        j = 1
        lines = list()
        while 1:
            if j == len(args) or args[j].startswith(('-', '/')):
                line = shlex.join(args[i:j])
                i = j
                lines.append(line)
                if j == len(args):
                    break
            j += 1
        lines = lines[:1] + sorted(lines[1:])
        ret = '\n'.join(lines)
        return ret
    a = canonalisee(a)
    b = canonalisee(b)
    log(f'a:')
    log(f'{textwrap.indent(a, "    ")}')
    log(f'b:')
    log(f'{textwrap.indent(b, "    ")}')
    return a == b


def python_version_tuple():
    '''
    Like platform.python_version_tuple() except converts items to integers if
    possible.
    '''
    ret = list(platform.python_version_tuple())
    for i, value in enumerate(ret):
        try:
            ret[i] = int(value)
        except Exception:
            pass
    return tuple(ret)


def venv_enter_cmd(venv_name):
    '''
    Returns command that will enter venv, different on Windows vs Unix.
    '''
    if windows():
        return f'{venv_name}/Scripts/activate'.replace('/', '\\')
    else:
        return f'. {venv_name}/bin/activate'


if __name__ == '__main__':
    # Internal-only limited command line support, used if
    # graal_legacy_python_config is true.
    #
    includes_, ldflags = sysconfig_python_flags()
    if len(sys.argv) == 1:
        run('sleep 4', ticker=0.5, timeout=0)
        run('sleep 3', ticker=0.25, timeout=0)
    elif sys.argv[1] == '--doctest':
        import doctest
        if sys.argv[2:]:
            for ff in sys.argv[2:]:
                fff = globals()[ff]
                doctest.run_docstring_examples(fff, globals())
                # Unfortunately doctest.run_docstring_examples() always seems
                # to return None even if test(s) failed.
        else:
            doctest.testmod(None)
    elif sys.argv[1:] == ['--graal-legacy-python-config', '--includes']:
        print(includes_)
    elif sys.argv[1:] == ['--graal-legacy-python-config', '--ldflags']:
        print(ldflags)
    elif sys.argv[1:] == ['--check-git_info']:
        git_info('aptest')
    else:
        assert 0, f'Expected `--graal-legacy-python-config --includes|--ldflags` but {sys.argv=}'
