prefect.utilities.callables

Utilities for working with Python callables.

ParameterSchema

Bases: BaseModel

Simple data model corresponding to an OpenAPI Schema.

Source code in src/prefect/utilities/callables.py

class ParameterSchema(pydantic.BaseModel):
    """Simple data model corresponding to an OpenAPI `Schema`."""

    title: Literal["Parameters"] = "Parameters"
    type: Literal["object"] = "object"
    properties: Dict[str, Any] = pydantic.Field(default_factory=dict)
    required: List[str] = pydantic.Field(default_factory=list)
    definitions: Dict[str, Any] = pydantic.Field(default_factory=dict)

    def model_dump_for_openapi(self) -> Dict[str, Any]:
        result = self.model_dump(mode="python", exclude_none=True)
        if "required" in result and not result["required"]:
            del result["required"]
        return result

call_with_parameters(fn, parameters)

Call a function with parameters extracted with get_call_parameters

The function must have an identical signature to the original function or this will fail. If you need to send to a function with a different signature, extract the args/kwargs using parameters_to_positional_and_keyword directly

Source code in src/prefect/utilities/callables.py

def call_with_parameters(fn: Callable, parameters: Dict[str, Any]):
    """
    Call a function with parameters extracted with `get_call_parameters`

    The function _must_ have an identical signature to the original function or this
    will fail. If you need to send to a function with a different signature, extract
    the args/kwargs using `parameters_to_positional_and_keyword` directly
    """
    args, kwargs = parameters_to_args_kwargs(fn, parameters)
    return fn(*args, **kwargs)

cloudpickle_wrapped_call(__fn, *args, **kwargs)

Serializes a function call using cloudpickle then returns a callable which will execute that call and return a cloudpickle serialized return value

This is particularly useful for sending calls to libraries that only use the Python built-in pickler (e.g. anyio.to_process and multiprocessing) but may require a wider range of pickling support.

Source code in src/prefect/utilities/callables.py

def cloudpickle_wrapped_call(
    __fn: Callable, *args: Any, **kwargs: Any
) -> Callable[[], bytes]:
    """
    Serializes a function call using cloudpickle then returns a callable which will
    execute that call and return a cloudpickle serialized return value

    This is particularly useful for sending calls to libraries that only use the Python
    built-in pickler (e.g. `anyio.to_process` and `multiprocessing`) but may require
    a wider range of pickling support.
    """
    payload = cloudpickle.dumps((__fn, args, kwargs))
    return partial(_run_serialized_call, payload)

collapse_variadic_parameters(fn, parameters)

Given a parameter dictionary, move any parameters stored not present in the signature into the variadic keyword argument.

Example:

```python
def foo(a, b, **kwargs):
    pass

parameters = {"a": 1, "b": 2, "c": 3, "d": 4}
collapse_variadic_parameters(foo, parameters)
# {"a": 1, "b": 2, "kwargs": {"c": 3, "d": 4}}
```

Source code in src/prefect/utilities/callables.py

def collapse_variadic_parameters(
    fn: Callable, parameters: Dict[str, Any]
) -> Dict[str, Any]:
    """
    Given a parameter dictionary, move any parameters stored not present in the
    signature into the variadic keyword argument.

    Example:

        ```python
        def foo(a, b, **kwargs):
            pass

        parameters = {"a": 1, "b": 2, "c": 3, "d": 4}
        collapse_variadic_parameters(foo, parameters)
        # {"a": 1, "b": 2, "kwargs": {"c": 3, "d": 4}}
        ```
    """
    signature_parameters = inspect.signature(fn).parameters
    variadic_key = None
    for key, parameter in signature_parameters.items():
        if parameter.kind == parameter.VAR_KEYWORD:
            variadic_key = key
            break

    missing_parameters = set(parameters.keys()) - set(signature_parameters.keys())

    if not variadic_key and missing_parameters:
        raise ValueError(
            f"Signature for {fn} does not include any variadic keyword argument "
            "but parameters were given that are not present in the signature."
        )

    if variadic_key and not missing_parameters:
        # variadic key is present but no missing parameters, return parameters unchanged
        return parameters

    new_parameters = parameters.copy()
    if variadic_key:
        new_parameters[variadic_key] = {}

    for key in missing_parameters:
        new_parameters[variadic_key][key] = new_parameters.pop(key)

    return new_parameters

expand_mapping_parameters(func, parameters)

Generates a list of call parameters to be used for individual calls in a mapping operation.

Parameters:

Name	Type	Description	Default
func	Callable	The function to be called	required
parameters	Dict[str, Any]	A dictionary of parameters with iterables to be mapped over	required

Returns:

Name	Type	Description
List	List[Dict[str, Any]]	A list of dictionaries to be used as parameters for each call in the mapping operation

Source code in src/prefect/utilities/callables.py

def expand_mapping_parameters(
    func: Callable, parameters: Dict[str, Any]
) -> List[Dict[str, Any]]:
    """
    Generates a list of call parameters to be used for individual calls in a mapping
    operation.

    Args:
        func: The function to be called
        parameters: A dictionary of parameters with iterables to be mapped over

    Returns:
        List: A list of dictionaries to be used as parameters for each
            call in the mapping operation
    """
    # Ensure that any parameters in kwargs are expanded before this check
    parameters = explode_variadic_parameter(func, parameters)

    iterable_parameters = {}
    static_parameters = {}
    annotated_parameters = {}
    for key, val in parameters.items():
        if isinstance(val, (allow_failure, quote)):
            # Unwrap annotated parameters to determine if they are iterable
            annotated_parameters[key] = val
            val = val.unwrap()

        if isinstance(val, unmapped):
            static_parameters[key] = val.value
        elif isiterable(val):
            iterable_parameters[key] = list(val)
        else:
            static_parameters[key] = val

    if not len(iterable_parameters):
        raise MappingMissingIterable(
            "No iterable parameters were received. Parameters for map must "
            f"include at least one iterable. Parameters: {parameters}"
        )

    iterable_parameter_lengths = {
        key: len(val) for key, val in iterable_parameters.items()
    }
    lengths = set(iterable_parameter_lengths.values())
    if len(lengths) > 1:
        raise MappingLengthMismatch(
            "Received iterable parameters with different lengths. Parameters for map"
            f" must all be the same length. Got lengths: {iterable_parameter_lengths}"
        )

    map_length = list(lengths)[0]

    call_parameters_list = []
    for i in range(map_length):
        call_parameters = {key: value[i] for key, value in iterable_parameters.items()}
        call_parameters.update({key: value for key, value in static_parameters.items()})

        # Add default values for parameters; these are skipped earlier since they should
        # not be mapped over
        for key, value in get_parameter_defaults(func).items():
            call_parameters.setdefault(key, value)

        # Re-apply annotations to each key again
        for key, annotation in annotated_parameters.items():
            call_parameters[key] = annotation.rewrap(call_parameters[key])

        # Collapse any previously exploded kwargs
        call_parameters_list.append(collapse_variadic_parameters(func, call_parameters))

    return call_parameters_list

explode_variadic_parameter(fn, parameters)

Given a parameter dictionary, move any parameters stored in a variadic keyword argument parameter (i.e. **kwargs) into the top level.

Example:

```python
def foo(a, b, **kwargs):
    pass

parameters = {"a": 1, "b": 2, "kwargs": {"c": 3, "d": 4}}
explode_variadic_parameter(foo, parameters)
# {"a": 1, "b": 2, "c": 3, "d": 4}
```

Source code in src/prefect/utilities/callables.py

def explode_variadic_parameter(
    fn: Callable, parameters: Dict[str, Any]
) -> Dict[str, Any]:
    """
    Given a parameter dictionary, move any parameters stored in a variadic keyword
    argument parameter (i.e. **kwargs) into the top level.

    Example:

        ```python
        def foo(a, b, **kwargs):
            pass

        parameters = {"a": 1, "b": 2, "kwargs": {"c": 3, "d": 4}}
        explode_variadic_parameter(foo, parameters)
        # {"a": 1, "b": 2, "c": 3, "d": 4}
        ```
    """
    variadic_key = None
    for key, parameter in inspect.signature(fn).parameters.items():
        if parameter.kind == parameter.VAR_KEYWORD:
            variadic_key = key
            break

    if not variadic_key:
        return parameters

    new_parameters = parameters.copy()
    for key, value in new_parameters.pop(variadic_key, {}).items():
        new_parameters[key] = value

    return new_parameters

generate_parameter_schema(signature, docstrings)

Generate a parameter schema from a function signature and docstrings.

To get a signature from a function, use inspect.signature(fn) or _generate_signature_from_source(source_code, func_name).

Parameters:

Name	Type	Description	Default
signature	Signature	The function signature.	required
docstrings	Dict[str, str]	A dictionary mapping parameter names to docstrings.	required

Returns:

Name	Type	Description
ParameterSchema	ParameterSchema	The parameter schema.

Source code in src/prefect/utilities/callables.py

def generate_parameter_schema(
    signature: inspect.Signature, docstrings: Dict[str, str]
) -> ParameterSchema:
    """
    Generate a parameter schema from a function signature and docstrings.

    To get a signature from a function, use `inspect.signature(fn)` or
    `_generate_signature_from_source(source_code, func_name)`.

    Args:
        signature: The function signature.
        docstrings: A dictionary mapping parameter names to docstrings.

    Returns:
        ParameterSchema: The parameter schema.
    """

    model_fields = {}
    aliases = {}

    if not has_v1_type_as_param(signature):
        create_schema = create_v2_schema
        process_params = process_v2_params

        config = pydantic.ConfigDict(arbitrary_types_allowed=True)
    else:
        create_schema = create_v1_schema
        process_params = process_v1_params

        class ModelConfig:
            arbitrary_types_allowed = True

        config = ModelConfig

    for position, param in enumerate(signature.parameters.values()):
        name, type_, field = process_params(
            param, position=position, docstrings=docstrings, aliases=aliases
        )
        # Generate a Pydantic model at each step so we can check if this parameter
        # type supports schema generation
        try:
            create_schema("CheckParameter", model_cfg=config, **{name: (type_, field)})
        except (ValueError, TypeError):
            # This field's type is not valid for schema creation, update it to `Any`
            type_ = Any
        model_fields[name] = (type_, field)

    # Generate the final model and schema
    schema = create_schema("Parameters", model_cfg=config, **model_fields)
    return ParameterSchema(**schema)

get_call_parameters(fn, call_args, call_kwargs, apply_defaults=True)

Bind a call to a function to get parameter/value mapping. Default values on the signature will be included if not overridden.

If the function has a __prefect_self__ attribute, it will be included as the first parameter. This attribute is set when Prefect decorates a bound method, so this approach allows Prefect to work with bound methods in a way that is consistent with how Python handles them (i.e. users don't have to pass the instance argument to the method) while still making the implicit self argument visible to all of Prefect's parameter machinery (such as cache key functions).

Raises a ParameterBindError if the arguments/kwargs are not valid for the function

Source code in src/prefect/utilities/callables.py

def get_call_parameters(
    fn: Callable,
    call_args: Tuple[Any, ...],
    call_kwargs: Dict[str, Any],
    apply_defaults: bool = True,
) -> Dict[str, Any]:
    """
    Bind a call to a function to get parameter/value mapping. Default values on
    the signature will be included if not overridden.

    If the function has a `__prefect_self__` attribute, it will be included as
    the first parameter. This attribute is set when Prefect decorates a bound
    method, so this approach allows Prefect to work with bound methods in a way
    that is consistent with how Python handles them (i.e. users don't have to
    pass the instance argument to the method) while still making the implicit self
    argument visible to all of Prefect's parameter machinery (such as cache key
    functions).

    Raises a ParameterBindError if the arguments/kwargs are not valid for the
    function
    """
    if hasattr(fn, "__prefect_self__"):
        call_args = (fn.__prefect_self__,) + call_args

    try:
        bound_signature = inspect.signature(fn).bind(*call_args, **call_kwargs)
    except TypeError as exc:
        raise ParameterBindError.from_bind_failure(fn, exc, call_args, call_kwargs)

    if apply_defaults:
        bound_signature.apply_defaults()

    # We cast from `OrderedDict` to `dict` because Dask will not convert futures in an
    # ordered dictionary to values during execution; this is the default behavior in
    # Python 3.9 anyway.
    return dict(bound_signature.arguments)

get_parameter_defaults(fn)

Get default parameter values for a callable.

Source code in src/prefect/utilities/callables.py

def get_parameter_defaults(
    fn: Callable,
) -> Dict[str, Any]:
    """
    Get default parameter values for a callable.
    """
    signature = inspect.signature(fn)

    parameter_defaults = {}

    for name, param in signature.parameters.items():
        if param.default is not signature.empty:
            parameter_defaults[name] = param.default

    return parameter_defaults

parameter_docstrings(docstring)

Given a docstring in Google docstring format, parse the parameter section and return a dictionary that maps parameter names to docstring.

Parameters:

Name	Type	Description	Default
docstring	Optional[str]	The function's docstring.	required

Returns:

Type	Description
Dict[str, str]	Mapping from parameter names to docstrings.

Source code in src/prefect/utilities/callables.py

def parameter_docstrings(docstring: Optional[str]) -> Dict[str, str]:
    """
    Given a docstring in Google docstring format, parse the parameter section
    and return a dictionary that maps parameter names to docstring.

    Args:
        docstring: The function's docstring.

    Returns:
        Mapping from parameter names to docstrings.
    """
    param_docstrings = {}

    if not docstring:
        return param_docstrings

    with disable_logger("griffe"):
        parsed = parse(Docstring(docstring), Parser.google)
        for section in parsed:
            if section.kind != DocstringSectionKind.parameters:
                continue
            param_docstrings = {
                parameter.name: parameter.description for parameter in section.value
            }

    return param_docstrings

parameter_schema(fn)

Given a function, generates an OpenAPI-compatible description of the function's arguments, including: - name - typing information - whether it is required - a default value - additional constraints (like possible enum values)

Parameters:

Name	Type	Description	Default
fn	Callable	The function whose arguments will be serialized	required

Returns:

Name	Type	Description
ParameterSchema	ParameterSchema	the argument schema

Source code in src/prefect/utilities/callables.py

def parameter_schema(fn: Callable) -> ParameterSchema:
    """Given a function, generates an OpenAPI-compatible description
    of the function's arguments, including:
        - name
        - typing information
        - whether it is required
        - a default value
        - additional constraints (like possible enum values)

    Args:
        fn (Callable): The function whose arguments will be serialized

    Returns:
        ParameterSchema: the argument schema
    """
    try:
        signature = inspect.signature(fn, eval_str=True)  # novm
    except (NameError, TypeError):
        # `eval_str` is not available in Python < 3.10
        signature = inspect.signature(fn)

    docstrings = parameter_docstrings(inspect.getdoc(fn))

    return generate_parameter_schema(signature, docstrings)

parameter_schema_from_entrypoint(entrypoint)

Generate a parameter schema from an entrypoint string.

Will load the source code of the function and extract the signature and docstring to generate the schema.

Useful for generating a schema for a function when instantiating the function may not be possible due to missing imports or other issues.

Parameters:

Name	Type	Description	Default
entrypoint	str	A string representing the entrypoint to a function. The string should be in the format of module.path.to.function:do_stuff.	required

Returns:

Name	Type	Description
ParameterSchema	ParameterSchema	The parameter schema for the function.

Source code in src/prefect/utilities/callables.py

def parameter_schema_from_entrypoint(entrypoint: str) -> ParameterSchema:
    """
    Generate a parameter schema from an entrypoint string.

    Will load the source code of the function and extract the signature and docstring
    to generate the schema.

    Useful for generating a schema for a function when instantiating the function may
    not be possible due to missing imports or other issues.

    Args:
        entrypoint: A string representing the entrypoint to a function. The string
            should be in the format of `module.path.to.function:do_stuff`.

    Returns:
        ParameterSchema: The parameter schema for the function.
    """
    filepath = None
    if ":" in entrypoint:
        # split by the last colon once to handle Windows paths with drive letters i.e C:\path\to\file.py:do_stuff
        path, func_name = entrypoint.rsplit(":", maxsplit=1)
        source_code = Path(path).read_text()
        filepath = path
    else:
        path, func_name = entrypoint.rsplit(".", maxsplit=1)
        spec = importlib.util.find_spec(path)
        if not spec or not spec.origin:
            raise ValueError(f"Could not find module {path!r}")
        source_code = Path(spec.origin).read_text()
    signature = _generate_signature_from_source(source_code, func_name, filepath)
    docstring = _get_docstring_from_source(source_code, func_name)
    return generate_parameter_schema(signature, parameter_docstrings(docstring))

parameters_to_args_kwargs(fn, parameters)

Convert a parameters dictionary to positional and keyword arguments

The function must have an identical signature to the original function or this will return an empty tuple and dict.

Source code in src/prefect/utilities/callables.py

def parameters_to_args_kwargs(
    fn: Callable,
    parameters: Dict[str, Any],
) -> Tuple[Tuple[Any, ...], Dict[str, Any]]:
    """
    Convert a `parameters` dictionary to positional and keyword arguments

    The function _must_ have an identical signature to the original function or this
    will return an empty tuple and dict.
    """
    function_params = dict(inspect.signature(fn).parameters).keys()
    # Check for parameters that are not present in the function signature
    unknown_params = parameters.keys() - function_params
    if unknown_params:
        raise SignatureMismatchError.from_bad_params(
            list(function_params), list(parameters.keys())
        )
    bound_signature = inspect.signature(fn).bind_partial()
    bound_signature.arguments = parameters

    return bound_signature.args, bound_signature.kwargs

raise_for_reserved_arguments(fn, reserved_arguments)

Raise a ReservedArgumentError if fn has any parameters that conflict with the names contained in reserved_arguments.

Source code in src/prefect/utilities/callables.py

def raise_for_reserved_arguments(fn: Callable, reserved_arguments: Iterable[str]):
    """Raise a ReservedArgumentError if `fn` has any parameters that conflict
    with the names contained in `reserved_arguments`."""
    function_paremeters = inspect.signature(fn).parameters

    for argument in reserved_arguments:
        if argument in function_paremeters:
            raise ReservedArgumentError(
                f"{argument!r} is a reserved argument name and cannot be used."
            )