Skip to content

RequirementsMixin

RequirementsMixin

Mixin for classes that have requirements attribute.

Used to add requirements to a Step and a Pipeline.

Source code in src/distilabel/mixins/requirements.py 
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
class RequirementsMixin:
    """Mixin for classes that have `requirements` attribute.

    Used to add requirements to a `Step` and a `Pipeline`.
    """

    _requirements: Union[List[Requirement], None] = []

    def _gather_requirements(self) -> List[str]:
        """This method will be overwritten in the `BasePipeline` class to gather the requirements
        from each step.
        """
        return []

    @property
    def requirements(self) -> List[str]:
        """Return a list of requirements that must be installed to run the `Pipeline`.

        The requirements in a Pipeline will include the requirements from all the steps (if any).

        Returns:
            List of requirements that must be installed to run the `Pipeline`, sorted alphabetically.
        """
        self.requirements = self._gather_requirements()

        return [str(r) for r in self._requirements]

    @requirements.setter
    def requirements(self, _requirements: List[str]) -> None:
        requirements = []
        if not isinstance(_requirements, list):
            _requirements = [_requirements]

        for r in _requirements:
            try:
                requirements.append(Requirement(r))
            except InvalidRequirement:
                self._logger.warning(f"Invalid requirement: `{r}`")

        self._requirements = sorted(
            set(self._requirements).union(set(requirements)), key=lambda x: str(x)
        )

    def requirements_to_install(self) -> List[str]:
        """Check if the requirements are installed in the current environment, and returns the ones that aren't.

        Returns:
            List of requirements required to run the pipeline that are not installed in the current environment.
        """

        to_install = []
        for req in self.requirements:
            requirement = Requirement(req)
            if importlib.util.find_spec(requirement.name):
                if (str(requirement.specifier) != "") and (
                    version(requirement.name) != str(requirement.specifier)
                ):
                    to_install.append(req)
            else:
                to_install.append(req)
        return to_install

requirements: List[str] property writable

Return a list of requirements that must be installed to run the Pipeline.

The requirements in a Pipeline will include the requirements from all the steps (if any).

Returns:

Type Description
List[str]

List of requirements that must be installed to run the Pipeline, sorted alphabetically.

requirements_to_install()

Check if the requirements are installed in the current environment, and returns the ones that aren't.

Returns:

Type Description
List[str]

List of requirements required to run the pipeline that are not installed in the current environment.
Source code in src/distilabel/mixins/requirements.py 
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
def requirements_to_install(self) -> List[str]:
    """Check if the requirements are installed in the current environment, and returns the ones that aren't.

    Returns:
        List of requirements required to run the pipeline that are not installed in the current environment.
    """

    to_install = []
    for req in self.requirements:
        requirement = Requirement(req)
        if importlib.util.find_spec(requirement.name):
            if (str(requirement.specifier) != "") and (
                version(requirement.name) != str(requirement.specifier)
            ):
                to_install.append(req)
        else:
            to_install.append(req)
    return to_install