Skip to content

Checks

Running previous version's tests on current version's code

One of the main risks with automatic semantic versioning is the risk accidentally mark a breaking change as a minor or patch release. Unfortunately, it is quite easy to forget to include the BREAKING CHANGE: marker. Stephan Bönnemann therefore suggests running the tests of the previous version against a new release candidate: If the tests from the previous version fail on the new release candidate, then that release candidate is likely a breaking change and should be a major version.

This is currently only implemented for tox:

RunPreviousVersionsTestsTox

Bases: VersionEstimator

Source code in /home/docs/checkouts/readthedocs.org/user_builds/semv/envs/latest/lib/python3.11/site-packages/semv/hooks.py 
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
class RunPreviousVersionsTestsTox(VersionEstimator):
    testenv: List[str]
    buildenv: str

    def __init__(
        self, testenv: Union[str, List[str]], buildenv: str = 'build'
    ):
        """Run tests from previous version on new version

        This is an attempt to automatically detect breaking versions. If
        the tests from the previous version fail, then we can assume
        that the version most likely contains a breaking change.

        Parameters:
            testenv:
                Name of the environment(s) that contain the actual tests to run.
            buildenv:
                To build the current version's package, we run this tox
                environment. It should create a wheel file in the
                `dist/` folder.
        """
        if isinstance(testenv, str):
            self.testenv = [testenv]
        else:
            self.testenv = testenv
        self.buildenv = buildenv

    def run(self, current_version: Version) -> VersionIncrement:
        source_dir = os.path.abspath(os.path.curdir)
        build_proc = subprocess.run(
            f'tox -e {self.buildenv}',
            shell=True,
            capture_output=True,
        )
        build_proc.check_returncode()
        package = os.path.join(source_dir, max(glob.glob('dist/*.whl')))
        with TemporaryDirectory() as tempdir:
            git_proc = subprocess.run(
                f'git clone --depth 1 --branch {current_version}'
                f' file://{source_dir}/.git .',
                shell=True,
                capture_output=True,
                cwd=tempdir,
            )
            git_proc.check_returncode()

            possible_misleading_imports = glob.glob(
                os.path.join(tempdir, f'{get_package_name()}.*')
            )
            if possible_misleading_imports:
                src_layout = os.path.join(tempdir, 'src')
                os.makedirs(src_layout, exist_ok=True)
                for fake_import in possible_misleading_imports:
                    shutil.move(fake_import, src_layout)

            envs = ','.join(self.testenv)
            test_proc = subprocess.run(
                f'tox --installpkg {package} -e "{envs}" -- -v',
                shell=True,
                cwd=tempdir,
                capture_output=True,
            )
            if test_proc.returncode:
                m = re.search(
                    r'=+ FAILURES =+(.*?).=+ \d+ failed in',
                    test_proc.stdout.decode('utf-8'),
                    re.DOTALL,
                )
                if m:
                    # This will trivially be true, because we already
                    # know that there was output due to the returncode
                    closing_line = (
                        '==========================='
                        '    end of test report   '
                        '============================\n'
                    )
                    sys.stderr.write(m.group(1).strip() + '\n' + closing_line)
                return VersionIncrement.major
        return VersionIncrement.skip

__init__(testenv, buildenv='build')

Run tests from previous version on new version

This is an attempt to automatically detect breaking versions. If the tests from the previous version fail, then we can assume that the version most likely contains a breaking change.

Parameters:

Name Type Description Default
testenv Union[str, List[str]]

Name of the environment(s) that contain the actual tests to run.

 required
buildenv str

To build the current version's package, we run this tox environment. It should create a wheel file in the dist/ folder.

 'build'
Source code in /home/docs/checkouts/readthedocs.org/user_builds/semv/envs/latest/lib/python3.11/site-packages/semv/hooks.py 
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
def __init__(
    self, testenv: Union[str, List[str]], buildenv: str = 'build'
):
    """Run tests from previous version on new version

    This is an attempt to automatically detect breaking versions. If
    the tests from the previous version fail, then we can assume
    that the version most likely contains a breaking change.

    Parameters:
        testenv:
            Name of the environment(s) that contain the actual tests to run.
        buildenv:
            To build the current version's package, we run this tox
            environment. It should create a wheel file in the
            `dist/` folder.
    """
    if isinstance(testenv, str):
        self.testenv = [testenv]
    else:
        self.testenv = testenv
    self.buildenv = buildenv