From patchwork Wed Mar 18 15:42:45 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Simon Glass X-Patchwork-Id: 243834 List-Id: U-Boot discussion From: sjg at chromium.org (Simon Glass) Date: Wed, 18 Mar 2020 09:42:45 -0600 Subject: [PATCH v3 07/23] buildman: Be more selective about which directories to remove In-Reply-To: <20200318154302.16389-1-sjg@chromium.org> References: <20200318154302.16389-1-sjg@chromium.org> Message-ID: <20200318154302.16389-5-sjg@chromium.org> At present buildman removes any directory it doesn't intend to write output into. This is overly expansive since if the output directory happens to be somewhere with existing files, they may be removed. Using an existing directory for buildman is not a good practice, but since the result might be catastrophic, it is best to guard against it. A previous commit[1] fixed this by refusing to write to a subdirectory of the current directory, assumed to have U-Boot source code. But we can do better by only removing directories that look like the ones buildman creates. Update the code to do this and add a test. Signed-off-by: Simon Glass [1] 409fc029c40 tools: buildman: Don't use the working dir as build dir --- Changes in v3: None Changes in v2: None tools/buildman/builder.py | 27 ++++++++++++++++++++++----- tools/buildman/test.py | 20 ++++++++++++++++++++ 2 files changed, 42 insertions(+), 5 deletions(-) diff --git a/tools/buildman/builder.py b/tools/buildman/builder.py index a41d0b316e..30ec4254f8 100644 --- a/tools/buildman/builder.py +++ b/tools/buildman/builder.py @@ -485,6 +485,7 @@ class Builder: if self.commits: commit = self.commits[commit_upto] subject = commit.subject.translate(trans_valid_chars) + # See _GetOutputSpaceRemovals() which parses this name commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1, self.commit_count, commit.hash, subject[:20])) elif not self.no_subdirs: @@ -1525,12 +1526,15 @@ class Builder: for thread in range(max_threads): self._PrepareThread(thread, setup_git) - def _PrepareOutputSpace(self): + def _GetOutputSpaceRemovals(self): """Get the output directories ready to receive files. - We delete any output directories which look like ones we need to - create. Having left over directories is confusing when the user wants - to check the output manually. + Figure out what needs to be deleted in the output directory before it + can be used. We only delete old buildman directories which have the + expected name pattern. See _GetOutputDir(). + + Returns: + List of full paths of directories to remove """ if not self.commits: return @@ -1541,7 +1545,20 @@ class Builder: to_remove = [] for dirname in glob.glob(os.path.join(self.base_dir, '*')): if dirname not in dir_list: - to_remove.append(dirname) + leaf = dirname[len(self.base_dir) + 1:] + m = re.match('[0-9]+_of_[0-9]+_g[0-9a-f]+_.*', leaf) + if m: + to_remove.append(dirname) + return to_remove + + def _PrepareOutputSpace(self): + """Get the output directories ready to receive files. + + We delete any output directories which look like ones we need to + create. Having left over directories is confusing when the user wants + to check the output manually. + """ + to_remove = self._GetOutputSpaceRemovals() if to_remove: Print('Removing %d old build directories' % len(to_remove), newline=False) diff --git a/tools/buildman/test.py b/tools/buildman/test.py index acd862b3b0..2aaedf44ac 100644 --- a/tools/buildman/test.py +++ b/tools/buildman/test.py @@ -22,6 +22,7 @@ import commit import terminal import test_util import toolchain +import tools use_network = True @@ -469,6 +470,25 @@ class TestBuild(unittest.TestCase): self.assertEqual('HOSTCC=clang CC=clang', tc.GetEnvArgs(toolchain.VAR_MAKE_ARGS)) + def testPrepareOutputSpace(self): + def _Touch(fname): + tools.WriteFile(os.path.join(base_dir, fname), b'') + + base_dir = tempfile.mkdtemp() + + # Add various files that we want removed and left alone + to_remove = ['01_of_22_g0982734987_title', '102_of_222_g92bf_title', + '01_of_22_g2938abd8_title'] + to_leave = ['something_else', '01-something.patch', '01_of_22_another'] + for name in to_remove + to_leave: + _Touch(name) + + build = builder.Builder(self.toolchains, base_dir, None, 1, 2) + build.commits = self.commits + build.commit_count = len(commits) + result = set(build._GetOutputSpaceRemovals()) + expected = set([os.path.join(base_dir, f) for f in to_remove]) + self.assertEqual(expected, result) if __name__ == "__main__": unittest.main()