From: Jelmer Vernooij Date: Fri, 12 Nov 2010 01:50:29 +0000 (+0100) Subject: work on history support using dulwich X-Git-Url: http://git.samba.org/?a=commitdiff_plain;h=10c6a0392671f0c4022c4b4decc2399c1d4a2e33;p=build-farm.git work on history support using dulwich --- diff --git a/buildfarm/__init__.py b/buildfarm/__init__.py index 3bbfe4fa..d866db4d 100644 --- a/buildfarm/__init__.py +++ b/buildfarm/__init__.py @@ -34,6 +34,10 @@ class Tree(object): self.srcdir = srcdir self.scm = scm + def get_branch(self): + from buildfarm.history import GitBranch + return GitBranch(self.repo, self.branch) + def __repr__(self): return "<%s %r>" % (self.__class__.__name__, self.name) diff --git a/buildfarm/history.py b/buildfarm/history.py index 6ca1af86..97a76569 100644 --- a/buildfarm/history.py +++ b/buildfarm/history.py @@ -20,10 +20,8 @@ # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. -from buildfarm import util - -import commands -import os +from dulwich.repo import Repo +import subprocess BASEDIR = "/home/build/master" HISTORYDIR = "/home/build/master/cache" @@ -31,69 +29,60 @@ TIMEZONE = "PST" TIMEOFFSET = 0 UNPACKED_DIR = "/home/ftp/pub/unpacked" -class History(object): - def __init__(self, db): - self.db = db +class Branch(object): + + def authors(self): + ret = set() + for rev in self.log(): + ret.add(rev.author) + return ret + + def log(self): + raise NotImplementedError(self.log) + + def diff(self, revision): + raise NotImplementedError(self.diff) - def _log(self, tree): - return util.LoadStructure(os.path.join(HISTORYDIR, "history.%s" % tree)) - def diff(self, author, date, tree, revision): - """get recent git entries""" - # validate the tree - t = self.db.trees[tree] +class Revision(object): - if t.scm == "git": - self._git_diff(t, revision, tree) - else: - raise Exception("Unknown VCS %s" % t.scm) + def __init__(self, revision, date, author, message, modified=[], added=[], removed=[]): + self.revision = revision + self.date = date + self.author = author + self.message = message + self.modified = modified + self.added = added + self.removed = removed - def _git_diff(self, t, revision, tree): - """show recent git entries""" - log = self._log(tree) +class GitBranch(object): - # backwards? why? well, usually our users are looking for the newest - # stuff, so it's most likely to be found sooner - for i in range(len(log), 0, -1): - if log[i]["REVISION"] == revision: - entry = log[i] - break - else: - raise Exception("Unable to locate commit information revision[%s]." % revision) + def __init__(self, path, branch="master"): + self.repo = Repo(path) + self.branch = branch - # get information about the current diff - title = "GIT Diff in %s:%s for revision %s" % ( - tree, t.branch, revision) + def _revision_from_commit(self, commit): + # FIXME: modified/added/removed + return Revision(commit.id, commit.commit_time, commit.author, commit.message) - pwd = os.environ["PWD"] - ret = None + def log(self): try: - os.chdir(os.path.join(UNPACKED_DIR, tree)) - cmd = "git diff %s^ %s ./" % (revision, revision) - ret = (title, entry, tree, [(cmd, commands.getoutput("%s 2> /dev/null" % cmd))]) - - finally: - os.chdir(pwd) - return ret - - def authors(self, tree): - log = self._log(tree) - authors = set() - for entry in log: - authors.add(entry["AUTHOR"]) - return authors - - def history(self, tree, author=None): - """get commit history for the given tree""" - log = self._log(tree) - - # what? backwards? why is that? oh... I know... we want the newest first - for i in range(len(log), 0, -1): - entry = log[i] - if (author is None or - (author == "") or - (author == "ALL") or - (author == entry["AUTHOR"])): - yield entry, tree + commit = self.repo["refs/heads/%s" % self.branch] + except KeyError: + return + done = set() + pending_commits = [commit.id] + while pending_commits != []: + commit_id = pending_commits.pop(0) + commit = self.repo[commit_id] + yield self._revision_from_commit(commit) + done.add(commit.id) + # FIXME: Add sorted by commit_time + pending_commits.extend(commit.parents) + + def diff(self, revision): + commit = self.repo[revision] + x = subprocess.Popen(["git", "show", revision], cwd=self.repo.path, stdout=subprocess.PIPE) + return (self._revision_from_commit(commit), x.communicate()[0]) diff --git a/buildfarm/tests/test_history.py b/buildfarm/tests/test_history.py index 7f802592..0d072417 100644 --- a/buildfarm/tests/test_history.py +++ b/buildfarm/tests/test_history.py @@ -15,6 +15,36 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. -import testtools +from buildfarm.history import GitBranch -from buildfarm import history +from dulwich.repo import Repo + +import tempfile +from testtools import TestCase +from testtools.testcase import TestSkipped + + +class GitBranchTests(TestCase): + + def setUp(self): + super(GitBranchTests, self).setUp() + self.repo = Repo.init(tempfile.mkdtemp()) + + def test_log_empty(self): + branch = GitBranch(self.repo.path, "master") + self.assertEquals([], list(branch.log())) + + def test_log_commits(self): + branch = GitBranch(self.repo.path, "master") + self.repo.do_commit("message", committer="Jelmer Vernooij") + log = list(branch.log()) + self.assertEquals(1, len(log)) + self.assertEquals("message", log[0].message) + + def test_empty_diff(self): + branch = GitBranch(self.repo.path, "master") + revid = self.repo.do_commit("message", committer="Jelmer Vernooij") + entry, diff = list(branch.diff(revid)) + self.assertEquals("message", entry.message) + raise TestSkipped("Must use alternative to git show") + self.assertEquals("", diff) diff --git a/web/build.py b/web/build.py index b28eb855..13700c1c 100755 --- a/web/build.py +++ b/web/build.py @@ -32,7 +32,6 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) from buildfarm import ( CachingBuildFarm, data, - history, util, ) @@ -48,7 +47,6 @@ basedir = os.path.abspath(os.path.join(webdir, "..")) buildfarm = CachingBuildFarm() db = data.BuildResultStore(basedir) -#history = history.History(db) hostsdb = buildfarm.hostdb compilers = buildfarm.compilers @@ -804,29 +802,20 @@ def diff_pretty(diff): def web_paths(t, paths): """change the given source paths into links""" - ret = "" - - fmt = None - if t.scm == "git": - r = t.repo - s = t.subdir - b = t.branch - fmt = " %%s" % (GITWEB_BASE, r, s, b, b) + ret = "" + for path in paths: + ret += " %s" % (GITWEB_BASE, t.repo, t.subdir, path, t.branch, t.branch, path) + return ret else: - return paths - - for m in re.finditer("\s*([^\s]+)", paths): - ret += fmt % (m.group(1), m.group(1)) - - return ret + raise Exception("Unknown scm %s" % t.scm) def history_row_html(myself, entry, tree): """show one row of history table""" - msg = cgi.escape(entry["MESSAGE"]) - t = time.asctime(time.gmtime(entry["DATE"])) - age = util.dhm_time(time()-entry["DATE"]) + msg = cgi.escape(entry.message) + t = time.asctime(time.gmtime(entry.date)) + age = util.dhm_time(time()-entry.date) t = t.replace(" ", " ") @@ -835,11 +824,11 @@ def history_row_html(myself, entry, tree):
%s
%s ago""" % (t, age) - if entry["REVISION"]: - yield " - %s
" % entry["REVISION"] - revision_url = "revision=%s" % entry["REVISION"] + if entry.revision: + yield " - %s
" % entry.revision + revision_url = "revision=%s" % entry.revision else: - revision_url = "author=%s" % entry["AUTHOR"] + revision_url = "author=%s" % entry.author yield """
show diffs @@ -852,9 +841,9 @@ def history_row_html(myself, entry, tree):
Author: %s -
""" % (myself, tree, entry["DATE"], revision_url, - myself, tree, entry["DATE"], revision_url, - msg, entry["AUTHOR"]) + """ % (myself, tree, entry.date, revision_url, + myself, tree, entry.date, revision_url, + msg, entry.author) t = db.trees.get(tree) @@ -862,45 +851,44 @@ def history_row_html(myself, entry, tree): yield "" return - if entry["FILES"]: + if entry.modified: yield "
Modified: " - yield web_paths(t, entry["FILES"]) + yield web_paths(t, entry.modified) yield "
\n" - if entry["ADDED"]: + if entry.added: yield "
Added: " - yield web_paths(t, entry["ADDED"]) + yield web_paths(t, entry.added) yield "
\n" - if entry["REMOVED"]: + if entry.removed: yield "
Removed: " - yield web_paths(t, entry["REMOVED"]) + yield web_paths(t, entry.removed) yield "
\n" yield "\n" + def history_row_text(entry, tree): """show one row of history table""" - msg = cgi.escape(entry["MESSAGE"]) - t = time.asctime(time.gmtime(entry["DATE"])) - age = util.dhm_time(time()-entry["DATE"]) - - yield "Author: %s\n" % entry["AUTHOR"] - if entry["REVISION"]: - yield "Revision: %s\n" % entry["REVISION"] - yield "Modified: %s\n" % entry["FILES"] - yield "Added: %s\n" % entry["ADDED"] - yield "Removed: %s\n" % entry["REMOVED"] + msg = cgi.escape(entry.message) + t = time.asctime(time.gmtime(entry.date)) + age = util.dhm_time(time()-entry.date) + + yield "Author: %s\n" % entry.author + if entry.revision: + yield "Revision: %s\n" % entry.revision + yield "Modified: %s\n" % entry.modified + yield "Added: %s\n" % entry.added + yield "Removed: %s\n" % entry.removed yield "\n\n%s\n\n\n" % msg -def show_diff(cmd, diff, text_html): +def show_diff(diff, text_html): if text_html == "html": diff = cgi.escape(diff) diff = diff_pretty(diff) - ret = "\n" % cmd - ret += "
%s
\n" % diff - return ret + return "
%s
\n" % diff else: return "%s\n" % diff @@ -928,13 +916,11 @@ def buildApp(environ, start_response): if fn_name == 'text_diff': start_response('200 OK', [('Content-type', 'application/x-diff')]) - (title, entry, tree, diffs) = history.diff(get_param(form, 'author'), - get_param(form, 'date'), - get_param(form, 'tree'), - get_param(form, 'revision')) + tree = get_param(form, 'tree') + t = db.trees[tree] + (entry, diff) = t.get_branch().diff(get_param(form, 'revision')) yield "".join(history_row_text(entry, tree)) - for (cmd, diff) in diffs: - yield show_diff(cmd, diff, "text") + yield show_diff(diff, "text") elif fn_name == 'Text_Summary': start_response('200 OK', [('Content-type', 'text/plain')]) yield "".join(view_summary(myself, 'text')) @@ -973,7 +959,7 @@ def buildApp(environ, start_response): tree = get_param(form, "tree") t = db.trees[tree] authors = set(["ALL"]) - authors.update(history.authors(tree)) + authors.update(t.get_branch().authors(tree)) yield "

Recent checkins for %s (%s branch %s)

\n" % ( tree, t.scm, t.branch) @@ -988,18 +974,23 @@ def buildApp(environ, start_response): yield "" yield "" - for entry, tree in history.history(get_param(form, 'tree'), get_param(form, 'author')): - yield "".join(history_row_html(myself, entry, tree)) + branch = t.get_branch() + author = get_param(form, 'author') + + for entry in branch.log(): + if author in ("ALL", "", entry.author): + yield "".join(history_row_html(myself, entry, tree)) yield "\n" elif fn_name == "diff": - (title, entry, tree, diffs) = history.diff(get_param(form, 'author'), - get_param(form, 'date'), - get_param(form, 'tree'), - get_param(form, 'revision')) + t = db.trees[tree] + revision = get_param(form, 'revision') + (entry, diff) = t.get_branch().diff(revision) + # get information about the current diff + title = "GIT Diff in %s:%s for revision %s" % ( + tree, t.branch, revision) yield "

%s

" % title yield "".join(history_row_html(myself, entry, tree)) - for (cmd, diff) in diffs: - yield show_diff(cmd, diff, "html") + yield show_diff(diff, "html") elif os.getenv("PATH_INFO") not in (None, "", "/"): paths = os.getenv("PATH_INFO").split('/') if paths[1] == "recent":