diff options
| -rwxr-xr-x | contrib/mtn2git/mtn2cache.py | 74 | ||||
| -rwxr-xr-x | contrib/mtn2git/mtn2git.py | 92 |
2 files changed, 102 insertions, 64 deletions
diff --git a/contrib/mtn2git/mtn2cache.py b/contrib/mtn2git/mtn2cache.py new file mode 100755 index 0000000000..323822a7fb --- /dev/null +++ b/contrib/mtn2git/mtn2cache.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python + +import os, processing +from child import Child + +MTN = ["mtn", "--db=~/oe/OE.mtn"] +DIFF = ["diff", "-u"] + +def handle_parents(revision): + parents = Child(*MTN+["au", "parents", revision]).stdout.strip().split('\n') + if not parents: + return + print "revision: %s (parents: %s)" % (revision, parents) + for parent in parents: + filename = os.path.join("mtn2cache", "patch", parent+"-"+revision) + curfilename = os.path.join("mtn2cache", "manifest", revision) + parfilename = os.path.join("mtn2cache", "manifest", parent) + if not os.path.exists(filename): + diff = Child(*DIFF+[parfilename, curfilename]).stdout + try: + file(filename, "w").write(diff) + except: + os.unlink(filename) + raise + +def handle_revision(revision): + print "revision: %s" % revision + filename = os.path.join("mtn2cache", "revision", revision) + if not os.path.exists(filename): + data = Child(*MTN+["au", "get_revision", revision]).stdout + try: + file(filename, "w").write(data) + except: + os.unlink(filename) + raise + + filename = os.path.join("mtn2cache", "manifest", revision) + if not os.path.exists(filename): + data = Child(*MTN+["au", "get_manifest_of", revision]).stdout + try: + file(filename, "w").write(data) + except: + os.unlink(filename) + raise + + filename = os.path.join("mtn2cache", "certs", revision) + if not os.path.exists(filename): + data = Child(*MTN+["au", "certs", revision]).stdout + try: + file(filename, "w").write(data) + except: + os.unlink(filename) + raise + +def handle_head(head): + print "head: %s" % head + ancestors = Child(*MTN+["au", "ancestors", head]).stdout.strip().split('\n') + pool.map(handle_revision, ancestors) + pool.map(handle_parents, ancestors) + +def handle_branch(branch): + print "branch: %s" % branch + heads = Child(*MTN+["au", "heads", branch]).stdout.strip().split('\n') + for head in heads: + handle_head(head) + +def main(): + branches = Child(*MTN+["au", "branches"]).stdout.strip().split('\n') + for branch in branches: + handle_branch(branch) + +pool = processing.Pool(12) +main() + diff --git a/contrib/mtn2git/mtn2git.py b/contrib/mtn2git/mtn2git.py index a4b43721d9..d6cfcf7891 100755 --- a/contrib/mtn2git/mtn2git.py +++ b/contrib/mtn2git/mtn2git.py @@ -59,25 +59,12 @@ def get_mark(revision): status.last_mark += 1 status.marks[revision] = status.last_mark print >> status.mark_file, "%d: %s" % (status.last_mark, revision) + status.mark_file.flush() return status.last_mark def has_mark(revision): return revision in status.marks - -def mark_empty_revision(revision, parent): - """Git does not like empty merges, just skip the revision""" - # TODO, FIXME, XXX, We might want to add a reset cmd here - print >> sys.stderr, "Found an empty revision, skipping '%s'" % revision - parent_mark = status.marks[parent] - status.marks[revision] = parent_mark - - # There is another mtn revision that is using this mark! - if not parent_mark in status.same_revisions: - status.same_revisions[parent_mark] = [] - status.same_revisions[parent_mark].append(revision) - - def get_branch_name(revision): """ TODO for unnamed branches (e.g. as we lack the certs) we might want to follow @@ -88,7 +75,7 @@ def get_branch_name(revision): branch = revision["branch"] else: #branch = "initial-%s" % revision["revision"] - branch = "mtn-unnamed-branch" + branch = "mtn-rev-%s" % revision["revision"] return branch def reset_git(ops, revision): @@ -103,6 +90,15 @@ def reset_git(ops, revision): cmd += [""] print "\n".join(cmd) +def checkpoint(): + """ + Force git to checkpoint the import + """ + cmd = [] + cmd += ["checkpoint"] + cmd += [""] + print "\n".join(cmd) + def get_git_date(revision): """ Convert the "date" cert of monotone to a time understandable by git. No timezone @@ -120,7 +116,6 @@ def is_executable_attribute_set(attributes, rev): return True return False - def build_tree(manifest, rev): """Assemble a filesystem tree from a given manifest""" @@ -192,10 +187,8 @@ def diff_manifest(old_tree, new_tree): if old != new: modified.add((file, new_tree.files[file][0])) - return (added, modified, deleted) - def fast_import(ops, revision): """Import a revision into git using git-fast-import. @@ -207,48 +200,21 @@ def fast_import(ops, revision): assert("committer" in revision) assert("parent" in revision) - branch = get_branch_name(revision) - # Okay: We sometimes have merged where the old manifest is the new one - # I have no idea how this can happen but there are at least two examples in the - # net.venge.monotone history. - # The problem ist git-fast-import will not let us create the same manifest again. - # So if we are in a merge, propagation and the old manifest is the new one we will - # do a git-reset. - # Examples in the mtn history: 6dc36d2cba722f500c06f33e225367461059d90e, dc661f0c25ee96a5a5cf5b5b60deafdf8ccaf286 - # and 7b8331681bf77cd8329662dbffed0311765e7547, 13b1a1e617a362c5735002937fead98d788737f7 - # aa05aa9171bac92766b769bbb703287f53e08693 is a merge of the same manifest... - # so we will just go with one of the two revisions.. - # We will have the same manifest if we propagate something from one branch to another. git does - # not have a special revision showing that copy but will only change the head. - # We will do the same and reset the branch to this revision. - for parent in revision["parent"]: - manifest_version = parse_revision(ops, parent)["manifest"] - if manifest_version == revision["manifest"]: - mark_empty_revision(revision["revision"], parent) - reset_git(ops, revision) - return - # Use the manifest to find dirs and files current_tree = get_and_cache_tree(ops, revision["revision"]) - all_added = set() - all_modifications = set() - all_deleted = set() - # Now diff the manifests - for parent in revision["parent"]: - (added, modified, deleted) = diff_manifest(get_and_cache_tree(ops, parent), current_tree) - all_added = all_added.union(added) - all_modifications = all_modifications.union(modified) - all_deleted = all_deleted.union(deleted) - if len(revision["parent"]) == 0: + merge_from = None + merge_other = [] (added, modified, deleted) = diff_manifest(build_tree([],""), current_tree) - all_added = all_added.union(added) - all_modifications = all_modifications.union(modified) - all_deleted = all_deleted.union(deleted) + else: + # The first parent is our from. + merge_from = revision["parent"][0] + merge_other = revision["parent"][1:] + (added, modified, deleted) = diff_manifest(get_and_cache_tree(ops, merge_from), current_tree) # TODO: # Readd the sanity check to see if we deleted and modified an entry. This @@ -264,27 +230,25 @@ def fast_import(ops, revision): cmd += ["data %d" % len(revision["changelog"])] cmd += ["%s" % revision["changelog"]] - if len(revision["parent"]) != 0: - cmd += ["from :%s" % get_mark(revision["parent"][0])] + if not merge_from is None: + cmd += ["from :%s" % get_mark(merge_from)] - # The first parent is our from. - for parent in revision["parent"][1:]: + for parent in merge_other: cmd += ["merge :%s" % get_mark(parent)] - - for dir_name in all_added: + for dir_name in added: cmd += ["M 644 inline %s" % os.path.join(dir_name, ".mtn2git_empty")] cmd += ["data <<EOF"] cmd += ["EOF"] cmd += [""] - for (file_name, file_revision) in all_modifications: + for (file_name, file_revision) in modified: (mode, file) = get_file_and_mode(ops, current_tree, file_name, file_revision, revision["revision"]) cmd += ["M %d inline %s" % (mode, file_name)] cmd += ["data %d" % len(file)] cmd += ["%s" % file] - for (path, is_dir) in all_deleted: + for (path, is_dir) in deleted: if is_dir: cmd += ["D %s" % os.path.join(path, ".mtn2git_empty")] else: @@ -293,7 +257,6 @@ def fast_import(ops, revision): cmd += [""] print "\n".join(cmd) - def is_trusted(operations, revision): for cert in operations.certs(revision): if cert[0] != 'key' or cert[3] != 'ok' or cert[8] != 'trust' or cert[9] != 'trusted': @@ -316,7 +279,6 @@ def get_file_and_mode(operations, file_tree, file_name, _file_revision, rev = No file = "".join([file for file in operations.get_file(file_revision)]) return (mode, file) - def parse_revision(operations, revision): """ Parse a revision as of mtn automate get_revision @@ -375,7 +337,6 @@ def parse_revision(operations, revision): return revision_description - def tests(ops, revs): """Load a bunch of revisions and exit""" for rev in revs: @@ -434,7 +395,7 @@ def main(mtn_cli, db, rev): all_revs += ops.ancestry_difference(head, old_heads) status.former_heads[branch] = heads - + counter = 0 sorted_revs = [rev for rev in ops.toposort(all_revs)] for rev in sorted_revs: if has_mark(rev): @@ -442,6 +403,9 @@ def main(mtn_cli, db, rev): else: print >> sys.stderr, "Going to import revision ", rev fast_import(ops, parse_revision(ops, rev)) + if counter % 1000 == 0: + checkpoint() + counter += 1 if __name__ == "__main__": import optparse |