path: root/contrib/mtn2git/mtn2git.py

#!/usr/bin/env python

"""
  Copyright (C) 2006, 2007 Holger Hans Peter Freyther

  Permission is hereby granted, free of charge, to any person obtaining a copy
  of this software and associated documentation files (the "Software"), to deal
  in the Software without restriction, including without limitation the rights
  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  copies of the Software, and to permit persons to whom the Software is
  furnished to do so, subject to the following conditions:

  The above copyright notice and this permission notice shall be included in
  all copies or substantial portions of the Software.

  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  THE SOFTWARE.
"""

####
# TODO:
#   -tag handling
#   -work with n-merges
#
# DISCUSSION:
#   -For some reason the get_revision information might be inaccurate
#    and I should consider just comparing the manifests.
#    I would use the manifests of the parents and consider all files deleted
#    and then remove every dir/file that is inside the new manifest from this
#    list.
#      Benefits:
#         - 1:1 match of the manifest regardles of get_revision information
#         - Renaming is handled by git anyway
#      Downsides:
#         - The size of the import will grow.
#

import mtn
import os
import sys
import datetime
import email.Utils

import status


def get_mark(revision):
    """
    Get a mark for a specific revision. If the revision is known the former
    mark will be returned. Otherwise a new mark will be allocated and stored
    to the mark file.
    """
    if revision in status.marks:
        return status.marks[revision]
    status.last_mark += 1
    status.marks[revision] = status.last_mark
    print >> status.mark_file, "%d: %s" % (status.last_mark, revision)
    return status.last_mark

def has_mark(revision):
    return revision in status.marks


def mark_empty_revision(revision, parent):
    """Git does not like empty merges, just skip the revision"""
    # TODO, FIXME, XXX, We might want to add a reset cmd here
    print >> sys.stderr, "Found an empty revision, skipping '%s'" % revision
    parent_mark = status.marks[parent]
    status.marks[revision] = parent_mark

    # There is another mtn revision that is using this mark!
    if not parent_mark in status.same_revisions:
        status.same_revisions[parent_mark] = []
    status.same_revisions[parent_mark].append(revision)


def get_branch_name(revision):
    """
    TODO for unnamed branches (e.g. as we lack the certs) we might want to follow
    the parents until we end up at a item with a branch name and then use the last
    item without a name...
    """
    if "branch" in revision:
        branch = revision["branch"]
    else:
        #branch = "initial-%s" % revision["revision"]
        branch = "mtn-unnamed-branch"
    return branch

def reset_git(ops, revision):
    """
    Find the name of the branch of this revision
    """
    branch = get_branch_name(revision)

    cmd = []
    cmd += ["reset refs/heads/%s" % branch]
    cmd += ["from :%s" % get_mark(revision["revision"])]
    cmd += [""]
    print "\n".join(cmd)

def filter_renamed(manifest, renamed):
    """
    If we base from a revision that has already done
    the move, git-fast-import will complain that the file
    has been already moved
    """
    if len(renamed) == 0:
        return renamed

    for line in manifest:
        if line[0] == "file":
            renamed = filter(lambda (to,from_,manifest): to != line[1], renamed)
            
    return renamed

def get_git_date(revision):
    """
    Convert the "date" cert of monotone to a time understandable by git. No timezone
    conversions are done.
    """
    dt = datetime.datetime.strptime(revision["date"], "%Y-%m-%dT%H:%M:%S").strftime("%a, %d %b %Y %H:%M:%S +0000")
    return dt

def recursively_delete(ops, manifest, revision, dir_name, to_delete):
    """
    Recursively delete all files that dir_name inside the name
    """
    for line in manifest:
        if line[0] == "dir" or line[0] == "file":
            if line[1].startswith(dir_name):
                print >> sys.stderr, "Deleting '%s'" % line[1]
                to_delete.add((line[1], revision))
        elif line[0] in ["format_version"]:
            assert(line[1] == "1")
        else:
            print >> sys.stderr, line[0]
            assert(False)

    return to_delete

def recursively_rename(ops, manifest, revision, old_name, new_name, to_add_dirs, to_add_files, to_remove_items, files_deleted, files_sticky):
    """
    mtn has a rename command and can rename entrie directories. For git we will have to do the recursive renaming
    ourselves. Basicly we will get all files and replace old_name with new_name but only:

        If the file of the old_manifest is not in our to be deleted list
    """
    old_dir = old_name + "/"
    for line in manifest:
        if line[1].startswith(old_dir) or line[1] == old_name:
            already_handled = False
            for (deleted,_) in files_deleted:
                if line[1] == deleted:
                    already_handled = True
                    break 

            # Don't rename files that should be in the same directory
            if line[1] in files_sticky:
                already_handled = True

            if already_handled:
                pass
            elif line[0] == "file":
                print >> sys.stderr, "Will add '%s' old: '%s' new: '%s' => result: '%s'" % (line[1], old_name, new_name, line[1].replace(old_name, new_name, 1))
                to_add_files.add((line[1].replace(old_name, new_name, 1), None, revision))
            elif line[0] == "dir":
                to_add_dirs.add((line[1].replace(old_name, new_name, 1), revision))
            elif line[0] in ["format_version"]:
                assert(line[1] == "1")
            else:
                print >> sys.stderr, line[0]
                assert(False)

    return (to_add_files, to_add_dirs)

#
#    We need to recursively rename the directories. Now the difficult part is to undo certain operations.
#    
#    e.g we rename the whole dir and then rename a file back. We could revive a directory that was marked
#    for deletion.
#
#    rename "weird/two/three"
#    to "unweird/four"
#
#    rename "weird/two/three/four"
#    to "weird/two/three"
#
#    Here we would schedule weird/two/three for deletion but then revive it again. So three does not
#    get copied to unweird/four/three
#    """
def recursively_rename_directories(ops, manifests, rename_commands, files_deleted, files_moved_sticky):
    to_add_directories = set()
    to_add_files = set()
    to_remove_items = set()

    for (old_name, new_name, old_revision) in rename_commands:
        # Check if we have the above case and rename a more specific directory
        # and then we will alter the result...
        inner_rename = False
        for (other_old_name, other_new_name, other_rev) in rename_commands:
            if old_name.startswith(other_old_name + "/") and other_old_name != old_name:
                inner_rename = True
                print >> sys.stderr, "Inner rename detected", old_name, other_old_name
                # Fixup the renaming
                def rename(filename, filerev, rev, would_be_new_name):
                    if filename.startswith(would_be_new_name + "/"):
                        return filename.replace(would_be_new_name, new_name, 1), filerev, rev
                    return filename, filerev, rev
                    
                would_be_new_name = other_new_name + "/" + old_name[len(other_old_name)+1:]
                to_remove_items = set(filter(lambda (item,_): item != new_name, to_remove_items))
                to_add_directories = set(filter(lambda (item,_): item != would_be_new_name, to_add_directories))
                to_add_directories.add((new_name, old_revision))
                to_add_files = set(map(lambda (fn, fr, r): rename(fn, fr, r, would_be_new_name), to_add_files))

        if not inner_rename:
            to_remove_items.add((old_name, old_revision))
            recursively_delete(ops, manifests[old_revision], old_revision, old_name + "/", to_remove_items)
            recursively_rename(ops, manifests[old_revision], old_revision, old_name, new_name, to_add_directories, to_add_files, to_remove_items, files_deleted, files_moved_sticky)

    return (to_add_directories, to_add_files, to_remove_items)


def build_tree(manifest):
    dirs = {}
    files = {}
    for line in manifest:
        if line[0] == "file":
            files[line[1]] = (line[3],line[4:])
        elif line[0] == "dir":
            dirs[line[1]] = 1
        elif line[0] != "format_version":
            print >> sys.stderr, line[0]
            assert(False)
    return (dirs,files)

def compare_with_manifest(all_added, all_modified, all_deleted, new_manifest, old_manifests):
    """
    Sanity check that the difference between the old and the new manifest is the one
    we have in all_added, all_modified, all_deleted
    """
    old_trees = {}
    really_added = {}
    really_modified = {}
    really_removed = {}

    current_dirs, current_files = build_tree(new_manifest)

    for parent in old_manifests.keys():
        old_trees[parent] = build_tree(old_manifests[parent])

    print >> sys.stderr, len(old_manifests)

def fast_import(ops, revision):
    """Import a revision into git using git-fast-import.

    First convert the revision to something git-fast-import
    can understand
    """
    assert("revision" in revision)
    assert("author" in revision)
    assert("committer" in revision)
    assert("parent" in revision)


    branch = get_branch_name(revision)

    # Okay: We sometimes have merged where the old manifest is the new one
    # I have no idea how this can happen but there are at least two examples in the
    # net.venge.monotone history.
    # The problem ist git-fast-import will not let us create the same manifest again.
    # So if we are in a merge, propagation and the old manifest is the new one we will
    # do a git-reset.
    # Examples in the mtn history: 6dc36d2cba722f500c06f33e225367461059d90e, dc661f0c25ee96a5a5cf5b5b60deafdf8ccaf286
    # and 7b8331681bf77cd8329662dbffed0311765e7547, 13b1a1e617a362c5735002937fead98d788737f7
    # aa05aa9171bac92766b769bbb703287f53e08693 is a merge of the same manifest...
    # so we will just go with one of the two revisions..
    # We will have the same manifest if we propagate something from one branch to another. git does
    # not have a special revision showing that copy but will only change the head.
    # We will do the same and reset the branch to this revision.
    for parent in revision["parent"]:
        manifest_version = parse_revision(ops, parent)["manifest"]
        if manifest_version == revision["manifest"]:
            mark_empty_revision(revision["revision"], parent)
            reset_git(ops, revision)
            return
        
    # Use the manifest to find dirs and files
    manifest = [line for line in ops.get_manifest_of(revision["revision"])]
    manifests = {}
    dirs = {}
    for parent in revision["parent"]:
        manifests[parent] = [line for line in ops.get_manifest_of(parent)]
        for line in manifests[parent]:
            if line[0] == "dir":
                if not parent in dirs:
                    dirs[parent] = {}
                dirs[parent][line[1]] = 1

    # We can not just change the mode of a file but we need to modifiy the whole file. We
    # will simply add it to the modified list and ask to retrieve the status from the manifest
    for (file, attribute, value, rev) in revision["set_attributes"]:
        if attribute == "mtn:execute":
            revision["modified"].append((file, None, rev))
    for (file, attribute, rev) in revision["clear_attributes"]:
        if attribute == "mtn:execute":
            revision["modified"].append((file, None, rev))



    cmd = []
    cmd += ["commit refs/heads/%s" % branch]
    cmd += ["mark :%s" % get_mark(revision["revision"])]
    cmd += ["author  <%s> %s" % (revision["author"], get_git_date(revision))]
    cmd += ["committer  <