ehlers
/
ova_import


								#!/usr/bin/env python3


								# Copyright (C) 2016-2018 Bernhard Ehlers

								#

								# This program is free software: you can redistribute it and/or modify

								# it under the terms of the GNU General Public License as published by

								# the Free Software Foundation, either version 3 of the License, or

								# (at your option) any later version.

								#

								# This program is distributed in the hope that it will be useful,

								# but WITHOUT ANY WARRANTY; without even the implied warranty of

								# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

								# GNU General Public License for more details.

								#

								# You should have received a copy of the GNU General Public License

								# along with this program.  If not, see <http://www.gnu.org/licenses/>.


								"""

								ova_import generates gns3a file to import ova appliance into GNS3.


								usage: ova_import [-h] [-d DIR] [-k] ova


								positional arguments:

								  ova                   .ova appliance


								optional arguments:

								  -h, --help            show this help message and exit

								  -d DIR, --dir DIR     directory for storing appliance and images

								  -k, --keepcompressed  keep images compressed

								"""


								import os

								import sys

								import argparse

								import hashlib

								import json

								import re

								import struct

								import tarfile

								import zlib

								import xml.etree.ElementTree as ElementTree

								from collections import defaultdict, OrderedDict


								def strip_namespace(tag):

								    """ strip namespace from tag """

								    if '}' in tag:

								        tag = tag.split('}', 1)[1]      # strip all namespaces

								    return tag


								def etree_to_dict(etree):

								    """

								    convert ElementTree to dictionary

								    see K3---rnc's answer of "Converting xml to dictionary using ElementTree"

								    http://stackoverflow.com/questions/7684333/converting-xml-to-dictionary-using-elementtree#answer-10076823

								    variable names changed to make it easier understandable

								    """

								    tag = strip_namespace(etree.tag)

								    node = {tag: {} if etree.attrib else None}

								    children = list(etree)

								    if children:

								        all_child_nodes = defaultdict(list)

								        for child_node in map(etree_to_dict, children):

								            for key, val in child_node.items():

								                all_child_nodes[key].append(val)

								        node = {tag: {key:val[0] if len(val) == 1 else val

								                      for key, val in all_child_nodes.items()}}

								    if etree.attrib:

								        node[tag].update((strip_namespace(key), val)

								                         for key, val in etree.attrib.items())

								    if etree.text:

								        text = etree.text.strip()

								        if children or etree.attrib:

								            if text:

								                node[tag]['#text'] = text

								        else:

								            node[tag] = text

								    return node


								def create_gns3a(fname, dname, file_info, ovf_data):

								    """ create GNS3 appliance file """

								    fbase = os.path.basename(fname)

								    gns3a = OrderedDict()

								    ovf = etree_to_dict(ElementTree.fromstring(ovf_data))['Envelope']


								    # base informations

								    try:

								        vm_name = ovf['VirtualSystem']['Name']

								    except KeyError:

								        vm_name = ovf['VirtualSystem']['id']

								    gns3a["name"]             = vm_name

								    gns3a["category"]         = "guest"

								    gns3a["description"]      = "ova import of " + fbase

								    gns3a["vendor_name"]      = "unknown"

								    gns3a["vendor_url"]       = "http://www.example.com"

								    gns3a["product_name"]     = vm_name

								    gns3a["registry_version"] = 3

								    gns3a["status"]           = "experimental"

								    gns3a["maintainer"]       = "GNS3 Team"

								    gns3a["maintainer_email"] = "developers@gns3.net"


								    # qemu

								    cores = 0

								    eth_adapters = 0

								    ram = 0

								    for item in ovf['VirtualSystem']['VirtualHardwareSection']['Item']:

								        if item['ResourceType'] == '3':

								            cores = int(item['VirtualQuantity'])

								        elif item['ResourceType'] == '4':

								            ram = int(item['VirtualQuantity'])

								        elif item['ResourceType'] == '10':

								            eth_adapters += 1

								    cores = max(cores, 1)

								    eth_adapters = max(eth_adapters, 1)

								    if ram == 0:

								        ram = 256

								    try:

								        vm_os = ovf['VirtualSystem']['OperatingSystemSection']['osType']

								    except KeyError:

								        vm_os = ovf['VirtualSystem']['OperatingSystemSection']['OSType']['#text']


								    qemu = OrderedDict()

								    qemu["adapter_type"] = "e1000"

								    qemu["adapters"] = eth_adapters

								    qemu["ram"] = ram

								    qemu["arch"] = "i386"

								    if "64" in vm_os:

								        qemu["arch"] = "x86_64"

								    qemu["console_type"] = "telnet"

								    qemu["kvm"] = "allow"

								    if cores > 1:

								        qemu["options"] = "-smp " + str(cores)

								    gns3a["qemu"] = qemu


								    # images

								    images = []

								    for name in sorted(file_info.keys()):

								        image = OrderedDict()

								        image["filename"] = name

								        image["version"]  = "0.0"

								        image["md5sum"]   = file_info[name]["md5"]

								        image["filesize"] = file_info[name]["len"]

								        images.append(image)

								    gns3a["images"] = images


								    # versions

								    images = OrderedDict()

								    cdrom = None

								    disk_id = 0

								    ovf_ref_file = ovf['References']['File']

								    if not isinstance(ovf_ref_file, (list, tuple)):

								        ovf_ref_file = [ovf_ref_file]

								    for image in ovf_ref_file:

								        img = image['href']

								        if img.endswith(".iso"):

								            if cdrom is None:

								                cdrom = img

								        else:

								            images["hd" + chr(ord("a")+disk_id) + "_disk_image"] = img

								            disk_id += 1

								    if cdrom is not None:

								        images["cdrom_image"] = cdrom

								    gns3a["versions"] = [OrderedDict([("name", "0.0"), ("images", images)])]


								    # write to file

								    ofile = os.path.join(dname, os.path.splitext(fbase)[0] + ".gns3a")

								    with open(ofile, "w") as f_out:

								        json.dump(gns3a, f_out, indent=4, separators=(',', ': '))

								        f_out.write("\n")


								SECTOR_SIZE = 512


								def div_ceil(dividend, divisor):

								    """ integer division, round up """

								    return -(-dividend // divisor)


								def copy_raw(f_in, f_out, initial_data=None):

								    """ copy input file to output file, return file info """

								    f_len = 0

								    f_md5 = hashlib.md5()


								    if initial_data:

								        f_out.write(initial_data)

								        f_len += len(initial_data)

								        f_md5.update(initial_data)


								    while True:

								        block = f_in.read(64 * 1024)

								        if not block:

								            break

								        f_out.write(block)

								        f_len += len(block)

								        f_md5.update(block)


								    return {'len': f_len, 'md5': f_md5.hexdigest()}


								def read_block(file, datalen):

								    """ read a block of data from file """

								    if datalen < 0:

								        raise ValueError("Can't seek backwards")

								    if datalen == 0:

								        data = b''

								    else:

								        data = file.read(datalen)

								        if len(data) != datalen:

								            raise ValueError('Premature EOF')

								    return data


								def copy_vmdk(f_in, f_out, out_name):

								    """

								    uncompress VMDK file, return file info


								    The VMDK disk format is documented in

								    https://www.vmware.com/support/developer/vddk/vmdk_50_technote.pdf

								    """


								    header = f_in.read(SECTOR_SIZE)

								    if len(header) != SECTOR_SIZE or header[0:4] != b'KDMV':

								        # unknown file type: extract unchanged

								        return copy_raw(f_in, f_out, header)


								    # decode header

								    (magic, _, flags, capacity, grain_size, descriptor_offset, descriptor_size,

								     num_gte_per_gt, _, in_gd_off, overhead, _, single_eol_char, non_eol_char,

								     double_eol_char1, double_eol_char2, compress_algorithm) = \

								        struct.unpack_from('<LLLQQQQLQQQBccccH', header)

								    if flags & 0x30000 != 0x30000 or compress_algorithm != 1:

								        # uncompressed VMDK file: extract unchanged

								        return copy_raw(f_in, f_out, header)


								    # grain directory / table

								    gt_size = div_ceil(num_gte_per_gt * 4, SECTOR_SIZE)

								    gt_count = div_ceil(capacity, grain_size * num_gte_per_gt)

								    gd_size = div_ceil(gt_count * 4, SECTOR_SIZE)

								    rgd_offset = 8

								    gd_offset = rgd_offset + gd_size + gt_count*gt_size

								    out_overhead = div_ceil(gd_offset + gd_size + gt_count*gt_size, 128) * 128


								    # read/modify descriptor

								    if descriptor_offset > 0 and descriptor_size > 0:

								        read_block(f_in, (descriptor_offset-1) * SECTOR_SIZE)

								        descriptor = read_block(f_in, descriptor_size * SECTOR_SIZE)

								        in_offset = descriptor_offset + descriptor_size

								        descriptor = descriptor.rstrip(b'\0')

								        descriptor = descriptor.replace(double_eol_char1 + double_eol_char2,

								                                        single_eol_char)

								        if single_eol_char != b'\n':

								            descriptor = descriptor.replace(single_eol_char, b'\n')

								        descriptor = re.sub(br'^([ \t]*createType[ \t]*=[ \t]*)"streamOptimized"',

								                            br'\1"monolithicSparse"',

								                            descriptor, flags=re.MULTILINE)

								        descriptor, count = re.subn(br'^([ \t]*)(RW|RDONLY)[ \t].*',

								                                    r'\1RW {:d} SPARSE "{}"'.format(capacity, out_name).encode(),

								                                    descriptor, flags=re.MULTILINE)

								        if count == 0:

								            raise ValueError('No extent defined in descriptor')

								        if count > 1:

								            raise ValueError('Multiple extents defined in descriptor')

								        if single_eol_char != b'\n':

								            descriptor = descriptor.replace(b'\n', single_eol_char)

								        out_desc_offset = 1

								        out_desc_size = rgd_offset - 1

								        if len(descriptor) > out_desc_size * SECTOR_SIZE:

								            raise ValueError('Descriptor too big')

								    else:

								        in_offset = 1

								        out_desc_offset = 0

								        out_desc_size = 0


								    # write new header

								    header = struct.pack('<LLLQQQQLQQQBccccH', \

								        magic, 1, 3, capacity, grain_size, out_desc_offset, out_desc_size, \

								        num_gte_per_gt, rgd_offset, gd_offset, out_overhead, 0, \

								        single_eol_char, non_eol_char, double_eol_char1, double_eol_char2, 0)

								    header += b'\0' * (SECTOR_SIZE - len(header))

								    f_out.write(header)


								    # write descriptor

								    if out_desc_offset > 0:

								        descriptor += b'\0' * (out_desc_size*SECTOR_SIZE - len(descriptor))

								        f_out.write(descriptor)


								    # initialize grain directory and tables

								    grain_directory = None

								    grain_table = []

								    grain_translation = {0:0, 1:1}

								    gt_translation = {0:-1}


								    # if available read grain directory and grain tables

								    if in_gd_off != 0xffffffffffffffff:

								        if in_gd_off < in_offset or \

								           in_gd_off + gd_size + gt_count*gt_size > overhead:

								            raise ValueError('Invalid grain directory offset')

								        read_block(f_in, (in_gd_off - in_offset) * SECTOR_SIZE)

								        in_offset = in_gd_off

								        data = read_block(f_in, gd_size * SECTOR_SIZE)

								        data = struct.unpack_from("<{:d}L".format(gt_count), data)

								        grain_directory = data

								        in_offset += gd_size

								        for index in range(0, gt_count):

								            gt_translation[in_offset] = index

								            data = read_block(f_in, gt_size * SECTOR_SIZE)

								            data = struct.unpack_from("<{:d}L".format(num_gte_per_gt), data)

								            grain_table.append(data)

								            in_offset += gt_size


								    # skip to data

								    read_block(f_in, (overhead - in_offset) * SECTOR_SIZE)

								    in_offset = overhead

								    out_offset = out_overhead

								    f_out.seek(out_offset * SECTOR_SIZE, 0)


								    # read, uncompress and write data

								    max_datasize = grain_size * SECTOR_SIZE

								    # add worst case compression overhead

								    max_datasize += div_ceil(max_datasize, 8) + div_ceil(max_datasize, 64) + 11

								    while True:

								        header = read_block(f_in, SECTOR_SIZE)

								        (val, size, marker) = struct.unpack_from('<QLL', header)


								        if size > 0:

								            # Compressed Grain

								            if size > max_datasize:

								                raise ValueError('Bad data block @{:08X}'.format(in_offset))

								            prev_offset = in_offset

								            grain_translation[in_offset] = out_offset

								            data = header[12:]

								            if size > (SECTOR_SIZE - 12):

								                data += read_block(f_in, size - (SECTOR_SIZE - 12))

								                read_block(f_in, -(size + 12) % SECTOR_SIZE)

								                in_offset += div_ceil(size + 12, SECTOR_SIZE)

								            else:

								                data = data[0:size]

								                in_offset += 1

								            try:

								                data = zlib.decompress(data)

								            except zlib.error:

								                raise ValueError('Bad data block @{:08X}'.format(prev_offset))

								            if len(data) != grain_size*SECTOR_SIZE:

								                raise ValueError('Data block @{:08X} has wrong size {:d}'.format(prev_offset, len(data)))

								            f_out.write(data)

								            out_offset += grain_size


								        elif marker == 0:

								            # End-of-Stream

								            if f_in.read(SECTOR_SIZE):

								                raise ValueError('No EOF after end of stream @{:08X}'.format(in_offset))

								            break


								        elif marker == 1:

								            # Grain Table

								            if len(grain_table) >= gt_count:

								                raise ValueError('Too many grain tables @{:08X}'.format(in_offset))

								            if val != gt_size:

								                raise ValueError('Grain table @{:08X} has wrong size {:d}'.format(in_offset, val * SECTOR_SIZE))

								            gt_translation[in_offset+1] = len(grain_table)

								            data = read_block(f_in, val * SECTOR_SIZE)

								            data = struct.unpack_from("<{:d}L".format(num_gte_per_gt), data)

								            grain_table.append(data)

								            in_offset += 1 + val


								        elif marker == 2:

								            # Grain Directory

								            if grain_directory is not None:

								                raise ValueError('Too many grain directories @{:08X}'.format(in_offset))

								            if val != gd_size:

								                raise ValueError('Grain directory @{:08X} has wrong size {:d}'.format(in_offset, val * SECTOR_SIZE))

								            data = read_block(f_in, val * SECTOR_SIZE)

								            data = struct.unpack_from("<{:d}L".format(gt_count), data)

								            grain_directory = data

								            in_offset += 1 + val


								        else:

								            # ignore footer or unknown marker

								            read_block(f_in, val * SECTOR_SIZE)

								            in_offset += 1 + val


								    # set table index in grain directory

								    if grain_directory is None:

								        raise ValueError('No grain directory')

								    try:

								        grain_directory = [gt_translation[x] for x in grain_directory]

								    except KeyError:

								        raise ValueError('Grain directory uses invalid table references')


								    # apply new offsets in grain tables

								    for index, table in enumerate(grain_table):

								        try:

								            grain_table[index] = [grain_translation[x] for x in table]

								        except KeyError:

								            raise ValueError('Grain table uses invalid data references')


								    # grain directory: sort grain tables, update table index to sector offset

								    gt_empty = [0] * num_gte_per_gt

								    unsorted_grain_table = grain_table

								    grain_table = []

								    gt_offset = rgd_offset + gd_size

								    for index in range(0, gt_count):

								        if grain_directory[index] < 0:

								            grain_table.append(gt_empty)

								        else:

								            grain_table.append(unsorted_grain_table[grain_directory[index]])

								        grain_directory[index] = gt_offset

								        gt_offset += gt_size


								    # save first/redundant grain descriptor

								    f_out.seek(rgd_offset * SECTOR_SIZE, 0)

								    data = struct.pack("<{:d}L".format(gt_count), *grain_directory)

								    data += b'\0' * (gd_size*SECTOR_SIZE - len(data))

								    f_out.write(data)

								    for table in grain_table:

								        data = struct.pack("<{:d}L".format(num_gte_per_gt), *table)

								        data += b'\0' * (gt_size*SECTOR_SIZE - len(data))

								        f_out.write(data)


								    # save second grain descriptor

								    second_directory = [x - rgd_offset + gd_offset for x in grain_directory]

								    data = struct.pack("<{:d}L".format(gt_count), *second_directory)

								    data += b'\0' * (gd_size*SECTOR_SIZE - len(data))

								    f_out.write(data)

								    for table in grain_table:

								        data = struct.pack("<{:d}L".format(num_gte_per_gt), *table)

								        data += b'\0' * (gt_size*SECTOR_SIZE - len(data))

								        f_out.write(data)


								    # get file information

								    f_out.seek(0, 0)

								    f_len = 0

								    f_md5 = hashlib.md5()

								    while True:

								        block = f_out.read(64 * 1024)

								        if not block:

								            break

								        f_len += len(block)

								        f_md5.update(block)


								    return {'len': f_len, 'md5': f_md5.hexdigest()}


								def main(argv):

								    """ main """


								    # parse command line

								    parser = argparse.ArgumentParser(description='%(prog)s generates gns3a file to import ova appliance into GNS3.')

								    parser.add_argument('-d', '--dir', default='',

								                        help='directory for storing appliance and images')

								    parser.add_argument('-k', '--keepcompressed', action='store_true',

								                        help='keep images compressed')

								    parser.add_argument('ova', help='.ova appliance')

								    args = parser.parse_args(argv[1:])


								    fname = args.ova

								    dname = args.dir


								    if dname != '' and not os.path.isdir(dname):

								        sys.exit("Directory '{}' doesn't exist.".format(dname))


								    # open ovf file

								    try:

								        tar = tarfile.open(fname)

								    except (IOError, tarfile.TarError) as err:

								        sys.exit("Error reading ova file: {}".format(err))


								    # get files from ova

								    file_info = {}

								    ovf_data = None


								    for tarinfo in tar:

								        name = tarinfo.name.split('/')[-1]

								        if not tarinfo.isfile():            # ova uses only regular files

								            pass

								        elif name.endswith(".ovf"):         # ovf file

								            f_in = tar.extractfile(tarinfo)

								            ovf_data = f_in.read()

								            f_in.close()

								        elif name.endswith(".mf"):          # ignore manifest file

								            pass

								        elif name.endswith(".cert"):        # ignore certificate

								            pass

								        elif name.endswith(".pem"):         # ignore certificate

								            pass

								        else:                               # save image file

								            print("Extracting image {}...".format(name))

								            out_name = os.path.join(dname, name)

								            f_in = tar.extractfile(tarinfo)

								            f_out = open(out_name, 'w+b')

								            try:

								                if not args.keepcompressed and name.endswith(".vmdk"):

								                    # uncompress VMDK file

								                    file_info[name] = copy_vmdk(f_in, f_out, name)

								                else:

								                    file_info[name] = copy_raw(f_in, f_out)

								            except ValueError as err:

								                sys.exit("{}: {}".format(name, err))

								            f_in.close()

								            f_out.close()

								            os.utime(out_name, (tarinfo.mtime, tarinfo.mtime))


								    tar.close()


								    if ovf_data is None:

								        sys.exit("No ovf information in ova.")


								    print("Creating appliance {}..."

								          .format(os.path.splitext(os.path.basename(fname))[0] + ".gns3a"))

								    create_gns3a(fname, dname, file_info, ovf_data)


								if __name__ == "__main__":

								    main(sys.argv)