|
- #!/usr/bin/env python3
-
- # Copyright (C) 2016-2018 Bernhard Ehlers
- #
- # This program is free software: you can redistribute it and/or modify
- # it under the terms of the GNU General Public License as published by
- # the Free Software Foundation, either version 3 of the License, or
- # (at your option) any later version.
- #
- # This program is distributed in the hope that it will be useful,
- # but WITHOUT ANY WARRANTY; without even the implied warranty of
- # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- # GNU General Public License for more details.
- #
- # You should have received a copy of the GNU General Public License
- # along with this program. If not, see <http://www.gnu.org/licenses/>.
-
- """
- ova_import generates gns3a file to import ova appliance into GNS3.
-
- usage: ova_import [-h] [-d DIR] [-k] ova
-
- positional arguments:
- ova .ova appliance
-
- optional arguments:
- -h, --help show this help message and exit
- -d DIR, --dir DIR directory for storing appliance and images
- -k, --keepcompressed keep images compressed
- """
-
- import os
- import sys
- import argparse
- import hashlib
- import json
- import re
- import struct
- import tarfile
- import zlib
- import xml.etree.ElementTree as ElementTree
- from collections import defaultdict, OrderedDict
-
-
- def strip_namespace(tag):
- """ strip namespace from tag """
- if '}' in tag:
- tag = tag.split('}', 1)[1] # strip all namespaces
- return tag
-
-
- def etree_to_dict(etree):
- """
- convert ElementTree to dictionary
- see K3---rnc's answer of "Converting xml to dictionary using ElementTree"
- http://stackoverflow.com/questions/7684333/converting-xml-to-dictionary-using-elementtree#answer-10076823
- variable names changed to make it easier understandable
- """
- tag = strip_namespace(etree.tag)
- node = {tag: {} if etree.attrib else None}
- children = list(etree)
- if children:
- all_child_nodes = defaultdict(list)
- for child_node in map(etree_to_dict, children):
- for key, val in child_node.items():
- all_child_nodes[key].append(val)
- node = {tag: {key:val[0] if len(val) == 1 else val
- for key, val in all_child_nodes.items()}}
- if etree.attrib:
- node[tag].update((strip_namespace(key), val)
- for key, val in etree.attrib.items())
- if etree.text:
- text = etree.text.strip()
- if children or etree.attrib:
- if text:
- node[tag]['#text'] = text
- else:
- node[tag] = text
- return node
-
-
- def create_gns3a(fname, dname, file_info, ovf_data):
- """ create GNS3 appliance file """
- fbase = os.path.basename(fname)
- gns3a = OrderedDict()
- ovf = etree_to_dict(ElementTree.fromstring(ovf_data))['Envelope']
-
- # base informations
- try:
- vm_name = ovf['VirtualSystem']['Name']
- except KeyError:
- vm_name = ovf['VirtualSystem']['id']
- gns3a["name"] = vm_name
- gns3a["category"] = "guest"
- gns3a["description"] = "ova import of " + fbase
- gns3a["vendor_name"] = "unknown"
- gns3a["vendor_url"] = "http://www.example.com"
- gns3a["product_name"] = vm_name
- gns3a["registry_version"] = 3
- gns3a["status"] = "experimental"
- gns3a["maintainer"] = "GNS3 Team"
- gns3a["maintainer_email"] = "developers@gns3.net"
-
- # qemu
- cores = 0
- eth_adapters = 0
- ram = 0
- for item in ovf['VirtualSystem']['VirtualHardwareSection']['Item']:
- if item['ResourceType'] == '3':
- cores = int(item['VirtualQuantity'])
- elif item['ResourceType'] == '4':
- ram = int(item['VirtualQuantity'])
- elif item['ResourceType'] == '10':
- eth_adapters += 1
- cores = max(cores, 1)
- eth_adapters = max(eth_adapters, 1)
- if ram == 0:
- ram = 256
- try:
- vm_os = ovf['VirtualSystem']['OperatingSystemSection']['osType']
- except KeyError:
- vm_os = ovf['VirtualSystem']['OperatingSystemSection']['OSType']['#text']
-
- qemu = OrderedDict()
- qemu["adapter_type"] = "e1000"
- qemu["adapters"] = eth_adapters
- qemu["ram"] = ram
- qemu["arch"] = "i386"
- if "64" in vm_os:
- qemu["arch"] = "x86_64"
- qemu["console_type"] = "telnet"
- qemu["kvm"] = "allow"
- if cores > 1:
- qemu["options"] = "-smp " + str(cores)
- gns3a["qemu"] = qemu
-
- # images
- images = []
- for name in sorted(file_info.keys()):
- image = OrderedDict()
- image["filename"] = name
- image["version"] = "0.0"
- image["md5sum"] = file_info[name]["md5"]
- image["filesize"] = file_info[name]["len"]
- images.append(image)
- gns3a["images"] = images
-
- # versions
- images = OrderedDict()
- cdrom = None
- disk_id = 0
- ovf_ref_file = ovf['References']['File']
- if not isinstance(ovf_ref_file, (list, tuple)):
- ovf_ref_file = [ovf_ref_file]
- for image in ovf_ref_file:
- img = image['href']
- if img.endswith(".iso"):
- if cdrom is None:
- cdrom = img
- else:
- images["hd" + chr(ord("a")+disk_id) + "_disk_image"] = img
- disk_id += 1
- if cdrom is not None:
- images["cdrom_image"] = cdrom
- gns3a["versions"] = [OrderedDict([("name", "0.0"), ("images", images)])]
-
- # write to file
- ofile = os.path.join(dname, os.path.splitext(fbase)[0] + ".gns3a")
- with open(ofile, "w") as f_out:
- json.dump(gns3a, f_out, indent=4, separators=(',', ': '))
- f_out.write("\n")
-
-
- SECTOR_SIZE = 512
-
- def div_ceil(dividend, divisor):
- """ integer division, round up """
- return -(-dividend // divisor)
-
-
- def copy_raw(f_in, f_out, initial_data=None):
- """ copy input file to output file, return file info """
- f_len = 0
- f_md5 = hashlib.md5()
-
- if initial_data:
- f_out.write(initial_data)
- f_len += len(initial_data)
- f_md5.update(initial_data)
-
- while True:
- block = f_in.read(64 * 1024)
- if not block:
- break
- f_out.write(block)
- f_len += len(block)
- f_md5.update(block)
-
- return {'len': f_len, 'md5': f_md5.hexdigest()}
-
-
- def read_block(file, datalen):
- """ read a block of data from file """
- if datalen < 0:
- raise ValueError("Can't seek backwards")
- if datalen == 0:
- data = b''
- else:
- data = file.read(datalen)
- if len(data) != datalen:
- raise ValueError('Premature EOF')
- return data
-
-
- def copy_vmdk(f_in, f_out, out_name):
- """
- uncompress VMDK file, return file info
-
- The VMDK disk format is documented in
- https://www.vmware.com/support/developer/vddk/vmdk_50_technote.pdf
- """
-
- header = f_in.read(SECTOR_SIZE)
- if len(header) != SECTOR_SIZE or header[0:4] != b'KDMV':
- # unknown file type: extract unchanged
- return copy_raw(f_in, f_out, header)
-
- # decode header
- (magic, _, flags, capacity, grain_size, descriptor_offset, descriptor_size,
- num_gte_per_gt, _, in_gd_off, overhead, _, single_eol_char, non_eol_char,
- double_eol_char1, double_eol_char2, compress_algorithm) = \
- struct.unpack_from('<LLLQQQQLQQQBccccH', header)
- if flags & 0x30000 != 0x30000 or compress_algorithm != 1:
- # uncompressed VMDK file: extract unchanged
- return copy_raw(f_in, f_out, header)
-
- # grain directory / table
- gt_size = div_ceil(num_gte_per_gt * 4, SECTOR_SIZE)
- gt_count = div_ceil(capacity, grain_size * num_gte_per_gt)
- gd_size = div_ceil(gt_count * 4, SECTOR_SIZE)
- rgd_offset = 8
- gd_offset = rgd_offset + gd_size + gt_count*gt_size
- out_overhead = div_ceil(gd_offset + gd_size + gt_count*gt_size, 128) * 128
-
- # read/modify descriptor
- if descriptor_offset > 0 and descriptor_size > 0:
- read_block(f_in, (descriptor_offset-1) * SECTOR_SIZE)
- descriptor = read_block(f_in, descriptor_size * SECTOR_SIZE)
- in_offset = descriptor_offset + descriptor_size
- descriptor = descriptor.rstrip(b'\0')
- descriptor = descriptor.replace(double_eol_char1 + double_eol_char2,
- single_eol_char)
- if single_eol_char != b'\n':
- descriptor = descriptor.replace(single_eol_char, b'\n')
- descriptor = re.sub(br'^([ \t]*createType[ \t]*=[ \t]*)"streamOptimized"',
- br'\1"monolithicSparse"',
- descriptor, flags=re.MULTILINE)
- descriptor, count = re.subn(br'^([ \t]*)(RW|RDONLY)[ \t].*',
- r'\1RW {:d} SPARSE "{}"'.format(capacity, out_name).encode(),
- descriptor, flags=re.MULTILINE)
- if count == 0:
- raise ValueError('No extent defined in descriptor')
- if count > 1:
- raise ValueError('Multiple extents defined in descriptor')
- if single_eol_char != b'\n':
- descriptor = descriptor.replace(b'\n', single_eol_char)
- out_desc_offset = 1
- out_desc_size = rgd_offset - 1
- if len(descriptor) > out_desc_size * SECTOR_SIZE:
- raise ValueError('Descriptor too big')
- else:
- in_offset = 1
- out_desc_offset = 0
- out_desc_size = 0
-
- # write new header
- header = struct.pack('<LLLQQQQLQQQBccccH', \
- magic, 1, 3, capacity, grain_size, out_desc_offset, out_desc_size, \
- num_gte_per_gt, rgd_offset, gd_offset, out_overhead, 0, \
- single_eol_char, non_eol_char, double_eol_char1, double_eol_char2, 0)
- header += b'\0' * (SECTOR_SIZE - len(header))
- f_out.write(header)
-
- # write descriptor
- if out_desc_offset > 0:
- descriptor += b'\0' * (out_desc_size*SECTOR_SIZE - len(descriptor))
- f_out.write(descriptor)
-
- # initialize grain directory and tables
- grain_directory = None
- grain_table = []
- grain_translation = {0:0, 1:1}
- gt_translation = {0:-1}
-
- # if available read grain directory and grain tables
- if in_gd_off != 0xffffffffffffffff:
- if in_gd_off < in_offset or \
- in_gd_off + gd_size + gt_count*gt_size > overhead:
- raise ValueError('Invalid grain directory offset')
- read_block(f_in, (in_gd_off - in_offset) * SECTOR_SIZE)
- in_offset = in_gd_off
- data = read_block(f_in, gd_size * SECTOR_SIZE)
- data = struct.unpack_from("<{:d}L".format(gt_count), data)
- grain_directory = data
- in_offset += gd_size
- for index in range(0, gt_count):
- gt_translation[in_offset] = index
- data = read_block(f_in, gt_size * SECTOR_SIZE)
- data = struct.unpack_from("<{:d}L".format(num_gte_per_gt), data)
- grain_table.append(data)
- in_offset += gt_size
-
- # skip to data
- read_block(f_in, (overhead - in_offset) * SECTOR_SIZE)
- in_offset = overhead
- out_offset = out_overhead
- f_out.seek(out_offset * SECTOR_SIZE, 0)
-
- # read, uncompress and write data
- max_datasize = grain_size * SECTOR_SIZE
- # add worst case compression overhead
- max_datasize += div_ceil(max_datasize, 8) + div_ceil(max_datasize, 64) + 11
- while True:
- header = read_block(f_in, SECTOR_SIZE)
- (val, size, marker) = struct.unpack_from('<QLL', header)
-
- if size > 0:
- # Compressed Grain
- if size > max_datasize:
- raise ValueError('Bad data block @{:08X}'.format(in_offset))
- prev_offset = in_offset
- grain_translation[in_offset] = out_offset
- data = header[12:]
- if size > (SECTOR_SIZE - 12):
- data += read_block(f_in, size - (SECTOR_SIZE - 12))
- read_block(f_in, -(size + 12) % SECTOR_SIZE)
- in_offset += div_ceil(size + 12, SECTOR_SIZE)
- else:
- data = data[0:size]
- in_offset += 1
- try:
- data = zlib.decompress(data)
- except zlib.error:
- raise ValueError('Bad data block @{:08X}'.format(prev_offset))
- if len(data) != grain_size*SECTOR_SIZE:
- raise ValueError('Data block @{:08X} has wrong size {:d}'.format(prev_offset, len(data)))
- f_out.write(data)
- out_offset += grain_size
-
- elif marker == 0:
- # End-of-Stream
- if f_in.read(SECTOR_SIZE):
- raise ValueError('No EOF after end of stream @{:08X}'.format(in_offset))
- break
-
- elif marker == 1:
- # Grain Table
- if len(grain_table) >= gt_count:
- raise ValueError('Too many grain tables @{:08X}'.format(in_offset))
- if val != gt_size:
- raise ValueError('Grain table @{:08X} has wrong size {:d}'.format(in_offset, val * SECTOR_SIZE))
- gt_translation[in_offset+1] = len(grain_table)
- data = read_block(f_in, val * SECTOR_SIZE)
- data = struct.unpack_from("<{:d}L".format(num_gte_per_gt), data)
- grain_table.append(data)
- in_offset += 1 + val
-
- elif marker == 2:
- # Grain Directory
- if grain_directory is not None:
- raise ValueError('Too many grain directories @{:08X}'.format(in_offset))
- if val != gd_size:
- raise ValueError('Grain directory @{:08X} has wrong size {:d}'.format(in_offset, val * SECTOR_SIZE))
- data = read_block(f_in, val * SECTOR_SIZE)
- data = struct.unpack_from("<{:d}L".format(gt_count), data)
- grain_directory = data
- in_offset += 1 + val
-
- else:
- # ignore footer or unknown marker
- read_block(f_in, val * SECTOR_SIZE)
- in_offset += 1 + val
-
- # set table index in grain directory
- if grain_directory is None:
- raise ValueError('No grain directory')
- try:
- grain_directory = [gt_translation[x] for x in grain_directory]
- except KeyError:
- raise ValueError('Grain directory uses invalid table references')
-
- # apply new offsets in grain tables
- for index, table in enumerate(grain_table):
- try:
- grain_table[index] = [grain_translation[x] for x in table]
- except KeyError:
- raise ValueError('Grain table uses invalid data references')
-
- # grain directory: sort grain tables, update table index to sector offset
- gt_empty = [0] * num_gte_per_gt
- unsorted_grain_table = grain_table
- grain_table = []
- gt_offset = rgd_offset + gd_size
- for index in range(0, gt_count):
- if grain_directory[index] < 0:
- grain_table.append(gt_empty)
- else:
- grain_table.append(unsorted_grain_table[grain_directory[index]])
- grain_directory[index] = gt_offset
- gt_offset += gt_size
-
- # save first/redundant grain descriptor
- f_out.seek(rgd_offset * SECTOR_SIZE, 0)
- data = struct.pack("<{:d}L".format(gt_count), *grain_directory)
- data += b'\0' * (gd_size*SECTOR_SIZE - len(data))
- f_out.write(data)
- for table in grain_table:
- data = struct.pack("<{:d}L".format(num_gte_per_gt), *table)
- data += b'\0' * (gt_size*SECTOR_SIZE - len(data))
- f_out.write(data)
-
- # save second grain descriptor
- second_directory = [x - rgd_offset + gd_offset for x in grain_directory]
- data = struct.pack("<{:d}L".format(gt_count), *second_directory)
- data += b'\0' * (gd_size*SECTOR_SIZE - len(data))
- f_out.write(data)
- for table in grain_table:
- data = struct.pack("<{:d}L".format(num_gte_per_gt), *table)
- data += b'\0' * (gt_size*SECTOR_SIZE - len(data))
- f_out.write(data)
-
- # get file information
- f_out.seek(0, 0)
- f_len = 0
- f_md5 = hashlib.md5()
- while True:
- block = f_out.read(64 * 1024)
- if not block:
- break
- f_len += len(block)
- f_md5.update(block)
-
- return {'len': f_len, 'md5': f_md5.hexdigest()}
-
-
- def main(argv):
- """ main """
-
- # parse command line
- parser = argparse.ArgumentParser(description='%(prog)s generates gns3a file to import ova appliance into GNS3.')
- parser.add_argument('-d', '--dir', default='',
- help='directory for storing appliance and images')
- parser.add_argument('-k', '--keepcompressed', action='store_true',
- help='keep images compressed')
- parser.add_argument('ova', help='.ova appliance')
- args = parser.parse_args(argv[1:])
-
- fname = args.ova
- dname = args.dir
-
- if dname != '' and not os.path.isdir(dname):
- sys.exit("Directory '{}' doesn't exist.".format(dname))
-
- # open ovf file
- try:
- tar = tarfile.open(fname)
- except (IOError, tarfile.TarError) as err:
- sys.exit("Error reading ova file: {}".format(err))
-
- # get files from ova
- file_info = {}
- ovf_data = None
-
- for tarinfo in tar:
- name = tarinfo.name.split('/')[-1]
- if not tarinfo.isfile(): # ova uses only regular files
- pass
- elif name.endswith(".ovf"): # ovf file
- f_in = tar.extractfile(tarinfo)
- ovf_data = f_in.read()
- f_in.close()
- elif name.endswith(".mf"): # ignore manifest file
- pass
- elif name.endswith(".cert"): # ignore certificate
- pass
- elif name.endswith(".pem"): # ignore certificate
- pass
- else: # save image file
- print("Extracting image {}...".format(name))
- out_name = os.path.join(dname, name)
- f_in = tar.extractfile(tarinfo)
- f_out = open(out_name, 'w+b')
- try:
- if not args.keepcompressed and name.endswith(".vmdk"):
- # uncompress VMDK file
- file_info[name] = copy_vmdk(f_in, f_out, name)
- else:
- file_info[name] = copy_raw(f_in, f_out)
- except ValueError as err:
- sys.exit("{}: {}".format(name, err))
- f_in.close()
- f_out.close()
- os.utime(out_name, (tarinfo.mtime, tarinfo.mtime))
-
- tar.close()
-
- if ovf_data is None:
- sys.exit("No ovf information in ova.")
-
- print("Creating appliance {}..."
- .format(os.path.splitext(os.path.basename(fname))[0] + ".gns3a"))
- create_gns3a(fname, dname, file_info, ovf_data)
-
-
- if __name__ == "__main__":
- main(sys.argv)
|