ehlers
/
ova_import
1
0
Fork 0
Code Issues Pull Requests Releases Activity
ova_import generates gns3a file to import ova appliance into GNS3
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
20 Commits
1 Branch
0 Tags
65 KiB
 
Branch: master
Branches Tags
${ item.name }
Create tag ${ searchTerm }
Create branch ${ searchTerm }
from 'master'
${ noResults }
ova_import/ova_import

516 lines
18 KiB
Raw Permalink Blame History

#!/usr/bin/env python3
# Copyright (C) 2016-2018 Bernhard Ehlers
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
ova_import generates gns3a file to import ova appliance into GNS3.
usage: ova_import [-h] [-d DIR] [-k] ova
positional arguments:
ova .ova appliance
optional arguments:
-h, --help show this help message and exit
-d DIR, --dir DIR directory for storing appliance and images
-k, --keepcompressed keep images compressed
"""
import os
import sys
import argparse
import hashlib
import json
import re
import struct
import tarfile
import zlib
import xml.etree.ElementTree as ElementTree
from collections import defaultdict, OrderedDict
def strip_namespace(tag):
""" strip namespace from tag """
if '}' in tag:
tag = tag.split('}', 1)[1] # strip all namespaces
return tag
def etree_to_dict(etree):
"""
convert ElementTree to dictionary
see K3---rnc's answer of "Converting xml to dictionary using ElementTree"
http://stackoverflow.com/questions/7684333/converting-xml-to-dictionary-using-elementtree#answer-10076823
variable names changed to make it easier understandable
"""
tag = strip_namespace(etree.tag)
node = {tag: {} if etree.attrib else None}
children = list(etree)
if children:
all_child_nodes = defaultdict(list)
for child_node in map(etree_to_dict, children):
for key, val in child_node.items():
all_child_nodes[key].append(val)
node = {tag: {key:val[0] if len(val) == 1 else val
for key, val in all_child_nodes.items()}}
if etree.attrib:
node[tag].update((strip_namespace(key), val)
for key, val in etree.attrib.items())
if etree.text:
text = etree.text.strip()
if children or etree.attrib:
if text:
node[tag]['#text'] = text
else:
node[tag] = text
return node
def create_gns3a(fname, dname, file_info, ovf_data):
""" create GNS3 appliance file """
fbase = os.path.basename(fname)
gns3a = OrderedDict()
ovf = etree_to_dict(ElementTree.fromstring(ovf_data))['Envelope']
# base informations
try:
vm_name = ovf['VirtualSystem']['Name']
except KeyError:
vm_name = ovf['VirtualSystem']['id']
gns3a["name"] = vm_name
gns3a["category"] = "guest"
gns3a["description"] = "ova import of " + fbase
gns3a["vendor_name"] = "unknown"
gns3a["vendor_url"] = "http://www.example.com"
gns3a["product_name"] = vm_name
gns3a["registry_version"] = 3
gns3a["status"] = "experimental"
gns3a["maintainer"] = "GNS3 Team"
gns3a["maintainer_email"] = "developers@gns3.net"
# qemu
cores = 0
eth_adapters = 0
ram = 0
for item in ovf['VirtualSystem']['VirtualHardwareSection']['Item']:
if item['ResourceType'] == '3':
cores = int(item['VirtualQuantity'])
elif item['ResourceType'] == '4':
ram = int(item['VirtualQuantity'])
elif item['ResourceType'] == '10':
eth_adapters += 1
cores = max(cores, 1)
eth_adapters = max(eth_adapters, 1)
if ram == 0:
ram = 256
try:
vm_os = ovf['VirtualSystem']['OperatingSystemSection']['osType']
except KeyError:
vm_os = ovf['VirtualSystem']['OperatingSystemSection']['OSType']['#text']
qemu = OrderedDict()
qemu["adapter_type"] = "e1000"
qemu["adapters"] = eth_adapters
qemu["ram"] = ram
qemu["arch"] = "i386"
if "64" in vm_os:
qemu["arch"] = "x86_64"
qemu["console_type"] = "telnet"
qemu["kvm"] = "allow"
if cores > 1:
qemu["options"] = "-smp " + str(cores)
gns3a["qemu"] = qemu
# images
images = []
for name in sorted(file_info.keys()):
image = OrderedDict()
image["filename"] = name
image["version"] = "0.0"
image["md5sum"] = file_info[name]["md5"]
image["filesize"] = file_info[name]["len"]
images.append(image)
gns3a["images"] = images
# versions
images = OrderedDict()
cdrom = None
disk_id = 0
ovf_ref_file = ovf['References']['File']
if not isinstance(ovf_ref_file, (list, tuple)):
ovf_ref_file = [ovf_ref_file]
for image in ovf_ref_file:
img = image['href']
if img.endswith(".iso"):
if cdrom is None:
cdrom = img
else:
images["hd" + chr(ord("a")+disk_id) + "_disk_image"] = img
disk_id += 1
if cdrom is not None:
images["cdrom_image"] = cdrom
gns3a["versions"] = [OrderedDict([("name", "0.0"), ("images", images)])]
# write to file
ofile = os.path.join(dname, os.path.splitext(fbase)[0] + ".gns3a")
with open(ofile, "w") as f_out:
json.dump(gns3a, f_out, indent=4, separators=(',', ': '))
f_out.write("\n")
SECTOR_SIZE = 512
def div_ceil(dividend, divisor):
""" integer division, round up """
return -(-dividend // divisor)
def copy_raw(f_in, f_out, initial_data=None):
""" copy input file to output file, return file info """
f_len = 0
f_md5 = hashlib.md5()
if initial_data:
f_out.write(initial_data)
f_len += len(initial_data)
f_md5.update(initial_data)
while True:
block = f_in.read(64 * 1024)
if not block:
break
f_out.write(block)
f_len += len(block)
f_md5.update(block)
return {'len': f_len, 'md5': f_md5.hexdigest()}
def read_block(file, datalen):
""" read a block of data from file """
if datalen < 0:
raise ValueError("Can't seek backwards")
if datalen == 0:
data = b''
else:
data = file.read(datalen)
if len(data) != datalen:
raise ValueError('Premature EOF')
return data
def copy_vmdk(f_in, f_out, out_name):
"""
uncompress VMDK file, return file info
The VMDK disk format is documented in
https://www.vmware.com/support/developer/vddk/vmdk_50_technote.pdf
"""
header = f_in.read(SECTOR_SIZE)
if len(header) != SECTOR_SIZE or header[0:4] != b'KDMV':
# unknown file type: extract unchanged
return copy_raw(f_in, f_out, header)
# decode header
(magic, _, flags, capacity, grain_size, descriptor_offset, descriptor_size,
num_gte_per_gt, _, in_gd_off, overhead, _, single_eol_char, non_eol_char,
double_eol_char1, double_eol_char2, compress_algorithm) = \
struct.unpack_from('<LLLQQQQLQQQBccccH', header)
if flags & 0x30000 != 0x30000 or compress_algorithm != 1:
# uncompressed VMDK file: extract unchanged
return copy_raw(f_in, f_out, header)
# grain directory / table
gt_size = div_ceil(num_gte_per_gt * 4, SECTOR_SIZE)
gt_count = div_ceil(capacity, grain_size * num_gte_per_gt)
gd_size = div_ceil(gt_count * 4, SECTOR_SIZE)
rgd_offset = 8
gd_offset = rgd_offset + gd_size + gt_count*gt_size
out_overhead = div_ceil(gd_offset + gd_size + gt_count*gt_size, 128) * 128
# read/modify descriptor
if descriptor_offset > 0 and descriptor_size > 0:
read_block(f_in, (descriptor_offset-1) * SECTOR_SIZE)
descriptor = read_block(f_in, descriptor_size * SECTOR_SIZE)
in_offset = descriptor_offset + descriptor_size
descriptor = descriptor.rstrip(b'\0')
descriptor = descriptor.replace(double_eol_char1 + double_eol_char2,
single_eol_char)
if single_eol_char != b'\n':
descriptor = descriptor.replace(single_eol_char, b'\n')
descriptor = re.sub(br'^([ \t]*createType[ \t]*=[ \t]*)"streamOptimized"',
br'\1"monolithicSparse"',
descriptor, flags=re.MULTILINE)
descriptor, count = re.subn(br'^([ \t]*)(RW|RDONLY)[ \t].*',
r'\1RW {:d} SPARSE "{}"'.format(capacity, out_name).encode(),
descriptor, flags=re.MULTILINE)
if count == 0:
raise ValueError('No extent defined in descriptor')
if count > 1:
raise ValueError('Multiple extents defined in descriptor')
if single_eol_char != b'\n':
descriptor = descriptor.replace(b'\n', single_eol_char)
out_desc_offset = 1
out_desc_size = rgd_offset - 1
if len(descriptor) > out_desc_size * SECTOR_SIZE:
raise ValueError('Descriptor too big')
else:
in_offset = 1
out_desc_offset = 0
out_desc_size = 0
# write new header
header = struct.pack('<LLLQQQQLQQQBccccH', \
magic, 1, 3, capacity, grain_size, out_desc_offset, out_desc_size, \
num_gte_per_gt, rgd_offset, gd_offset, out_overhead, 0, \
single_eol_char, non_eol_char, double_eol_char1, double_eol_char2, 0)
header += b'\0' * (SECTOR_SIZE - len(header))
f_out.write(header)
# write descriptor
if out_desc_offset > 0:
descriptor += b'\0' * (out_desc_size*SECTOR_SIZE - len(descriptor))
f_out.write(descriptor)
# initialize grain directory and tables
grain_directory = None
grain_table = []
grain_translation = {0:0, 1:1}
gt_translation = {0:-1}
# if available read grain directory and grain tables
if in_gd_off != 0xffffffffffffffff:
if in_gd_off < in_offset or \
in_gd_off + gd_size + gt_count*gt_size > overhead:
raise ValueError('Invalid grain directory offset')
read_block(f_in, (in_gd_off - in_offset) * SECTOR_SIZE)
in_offset = in_gd_off
data = read_block(f_in, gd_size * SECTOR_SIZE)
data = struct.unpack_from("<{:d}L".format(gt_count), data)
grain_directory = data
in_offset += gd_size
for index in range(0, gt_count):
gt_translation[in_offset] = index
data = read_block(f_in, gt_size * SECTOR_SIZE)
data = struct.unpack_from("<{:d}L".format(num_gte_per_gt), data)
grain_table.append(data)
in_offset += gt_size
# skip to data
read_block(f_in, (overhead - in_offset) * SECTOR_SIZE)
in_offset = overhead
out_offset = out_overhead
f_out.seek(out_offset * SECTOR_SIZE, 0)
# read, uncompress and write data
max_datasize = grain_size * SECTOR_SIZE
# add worst case compression overhead
max_datasize += div_ceil(max_datasize, 8) + div_ceil(max_datasize, 64) + 11
while True:
header = read_block(f_in, SECTOR_SIZE)
(val, size, marker) = struct.unpack_from('<QLL', header)
if size > 0:
# Compressed Grain
if size > max_datasize:
raise ValueError('Bad data block @{:08X}'.format(in_offset))
prev_offset = in_offset
grain_translation[in_offset] = out_offset
data = header[12:]
if size > (SECTOR_SIZE - 12):
data += read_block(f_in, size - (SECTOR_SIZE - 12))
read_block(f_in, -(size + 12) % SECTOR_SIZE)
in_offset += div_ceil(size + 12, SECTOR_SIZE)
else:
data = data[0:size]
in_offset += 1
try:
data = zlib.decompress(data)
except zlib.error:
raise ValueError('Bad data block @{:08X}'.format(prev_offset))
if len(data) != grain_size*SECTOR_SIZE:
raise ValueError('Data block @{:08X} has wrong size {:d}'.format(prev_offset, len(data)))
f_out.write(data)
out_offset += grain_size
elif marker == 0:
# End-of-Stream
if f_in.read(SECTOR_SIZE):
raise ValueError('No EOF after end of stream @{:08X}'.format(in_offset))
break
elif marker == 1:
# Grain Table
if len(grain_table) >= gt_count:
raise ValueError('Too many grain tables @{:08X}'.format(in_offset))
if val != gt_size:
raise ValueError('Grain table @{:08X} has wrong size {:d}'.format(in_offset, val * SECTOR_SIZE))
gt_translation[in_offset+1] = len(grain_table)
data = read_block(f_in, val * SECTOR_SIZE)
data = struct.unpack_from("<{:d}L".format(num_gte_per_gt), data)
grain_table.append(data)
in_offset += 1 + val
elif marker == 2:
# Grain Directory
if grain_directory is not None:
raise ValueError('Too many grain directories @{:08X}'.format(in_offset))
if val != gd_size:
raise ValueError('Grain directory @{:08X} has wrong size {:d}'.format(in_offset, val * SECTOR_SIZE))
data = read_block(f_in, val * SECTOR_SIZE)
data = struct.unpack_from("<{:d}L".format(gt_count), data)
grain_directory = data
in_offset += 1 + val
else:
# ignore footer or unknown marker
read_block(f_in, val * SECTOR_SIZE)
in_offset += 1 + val
# set table index in grain directory
if grain_directory is None:
raise ValueError('No grain directory')
try:
grain_directory = [gt_translation[x] for x in grain_directory]
except KeyError:
raise ValueError('Grain directory uses invalid table references')
# apply new offsets in grain tables
for index, table in enumerate(grain_table):
try:
grain_table[index] = [grain_translation[x] for x in table]
except KeyError:
raise ValueError('Grain table uses invalid data references')
# grain directory: sort grain tables, update table index to sector offset
gt_empty = [0] * num_gte_per_gt
unsorted_grain_table = grain_table
grain_table = []
gt_offset = rgd_offset + gd_size
for index in range(0, gt_count):
if grain_directory[index] < 0:
grain_table.append(gt_empty)
else:
grain_table.append(unsorted_grain_table[grain_directory[index]])
grain_directory[index] = gt_offset
gt_offset += gt_size
# save first/redundant grain descriptor
f_out.seek(rgd_offset * SECTOR_SIZE, 0)
data = struct.pack("<{:d}L".format(gt_count), *grain_directory)
data += b'\0' * (gd_size*SECTOR_SIZE - len(data))
f_out.write(data)
for table in grain_table:
data = struct.pack("<{:d}L".format(num_gte_per_gt), *table)
data += b'\0' * (gt_size*SECTOR_SIZE - len(data))
f_out.write(data)
# save second grain descriptor
second_directory = [x - rgd_offset + gd_offset for x in grain_directory]
data = struct.pack("<{:d}L".format(gt_count), *second_directory)
data += b'\0' * (gd_size*SECTOR_SIZE - len(data))
f_out.write(data)
for table in grain_table:
data = struct.pack("<{:d}L".format(num_gte_per_gt), *table)
data += b'\0' * (gt_size*SECTOR_SIZE - len(data))
f_out.write(data)
# get file information
f_out.seek(0, 0)
f_len = 0
f_md5 = hashlib.md5()
while True:
block = f_out.read(64 * 1024)
if not block:
break
f_len += len(block)
f_md5.update(block)
return {'len': f_len, 'md5': f_md5.hexdigest()}
def main(argv):
""" main """
# parse command line
parser = argparse.ArgumentParser(description='%(prog)s generates gns3a file to import ova appliance into GNS3.')
parser.add_argument('-d', '--dir', default='',
help='directory for storing appliance and images')
parser.add_argument('-k', '--keepcompressed', action='store_true',
help='keep images compressed')
parser.add_argument('ova', help='.ova appliance')
args = parser.parse_args(argv[1:])
fname = args.ova
dname = args.dir
if dname != '' and not os.path.isdir(dname):
sys.exit("Directory '{}' doesn't exist.".format(dname))
# open ovf file
try:
tar = tarfile.open(fname)
except (IOError, tarfile.TarError) as err:
sys.exit("Error reading ova file: {}".format(err))
# get files from ova
file_info = {}
ovf_data = None
for tarinfo in tar:
name = tarinfo.name.split('/')[-1]
if not tarinfo.isfile(): # ova uses only regular files
pass
elif name.endswith(".ovf"): # ovf file
f_in = tar.extractfile(tarinfo)
ovf_data = f_in.read()
f_in.close()
elif name.endswith(".mf"): # ignore manifest file
pass
elif name.endswith(".cert"): # ignore certificate
pass
elif name.endswith(".pem"): # ignore certificate
pass
else: # save image file
print("Extracting image {}...".format(name))
out_name = os.path.join(dname, name)
f_in = tar.extractfile(tarinfo)
f_out = open(out_name, 'w+b')
try:
if not args.keepcompressed and name.endswith(".vmdk"):
# uncompress VMDK file
file_info[name] = copy_vmdk(f_in, f_out, name)
else:
file_info[name] = copy_raw(f_in, f_out)
except ValueError as err:
sys.exit("{}: {}".format(name, err))
f_in.close()
f_out.close()
os.utime(out_name, (tarinfo.mtime, tarinfo.mtime))
tar.close()
if ovf_data is None:
sys.exit("No ovf information in ova.")
print("Creating appliance {}..."
.format(os.path.splitext(os.path.basename(fname))[0] + ".gns3a"))
create_gns3a(fname, dname, file_info, ovf_data)
if __name__ == "__main__":
main(sys.argv)